fallow_api/mutation/
verdicts.rs

1//! Joins a mutation report with a separate verdict document.
2//!
3//! A verdict marks one mutant as equivalent to the original code. This
4//! replaces an inline suppression comment: the report stays the evidence, and the verdict is a separate,
5//! reviewable judgment. The join key is the `review_key` of the verdict, which
6//! is the `mutant_id` of the report. The id hashes the enclosing function
7//! text, so a verdict stays valid until that function changes.
8
9use std::collections::BTreeSet;
10
11use fallow_output::{
12    MutantStatus, MutationReport, MutationStatusCounts, MutationSummary, MutationVerdictInput,
13    MutationVerdictJoin,
14};
15use rustc_hash::FxHashSet;
16use std::fmt::Write as _;
17
18use sha2::{Digest, Sha256};
19
20use crate::ProgrammaticError;
21
22/// The largest verdict document that the join reads.
23pub const MAX_VERDICT_INPUT_BYTES: usize = 16 * 1024 * 1024;
24
25fn verdict_error(message: impl Into<String>) -> ProgrammaticError {
26    ProgrammaticError::new(message, 2)
27        .with_code("FALLOW_MUTATION_VERDICTS_INVALID")
28        .with_context("mutation.verdicts")
29}
30
31fn sha256_hex(bytes: &[u8]) -> String {
32    Sha256::digest(bytes)
33        .iter()
34        .fold(String::with_capacity(64), |mut hex, byte| {
35            let _ = write!(hex, "{byte:02x}");
36            hex
37        })
38}
39
40/// The effect of one verdict on a mutant with `status`.
41enum Effect {
42    Applied,
43    Contradicted,
44    Deferred,
45}
46
47const fn effect(status: MutantStatus) -> Effect {
48    match status {
49        MutantStatus::Survived
50        | MutantStatus::NoCoverage
51        | MutantStatus::Flaky
52        | MutantStatus::Equivalent => Effect::Applied,
53        MutantStatus::Killed | MutantStatus::Timeout | MutantStatus::CompileError => {
54            Effect::Contradicted
55        }
56        MutantStatus::Pending | MutantStatus::RuntimeError | MutantStatus::Pruned => {
57            Effect::Deferred
58        }
59    }
60}
61
62/// Moves one mutant from `from` to `equivalent` in the function group that
63/// holds it.
64fn move_in_function(report: &mut MutationReport, index: usize, from: MutantStatus) {
65    let location = &report.mutants[index].location;
66    let key = (
67        location.file.as_str(),
68        location.function.as_deref(),
69        location.function_line,
70    );
71    let group = report.functions.iter_mut().find(|group| {
72        (group.file.as_str(), group.name.as_deref(), group.line) == key
73            && count_of(&group.counts, from) > 0
74    });
75    let Some(group) = group else {
76        return;
77    };
78    group.counts.remove(from);
79    group.counts.add(MutantStatus::Equivalent);
80    group.survived.retain(|id| *id != location.mutant_id);
81    group.score = group.counts.score();
82}
83
84const fn count_of(counts: &MutationStatusCounts, status: MutantStatus) -> usize {
85    match status {
86        MutantStatus::Killed => counts.killed,
87        MutantStatus::Timeout => counts.timeout,
88        MutantStatus::Survived => counts.survived,
89        MutantStatus::NoCoverage => counts.no_coverage,
90        MutantStatus::RuntimeError => counts.runtime_error,
91        MutantStatus::Flaky => counts.flaky,
92        MutantStatus::Pruned => counts.pruned,
93        MutantStatus::Pending => counts.pending,
94        MutantStatus::Equivalent => counts.equivalent,
95        MutantStatus::CompileError => counts.compile_error,
96    }
97}
98
99/// Parses a verdict document and checks its keys.
100fn parse_verdicts(verdict_json: &[u8]) -> Result<MutationVerdictInput, ProgrammaticError> {
101    if verdict_json.len() > MAX_VERDICT_INPUT_BYTES {
102        return Err(verdict_error(
103            "the mutation verdict document is larger than 16 MiB",
104        ));
105    }
106    let input: MutationVerdictInput = serde_json::from_slice(verdict_json)
107        .map_err(|error| verdict_error(format!("invalid mutation verdict document: {error}")))?;
108    let mut keys = FxHashSet::default();
109    for verdict in &input.verdicts {
110        if verdict.review_key.trim().is_empty() {
111            return Err(verdict_error("a mutation verdict has an empty review_key"));
112        }
113        if !keys.insert(verdict.review_key.as_str()) {
114            return Err(verdict_error(format!(
115                "the review_key `{}` occurs more than one time",
116                verdict.review_key
117            )));
118        }
119    }
120    Ok(input)
121}
122
123/// Checks a verdict document without a report.
124///
125/// A caller runs this check before a mutation run, so an invalid document
126/// stops the command before any test runs.
127///
128/// # Errors
129///
130/// Returns the same errors as [`apply_mutation_verdicts`].
131pub fn validate_mutation_verdicts(verdict_json: &[u8]) -> Result<(), ProgrammaticError> {
132    parse_verdicts(verdict_json).map(|_| ())
133}
134
135/// Applies a verdict document to `report`.
136///
137/// Each verdict for a `survived`, `no_coverage` or `flaky` mutant sets the
138/// status `equivalent`, which does not count in the score. A verdict for a
139/// killed or timed-out mutant is contradicted by a test, so the status stays.
140/// A verdict for a mutant with no tested result (`pending`, `runtime_error`,
141/// `pruned`) is deferred. A verdict that matches no mutant is unmatched; it
142/// is not an error, because one verdict document serves many scoped runs.
143/// The summary and the function results are recomputed.
144///
145/// # Errors
146///
147/// Returns a fail-closed error for a document over 16 MiB, a malformed
148/// document, an empty `review_key`, or a `review_key` that occurs twice.
149pub fn apply_mutation_verdicts(
150    report: &mut MutationReport,
151    verdict_json: &[u8],
152) -> Result<(), ProgrammaticError> {
153    let input = parse_verdicts(verdict_json)?;
154    let keys: FxHashSet<&str> = input
155        .verdicts
156        .iter()
157        .map(|verdict| verdict.review_key.as_str())
158        .collect();
159
160    let mut applied = BTreeSet::new();
161    let mut contradicted = BTreeSet::new();
162    let mut deferred = BTreeSet::new();
163    let mut matched = FxHashSet::default();
164    for index in 0..report.mutants.len() {
165        let id = report.mutants[index].location.mutant_id.clone();
166        if !keys.contains(id.as_str()) {
167            continue;
168        }
169        matched.insert(id.clone());
170        let status = report.mutants[index].status;
171        match effect(status) {
172            Effect::Applied => {
173                if status != MutantStatus::Equivalent {
174                    report.mutants[index].status = MutantStatus::Equivalent;
175                    move_in_function(report, index, status);
176                }
177                applied.insert(id);
178            }
179            Effect::Contradicted => {
180                contradicted.insert(id);
181            }
182            Effect::Deferred => {
183                deferred.insert(id);
184            }
185        }
186    }
187    let unmatched: BTreeSet<String> = input
188        .verdicts
189        .iter()
190        .filter(|verdict| !matched.contains(&verdict.review_key))
191        .map(|verdict| verdict.review_key.clone())
192        .collect();
193
194    report.summary = MutationSummary::from_results(&report.mutants);
195    report.verdicts = Some(MutationVerdictJoin {
196        verdicts_sha256: sha256_hex(verdict_json),
197        applied: applied.into_iter().collect(),
198        contradicted: contradicted.into_iter().collect(),
199        deferred: deferred.into_iter().collect(),
200        unmatched: unmatched.into_iter().collect(),
201    });
202    Ok(())
203}
204
205#[cfg(test)]
206#[expect(clippy::unwrap_used, reason = "tests unwrap to keep assertions short")]
207mod tests {
208    use fallow_output::{
209        MutantLocation, MutantResult, MutationCompletion, MutationFunctionResult,
210        MutationSchemaVersion, MutationScopeKind, MutationScopeOutput, MutationSummary,
211        MutationTimings, MutationWorktreeCheck,
212    };
213    use fallow_types::envelope::{ElapsedMs, ToolVersion};
214
215    use super::*;
216
217    fn mutant(id: &str, status: MutantStatus) -> MutantResult {
218        MutantResult {
219            location: MutantLocation {
220                mutant_id: id.to_owned(),
221                file: "src/a.ts".to_owned(),
222                line: 2,
223                column: 3,
224                span: (10, 15),
225                operator: "arithmetic-operator".to_owned(),
226                function: Some("add".to_owned()),
227                function_line: Some(1),
228                rank: Some(1),
229                change: None,
230            },
231            status,
232            prune_reason: None,
233            killed_by: None,
234            covering_tests: match status {
235                MutantStatus::NoCoverage | MutantStatus::Pending | MutantStatus::Pruned => {
236                    Vec::new()
237                }
238                _ => vec!["test/a.test.ts::t1".to_owned()],
239            },
240            static_mutant: false,
241            duration_ms: None,
242            test_reach: None,
243            retried: false,
244            attempts: Vec::new(),
245            cached: false,
246            type_error_codes: Vec::new(),
247        }
248    }
249
250    fn report(mutants: Vec<MutantResult>) -> MutationReport {
251        let mut counts = MutationStatusCounts::default();
252        let mut survived = Vec::new();
253        for result in &mutants {
254            counts.add(result.status);
255            if result.status == MutantStatus::Survived {
256                survived.push(result.location.mutant_id.clone());
257            }
258        }
259        MutationReport {
260            schema_version: MutationSchemaVersion::V1,
261            version: ToolVersion("0.0.0".to_owned()),
262            elapsed_ms: ElapsedMs(0),
263            completion: MutationCompletion::from_reasons(Vec::new()),
264            scope: MutationScopeOutput {
265                kind: MutationScopeKind::All,
266                base_ref: None,
267            },
268            runner: None,
269            cache: None,
270            summary: MutationSummary::from_results(&mutants),
271            functions: vec![MutationFunctionResult {
272                file: "src/a.ts".to_owned(),
273                name: Some("add".to_owned()),
274                line: Some(1),
275                end_line: Some(4),
276                in_diff: true,
277                cyclomatic: Some(1),
278                counts,
279                score: counts.score(),
280                survived,
281            }],
282            mutants,
283            no_test_files: false,
284            baseline_failures: Vec::new(),
285            skipped_files: Vec::new(),
286            unmatched_paths: Vec::new(),
287            unloaded_paths: Vec::new(),
288            worktree: MutationWorktreeCheck {
289                checked: true,
290                allow_dirty: false,
291                dirty_at_start: Vec::new(),
292                changed_paths: Vec::new(),
293            },
294            errors: Vec::new(),
295            timings: MutationTimings::default(),
296            files: Vec::new(),
297            verdicts: None,
298            typecheck: None,
299        }
300    }
301
302    fn document(keys: &[&str]) -> Vec<u8> {
303        let verdicts: Vec<serde_json::Value> = keys
304            .iter()
305            .map(|key| serde_json::json!({ "review_key": key, "verdict": "equivalent" }))
306            .collect();
307        serde_json::to_vec(&serde_json::json!({ "schema_version": "1", "verdicts": verdicts }))
308            .unwrap()
309    }
310
311    #[test]
312    fn equivalent_verdicts_leave_the_score_and_the_survivor_list() {
313        let mut report = report(vec![
314            mutant("m1", MutantStatus::Killed),
315            mutant("m2", MutantStatus::Survived),
316            mutant("m3", MutantStatus::Survived),
317            mutant("m4", MutantStatus::NoCoverage),
318        ]);
319        assert_eq!(report.summary.score, Some(1.0 / 3.0));
320        assert_eq!(report.summary.coverage, Some(0.75));
321
322        apply_mutation_verdicts(&mut report, &document(&["m2", "m4"])).unwrap();
323
324        let statuses: Vec<MutantStatus> = report.mutants.iter().map(|m| m.status).collect();
325        assert_eq!(
326            statuses,
327            vec![
328                MutantStatus::Killed,
329                MutantStatus::Equivalent,
330                MutantStatus::Survived,
331                MutantStatus::Equivalent
332            ]
333        );
334        assert_eq!(report.summary.counts.equivalent, 2);
335        assert_eq!(report.summary.counts.survived, 1);
336        assert_eq!(report.summary.counts.no_coverage, 0);
337        assert_eq!(report.summary.counts.total, 4);
338        assert_eq!(report.summary.score, Some(0.5));
339        assert_eq!(report.summary.coverage, Some(1.0));
340        let function = &report.functions[0];
341        assert_eq!(function.counts.equivalent, 2);
342        assert_eq!(function.counts.no_coverage, 0);
343        assert_eq!(function.survived, vec!["m3".to_owned()]);
344        assert_eq!(function.score, Some(0.5));
345        let join = report.verdicts.unwrap();
346        assert_eq!(join.applied, vec!["m2".to_owned(), "m4".to_owned()]);
347        assert!(join.unmatched.is_empty());
348        assert_eq!(join.verdicts_sha256.len(), 64);
349    }
350
351    #[test]
352    fn a_kill_contradicts_a_verdict_and_unknown_keys_stay_unmatched() {
353        let mut report = report(vec![
354            mutant("m1", MutantStatus::Killed),
355            mutant("m2", MutantStatus::Pending),
356            mutant("m3", MutantStatus::Survived),
357        ]);
358
359        apply_mutation_verdicts(&mut report, &document(&["gone", "m1", "m2"])).unwrap();
360
361        assert_eq!(report.mutants[0].status, MutantStatus::Killed);
362        assert_eq!(report.mutants[1].status, MutantStatus::Pending);
363        assert_eq!(report.mutants[2].status, MutantStatus::Survived);
364        assert_eq!(report.summary.score, Some(0.5));
365        let join = report.verdicts.unwrap();
366        assert!(join.applied.is_empty());
367        assert_eq!(join.contradicted, vec!["m1".to_owned()]);
368        assert_eq!(join.deferred, vec!["m2".to_owned()]);
369        assert_eq!(join.unmatched, vec!["gone".to_owned()]);
370    }
371
372    #[test]
373    fn malformed_and_duplicate_verdicts_fail_closed() {
374        let cases: [(&[u8], &str); 4] = [
375            (b"{", "invalid mutation verdict document"),
376            (
377                br#"{"schema_version":"1","verdicts":[{"review_key":"m1","verdict":"survived"}]}"#,
378                "invalid mutation verdict document",
379            ),
380            (
381                br#"{"schema_version":"1","verdicts":[{"review_key":" ","verdict":"equivalent"}]}"#,
382                "empty review_key",
383            ),
384            (&document(&["m1", "m1"]), "more than one time"),
385        ];
386        for (json, needle) in cases {
387            let checked = validate_mutation_verdicts(json).unwrap_err();
388            assert_eq!(checked.exit_code, 2);
389            assert!(checked.message.contains(needle), "{}", checked.message);
390            let mut report = report(vec![mutant("m1", MutantStatus::Survived)]);
391            let error = apply_mutation_verdicts(&mut report, json).unwrap_err();
392            assert_eq!(
393                error.code.as_deref(),
394                Some("FALLOW_MUTATION_VERDICTS_INVALID")
395            );
396            assert_eq!(error.exit_code, 2);
397            assert!(error.message.contains(needle), "{}", error.message);
398            assert_eq!(report.mutants[0].status, MutantStatus::Survived);
399            assert!(report.verdicts.is_none());
400        }
401        assert!(validate_mutation_verdicts(&document(&["m1", "m2"])).is_ok());
402    }
403}
fallow_api/mutation/verdicts.rs

fallow_api/mutation/
verdicts.rs
