fallow_api/mutation/
verdicts.rs

1//! Joins a mutation report with a separate verdict document.
2//!
3//! A verdict marks one mutant as equivalent to the original code. This
4//! replaces an inline suppression comment: the report stays the evidence, and the verdict is a separate,
5//! reviewable judgment. The join key is the `review_key` of the verdict, which
6//! is the `mutant_id` of the report. The id hashes the enclosing function
7//! text, so a verdict stays valid until that function changes.
8
9use std::collections::BTreeSet;
10
11use fallow_output::{
12    MutantStatus, MutationReport, MutationStatusCounts, MutationSummary, MutationVerdictInput,
13    MutationVerdictJoin,
14};
15use rustc_hash::FxHashSet;
16use std::fmt::Write as _;
17
18use sha2::{Digest, Sha256};
19
20use crate::ProgrammaticError;
21
22/// The largest verdict document that the join reads.
23pub const MAX_VERDICT_INPUT_BYTES: usize = 16 * 1024 * 1024;
24
25fn verdict_error(message: impl Into<String>) -> ProgrammaticError {
26    ProgrammaticError::new(message, 2)
27        .with_code("FALLOW_MUTATION_VERDICTS_INVALID")
28        .with_context("mutation.verdicts")
29}
30
31fn sha256_hex(bytes: &[u8]) -> String {
32    Sha256::digest(bytes)
33        .iter()
34        .fold(String::with_capacity(64), |mut hex, byte| {
35            let _ = write!(hex, "{byte:02x}");
36            hex
37        })
38}
39
40/// The effect of one verdict on a mutant with `status`.
41enum Effect {
42    Applied,
43    Contradicted,
44    Deferred,
45}
46
47const fn effect(status: MutantStatus) -> Effect {
48    match status {
49        MutantStatus::Survived
50        | MutantStatus::NoCoverage
51        | MutantStatus::Flaky
52        | MutantStatus::Equivalent => Effect::Applied,
53        MutantStatus::Killed | MutantStatus::Timeout | MutantStatus::CompileError => {
54            Effect::Contradicted
55        }
56        MutantStatus::Pending | MutantStatus::RuntimeError | MutantStatus::Pruned => {
57            Effect::Deferred
58        }
59    }
60}
61
62/// Moves one mutant from `from` to `equivalent` in the function group that
63/// holds it.
64fn move_in_function(report: &mut MutationReport, index: usize, from: MutantStatus) {
65    let location = &report.mutants[index].location;
66    let key = (
67        location.file.as_str(),
68        location.function.as_deref(),
69        location.function_line,
70    );
71    let group = report.functions.iter_mut().find(|group| {
72        (group.file.as_str(), group.name.as_deref(), group.line) == key
73            && count_of(&group.counts, from) > 0
74    });
75    let Some(group) = group else {
76        return;
77    };
78    group.counts.remove(from);
79    group.counts.add(MutantStatus::Equivalent);
80    group.survived.retain(|id| *id != location.mutant_id);
81    group.score = group.counts.score();
82}
83
84const fn count_of(counts: &MutationStatusCounts, status: MutantStatus) -> usize {
85    match status {
86        MutantStatus::Killed => counts.killed,
87        MutantStatus::Timeout => counts.timeout,
88        MutantStatus::Survived => counts.survived,
89        MutantStatus::NoCoverage => counts.no_coverage,
90        MutantStatus::RuntimeError => counts.runtime_error,
91        MutantStatus::Flaky => counts.flaky,
92        MutantStatus::Pruned => counts.pruned,
93        MutantStatus::Pending => counts.pending,
94        MutantStatus::Equivalent => counts.equivalent,
95        MutantStatus::CompileError => counts.compile_error,
96    }
97}
98
99/// Parses a verdict document and checks its keys.
100fn parse_verdicts(verdict_json: &[u8]) -> Result<MutationVerdictInput, ProgrammaticError> {
101    if verdict_json.len() > MAX_VERDICT_INPUT_BYTES {
102        return Err(verdict_error(
103            "the mutation verdict document is larger than 16 MiB",
104        ));
105    }
106    let input: MutationVerdictInput = serde_json::from_slice(verdict_json)
107        .map_err(|error| verdict_error(format!("invalid mutation verdict document: {error}")))?;
108    let mut keys = FxHashSet::default();
109    for verdict in &input.verdicts {
110        if verdict.review_key.trim().is_empty() {
111            return Err(verdict_error("a mutation verdict has an empty review_key"));
112        }
113        if !keys.insert(verdict.review_key.as_str()) {
114            return Err(verdict_error(format!(
115                "the review_key `{}` occurs more than one time",
116                verdict.review_key
117            )));
118        }
119    }
120    Ok(input)
121}
122
123/// Checks a verdict document without a report.
124///
125/// A caller runs this check before a mutation run, so an invalid document
126/// stops the command before any test runs.
127///
128/// # Errors
129///
130/// Returns the same errors as [`apply_mutation_verdicts`].
131pub fn validate_mutation_verdicts(verdict_json: &[u8]) -> Result<(), ProgrammaticError> {
132    parse_verdicts(verdict_json).map(|_| ())
133}
134
135/// Applies a verdict document to `report`.
136///
137/// Each verdict for a `survived`, `no_coverage` or `flaky` mutant sets the
138/// status `equivalent`, which does not count in the score. A verdict for a
139/// killed or timed-out mutant is contradicted by a test, so the status stays.
140/// A verdict for a mutant with no tested result (`pending`, `runtime_error`,
141/// `pruned`) is deferred. A verdict that matches no mutant is unmatched; it
142/// is not an error, because one verdict document serves many scoped runs.
143/// The summary and the function results are recomputed.
144///
145/// # Errors
146///
147/// Returns a fail-closed error for a document over 16 MiB, a malformed
148/// document, an empty `review_key`, or a `review_key` that occurs twice.
149pub fn apply_mutation_verdicts(
150    report: &mut MutationReport,
151    verdict_json: &[u8],
152) -> Result<(), ProgrammaticError> {
153    let input = parse_verdicts(verdict_json)?;
154    let keys: FxHashSet<&str> = input
155        .verdicts
156        .iter()
157        .map(|verdict| verdict.review_key.as_str())
158        .collect();
159
160    let mut applied = BTreeSet::new();
161    let mut contradicted = BTreeSet::new();
162    let mut deferred = BTreeSet::new();
163    let mut matched = FxHashSet::default();
164    for index in 0..report.mutants.len() {
165        let id = report.mutants[index].location.mutant_id.clone();
166        if !keys.contains(id.as_str()) {
167            continue;
168        }
169        matched.insert(id.clone());
170        let status = report.mutants[index].status;
171        match effect(status) {
172            Effect::Applied => {
173                if status != MutantStatus::Equivalent {
174                    report.mutants[index].status = MutantStatus::Equivalent;
175                    move_in_function(report, index, status);
176                }
177                applied.insert(id);
178            }
179            Effect::Contradicted => {
180                contradicted.insert(id);
181            }
182            Effect::Deferred => {
183                deferred.insert(id);
184            }
185        }
186    }
187    let unmatched: BTreeSet<String> = input
188        .verdicts
189        .iter()
190        .filter(|verdict| !matched.contains(&verdict.review_key))
191        .map(|verdict| verdict.review_key.clone())
192        .collect();
193
194    report.summary = MutationSummary::from_results(&report.mutants);
195    report.verdicts = Some(MutationVerdictJoin {
196        verdicts_sha256: sha256_hex(verdict_json),
197        applied: applied.into_iter().collect(),
198        contradicted: contradicted.into_iter().collect(),
199        deferred: deferred.into_iter().collect(),
200        unmatched: unmatched.into_iter().collect(),
201    });
202    Ok(())
203}
204
205#[cfg(test)]
206#[expect(clippy::unwrap_used, reason = "tests unwrap to keep assertions short")]
207mod tests {
208    use fallow_output::{
209        MutantLocation, MutantResult, MutationCompletion, MutationFunctionResult,
210        MutationSchemaVersion, MutationScopeKind, MutationScopeOutput, MutationSummary,
211        MutationTimings, MutationWorktreeCheck,
212    };
213    use fallow_types::envelope::{ElapsedMs, ToolVersion};
214
215    use super::*;
216
217    fn mutant(id: &str, status: MutantStatus) -> MutantResult {
218        MutantResult {
219            location: MutantLocation {
220                mutant_id: id.to_owned(),
221                file: "src/a.ts".to_owned(),
222                line: 2,
223                column: 3,
224                span: (10, 15),
225                operator: "arithmetic-operator".to_owned(),
226                function: Some("add".to_owned()),
227                function_line: Some(1),
228                rank: Some(1),
229                change: None,
230            },
231            status,
232            prune_reason: None,
233            killed_by: None,
234            covering_tests: match status {
235                MutantStatus::NoCoverage | MutantStatus::Pending | MutantStatus::Pruned => {
236                    Vec::new()
237                }
238                _ => vec!["test/a.test.ts::t1".to_owned()],
239            },
240            static_mutant: false,
241            duration_ms: None,
242            test_reach: None,
243            retried: false,
244            attempts: Vec::new(),
245            cached: false,
246            resumed: false,
247            type_error_codes: Vec::new(),
248        }
249    }
250
251    fn report(mutants: Vec<MutantResult>) -> MutationReport {
252        let mut counts = MutationStatusCounts::default();
253        let mut survived = Vec::new();
254        for result in &mutants {
255            counts.add(result.status);
256            if result.status == MutantStatus::Survived {
257                survived.push(result.location.mutant_id.clone());
258            }
259        }
260        MutationReport {
261            schema_version: MutationSchemaVersion::V1,
262            version: ToolVersion("0.0.0".to_owned()),
263            elapsed_ms: ElapsedMs(0),
264            completion: MutationCompletion::from_reasons(Vec::new()),
265            scope: MutationScopeOutput {
266                kind: MutationScopeKind::All,
267                base_ref: None,
268            },
269            runner: None,
270            cache: None,
271            resume: None,
272            summary: MutationSummary::from_results(&mutants),
273            functions: vec![MutationFunctionResult {
274                file: "src/a.ts".to_owned(),
275                name: Some("add".to_owned()),
276                line: Some(1),
277                end_line: Some(4),
278                in_diff: true,
279                cyclomatic: Some(1),
280                counts,
281                score: counts.score(),
282                survived,
283            }],
284            mutants,
285            no_test_files: false,
286            baseline_failures: Vec::new(),
287            skipped_files: Vec::new(),
288            unmatched_paths: Vec::new(),
289            unloaded_paths: Vec::new(),
290            worktree: MutationWorktreeCheck {
291                checked: true,
292                allow_dirty: false,
293                dirty_at_start: Vec::new(),
294                changed_paths: Vec::new(),
295            },
296            errors: Vec::new(),
297            timings: MutationTimings::default(),
298            files: Vec::new(),
299            verdicts: None,
300            typecheck: None,
301        }
302    }
303
304    fn document(keys: &[&str]) -> Vec<u8> {
305        let verdicts: Vec<serde_json::Value> = keys
306            .iter()
307            .map(|key| serde_json::json!({ "review_key": key, "verdict": "equivalent" }))
308            .collect();
309        serde_json::to_vec(&serde_json::json!({ "schema_version": "1", "verdicts": verdicts }))
310            .unwrap()
311    }
312
313    #[test]
314    fn equivalent_verdicts_leave_the_score_and_the_survivor_list() {
315        let mut report = report(vec![
316            mutant("m1", MutantStatus::Killed),
317            mutant("m2", MutantStatus::Survived),
318            mutant("m3", MutantStatus::Survived),
319            mutant("m4", MutantStatus::NoCoverage),
320        ]);
321        assert_eq!(report.summary.score, Some(1.0 / 3.0));
322        assert_eq!(report.summary.coverage, Some(0.75));
323
324        apply_mutation_verdicts(&mut report, &document(&["m2", "m4"])).unwrap();
325
326        let statuses: Vec<MutantStatus> = report.mutants.iter().map(|m| m.status).collect();
327        assert_eq!(
328            statuses,
329            vec![
330                MutantStatus::Killed,
331                MutantStatus::Equivalent,
332                MutantStatus::Survived,
333                MutantStatus::Equivalent
334            ]
335        );
336        assert_eq!(report.summary.counts.equivalent, 2);
337        assert_eq!(report.summary.counts.survived, 1);
338        assert_eq!(report.summary.counts.no_coverage, 0);
339        assert_eq!(report.summary.counts.total, 4);
340        assert_eq!(report.summary.score, Some(0.5));
341        assert_eq!(report.summary.coverage, Some(1.0));
342        let function = &report.functions[0];
343        assert_eq!(function.counts.equivalent, 2);
344        assert_eq!(function.counts.no_coverage, 0);
345        assert_eq!(function.survived, vec!["m3".to_owned()]);
346        assert_eq!(function.score, Some(0.5));
347        let join = report.verdicts.unwrap();
348        assert_eq!(join.applied, vec!["m2".to_owned(), "m4".to_owned()]);
349        assert!(join.unmatched.is_empty());
350        assert_eq!(join.verdicts_sha256.len(), 64);
351    }
352
353    #[test]
354    fn a_kill_contradicts_a_verdict_and_unknown_keys_stay_unmatched() {
355        let mut report = report(vec![
356            mutant("m1", MutantStatus::Killed),
357            mutant("m2", MutantStatus::Pending),
358            mutant("m3", MutantStatus::Survived),
359        ]);
360
361        apply_mutation_verdicts(&mut report, &document(&["gone", "m1", "m2"])).unwrap();
362
363        assert_eq!(report.mutants[0].status, MutantStatus::Killed);
364        assert_eq!(report.mutants[1].status, MutantStatus::Pending);
365        assert_eq!(report.mutants[2].status, MutantStatus::Survived);
366        assert_eq!(report.summary.score, Some(0.5));
367        let join = report.verdicts.unwrap();
368        assert!(join.applied.is_empty());
369        assert_eq!(join.contradicted, vec!["m1".to_owned()]);
370        assert_eq!(join.deferred, vec!["m2".to_owned()]);
371        assert_eq!(join.unmatched, vec!["gone".to_owned()]);
372    }
373
374    #[test]
375    fn malformed_and_duplicate_verdicts_fail_closed() {
376        let cases: [(&[u8], &str); 4] = [
377            (b"{", "invalid mutation verdict document"),
378            (
379                br#"{"schema_version":"1","verdicts":[{"review_key":"m1","verdict":"survived"}]}"#,
380                "invalid mutation verdict document",
381            ),
382            (
383                br#"{"schema_version":"1","verdicts":[{"review_key":" ","verdict":"equivalent"}]}"#,
384                "empty review_key",
385            ),
386            (&document(&["m1", "m1"]), "more than one time"),
387        ];
388        for (json, needle) in cases {
389            let checked = validate_mutation_verdicts(json).unwrap_err();
390            assert_eq!(checked.exit_code, 2);
391            assert!(checked.message.contains(needle), "{}", checked.message);
392            let mut report = report(vec![mutant("m1", MutantStatus::Survived)]);
393            let error = apply_mutation_verdicts(&mut report, json).unwrap_err();
394            assert_eq!(
395                error.code.as_deref(),
396                Some("FALLOW_MUTATION_VERDICTS_INVALID")
397            );
398            assert_eq!(error.exit_code, 2);
399            assert!(error.message.contains(needle), "{}", error.message);
400            assert_eq!(report.mutants[0].status, MutantStatus::Survived);
401            assert!(report.verdicts.is_none());
402        }
403        assert!(validate_mutation_verdicts(&document(&["m1", "m2"])).is_ok());
404    }
405}
fallow_api/mutation/verdicts.rs

fallow_api/mutation/
verdicts.rs
