fallow_api/mutation/
verdicts.rs

1//! Joins a mutation report with a separate verdict document.
2//!
3//! A verdict marks one mutant as equivalent to the original code. This
4//! replaces an inline suppression comment: the report stays the evidence, and the verdict is a separate,
5//! reviewable judgment. The join key is the `review_key` of the verdict, which
6//! is the `mutant_id` of the report. The id hashes the enclosing function
7//! text, so a verdict stays valid until that function changes.
8
9use std::collections::BTreeSet;
10
11use fallow_output::{
12    MutantStatus, MutationReport, MutationStatusCounts, MutationSummary, MutationVerdictInput,
13    MutationVerdictJoin,
14};
15use rustc_hash::FxHashSet;
16use std::fmt::Write as _;
17
18use sha2::{Digest, Sha256};
19
20use crate::ProgrammaticError;
21
22/// The largest verdict document that the join reads.
23pub const MAX_VERDICT_INPUT_BYTES: usize = 16 * 1024 * 1024;
24
25fn verdict_error(message: impl Into<String>) -> ProgrammaticError {
26    ProgrammaticError::new(message, 2)
27        .with_code("FALLOW_MUTATION_VERDICTS_INVALID")
28        .with_context("mutation.verdicts")
29}
30
31fn sha256_hex(bytes: &[u8]) -> String {
32    Sha256::digest(bytes)
33        .iter()
34        .fold(String::with_capacity(64), |mut hex, byte| {
35            let _ = write!(hex, "{byte:02x}");
36            hex
37        })
38}
39
40/// The effect of one verdict on a mutant with `status`.
41enum Effect {
42    Applied,
43    Contradicted,
44    Deferred,
45}
46
47const fn effect(status: MutantStatus) -> Effect {
48    match status {
49        MutantStatus::Survived
50        | MutantStatus::NoCoverage
51        | MutantStatus::Flaky
52        | MutantStatus::Equivalent => Effect::Applied,
53        MutantStatus::Killed | MutantStatus::Timeout | MutantStatus::CompileError => {
54            Effect::Contradicted
55        }
56        MutantStatus::Pending | MutantStatus::RuntimeError | MutantStatus::Pruned => {
57            Effect::Deferred
58        }
59    }
60}
61
62/// Moves one mutant from `from` to `equivalent` in the function group that
63/// holds it.
64fn move_in_function(report: &mut MutationReport, index: usize, from: MutantStatus) {
65    let location = &report.mutants[index].location;
66    let key = (
67        location.file.as_str(),
68        location.function.as_deref(),
69        location.function_line,
70    );
71    let group = report.functions.iter_mut().find(|group| {
72        (group.file.as_str(), group.name.as_deref(), group.line) == key
73            && count_of(&group.counts, from) > 0
74    });
75    let Some(group) = group else {
76        return;
77    };
78    group.counts.remove(from);
79    group.counts.add(MutantStatus::Equivalent);
80    group.survived.retain(|id| *id != location.mutant_id);
81    group.score = group.counts.score();
82}
83
84const fn count_of(counts: &MutationStatusCounts, status: MutantStatus) -> usize {
85    match status {
86        MutantStatus::Killed => counts.killed,
87        MutantStatus::Timeout => counts.timeout,
88        MutantStatus::Survived => counts.survived,
89        MutantStatus::NoCoverage => counts.no_coverage,
90        MutantStatus::RuntimeError => counts.runtime_error,
91        MutantStatus::Flaky => counts.flaky,
92        MutantStatus::Pruned => counts.pruned,
93        MutantStatus::Pending => counts.pending,
94        MutantStatus::Equivalent => counts.equivalent,
95        MutantStatus::CompileError => counts.compile_error,
96    }
97}
98
99/// Parses a verdict document and checks its keys.
100fn parse_verdicts(verdict_json: &[u8]) -> Result<MutationVerdictInput, ProgrammaticError> {
101    if verdict_json.len() > MAX_VERDICT_INPUT_BYTES {
102        return Err(verdict_error(
103            "the mutation verdict document is larger than 16 MiB",
104        ));
105    }
106    let input: MutationVerdictInput = serde_json::from_slice(verdict_json)
107        .map_err(|error| verdict_error(format!("invalid mutation verdict document: {error}")))?;
108    let mut keys = FxHashSet::default();
109    for verdict in &input.verdicts {
110        if verdict.review_key.trim().is_empty() {
111            return Err(verdict_error("a mutation verdict has an empty review_key"));
112        }
113        if !keys.insert(verdict.review_key.as_str()) {
114            return Err(verdict_error(format!(
115                "the review_key `{}` occurs more than one time",
116                verdict.review_key
117            )));
118        }
119    }
120    Ok(input)
121}
122
123/// Checks a verdict document without a report.
124///
125/// A caller runs this check before a mutation run, so an invalid document
126/// stops the command before any test runs.
127///
128/// # Errors
129///
130/// Returns the same errors as [`apply_mutation_verdicts`].
131pub fn validate_mutation_verdicts(verdict_json: &[u8]) -> Result<(), ProgrammaticError> {
132    parse_verdicts(verdict_json).map(|_| ())
133}
134
135/// Applies a verdict document to `report`.
136///
137/// Each verdict for a `survived`, `no_coverage` or `flaky` mutant sets the
138/// status `equivalent`, which does not count in the score. A verdict for a
139/// killed or timed-out mutant is contradicted by a test, so the status stays.
140/// A verdict for a mutant with no tested result (`pending`, `runtime_error`,
141/// `pruned`) is deferred. A verdict that matches no mutant is unmatched; it
142/// is not an error, because one verdict document serves many scoped runs.
143/// The summary and the function results are recomputed.
144///
145/// # Errors
146///
147/// Returns a fail-closed error for a document over 16 MiB, a malformed
148/// document, an empty `review_key`, or a `review_key` that occurs twice.
149pub fn apply_mutation_verdicts(
150    report: &mut MutationReport,
151    verdict_json: &[u8],
152) -> Result<(), ProgrammaticError> {
153    let input = parse_verdicts(verdict_json)?;
154    let keys: FxHashSet<&str> = input
155        .verdicts
156        .iter()
157        .map(|verdict| verdict.review_key.as_str())
158        .collect();
159
160    let mut applied = BTreeSet::new();
161    let mut contradicted = BTreeSet::new();
162    let mut deferred = BTreeSet::new();
163    let mut matched = FxHashSet::default();
164    for index in 0..report.mutants.len() {
165        let id = report.mutants[index].location.mutant_id.clone();
166        if !keys.contains(id.as_str()) {
167            continue;
168        }
169        matched.insert(id.clone());
170        let status = report.mutants[index].status;
171        match effect(status) {
172            Effect::Applied => {
173                if status != MutantStatus::Equivalent {
174                    report.mutants[index].status = MutantStatus::Equivalent;
175                    move_in_function(report, index, status);
176                }
177                applied.insert(id);
178            }
179            Effect::Contradicted => {
180                contradicted.insert(id);
181            }
182            Effect::Deferred => {
183                deferred.insert(id);
184            }
185        }
186    }
187    let unmatched: BTreeSet<String> = input
188        .verdicts
189        .iter()
190        .filter(|verdict| !matched.contains(&verdict.review_key))
191        .map(|verdict| verdict.review_key.clone())
192        .collect();
193
194    report.summary = MutationSummary::from_results(&report.mutants);
195    report.verdicts = Some(MutationVerdictJoin {
196        verdicts_sha256: sha256_hex(verdict_json),
197        applied: applied.into_iter().collect(),
198        contradicted: contradicted.into_iter().collect(),
199        deferred: deferred.into_iter().collect(),
200        unmatched: unmatched.into_iter().collect(),
201    });
202    Ok(())
203}
204
205#[cfg(test)]
206#[expect(clippy::unwrap_used, reason = "tests unwrap to keep assertions short")]
207mod tests {
208    use fallow_output::{
209        MutantLocation, MutantResult, MutationCompletion, MutationFunctionResult,
210        MutationSchemaVersion, MutationScopeKind, MutationScopeOutput, MutationSummary,
211        MutationTimings, MutationWorktreeCheck,
212    };
213    use fallow_types::envelope::{ElapsedMs, ToolVersion};
214
215    use super::*;
216
217    fn mutant(id: &str, status: MutantStatus) -> MutantResult {
218        MutantResult {
219            location: MutantLocation {
220                mutant_id: id.to_owned(),
221                file: "src/a.ts".to_owned(),
222                line: 2,
223                column: 3,
224                span: (10, 15),
225                operator: "arithmetic-operator".to_owned(),
226                function: Some("add".to_owned()),
227                function_line: Some(1),
228                rank: Some(1),
229            },
230            status,
231            prune_reason: None,
232            killed_by: None,
233            covering_tests: match status {
234                MutantStatus::NoCoverage | MutantStatus::Pending | MutantStatus::Pruned => {
235                    Vec::new()
236                }
237                _ => vec!["test/a.test.ts::t1".to_owned()],
238            },
239            static_mutant: false,
240            duration_ms: None,
241            test_reach: None,
242            retried: false,
243            cached: false,
244            type_error_codes: Vec::new(),
245        }
246    }
247
248    fn report(mutants: Vec<MutantResult>) -> MutationReport {
249        let mut counts = MutationStatusCounts::default();
250        let mut survived = Vec::new();
251        for result in &mutants {
252            counts.add(result.status);
253            if result.status == MutantStatus::Survived {
254                survived.push(result.location.mutant_id.clone());
255            }
256        }
257        MutationReport {
258            schema_version: MutationSchemaVersion::V1,
259            version: ToolVersion("0.0.0".to_owned()),
260            elapsed_ms: ElapsedMs(0),
261            completion: MutationCompletion::from_reasons(Vec::new()),
262            scope: MutationScopeOutput {
263                kind: MutationScopeKind::All,
264                base_ref: None,
265            },
266            runner: None,
267            cache: None,
268            summary: MutationSummary::from_results(&mutants),
269            functions: vec![MutationFunctionResult {
270                file: "src/a.ts".to_owned(),
271                name: Some("add".to_owned()),
272                line: Some(1),
273                end_line: Some(4),
274                in_diff: true,
275                cyclomatic: Some(1),
276                counts,
277                score: counts.score(),
278                survived,
279            }],
280            mutants,
281            no_test_files: false,
282            baseline_failures: Vec::new(),
283            skipped_files: Vec::new(),
284            unmatched_paths: Vec::new(),
285            unloaded_paths: Vec::new(),
286            worktree: MutationWorktreeCheck {
287                checked: true,
288                allow_dirty: false,
289                dirty_at_start: Vec::new(),
290                changed_paths: Vec::new(),
291            },
292            errors: Vec::new(),
293            timings: MutationTimings::default(),
294            files: Vec::new(),
295            verdicts: None,
296            typecheck: None,
297        }
298    }
299
300    fn document(keys: &[&str]) -> Vec<u8> {
301        let verdicts: Vec<serde_json::Value> = keys
302            .iter()
303            .map(|key| serde_json::json!({ "review_key": key, "verdict": "equivalent" }))
304            .collect();
305        serde_json::to_vec(&serde_json::json!({ "schema_version": "1", "verdicts": verdicts }))
306            .unwrap()
307    }
308
309    #[test]
310    fn equivalent_verdicts_leave_the_score_and_the_survivor_list() {
311        let mut report = report(vec![
312            mutant("m1", MutantStatus::Killed),
313            mutant("m2", MutantStatus::Survived),
314            mutant("m3", MutantStatus::Survived),
315            mutant("m4", MutantStatus::NoCoverage),
316        ]);
317        assert_eq!(report.summary.score, Some(1.0 / 3.0));
318        assert_eq!(report.summary.coverage, Some(0.75));
319
320        apply_mutation_verdicts(&mut report, &document(&["m2", "m4"])).unwrap();
321
322        let statuses: Vec<MutantStatus> = report.mutants.iter().map(|m| m.status).collect();
323        assert_eq!(
324            statuses,
325            vec![
326                MutantStatus::Killed,
327                MutantStatus::Equivalent,
328                MutantStatus::Survived,
329                MutantStatus::Equivalent
330            ]
331        );
332        assert_eq!(report.summary.counts.equivalent, 2);
333        assert_eq!(report.summary.counts.survived, 1);
334        assert_eq!(report.summary.counts.no_coverage, 0);
335        assert_eq!(report.summary.counts.total, 4);
336        assert_eq!(report.summary.score, Some(0.5));
337        assert_eq!(report.summary.coverage, Some(1.0));
338        let function = &report.functions[0];
339        assert_eq!(function.counts.equivalent, 2);
340        assert_eq!(function.counts.no_coverage, 0);
341        assert_eq!(function.survived, vec!["m3".to_owned()]);
342        assert_eq!(function.score, Some(0.5));
343        let join = report.verdicts.unwrap();
344        assert_eq!(join.applied, vec!["m2".to_owned(), "m4".to_owned()]);
345        assert!(join.unmatched.is_empty());
346        assert_eq!(join.verdicts_sha256.len(), 64);
347    }
348
349    #[test]
350    fn a_kill_contradicts_a_verdict_and_unknown_keys_stay_unmatched() {
351        let mut report = report(vec![
352            mutant("m1", MutantStatus::Killed),
353            mutant("m2", MutantStatus::Pending),
354            mutant("m3", MutantStatus::Survived),
355        ]);
356
357        apply_mutation_verdicts(&mut report, &document(&["gone", "m1", "m2"])).unwrap();
358
359        assert_eq!(report.mutants[0].status, MutantStatus::Killed);
360        assert_eq!(report.mutants[1].status, MutantStatus::Pending);
361        assert_eq!(report.mutants[2].status, MutantStatus::Survived);
362        assert_eq!(report.summary.score, Some(0.5));
363        let join = report.verdicts.unwrap();
364        assert!(join.applied.is_empty());
365        assert_eq!(join.contradicted, vec!["m1".to_owned()]);
366        assert_eq!(join.deferred, vec!["m2".to_owned()]);
367        assert_eq!(join.unmatched, vec!["gone".to_owned()]);
368    }
369
370    #[test]
371    fn malformed_and_duplicate_verdicts_fail_closed() {
372        let cases: [(&[u8], &str); 4] = [
373            (b"{", "invalid mutation verdict document"),
374            (
375                br#"{"schema_version":"1","verdicts":[{"review_key":"m1","verdict":"survived"}]}"#,
376                "invalid mutation verdict document",
377            ),
378            (
379                br#"{"schema_version":"1","verdicts":[{"review_key":" ","verdict":"equivalent"}]}"#,
380                "empty review_key",
381            ),
382            (&document(&["m1", "m1"]), "more than one time"),
383        ];
384        for (json, needle) in cases {
385            let checked = validate_mutation_verdicts(json).unwrap_err();
386            assert_eq!(checked.exit_code, 2);
387            assert!(checked.message.contains(needle), "{}", checked.message);
388            let mut report = report(vec![mutant("m1", MutantStatus::Survived)]);
389            let error = apply_mutation_verdicts(&mut report, json).unwrap_err();
390            assert_eq!(
391                error.code.as_deref(),
392                Some("FALLOW_MUTATION_VERDICTS_INVALID")
393            );
394            assert_eq!(error.exit_code, 2);
395            assert!(error.message.contains(needle), "{}", error.message);
396            assert_eq!(report.mutants[0].status, MutantStatus::Survived);
397            assert!(report.verdicts.is_none());
398        }
399        assert!(validate_mutation_verdicts(&document(&["m1", "m2"])).is_ok());
400    }
401}
fallow_api/mutation/verdicts.rs

fallow_api/mutation/
verdicts.rs
