fallow_api/mutation/scheduler/
control.rs

1//! Control runs: runs of covering tests with no active mutant. A kill with
2//! no executed test, and a kill in a survivor retry by a test that the first
3//! run did not execute, need a control run of their tests.
4
5use std::collections::BTreeMap;
6use std::sync::{Arc, Mutex};
7
8use rustc_hash::{FxHashMap, FxHashSet};
9
10use super::super::protocol::{RunResult, RunStatus, TestStatus};
11
12/// Why a control run did not pass.
13#[derive(Debug, Clone, PartialEq, Eq)]
14pub(super) struct ControlFault {
15    /// The run executed no test: the run path is broken.
16    pub(super) no_tests: bool,
17    /// The requested tests that ran and failed. A failed test is a baseline
18    /// failure of the run, not a fault of the run path.
19    pub(super) failed_tests: Vec<FailedTest>,
20    pub(super) detail: String,
21}
22
23/// A test that failed in a control run.
24#[derive(Debug, Clone, PartialEq, Eq)]
25pub(super) struct FailedTest {
26    pub(super) test_id: String,
27    /// The error of the test from the companion. An older companion sends
28    /// none.
29    pub(super) error: Option<String>,
30}
31
32impl FailedTest {
33    /// The message of the baseline failure of this test: `reason`, then the
34    /// error of the test. The first line then has the reason and the first
35    /// line of the error, and the stack follows. With no error, the message
36    /// is `reason`.
37    pub(super) fn message(&self, reason: &str) -> String {
38        let Some(error) = self
39            .error
40            .as_deref()
41            .map(str::trim)
42            .filter(|e| !e.is_empty())
43        else {
44            return reason.to_owned();
45        };
46        let kept = &error[..error.floor_char_boundary(MAX_TEST_ERROR_BYTES)];
47        let ellipsis = if kept.len() < error.len() { "..." } else { "" };
48        format!("{reason}: {kept}{ellipsis}")
49    }
50}
51
52impl ControlFault {
53    pub(super) fn untrusted(detail: String) -> Self {
54        Self {
55            no_tests: false,
56            failed_tests: Vec::new(),
57            detail,
58        }
59    }
60}
61
62/// Checks the result of a control run of `tests` with no active mutant. Each
63/// requested test must run and pass, with no error.
64pub(super) fn control_fault(tests: &[String], result: &RunResult) -> Option<ControlFault> {
65    let first_error = result
66        .unhandled_errors
67        .first()
68        .map(|error| format!("; first error: {}", first_line(&error.message)));
69    if result.tests_executed == 0 {
70        return Some(ControlFault {
71            no_tests: true,
72            failed_tests: Vec::new(),
73            detail: format!(
74                "0 of {} requested tests ran{}",
75                tests.len(),
76                first_error.unwrap_or_default()
77            ),
78        });
79    }
80    if let Some(id) = &result.mutant_id {
81        return Some(ControlFault::untrusted(format!(
82            "the run result names mutant `{id}`, expected none"
83        )));
84    }
85    let requested: FxHashSet<&str> = tests.iter().map(String::as_str).collect();
86    let failed_tests: Vec<FailedTest> = result
87        .tests
88        .iter()
89        .filter(|test| test.status == TestStatus::Fail && requested.contains(test.test_id.as_str()))
90        .map(|test| FailedTest {
91            test_id: test.test_id.clone(),
92            error: test.message.clone(),
93        })
94        .collect();
95    if let Some(first) = failed_tests.first() {
96        return Some(ControlFault {
97            no_tests: false,
98            detail: format!("the test `{}` failed", first.test_id),
99            failed_tests,
100        });
101    }
102    if let Some(first_error) = first_error {
103        return Some(ControlFault::untrusted(format!(
104            "the run reported an error{first_error}"
105        )));
106    }
107    let passed: FxHashSet<&str> = result
108        .tests
109        .iter()
110        .filter(|test| test.status == TestStatus::Pass)
111        .map(|test| test.test_id.as_str())
112        .collect();
113    if result.status != RunStatus::Survived
114        || result.tests_executed != tests.len()
115        || tests.iter().any(|test| !passed.contains(test.as_str()))
116    {
117        return Some(ControlFault::untrusted(format!(
118            "{} of {} requested tests ran and passed",
119            passed.len(),
120            tests.len()
121        )));
122    }
123    None
124}
125
126/// The message of a test that fails in a control run. The dry run runs the
127/// whole test file, and a mutant run runs only the covering tests.
128pub(super) const CONTROL_RUN_FAILURE: &str = "the test fails in a mutant run with no active mutant, but passed in the dry run of its whole file; it can depend on another test of its file";
129
130/// The message of a test that fails in the control run of a survivor retry.
131/// That run already has each passing test of the file, so a missing covering
132/// test is not the cause.
133pub(super) const CONFIRMATION_RUN_FAILURE: &str = "the test fails with no active mutant in a run of each test of its file that passed in the dry run, but passed in the dry run of its whole file; it can depend on a test of its file that failed or was skipped in the dry run, or it can be flaky";
134
135/// The longest test error that a baseline failure keeps, in bytes. The
136/// companion bounds the error too. This bound guards the pipe from the
137/// companion.
138const MAX_TEST_ERROR_BYTES: usize = 8 * 1024;
139
140/// Splits the message of a control-run baseline failure into its fixed
141/// reason and the test error after it. Gives `None` for another message or
142/// for a reason with no test error.
143#[must_use]
144pub fn split_control_run_failure(message: &str) -> Option<(&str, &str)> {
145    [CONTROL_RUN_FAILURE, CONFIRMATION_RUN_FAILURE]
146        .into_iter()
147        .find_map(|reason| {
148            let error = message.strip_prefix(reason)?.strip_prefix(": ")?;
149            Some((reason, error))
150        })
151}
152
153/// The first line of `message`.
154pub(super) fn first_line(message: &str) -> &str {
155    message.lines().next().unwrap_or_default()
156}
157
158/// The result of a control run.
159pub(super) enum ControlOutcome {
160    Passed,
161    Failed(ControlFault),
162    /// The companion did not give a run result.
163    Fault(String),
164    /// A test or the companion passed the timeout of the control run.
165    TimedOut(String),
166}
167
168/// `tests`, sorted.
169pub(super) fn sorted_tests(tests: &[String]) -> Vec<&str> {
170    let mut sorted: Vec<&str> = tests.iter().map(String::as_str).collect();
171    sorted.sort_unstable();
172    sorted
173}
174
175/// The stored control result of one test set.
176#[derive(Debug, Clone, PartialEq, Eq)]
177pub(super) enum ControlVerdict {
178    /// The tests pass with no active mutant.
179    Passed,
180    /// A test failed with no active mutant.
181    Failed,
182    /// The control run timed out, with the reason. A later control run of
183    /// the same tests would wait as long again.
184    TimedOut(String),
185}
186
187/// The control evidence about one test set, from the memo or from a new
188/// control run.
189pub(super) enum ControlCheck {
190    /// The tests pass with no active mutant.
191    Passed,
192    /// The tests do not pass with no active mutant, with the reason. A
193    /// `settled` check is a fact about the tests. A fault of the companion is
194    /// not settled.
195    Failed { detail: String, settled: bool },
196    /// The time budget ended before a control run.
197    NoTime,
198}
199
200/// The control result of one test set: `None` before a control run gave a
201/// result.
202pub(super) type ControlCell = Arc<Mutex<Option<ControlVerdict>>>;
203
204/// The control result of each test set within one run. A mutant kill with
205/// no executed test needs a control run of its tests, and each test set gets
206/// at most one. A timeout is stored, so a hanging test set costs one control
207/// run. A fault of the companion is not stored, so a later mutant with the
208/// same tests runs the control again. The memo also keeps the tests
209/// that failed in a control run, each with the message of its baseline
210/// failure: each later mutant run leaves them out.
211#[derive(Default)]
212pub(super) struct ControlMemo {
213    entries: Mutex<FxHashMap<Vec<String>, ControlCell>>,
214    pub(super) failed_tests: Mutex<BTreeMap<String, String>>,
215}
216
217impl ControlMemo {
218    /// The cell of `tests`. A thread that holds the lock of the cell runs
219    /// the control, and each other thread with the same tests waits for it.
220    pub(super) fn cell(&self, tests: &[String]) -> ControlCell {
221        let mut key = tests.to_vec();
222        key.sort_unstable();
223        let Ok(mut entries) = self.entries.lock() else {
224            return Arc::new(Mutex::new(None));
225        };
226        Arc::clone(entries.entry(key).or_default())
227    }
228
229    pub(super) fn store(&self, tests: &[String], verdict: ControlVerdict) {
230        if let Ok(mut cell) = self.cell(tests).lock() {
231            *cell = Some(verdict);
232        }
233    }
234
235    /// Marks `tests` as failed for `reason`. The first message of a test
236    /// stays.
237    pub(super) fn mark_failed(&self, tests: &[FailedTest], reason: &str) {
238        if let Ok(mut failed) = self.failed_tests.lock() {
239            for test in tests {
240                failed
241                    .entry(test.test_id.clone())
242                    .or_insert_with(|| test.message(reason));
243            }
244        }
245    }
246
247    pub(super) fn is_failed(&self, test: &str) -> bool {
248        self.failed_tests
249            .lock()
250            .is_ok_and(|failed| failed.contains_key(test))
251    }
252
253    /// Each failed test with the message of its baseline failure.
254    pub(super) fn failed(&self) -> BTreeMap<String, String> {
255        self.failed_tests
256            .lock()
257            .map(|failed| failed.clone())
258            .unwrap_or_default()
259    }
260}
261
262#[cfg(test)]
263mod tests {
264    use super::*;
265
266    #[test]
267    fn a_control_run_failure_message_splits_into_its_reason_and_the_test_error() {
268        let error = "AssertionError: expected 1 to be 2\n    at test/a.test.ts:3:5";
269        let test = FailedTest {
270            test_id: "test/a.test.ts::adds".to_owned(),
271            error: Some(error.to_owned()),
272        };
273        for reason in [CONTROL_RUN_FAILURE, CONFIRMATION_RUN_FAILURE] {
274            let message = test.message(reason);
275            assert_eq!(split_control_run_failure(&message), Some((reason, error)));
276        }
277        let no_error = FailedTest {
278            error: None,
279            ..test
280        };
281        assert_eq!(
282            split_control_run_failure(&no_error.message(CONTROL_RUN_FAILURE)),
283            None
284        );
285        assert_eq!(
286            split_control_run_failure("Cannot find module './setup'"),
287            None
288        );
289    }
290}
fallow_api/mutation/scheduler/control.rs

fallow_api/mutation/scheduler/
control.rs
