supercov_engine/
run_currency.rs

1//! Whether a run is still a record of a checkout whose test-side files have
2//! moved on, judged file by file.
3//!
4//! A run is fingerprinted by domain -- source, tests, dependencies,
5//! configuration -- and any change in a domain made it out of date. For the
6//! source and the dependencies that is right. For the rest it was too blunt:
7//! the end-to-end suite's Playwright configuration or helpers changing made
8//! every unit run out of date, and a helper that only gained a function for a
9//! new test made every run that loads it one. Each file that differs is
10//! judged here by what it is and what the run's tests did with it.
11use std::{
12    collections::{BTreeMap, BTreeSet},
13    path::Path,
14};
15
16use serde_json::Value;
17
18use crate::{
19    evidence_archive::{EvidenceArchiveEntry, read_archive_selected},
20    run_store::StoredRun,
21    source_manifest::{FileFingerprint, FileManifest, describe},
22    source_units::only_added,
23    test_loads::{Imports, configuration_runner, reads_imports},
24};
25
26/// The archive entry holding the Playwright configuration a run resolved.
27pub const PLAYWRIGHT_CONFIGURATION: &str = "playwright-configuration.json";
28
29/// What a run's archive says about how it ran, beyond its files.
30#[derive(Debug, Clone, Default)]
31pub struct Ran {
32    /// The runners its tests ran under, where its archive says.
33    pub runners: Option<BTreeSet<String>>,
34    /// The Playwright configuration as it resolved, where it ran Playwright.
35    pub playwright: Option<Value>,
36}
37
38impl Ran {
39    pub fn of(run: &StoredRun) -> Self {
40        let merged = run.metadata.merged == Some(true);
41        match read_archive_selected(&run.evidence_path, |path| {
42            Self::wanted(path, merged).is_some()
43        }) {
44            Ok(entries) => Self::from_entries(&entries, merged),
45            Err(_) => Self::default(),
46        }
47    }
48
49    /// The same, from an archive already read.
50    pub fn from_entries(entries: &[EvidenceArchiveEntry], merged: bool) -> Self {
51        let mut ran = Self::default();
52        for entry in entries {
53            let Some(name) = Self::wanted(&entry.path, merged) else {
54                continue;
55            };
56            let Ok(value) = serde_json::from_slice::<Value>(&entry.contents) else {
57                continue;
58            };
59            if name == PLAYWRIGHT_CONFIGURATION {
60                ran.playwright = Some(value);
61            } else if let Some(runners) = value["runners"].as_array() {
62                ran.runners.get_or_insert_with(BTreeSet::new).extend(
63                    runners
64                        .iter()
65                        .filter_map(|runner| runner["runner"].as_str())
66                        .filter(|runner| !matches!(*runner, "command" | "background"))
67                        .map(str::to_owned),
68                );
69            }
70        }
71        ran
72    }
73
74    /// Which of the two entries a path is, in a run or a merge of runs.
75    fn wanted(path: &str, merged: bool) -> Option<&'static str> {
76        let name = if merged {
77            path.rsplit('/').next()?
78        } else {
79            path
80        };
81        [PLAYWRIGHT_CONFIGURATION, "frontend.json"]
82            .into_iter()
83            .find(|wanted| *wanted == name)
84    }
85}
86
87/// The verdict on one run.
88#[derive(Debug, Clone, Default, PartialEq, Eq)]
89pub struct Judgment {
90    /// Files whose change makes the run a record of something else, each with
91    /// what moved in it.
92    pub outdated: Vec<String>,
93    /// Whether one of those is a file of the run's own tests: no option makes
94    /// a changed test's record its own.
95    pub own_tests: bool,
96    /// Files that differ and do not reach the run, with why; and those the
97    /// user accepted.
98    pub across: Vec<String>,
99}
100
101/// One checkout's files, read once for every run judged against it.
102pub struct Checkout<'a> {
103    pub root: &'a Path,
104    /// Every file known to hold tests, from every run there is.
105    pub tests: &'a BTreeSet<String>,
106    imports: Imports,
107    /// What all the known test files load: a helper none of them loads is one
108    /// Supercov cannot place, so it counts against every run. Read when a
109    /// helper first needs placing.
110    loaded_by_tests: std::cell::OnceCell<crate::test_loads::Loads>,
111}
112
113impl<'a> Checkout<'a> {
114    pub fn new(root: &'a Path, tests: &'a BTreeSet<String>) -> Self {
115        Self {
116            root,
117            tests,
118            imports: Imports::new(root),
119            loaded_by_tests: std::cell::OnceCell::new(),
120        }
121    }
122
123    /// Whether some known test file imports `file`: one that none does is
124    /// loaded from where Supercov cannot see -- a pattern in a configuration,
125    /// a command -- and may be loaded by any test. A run whose own imports
126    /// are all followed and miss it does not load it.
127    pub fn places(&self, file: &str) -> bool {
128        self.loaded_by_tests
129            .get_or_init(|| {
130                self.imports.loads(
131                    self.tests
132                        .iter()
133                        .filter(|file| reads_imports(file))
134                        .map(String::as_str),
135                )
136            })
137            .files
138            .contains(file)
139    }
140
141    /// What a run's tests load: their files, and the configuration of each
142    /// runner they ran under with what it names. `None` where a file of
143    /// theirs is not one Supercov reads imports from.
144    pub fn loads(
145        &self,
146        own: &BTreeSet<String>,
147        runners: Option<&BTreeSet<String>>,
148        configurations: &[String],
149    ) -> Option<crate::test_loads::Loads> {
150        if own.is_empty() || !own.iter().all(|file| reads_imports(file)) {
151            return None;
152        }
153        let mut entries = own.iter().cloned().collect::<Vec<_>>();
154        for configuration in configurations {
155            let used = configuration_runner(configuration)
156                .is_some_and(|runner| runners.is_some_and(|runners| runners.contains(runner)));
157            if used {
158                entries.push(configuration.clone());
159                entries.extend(self.imports.named_by(configuration));
160            }
161        }
162        Some(self.imports.loads(entries.iter().map(String::as_str)))
163    }
164}
165
166/// A file that sets how tests or builds run, by the rule the fingerprint's
167/// configuration domain uses.
168pub fn configuration_file(file: &str) -> bool {
169    crate::integrity::configuration_file(Path::new(file))
170}
171
172/// A file that sits with the tests by its path, as the fingerprint's test
173/// domain finds them: in a `test`, `tests` or `__tests__` directory, or named
174/// for tests.
175pub fn beside_the_tests(file: &str) -> bool {
176    let lower = file.to_ascii_lowercase();
177    lower
178        .split('/')
179        .next()
180        .is_some_and(|top| matches!(top, "test" | "tests" | "__tests__"))
181        || lower
182            .split(['/', '_', '.', '-'])
183            .any(|part| matches!(part, "test" | "spec"))
184}
185
186pub fn dependency_file(file: &str) -> bool {
187    let name = file.rsplit('/').next().unwrap_or(file);
188    matches!(
189        name,
190        "package.json"
191            | "package-lock.json"
192            | "npm-shrinkwrap.json"
193            | "pnpm-lock.yaml"
194            | "yarn.lock"
195            | "bun.lock"
196            | "bun.lockb"
197    )
198}
199
200/// The Playwright settings that decide what a test does, as opposed to when
201/// and how often it runs: the rest -- timeouts, retries, workers, reporters,
202/// which files and names to run -- schedules tests and changes nothing a test
203/// that ran reaches.
204fn playwright_shape(configuration: &Value) -> Value {
205    const PROJECT: [&str; 5] = ["name", "use", "testDir", "dependencies", "teardown"];
206    // At the top level `use` and `testDir` are what every project inherits.
207    const CONFIG: [&str; 5] = [
208        "globalSetup",
209        "globalTeardown",
210        "webServer",
211        "use",
212        "testDir",
213    ];
214    let pick = |value: &Value, keys: &[&str]| {
215        Value::Object(
216            keys.iter()
217                .filter_map(|key| value.get(*key).map(|v| ((*key).to_owned(), v.clone())))
218                .collect(),
219        )
220    };
221    let mut shape = pick(configuration, &CONFIG);
222    shape["projects"] = Value::Array(
223        configuration["projects"]
224            .as_array()
225            .into_iter()
226            .flatten()
227            .map(|project| pick(project, &PROJECT))
228            .collect(),
229    );
230    shape
231}
232
233/// Whether the run's Playwright configuration and the reference's do the same
234/// to the run's tests: every project it had is there and set alike, and the
235/// servers and global setup are the same. A project added since runs none of
236/// its tests. Where they differ, which settings did.
237pub fn playwright_differences(run: &Value, reference: &Value) -> Vec<String> {
238    let (run, reference) = (playwright_shape(run), playwright_shape(reference));
239    let mut differences = Vec::new();
240    for key in [
241        "globalSetup",
242        "globalTeardown",
243        "webServer",
244        "use",
245        "testDir",
246    ] {
247        if run.get(key) != reference.get(key) {
248            differences.push(key.to_owned());
249        }
250    }
251    let projects = |shape: &Value| {
252        shape["projects"]
253            .as_array()
254            .into_iter()
255            .flatten()
256            .map(|project| {
257                (
258                    project["name"].as_str().unwrap_or("").to_owned(),
259                    project.clone(),
260                )
261            })
262            .collect::<BTreeMap<_, _>>()
263    };
264    let now = projects(&reference);
265    for (name, project) in projects(&run) {
266        let label = if name.is_empty() {
267            String::new()
268        } else {
269            format!("project {name}: ")
270        };
271        match now.get(&name) {
272            None => differences.push(format!("{label}removed")),
273            Some(current) => {
274                for key in ["use", "testDir", "dependencies", "teardown"] {
275                    if project.get(key) != current.get(key) {
276                        differences.push(format!("{label}{key}"));
277                    }
278                }
279            }
280        }
281    }
282    differences
283}
284
285/// What the files a run was recorded with say against `now`, the files of the
286/// checkout or of the later run it is merged into.
287///
288/// `text` gives a file's current text where it is the text `now` describes;
289/// `reference` is how Playwright resolved for the later run, where one did;
290/// `label` names the run in what is said about it.
291#[allow(clippy::too_many_arguments)]
292pub fn judge(
293    checkout: &Checkout<'_>,
294    label: &str,
295    recorded: &FileManifest,
296    now: &FileManifest,
297    text: &dyn Fn(&str) -> Option<String>,
298    own: Option<&BTreeSet<String>>,
299    ran: &Ran,
300    reference: Option<&Value>,
301    allow_changed: bool,
302) -> Judgment {
303    let mut judgment = Judgment::default();
304    let configurations = recorded
305        .keys()
306        .chain(now.keys())
307        .filter(|file| configuration_file(file))
308        .cloned()
309        .collect::<BTreeSet<_>>()
310        .into_iter()
311        .collect::<Vec<_>>();
312    let loads = own.and_then(|own| checkout.loads(own, ran.runners.as_ref(), &configurations));
313    let mut changed = recorded
314        .iter()
315        .filter(|(file, was)| now.get(*file).is_none_or(|is| !is.same_bytes(was)))
316        .map(|(file, _)| file.clone())
317        .collect::<BTreeSet<_>>();
318    // A configuration that did not exist then can change a runner too.
319    changed.extend(
320        now.keys()
321            .filter(|file| !recorded.contains_key(*file) && configuration_file(file))
322            .cloned(),
323    );
324    let refuse = |judgment: &mut Judgment, file: &str, what: String| {
325        if allow_changed {
326            judgment.across.push(format!("{file} (accepted)"));
327        } else {
328            judgment.outdated.push(what);
329        }
330    };
331    for file in changed {
332        let gone = !now.contains_key(&file);
333        if own.is_some_and(|own| own.contains(&file)) {
334            judgment.own_tests = true;
335            judgment.outdated.push(format!(
336                "{file}, a file of its own tests, {}",
337                if gone { "is gone" } else { "has changed" }
338            ));
339            continue;
340        }
341        // Another run's test: this run did not run it. Where the run does not
342        // say which files are its own, every test file counts.
343        if checkout.tests.contains(&file) && own.is_some() {
344            continue;
345        }
346        if dependency_file(&file) {
347            judgment.outdated.push(format!("{file} has changed"));
348            continue;
349        }
350        if configuration_file(&file) {
351            let runner = configuration_runner(&file);
352            match (runner, ran.runners.as_ref()) {
353                (Some(runner), Some(runners)) if !runners.contains(runner) => {
354                    judgment.across.push(format!(
355                        "{file} ({runner} settings; `{label}` ran under {})",
356                        runners.iter().cloned().collect::<Vec<_>>().join(", ")
357                    ));
358                }
359                (Some("playwright"), Some(_)) => match (&ran.playwright, reference) {
360                    (Some(then), Some(reference)) => {
361                        let differences = playwright_differences(then, reference);
362                        if differences.is_empty() {
363                            judgment
364                                .across
365                                .push(format!("{file} (only settings that schedule tests)"));
366                        } else {
367                            refuse(
368                                &mut judgment,
369                                &file,
370                                format!("{file}: {} changed", differences.join(", ")),
371                            );
372                        }
373                    }
374                    _ => refuse(&mut judgment, &file, format!("{file} has changed")),
375                },
376                _ => refuse(&mut judgment, &file, format!("{file} has changed")),
377            }
378            continue;
379        }
380        // A file tests may load: a helper, a fixture, a support module.
381        let placed = checkout.places(&file);
382        if let Some(loads) = &loads
383            && loads.open.is_none()
384            && placed
385            && !loads.files.contains(&file)
386        {
387            judgment
388                .across
389                .push(format!("{file} (`{label}` does not load it)"));
390            continue;
391        }
392        if gone {
393            refuse(&mut judgment, &file, format!("{file} is gone"));
394            continue;
395        }
396        let before = recorded.get(&file).and_then(|was| was.code.as_ref());
397        let after = now.get(&file).and_then(|is| is.code.as_ref());
398        if let (Some(before), Some(source)) = (before, text(&file))
399            && let Some(added) = only_added(before, &file, &source)
400        {
401            judgment
402                .across
403                .push(format!("{file} (only added {})", added.join(", ")));
404            continue;
405        }
406        let what = match (before, after) {
407            (Some(before), Some(after)) => {
408                format!(
409                    "{file} changed: {}",
410                    describe(before, after, &before.diff(after))
411                )
412            }
413            _ => format!("{file} has changed"),
414        };
415        refuse(&mut judgment, &file, what);
416    }
417    judgment
418}
419
420/// What several judgments let through, one entry per file: `file (why;
421/// why)`.
422pub fn across(items: impl IntoIterator<Item = String>) -> Vec<String> {
423    let mut files = BTreeMap::<String, Vec<String>>::new();
424    for item in items {
425        let (file, why) = match item.split_once(" (") {
426            Some((file, why)) => (file.to_owned(), why.trim_end_matches(')').to_owned()),
427            None => (item, String::new()),
428        };
429        let whys = files.entry(file).or_default();
430        if !why.is_empty() && !whys.contains(&why) {
431            whys.push(why);
432        }
433    }
434    files
435        .into_iter()
436        .map(|(file, whys)| {
437            if whys.is_empty() {
438                file
439            } else {
440                format!("{file} ({})", whys.join("; "))
441            }
442        })
443        .collect()
444}
445
446/// The checkout's own fingerprints of the files a run was recorded with, and
447/// of every configuration file there is now.
448pub fn checkout_files(
449    root: &Path,
450    recorded: &FileManifest,
451    configurations: &[String],
452) -> FileManifest {
453    recorded
454        .keys()
455        .chain(configurations)
456        .filter_map(|file| {
457            let text = std::fs::read_to_string(root.join(file)).ok()?;
458            Some((file.clone(), FileFingerprint::read(file, &text)))
459        })
460        .collect()
461}
462
463#[cfg(test)]
464mod tests {
465    use super::*;
466    use serde_json::json;
467
468    #[test]
469    fn scheduling_settings_do_not_change_what_a_test_reaches() {
470        let then = json!({
471            "webServer": [{"command": "npm start", "url": "http://127.0.0.1:3000"}],
472            "projects": [{"name": "chromium", "use": {"baseURL": "http://127.0.0.1:3000"}, "timeout": 30000, "retries": 0, "testMatch": "*.spec.ts"}],
473            "workers": 1,
474        });
475        let mut later = then.clone();
476        later["workers"] = json!(4);
477        later["projects"][0]["timeout"] = json!(45000);
478        later["projects"][0]["testMatch"] = json!(["*.spec.ts", "new/*.spec.ts"]);
479        later["projects"]
480            .as_array_mut()
481            .unwrap()
482            .push(json!({"name": "firefox", "use": {"browserName": "firefox"}}));
483        assert_eq!(playwright_differences(&then, &later), Vec::<String>::new());
484
485        let mut shaped = then.clone();
486        shaped["projects"][0]["use"]["extraHTTPHeaders"] = json!({"x-suite": "e2e"});
487        shaped["webServer"][0]["command"] = json!("npm run start:mock");
488        assert_eq!(
489            playwright_differences(&then, &shaped),
490            ["webServer", "project chromium: use"]
491        );
492        // A configuration without projects sets `use` for all of them.
493        let single = json!({"use": {"baseURL": "http://127.0.0.1:3000"}, "timeout": 30000});
494        let mut longer = single.clone();
495        longer["timeout"] = json!(45000);
496        assert_eq!(
497            playwright_differences(&single, &longer),
498            Vec::<String>::new()
499        );
500        longer["use"]["locale"] = json!("de-DE");
501        assert_eq!(playwright_differences(&single, &longer), ["use"]);
502    }
503}
supercov_engine/run_currency.rs

supercov_engine/
run_currency.rs
