Skip to main content

unityrs_testkit/
scratch.rs

1//! Scratch folders: the rule every Rust test in the workspace keeps for
2//! what it writes outside its own memory. Its scratch is removed whatever
3//! happens: when it passes, fails or panics, and when a child it runs
4//! ends, aborts, overflows its stack or is killed. (Review round 9: a stack
5//! probe's folder, made in a child its bisection overflows by design,
6//! outlived every such child; a test whose own process a mutant aborted
7//! left a chain of 996 folders; two tests removed their files only after
8//! their asserts.)
9//!
10//! - **Every scratch folder is a [`Scratch`]**, removed with what it holds
11//!   when dropped, which a panic's unwinding does too. No test names
12//!   [`std::env::temp_dir`] but this module ([`direct_scratch`] scans for
13//!   it).
14//! - **A child's scratch is its parent's.** [`crate::child::output`] and
15//!   [`crate::child::status`] make a folder for each run of a child, name
16//!   it in the child's [`VAR`], and remove it when the run ends, however it
17//!   ends. A child that makes a [`Scratch`] (this test binary run again as
18//!   a probe, or a Python test through `tools/parity/scratch.py`, which
19//!   keeps the same rule) makes it inside that folder. A child that aborts
20//!   or is killed runs no drop of its own, so its parent's removal is the
21//!   one that runs.
22//! - **A process with no parent's folder** makes its folders in a root of
23//!   its own in the temp folder, `unityrs-test-<pid namespace>-<pid>`,
24//!   which goes with the last of them. A process that dies in its own code
25//!   (an abort, a stack overflow, a kill) leaves its root behind, so the
26//!   first folder any process makes removes every such root whose process
27//!   no longer runs ([`sweep`]): a root is its own process's alone, and
28//!   only roots of this pid namespace are compared with its processes.
29//!
30//! Owner: package B.
31
32use std::path::{Path, PathBuf};
33use std::sync::atomic::{AtomicU64, Ordering};
34use std::sync::{Mutex, PoisonError};
35
36/// The variable naming the folder a parent made for its child to make its
37/// scratch in (module docs).
38pub const VAR: &str = "UNITYRS_TEST_SCRATCH";
39
40/// The name of every process's own root, before its pid namespace and pid.
41pub const PREFIX: &str = "unityrs-test-";
42
43struct Root {
44    path: PathBuf,
45    /// The parent's folder ([`VAR`]), which the parent removes.
46    given: bool,
47    /// The folders made in it and not yet dropped.
48    live: usize,
49}
50
51static ROOT: Mutex<Option<Root>> = Mutex::new(None);
52static NEXT: AtomicU64 = AtomicU64::new(0);
53
54/// A new empty folder, removed with everything in it when dropped (module
55/// docs).
56#[derive(Debug)]
57pub struct Scratch(PathBuf);
58
59impl Scratch {
60    /// A new empty folder named after `tag`, inside the parent's folder or
61    /// this process's root (module docs).
62    ///
63    /// # Panics
64    /// When the folder cannot be made.
65    #[must_use]
66    pub fn new(tag: &str) -> Self {
67        let mut root = ROOT.lock().unwrap_or_else(PoisonError::into_inner);
68        let root = root.get_or_insert_with(open_root);
69        let path = root.path.join(format!(
70            "{tag}-{}-{}",
71            std::process::id(),
72            NEXT.fetch_add(1, Ordering::Relaxed)
73        ));
74        std::fs::create_dir(&path).unwrap_or_else(|e| panic!("{}: {e}", path.display()));
75        root.live += 1;
76        Self(path)
77    }
78
79    /// The folder.
80    #[must_use]
81    pub fn path(&self) -> &Path {
82        &self.0
83    }
84
85    /// `path` inside the folder.
86    #[must_use]
87    pub fn join(&self, path: impl AsRef<Path>) -> PathBuf {
88        self.0.join(path)
89    }
90}
91
92impl std::ops::Deref for Scratch {
93    type Target = Path;
94
95    fn deref(&self) -> &Path {
96        &self.0
97    }
98}
99
100impl AsRef<Path> for Scratch {
101    fn as_ref(&self) -> &Path {
102        &self.0
103    }
104}
105
106impl Drop for Scratch {
107    fn drop(&mut self) {
108        remove_tree(&self.0);
109        let mut root = ROOT.lock().unwrap_or_else(PoisonError::into_inner);
110        if let Some(r) = root.as_mut() {
111            r.live -= 1;
112            if r.live == 0 {
113                if !r.given {
114                    let _ = std::fs::remove_dir(&r.path);
115                }
116                *root = None;
117            }
118        }
119    }
120}
121
122/// This process's root: the parent's folder when [`VAR`] names one, else a
123/// new one of its own in the temp folder, after a [`sweep`].
124fn open_root() -> Root {
125    if let Some(given) = std::env::var_os(VAR).filter(|v| !v.is_empty()) {
126        let path = PathBuf::from(given);
127        if path.is_dir() {
128            return Root {
129                path,
130                given: true,
131                live: 0,
132            };
133        }
134    }
135    sweep();
136    let path = std::env::temp_dir().join(format!(
137        "{PREFIX}{}-{}",
138        namespace().unwrap_or(0),
139        std::process::id()
140    ));
141    // A dead process's of the same pid, which only this process can have now.
142    remove_tree(&path);
143    std::fs::create_dir(&path).unwrap_or_else(|e| panic!("{}: {e}", path.display()));
144    Root {
145        path,
146        given: false,
147        live: 0,
148    }
149}
150
151/// Removes `path` and everything under it a folder at a time, without
152/// following a link: a chain of folders past the stack or the open-file
153/// limit, and a folder of mode 000, go too. Nothing when it is not there.
154pub fn remove_tree(path: &Path) {
155    let Ok(meta) = std::fs::symlink_metadata(path) else {
156        return;
157    };
158    if !meta.is_dir() {
159        let _ = std::fs::remove_file(path);
160        return;
161    }
162    let mut pending = vec![path.to_path_buf()];
163    let mut folders = Vec::new();
164    while let Some(dir) = pending.pop() {
165        open_up(&dir);
166        let entries = std::fs::read_dir(&dir);
167        folders.push(dir);
168        let Ok(entries) = entries else {
169            continue;
170        };
171        for e in entries.flatten() {
172            if e.file_type().is_ok_and(|t| t.is_dir()) {
173                pending.push(e.path());
174            } else {
175                let _ = std::fs::remove_file(e.path());
176            }
177        }
178    }
179    for dir in folders.iter().rev() {
180        let _ = std::fs::remove_dir(dir);
181    }
182}
183
184#[cfg(unix)]
185fn open_up(dir: &Path) {
186    use std::os::unix::fs::PermissionsExt;
187    let _ = std::fs::set_permissions(dir, std::fs::Permissions::from_mode(0o700));
188}
189
190#[cfg(not(unix))]
191fn open_up(_: &Path) {}
192
193/// Removes every root in the temp folder of a process of this pid
194/// namespace that no longer runs (module docs). Nothing on a system
195/// without `/proc`'s pid namespaces.
196pub fn sweep() {
197    let Some(ns) = namespace() else {
198        return;
199    };
200    let own = format!("{PREFIX}{ns}-");
201    let Ok(entries) = std::fs::read_dir(std::env::temp_dir()) else {
202        return;
203    };
204    for e in entries.flatten() {
205        let name = e.file_name();
206        let Some(pid) = name.to_str().and_then(|n| n.strip_prefix(&own)) else {
207            continue;
208        };
209        if pid.is_empty() || !pid.bytes().all(|b| b.is_ascii_digit()) {
210            continue;
211        }
212        if pid != std::process::id().to_string() && !Path::new("/proc").join(pid).exists() {
213            remove_tree(&e.path());
214        }
215    }
216}
217
218/// This process's pid namespace, as `/proc/self/ns/pid` names it.
219fn namespace() -> Option<u64> {
220    let link = std::fs::read_link("/proc/self/ns/pid").ok()?;
221    let link = link.to_str()?;
222    link.strip_prefix("pid:[")?.strip_suffix(']')?.parse().ok()
223}
224
225/// Every place under `root` (its `.rs` files, `target` directories left
226/// out, this file too) that names the temp folder other than through this
227/// module: each line outside a `//` comment that names `temp_dir()`.
228///
229/// # Panics
230/// When a directory or a file under `root` cannot be read.
231#[must_use]
232pub fn direct_scratch(root: &Path) -> Vec<String> {
233    let mut files = Vec::new();
234    crate::guard::rust_files(root, &mut files);
235    files.sort();
236    let mut found = Vec::new();
237    for path in files {
238        if path.ends_with("unityrs-testkit/src/scratch.rs") {
239            continue;
240        }
241        let text =
242            std::fs::read_to_string(&path).unwrap_or_else(|e| panic!("{}: {e}", path.display()));
243        for (i, line) in text.lines().enumerate() {
244            let code = line.trim_start();
245            if !code.starts_with("//") && code.contains("temp_dir()") {
246                found.push(format!("{}:{}: {}", path.display(), i + 1, code));
247            }
248        }
249    }
250    found
251}
252
253#[cfg(all(test, unix))]
254mod tests {
255    use std::path::{Path, PathBuf};
256    use std::process::Command;
257
258    use super::{PoisonError, ROOT, Scratch, VAR, direct_scratch, remove_tree, sweep};
259    use crate::child::{self, Marks};
260
261    const PROBE_VAR: &str = "UNITYRS_TESTKIT_SCRATCH_PROBE";
262
263    /// Run by the tests below, as `WHAT:DIR`. `write` makes a [`Scratch`],
264    /// records its path, the parent's folder and this process's root in a
265    /// file of `DIR` named by its run, writes into the folder and dies
266    /// half way: killed by `SIGKILL` on its first run, aborted on the next.
267    /// `last` makes two folders, records its root and whether it is there
268    /// after the first goes and after the second, then passes.
269    #[test]
270    #[ignore = "run by the scratch tests"]
271    fn probe() {
272        let Ok(spec) = std::env::var(PROBE_VAR) else {
273            return;
274        };
275        let (what, dir) = spec.split_once(':').unwrap();
276        let dir = Path::new(dir);
277        let runs = std::fs::read_dir(dir).unwrap().count();
278        let first = Scratch::new("probe");
279        let root = root();
280        let parent = std::env::var(VAR).unwrap_or_default();
281        let mut record = format!("{}\n{parent}\n{}\n", first.display(), root.display());
282        if what == "last" {
283            let second = Scratch::new("probe");
284            drop(first);
285            record.push_str(if root.exists() { "true\n" } else { "false\n" });
286            drop(second);
287            record.push_str(if root.exists() { "true\n" } else { "false\n" });
288            std::fs::write(dir.join(runs.to_string()), record).unwrap();
289            return;
290        }
291        std::fs::write(dir.join(runs.to_string()), record).unwrap();
292        std::fs::write(first.join("half"), vec![7_u8; 4096]).unwrap();
293        if runs == 0 {
294            child::kill_self();
295        }
296        std::process::abort();
297    }
298
299    fn root() -> PathBuf {
300        let root = ROOT.lock().unwrap_or_else(PoisonError::into_inner);
301        root.as_ref().unwrap().path.clone()
302    }
303
304    /// The probe run as `what`, its marks in `marks`, with no parent's
305    /// folder when `own_root`: its last run's status and each run's
306    /// record, as lines.
307    fn probe_run(
308        what: &str,
309        marks: &Marks,
310        own_root: bool,
311    ) -> (std::process::ExitStatus, Vec<Vec<String>>) {
312        let exe = std::env::current_exe().unwrap();
313        let mut cmd = Command::new(exe);
314        cmd.args([
315            "--exact",
316            "scratch::tests::probe",
317            "--ignored",
318            "--test-threads=1",
319        ])
320        .env(PROBE_VAR, format!("{what}:{}", marks.0.display()));
321        if own_root {
322            cmd.env_remove(VAR);
323        }
324        let o = child::output(&mut cmd).unwrap();
325        let mut records = Vec::new();
326        for run in 0..marks.runs() {
327            let text = std::fs::read_to_string(marks.0.join(run.to_string())).unwrap();
328            records.push(text.lines().map(str::to_owned).collect());
329        }
330        (o.status, records)
331    }
332
333    /// A folder goes when it is dropped, a panic's unwinding included.
334    #[test]
335    fn a_scratch_goes_on_drop_and_on_a_panic() {
336        let s = Scratch::new("drop");
337        std::fs::write(s.join("f"), b"x").unwrap();
338        let path = s.to_path_buf();
339        drop(s);
340        assert!(!path.exists());
341        let kept = std::sync::Mutex::new(PathBuf::new());
342        let r = std::panic::catch_unwind(|| {
343            let s = Scratch::new("panic");
344            std::fs::write(s.join("f"), b"x").unwrap();
345            *kept.lock().unwrap() = s.to_path_buf();
346            panic!("as designed");
347        });
348        assert!(r.is_err());
349        assert!(!kept.into_inner().unwrap().exists());
350    }
351
352    /// The parent removes the folder it gave a child however the child
353    /// ends: killed by `SIGKILL`, then aborted, each half way through a
354    /// write, neither ran a drop (review round 9: a stack probe's folder
355    /// outlived each child its bisection overflowed).
356    #[test]
357    fn a_child_that_dies_mid_write_leaves_nothing() {
358        use std::os::unix::process::ExitStatusExt;
359        let marks = Marks::new("mid-write");
360        let (status, records) = probe_run("write", &marks, false);
361        // A child's own root, were it given none, is left there: removed on this test's failure.
362        let _roots: Vec<Guard> = records
363            .iter()
364            .map(|r| Guard(PathBuf::from(&r[2])))
365            .collect();
366        assert_eq!(status.signal(), Some(6), "{status:?}");
367        assert_eq!(records.len(), 2, "killed once, then aborted");
368        for r in &records {
369            let (folder, parent, root) = (Path::new(&r[0]), Path::new(&r[1]), Path::new(&r[2]));
370            assert_eq!(root, parent, "a child's root is its parent's folder");
371            assert!(folder.starts_with(parent), "{r:?}");
372            assert!(!parent.exists(), "{r:?}");
373        }
374        assert_ne!(
375            records[0][1], records[1][1],
376            "each run has a folder of its own"
377        );
378    }
379
380    /// A process with no parent's folder that dies leaves its root, and the
381    /// next sweep removes it (review round 9: a test whose own process a
382    /// mutant aborted left a chain of 996 folders).
383    #[test]
384    fn a_process_that_dies_with_no_parent_folder_has_its_root_swept() {
385        let marks = Marks::new("swept");
386        let (status, records) = probe_run("write", &marks, true);
387        // Left there, on this test's failure.
388        let roots: Vec<Guard> = records
389            .iter()
390            .map(|r| Guard(PathBuf::from(&r[2])))
391            .collect();
392        assert!(!status.success());
393        for r in &records {
394            assert_eq!(r[1], "", "no parent's folder");
395            let root = Path::new(&r[2]);
396            assert_eq!(root.parent(), Some(std::env::temp_dir().as_path()));
397            let name = root.file_name().unwrap().to_str().unwrap();
398            let rest = name.strip_prefix(super::PREFIX).unwrap();
399            assert!(rest.split('-').all(|n| n.parse::<u64>().is_ok()), "{name}");
400        }
401        sweep();
402        for g in &roots {
403            assert!(
404                !g.0.exists(),
405                "a dead process's root outlived a sweep: {}",
406                g.0.display()
407            );
408        }
409    }
410
411    struct Guard(PathBuf);
412
413    impl Drop for Guard {
414        fn drop(&mut self) {
415            remove_tree(&self.0);
416        }
417    }
418
419    /// A process's own root goes with its last folder.
420    #[test]
421    fn the_root_goes_with_its_last_folder() {
422        let marks = Marks::new("last");
423        let (status, records) = probe_run("last", &marks, true);
424        let _root = Guard(PathBuf::from(&records[0][2]));
425        assert!(status.success(), "{status:?}");
426        assert_eq!(records[0][3..], ["true", "false"]);
427    }
428
429    /// The removal takes a chain of folders deeper than a recursive walk's
430    /// stack and a folder of mode 000, which `remove_dir_all` cannot.
431    #[test]
432    fn remove_tree_takes_a_deep_chain_and_a_closed_folder() {
433        use std::os::unix::fs::PermissionsExt;
434        let s = Scratch::new("deep");
435        let mut deepest = s.to_path_buf();
436        for _ in 0..1100 {
437            deepest.push("d");
438            std::fs::create_dir(&deepest).unwrap();
439        }
440        std::fs::set_permissions(&deepest, std::fs::Permissions::from_mode(0o000)).unwrap();
441        let path = s.to_path_buf();
442        remove_tree(&path);
443        assert!(!path.exists());
444    }
445
446    /// Every test in the workspace makes its scratch here (review round 9:
447    /// tests that removed theirs after their asserts left it on a failure).
448    #[test]
449    fn every_scratch_in_the_workspace_is_made_here() {
450        let crates = Path::new(env!("CARGO_MANIFEST_DIR")).join("..");
451        let found = direct_scratch(&crates);
452        assert!(
453            found.is_empty(),
454            "scratch made outside unityrs_testkit::scratch, which a failure or a \
455             dying child leaves behind:\n{}",
456            found.join("\n")
457        );
458    }
459
460    /// The scan finds a temp folder named in code, and passes a comment.
461    #[test]
462    fn the_scan_finds_each_direct_temp_dir() {
463        let s = Scratch::new("scan");
464        std::fs::write(
465            s.join("a.rs"),
466            "// std::env::temp_dir() in a comment\nlet d = std::env::temp_dir().join(\"x\");\n",
467        )
468        .unwrap();
469        std::fs::write(s.join("b.rs"), "let s = Scratch::new(\"x\");\n").unwrap();
470        let found = direct_scratch(s.path());
471        let at = s.join("a.rs").display().to_string();
472        assert_eq!(
473            found,
474            [format!("{at}:2: let d = std::env::temp_dir().join(\"x\");")]
475        );
476    }
477}