Skip to main content

unityrs_testkit/
guard.rs

1//! Deadlock guards: how a test that hands work between threads turns a
2//! hang into a failure without timing the code it tests.
3//!
4//! A test of a lock rule (a callback that re-enters the core, a thread that
5//! holds a lock another thread's callback needs, a rendezvous) hangs for
6//! ever when the code under test deadlocks or never ends, so it waits with
7//! a limit and fails past it. The limit is no measure of the code: a
8//! correct run answers at once, and [`DEADLOCK_GUARD`] is ten minutes,
9//! beyond any delay a loaded machine's scheduler adds, so only a hang
10//! reaches it, and each failure says that it is this guard. A hook that
11//! holds a thread at a point until the test lets it go ([`hold`]) waits as
12//! long, so load never lets it go early. [`unguarded`] finds a timed wait
13//! that does not take its limit from here (review round 3: guards of 10 s
14//! could fail on a loaded machine).
15
16use std::path::Path;
17use std::sync::mpsc::{self, Receiver, RecvTimeoutError};
18use std::time::Duration;
19
20/// How long a deadlock guard waits: ten minutes.
21pub const DEADLOCK_GUARD: Duration = Duration::from_secs(600);
22
23/// `rx`'s next message, waited for at most [`DEADLOCK_GUARD`]; `what` names
24/// what the test waits for, and what a lock held where it must not be
25/// would deadlock.
26///
27/// # Panics
28/// Past the guard with no message, or when every sender is gone without
29/// sending (the thread that held one panicked, and its own message says
30/// why).
31#[track_caller]
32#[must_use]
33pub fn recv<T>(rx: &Receiver<T>, what: &str) -> T {
34    match rx.recv_timeout(DEADLOCK_GUARD) {
35        Ok(v) => v,
36        Err(RecvTimeoutError::Timeout) => hung(what),
37        Err(RecvTimeoutError::Disconnected) => {
38            panic!(
39                "the thread that would answer panicked without answering (its own message says \
40                 why); waiting for {what}"
41            )
42        }
43    }
44}
45
46/// Waits for a signal on `rx` (a thread has reached a point), as [`recv`].
47///
48/// # Panics
49/// As [`recv`].
50#[track_caller]
51pub fn wait(rx: &Receiver<()>, what: &str) {
52    let () = recv(rx, what);
53}
54
55/// Holds the calling thread (a test's hook) until the test lets it go, a
56/// message on `rx`, or drops the sender (it is done with the thread).
57///
58/// # Panics
59/// Past [`DEADLOCK_GUARD`] with neither: the test is waiting on this thread.
60#[track_caller]
61pub fn hold(rx: &Receiver<()>, what: &str) {
62    if let Err(RecvTimeoutError::Timeout) = rx.recv_timeout(DEADLOCK_GUARD) {
63        hung(what);
64    }
65}
66
67#[track_caller]
68fn hung(what: &str) -> ! {
69    panic!(
70        "deadlock guard: no answer in {} s, which only a hang takes, not load; waiting for {what}",
71        DEADLOCK_GUARD.as_secs()
72    )
73}
74
75/// Runs `f` on a thread of its own and returns what it returns, failing as
76/// [`recv`] does; past the guard the thread is left behind, not waited for.
77///
78/// # Panics
79/// As [`recv`].
80#[track_caller]
81#[must_use]
82pub fn finishes<T: Send + 'static>(what: &str, f: impl FnOnce() -> T + Send + 'static) -> T {
83    let (tx, rx) = mpsc::channel();
84    std::thread::spawn(move || {
85        let _ = tx.send(f());
86    });
87    recv(&rx, what)
88}
89
90/// The calls that wait with a time limit: a channel's, a condition
91/// variable's, a park's.
92pub const TIMED_WAITS: &[&str] = &[
93    ".recv_timeout(",
94    ".wait_timeout(",
95    ".wait_timeout_while(",
96    "::park_timeout(",
97];
98
99/// Every `file:line` of a `.rs` file under `root` (`target` directories
100/// left out) that calls a [`TIMED_WAITS`] wait in a file that never names
101/// [`DEADLOCK_GUARD`], whose limit therefore comes from somewhere else.
102/// Comment lines are not scanned. The engine's early `exit` line
103/// (`unityrs_batch::progress::early`), whose wait is the contract's own
104/// bound, is left out for its one wait only: a count there other than one
105/// is reported as `<file>: <n> timed waits, the exemption is for one`; so
106/// is the same line's writer on a descriptor it could not duplicate
107/// (`unityrs_batch::sys::borrowed`). Execution's two production startup deadlines, two channel-priming
108/// waits and one startup-gate priming wait are exempted by their exact
109/// count; a sixth wait is reported. Normal progress (`progress/live.rs`)
110/// is exempted for its heartbeat, its final acknowledgment and one startup
111/// priming wait.
112///
113/// # Panics
114/// When `root` cannot be read (a test helper).
115#[must_use]
116pub fn unguarded(root: &Path) -> Vec<String> {
117    let mut files = Vec::new();
118    rust_files(root, &mut files);
119    files.sort();
120    let mut found = Vec::new();
121    for path in files {
122        let text =
123            std::fs::read_to_string(&path).unwrap_or_else(|e| panic!("{}: {e}", path.display()));
124        if text.contains("DEADLOCK_GUARD") {
125            continue;
126        }
127        let hits: Vec<String> = text
128            .lines()
129            .enumerate()
130            .filter(|(_, line)| {
131                !line.trim_start().starts_with("//") && TIMED_WAITS.iter().any(|t| line.contains(t))
132            })
133            .map(|(i, _)| format!("{}:{}", path.display(), i + 1))
134            .collect();
135        // The engine's own early `exit` line waits on its writer thread
136        // for the contract's bound (`docs/BATCH_SPEC.md` section 10: at
137        // most 1 s, or the time left to the deadline), not for a test's:
138        // that one wait, and no other.
139        if path.ends_with("unityrs-batch/src/progress/early.rs") {
140            if hits.len() != 1 {
141                found.push(exemption_broken(&path, hits.len(), "timed waits"));
142            }
143            continue;
144        }
145        // The same line on a descriptor the main thread could not
146        // duplicate (`unityrs_batch::sys::borrowed`): its writer thread
147        // waited on for the same bound, that one wait.
148        if path.ends_with("unityrs-batch/src/sys/borrowed.rs") {
149            if hits.len() != 1 {
150                found.push(exemption_broken(&path, hits.len(), "timed waits"));
151            }
152            continue;
153        }
154        // Two production readiness deadlines (the workers' and the pool
155        // prewarm's one second) plus two 1 ms waits that initialize the end
156        // and dispatch channels' lazy wait contexts before the running head.
157        // These are not test deadlock guards.
158        if path.ends_with("unityrs-batch/src/execution.rs") {
159            if hits.len() != 5 {
160                found.push(format!(
161                    "{}: {} timed waits, the exemption is for two startup deadlines, two channel-priming waits and one startup-gate priming wait",
162                    path.display(),
163                    hits.len()
164                ));
165            }
166            continue;
167        }
168        // W20: both pool tasks and main use the same absolute startup bound.
169        if path.ends_with("unityrs-batch/src/lifetime/runtimes.rs") {
170            if hits.len() != 2 {
171                found.push(exemption_broken(&path, hits.len(), "W20 startup waits"));
172            }
173            continue;
174        }
175        // Normal progress has two contractual waits (heartbeat and final
176        // acknowledgment) plus one zero-duration startup condvar priming wait.
177        if path.ends_with("unityrs-batch/src/progress/live.rs") {
178            if hits.len() != 3 {
179                found.push(format!(
180                    "{}: {} timed waits, the exemption is for two normal progress waits and one startup priming wait",
181                    path.display(),
182                    hits.len()
183                ));
184            }
185            continue;
186        }
187        found.extend(hits);
188    }
189    found
190}
191
192/// A scan's report for an exempt file whose matches are not the one its
193/// exemption names.
194pub(crate) fn exemption_broken(path: &Path, n: usize, what: &str) -> String {
195    format!("{}: {n} {what}, the exemption is for one", path.display())
196}
197
198/// Every `.rs` file under `dir`, `target` directories left out.
199pub(crate) fn rust_files(dir: &Path, out: &mut Vec<std::path::PathBuf>) {
200    for entry in std::fs::read_dir(dir).unwrap_or_else(|e| panic!("{}: {e}", dir.display())) {
201        let path = entry
202            .unwrap_or_else(|e| panic!("{}: {e}", dir.display()))
203            .path();
204        if path.is_dir() {
205            if path.file_name().is_none_or(|n| n != "target") {
206                rust_files(&path, out);
207            }
208        } else if path.extension().is_some_and(|e| e == "rs") {
209            out.push(path);
210        }
211    }
212}
213
214#[cfg(test)]
215mod tests {
216    use super::{finishes, hold, recv, unguarded, wait};
217
218    #[test]
219    fn an_exempt_writer_with_two_waits_is_reported() {
220        let dir = crate::scratch::Scratch::new("guard-exemption");
221        let file = dir.join("unityrs-batch/src/progress/early.rs");
222        std::fs::create_dir_all(file.parent().unwrap()).unwrap();
223        std::fs::write(&file, "rx.recv_timeout(a);\nrx.recv_timeout(b);\n").unwrap();
224        assert_eq!(
225            unguarded(dir.path()),
226            [format!(
227                "{}: 2 timed waits, the exemption is for one",
228                file.display()
229            )]
230        );
231    }
232
233    #[test]
234    fn the_borrowed_writer_s_exemption_is_for_one_wait() {
235        let dir = crate::scratch::Scratch::new("guard-borrowed");
236        let file = dir.join("unityrs-batch/src/sys/borrowed.rs");
237        std::fs::create_dir_all(file.parent().unwrap()).unwrap();
238        for n in [1, 2] {
239            std::fs::write(&file, "rx.recv_timeout(a);\n".repeat(n)).unwrap();
240            assert_eq!(unguarded(dir.path()).len(), usize::from(n != 1));
241        }
242    }
243
244    #[test]
245    fn the_startup_exemption_counts_two_deadlines_and_three_priming_waits() {
246        let dir = crate::scratch::Scratch::new("startup-waits");
247        let file = dir.join("unityrs-batch/src/execution.rs");
248        std::fs::create_dir_all(file.parent().unwrap()).unwrap();
249        for n in [4, 5, 6] {
250            std::fs::write(&file, "rx.recv_timeout(a);\n".repeat(n)).unwrap();
251            let found = unguarded(dir.path());
252            if n == 5 {
253                assert_eq!(found.len(), 0);
254            } else {
255                assert_eq!(found.len(), 1);
256            }
257        }
258    }
259
260    #[test]
261    fn progress_exemption_counts_heartbeat_acknowledgment_and_startup_priming() {
262        let dir = crate::scratch::Scratch::new("progress-waits");
263        let file = dir.join("unityrs-batch/src/progress/live.rs");
264        std::fs::create_dir_all(file.parent().unwrap()).unwrap();
265        for n in [2, 3, 4] {
266            std::fs::write(&file, "changed.wait_timeout(state, wait);\n".repeat(n)).unwrap();
267            let found = unguarded(dir.path());
268            assert_eq!(found.len(), usize::from(n != 3));
269        }
270    }
271
272    /// Every crate's timed waits take their limit from [`super::DEADLOCK_GUARD`].
273    #[test]
274    fn every_timed_wait_in_the_workspace_takes_the_guard() {
275        let crates = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("..");
276        let found = unguarded(&crates);
277        assert!(
278            found.is_empty(),
279            "timed waits with a limit of their own, which load can outlast:\n{}",
280            found.join("\n")
281        );
282    }
283
284    #[test]
285    fn an_answer_comes_back() {
286        assert_eq!(finishes("a sum", || 1 + 1), 2);
287        let (tx, rx) = std::sync::mpsc::channel();
288        tx.send(3).unwrap();
289        assert_eq!(recv(&rx, "a sent value"), 3);
290    }
291
292    #[test]
293    #[should_panic(expected = "panicked without answering")]
294    fn a_thread_that_panics_fails_at_once_and_says_so() {
295        let (tx, rx) = std::sync::mpsc::channel::<()>();
296        drop(tx);
297        wait(&rx, "a signal");
298    }
299
300    #[test]
301    fn a_hold_ends_when_the_test_lets_it_go_or_drops_the_sender() {
302        let (tx, rx) = std::sync::mpsc::channel();
303        tx.send(()).unwrap();
304        hold(&rx, "the test to let go");
305        drop(tx);
306        hold(&rx, "the test to drop its sender");
307    }
308}