Skip to main content

unityrs_testkit/child/
live.rs

1//! One closure owns the WHOLE scene: servers, FIFOs, scratch and engine. An
2//! outside SIGKILL restarts that closure, even when the killed child caused an
3//! assertion or a broken pipe. Deliberate kills are explicit and never retried.
4//! Spawn and reap on the calling, long-lived thread: Linux PDEATHSIG follows
5//! the spawning thread. Every child has its own process group; cleanup kills
6//! the group before reaping the child, including namespace wrappers/services.
7use crate::{
8    guard::{self, DEADLOCK_GUARD},
9    scratch::Scratch,
10};
11use std::{
12    fs::{File, OpenOptions},
13    io::{self, Read},
14    os::unix::{fs::OpenOptionsExt, process::CommandExt},
15    path::{Path, PathBuf},
16    process::{Child, ChildStderr, ChildStdin, ChildStdout, Command, ExitStatus},
17    sync::mpsc::{self, Receiver},
18    thread::JoinHandle,
19    time::{Duration, Instant},
20};
21
22#[derive(Clone, Copy)]
23pub struct Live(usize);
24struct Owned {
25    /// `None` for an adopted descendant ([`Scene::adopt`]), reaped by pid.
26    child: Option<Child>,
27    pid: u32,
28    status: Option<ExitStatus>,
29    deliberate: bool,
30    service: bool,
31    group: bool,
32}
33pub struct Scene {
34    pub scratch: Scratch,
35    children: Vec<Owned>,
36    nested: bool,
37}
38/// Rebuild the complete scene after outside kills. Cleanup also runs on panic.
39///
40/// # Panics
41/// Repeated outside SIGKILL or a scene assertion failure.
42pub fn scene<T>(mut run: impl FnMut(&mut Scene) -> io::Result<T>) -> io::Result<T> {
43    for attempt in 1..=super::KILLED_RUNS {
44        let mut scene = Scene {
45            scratch: Scratch::new("scene"),
46            children: Vec::new(),
47            nested: std::env::var_os("UNITYRS_TEST_INHERIT_GROUP").is_some(),
48        };
49        let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| run(&mut scene)));
50        // Observe endings before deliberate cleanup can disguise an outside kill.
51        let (outside, unfinished) = scene.finish()?;
52        if outside {
53            assert!(
54                attempt < super::KILLED_RUNS,
55                "scene killed by SIGKILL on every run: the system ended it from outside"
56            );
57            continue;
58        }
59        return match result {
60            Ok(Ok(_)) if unfinished => Err(io::Error::other(
61                "scene returned while its engine was still alive",
62            )),
63            Ok(value) => value,
64            Err(panic) => std::panic::resume_unwind(panic),
65        };
66    }
67    unreachable!("the final killed scene fails above")
68}
69/// Spawn an engine or service in the current scene. Use `service=true` only
70/// for a process intentionally retained until scene cleanup. All streams must
71/// be drained by the caller when piped; `drain` supplies a continuous reader.
72pub fn live(scene: &mut Scene, cmd: &mut Command, service: bool) -> io::Result<Live> {
73    if !cmd.get_envs().any(|(k, _)| k == crate::scratch::VAR) {
74        cmd.env(crate::scratch::VAR, scene.scratch.path());
75    }
76    if !scene.nested {
77        cmd.process_group(0);
78    }
79    cmd.env("UNITYRS_TEST_INHERIT_GROUP", "1");
80    let child = cmd
81        .spawn()
82        .map_err(|e| io::Error::new(e.kind(), format!("scene spawn: {e}")))?;
83    let token = Live(scene.children.len());
84    scene.children.push(Owned {
85        pid: child.id(),
86        child: Some(child),
87        status: None,
88        deliberate: false,
89        service,
90        group: !scene.nested,
91    });
92    Ok(token)
93}
94impl Scene {
95    #[must_use]
96    pub fn id(&self, live: Live) -> u32 {
97        self.children[live.0].pid
98    }
99    /// The child's piped stdin, to write to. The scene hands out only stdio
100    /// handles, never the `Child`: observing, killing and reaping go through
101    /// the scene, so no test can reap a root whose group cleanup then kills.
102    pub fn stdin(&mut self, live: Live) -> Option<&mut ChildStdin> {
103        self.children[live.0].child.as_mut()?.stdin.as_mut()
104    }
105    /// Take the piped stdin (dropping it closes the child's stdin).
106    pub fn take_stdin(&mut self, live: Live) -> Option<ChildStdin> {
107        self.children[live.0].child.as_mut()?.stdin.take()
108    }
109    /// Take the piped stdout, for [`drain`].
110    pub fn take_stdout(&mut self, live: Live) -> Option<ChildStdout> {
111        self.children[live.0].child.as_mut()?.stdout.take()
112    }
113    /// Take the piped stderr, for [`drain`].
114    pub fn take_stderr(&mut self, live: Live) -> Option<ChildStderr> {
115        self.children[live.0].child.as_mut()?.stderr.take()
116    }
117    /// Adopt a descendant re-parented to this process (a Linux subreaper's
118    /// orphan): the scene observes, reaps and on cleanup kills it as one of
119    /// its own, so its outside SIGKILL is the scene's retry case. Its group
120    /// is never signalled, since it may be this process's own.
121    ///
122    /// # Errors
123    /// `ECHILD` when `pid` is not (yet) this process's child.
124    pub fn adopt(&mut self, pid: u32, service: bool) -> io::Result<Live> {
125        sys::exited(pid).map_err(|e| io::Error::new(e.kind(), format!("scene adopt: {e}")))?;
126        let token = Live(self.children.len());
127        self.children.push(Owned {
128            child: None,
129            pid,
130            status: None,
131            deliberate: false,
132            service,
133            group: false,
134        });
135        Ok(token)
136    }
137    /// Observe/reap through the scene so descendants are killed before the
138    /// root pid can be recycled.
139    pub fn running(&mut self, live: Live) -> io::Result<bool> {
140        let owned = &mut self.children[live.0];
141        if owned.status.is_none()
142            && sys::exited(owned.pid)
143                .map_err(|e| io::Error::new(e.kind(), format!("scene waitid: {e}")))?
144        {
145            if owned.group {
146                sys::cleanup_exited_group(owned.pid)?;
147            }
148            owned.status = Some(match &mut owned.child {
149                Some(child) => child.wait()?,
150                None => sys::reap(owned.pid)?,
151            });
152        }
153        Ok(owned.status.is_none())
154    }
155    /// Wait for a filesystem mark (a file another process creates) while
156    /// watching `watched`. One that ends first returns `BrokenPipe` at once,
157    /// so an outside SIGKILL reaches scene arbitration, not the guard.
158    ///
159    /// # Panics
160    /// When the mark outlasts the testkit deadlock guard.
161    pub fn mark(&mut self, path: &Path, watched: &[Live]) -> io::Result<()> {
162        let deadline = Instant::now() + DEADLOCK_GUARD;
163        while !path.exists() {
164            for live in watched {
165                if !self.running(*live)? {
166                    return Err(io::Error::new(
167                        io::ErrorKind::BrokenPipe,
168                        format!("scene child ended before mark {}", path.display()),
169                    ));
170                }
171            }
172            assert!(
173                Instant::now() < deadline,
174                "deadlock guard: scene mark {} missing",
175                path.display()
176            );
177            std::thread::sleep(Duration::from_millis(1));
178        }
179        Ok(())
180    }
181    pub fn signal(&mut self, live: Live, signal: i32) -> io::Result<()> {
182        if signal == libc::SIGKILL {
183            return Err(io::ErrorKind::InvalidInput.into());
184        }
185        if !self.running(live)? {
186            return Err(io::ErrorKind::NotFound.into());
187        }
188        sys::signal(self.id(live), signal, false)
189    }
190    /// A test's deliberate SIGKILL never counts as an outside kill.
191    pub fn kill(&mut self, live: Live) -> io::Result<()> {
192        if !self.running(live)? {
193            return Ok(());
194        }
195        let owned = &mut self.children[live.0];
196        owned.deliberate = true;
197        sys::signal(owned.pid, libc::SIGKILL, owned.group)
198    }
199    /// # Panics
200    /// When the child outlasts the testkit deadlock guard.
201    pub fn wait(&mut self, live: Live) -> io::Result<ExitStatus> {
202        let deadline = Instant::now() + DEADLOCK_GUARD;
203        loop {
204            if let Some(status) = self.children[live.0].status {
205                return Ok(status);
206            }
207            if !self.running(live)? {
208                continue;
209            }
210            assert!(
211                Instant::now() < deadline,
212                "deadlock guard: child did not exit"
213            );
214            std::thread::sleep(Duration::from_millis(1));
215        }
216    }
217    fn finish(&mut self) -> io::Result<(bool, bool)> {
218        let mut outside = false;
219        let mut unfinished = false;
220        for i in 0..self.children.len() {
221            let _ = self.running(Live(i))?;
222            if self.children[i].status.is_none() {
223                unfinished |= !self.children[i].service;
224                // Failure paths must not wait ten minutes for a held engine.
225                self.kill(Live(i))?;
226            }
227            let status = self.wait(Live(i))?;
228            outside |= super::killed(status) && !self.children[i].deliberate;
229        }
230        Ok((outside, unfinished))
231    }
232}
233impl Drop for Scene {
234    fn drop(&mut self) {
235        for owned in &mut self.children {
236            if owned.status.is_none() {
237                // Before reaping, this pid/group cannot name an unrelated run.
238                let _ = sys::signal(owned.pid, libc::SIGKILL, owned.group);
239                let _ = match &mut owned.child {
240                    Some(child) => child.wait(),
241                    None => sys::reap(owned.pid),
242                };
243            }
244        }
245    }
246}
247/// Continuously drain a pipe before waiting for the next protocol event.
248#[must_use]
249pub fn drain(mut input: impl Read + Send + 'static) -> (Receiver<Vec<u8>>, JoinHandle<()>) {
250    let (tx, rx) = mpsc::channel();
251    let thread = std::thread::spawn(move || {
252        loop {
253            let mut bytes = vec![0; 4096];
254            match input.read(&mut bytes) {
255                Ok(0) => break,
256                Ok(n) => {
257                    bytes.truncate(n);
258                    // Dropping the observer must not close the child's pipe or
259                    // turn its later diagnostics into EPIPE. Drain until EOF even
260                    // when the scene only wanted an early protocol observation.
261                    let _ = tx.send(bytes);
262                }
263                Err(e) if e.kind() == io::ErrorKind::Interrupted => {}
264                Err(_) => break,
265            }
266        }
267    });
268    (rx, thread)
269}
270/// A fresh scene-local FIFO opened before engine spawn. Its dedicated reader
271/// drains continuously, preserving partial lines; no stale file can satisfy a
272/// later scene's wait. Drop ends the reader without waiting for engine exit.
273pub struct Fifo {
274    path: PathBuf,
275    rx: Receiver<String>,
276    stop: std::sync::Arc<std::sync::atomic::AtomicBool>,
277    /// Reads the reader found the FIFO empty: [`Self::line_while`] waits for
278    /// two more once its child has ended, so every line the child wrote has
279    /// been sent before it calls the FIFO done.
280    empty: std::sync::Arc<std::sync::atomic::AtomicU64>,
281    thread: Option<JoinHandle<()>>,
282}
283impl Fifo {
284    pub fn new(scene: &Scene, name: &str) -> io::Result<Self> {
285        let path = scene.scratch.path().join(name);
286        sys::fifo(&path).map_err(|e| io::Error::new(e.kind(), format!("scene mkfifo: {e}")))?;
287        let mut file: File = OpenOptions::new()
288            .read(true)
289            .write(true)
290            .custom_flags(libc::O_NONBLOCK | libc::O_CLOEXEC)
291            .open(&path)?;
292        let stop = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false));
293        let stopping = stop.clone();
294        let empty = std::sync::Arc::new(std::sync::atomic::AtomicU64::new(0));
295        let emptied = empty.clone();
296        let (tx, rx) = mpsc::channel();
297        let thread = std::thread::spawn(move || {
298            let mut pending = Vec::new();
299            let mut bytes = [0; 4096];
300            while !stopping.load(std::sync::atomic::Ordering::Acquire) {
301                match file.read(&mut bytes) {
302                    Ok(n) => {
303                        for b in &bytes[..n] {
304                            if *b == b'\n' {
305                                if tx
306                                    .send(String::from_utf8_lossy(&pending).into_owned())
307                                    .is_err()
308                                {
309                                    return;
310                                }
311                                pending.clear();
312                            } else {
313                                pending.push(*b);
314                            }
315                        }
316                    }
317                    Err(e) if e.kind() == io::ErrorKind::WouldBlock => {
318                        emptied.fetch_add(1, std::sync::atomic::Ordering::Release);
319                        std::thread::sleep(Duration::from_millis(1));
320                    }
321                    Err(e) if e.kind() == io::ErrorKind::Interrupted => {}
322                    Err(_) => return,
323                }
324            }
325        });
326        Ok(Self {
327            path,
328            rx,
329            stop,
330            empty,
331            thread: Some(thread),
332        })
333    }
334    #[must_use]
335    pub fn path(&self) -> &Path {
336        &self.path
337    }
338    #[must_use]
339    pub fn line(&self) -> String {
340        guard::recv(&self.rx, "fresh lifecycle FIFO line")
341    }
342    /// Wait for a trace while also observing child death. An outside SIGKILL
343    /// returns to scene arbitration immediately instead of spending the guard
344    /// waiting on a FIFO whose read/write keeper intentionally prevents EOF.
345    /// A line the child wrote before it ended is still returned: the reader
346    /// polls, so it can lag an exit, and calling the FIFO done then lost a
347    /// stopped engine's last trace (URS-33).
348    ///
349    /// # Panics
350    /// When no line arrives before the testkit deadlock guard.
351    pub fn line_while(&self, scene: &mut Scene, live: Live) -> io::Result<String> {
352        let deadline = Instant::now() + DEADLOCK_GUARD;
353        loop {
354            match self.rx.recv_timeout(Duration::from_millis(1)) {
355                Ok(text) => return Ok(text),
356                Err(mpsc::RecvTimeoutError::Disconnected) => {
357                    return Err(io::ErrorKind::BrokenPipe.into());
358                }
359                Err(mpsc::RecvTimeoutError::Timeout) => {}
360            }
361            if !scene.running(live)? {
362                return self.rest(deadline);
363            }
364            assert!(
365                Instant::now() < deadline,
366                "deadlock guard: lifecycle FIFO line missing"
367            );
368        }
369    }
370    /// A line its ended writer left, or `BrokenPipe` when there is none,
371    /// within the caller's `deadline`. The read that bumps the empty count to
372    /// `seen + 1` may have started before the exit; the one after it started
373    /// later, so it found every byte the writer wrote, and the reader sent
374    /// their lines before it.
375    fn rest(&self, deadline: Instant) -> io::Result<String> {
376        use std::sync::atomic::Ordering;
377        let seen = self.empty.load(Ordering::Acquire);
378        while self.empty.load(Ordering::Acquire) < seen + 2 {
379            match self.rx.recv_timeout(Duration::from_millis(1)) {
380                Ok(text) => return Ok(text),
381                Err(mpsc::RecvTimeoutError::Disconnected) => break,
382                Err(mpsc::RecvTimeoutError::Timeout) => {}
383            }
384            assert!(
385                Instant::now() < deadline,
386                "deadlock guard: FIFO reader after its writer ended"
387            );
388        }
389        self.rx
390            .try_recv()
391            .map_err(|_| io::ErrorKind::BrokenPipe.into())
392    }
393}
394impl Drop for Fifo {
395    fn drop(&mut self) {
396        self.stop.store(true, std::sync::atomic::Ordering::Release);
397        if let Some(thread) = self.thread.take() {
398            let _ = thread.join();
399        }
400    }
401}
402#[path = "sys.rs"]
403mod sys;
404
405#[cfg(test)]
406#[path = "live_tests.rs"]
407mod tests;