use std::fs::File;
use std::io::Read;
#[cfg(any(target_os = "macos", target_os = "linux"))]
use std::os::unix::process::CommandExt;
use std::process::{Child, Command, Stdio};
use std::time::{Duration, Instant};
#[cfg(any(target_os = "macos", target_os = "linux"))]
use rustix::process::{
Pid, Signal, WaitId, WaitIdOptions, kill_process, kill_process_group, waitid,
};
pub struct Process {
child: Child,
#[cfg(any(target_os = "macos", target_os = "linux"))]
discover: bool,
#[cfg(windows)]
job: job::Job,
output: tempfile::NamedTempFile,
diagnostics: tempfile::NamedTempFile,
graceful: bool,
}
impl Process {
#[allow(dead_code)]
pub fn spawn(command: &mut Command, graceful: bool) -> Self {
Self::start(command, graceful, true)
}
#[allow(dead_code)]
pub fn spawn_grouped(command: &mut Command, graceful: bool) -> Self {
Self::start(command, graceful, false)
}
fn start(command: &mut Command, graceful: bool, discover: bool) -> Self {
#[cfg(not(any(target_os = "macos", target_os = "linux")))]
let _ = discover;
let output = tempfile::NamedTempFile::new().unwrap();
let diagnostics = tempfile::NamedTempFile::new().unwrap();
#[cfg(any(target_os = "macos", target_os = "linux"))]
command.process_group(0);
command
.stdin(Stdio::piped())
.stdout(output.reopen().unwrap())
.stderr(diagnostics.reopen().unwrap());
#[cfg(windows)]
let child = job::spawn_suspended(command);
#[cfg(not(windows))]
let child = command.spawn();
let child = child.unwrap_or_else(|error| panic!("fixture spawn {command:?}: {error}"));
#[cfg(windows)]
let (child, job) = job::Job::own(child);
Self {
child,
#[cfg(any(target_os = "macos", target_os = "linux"))]
discover,
#[cfg(windows)]
job,
output,
diagnostics,
graceful,
}
}
pub fn ready<T>(&mut self, parse: impl FnOnce(&str) -> Result<T, String>) -> T {
let result = self
.line(Duration::from_secs(10))
.and_then(|line| parse(&line));
result.unwrap_or_else(|error| panic!("fixture startup: {error}\n{}", self.diagnostics()))
}
#[allow(dead_code)]
pub fn complete(&mut self, timeout: Duration) {
let end = Instant::now() + timeout;
while matches!(self.exited(), Ok(false)) && Instant::now() < end {
std::thread::sleep(Duration::from_millis(10));
}
assert!(
matches!(self.exited(), Ok(true)),
"fixture completion deadline expired\n{}",
self.diagnostics()
);
#[cfg(any(target_os = "macos", target_os = "linux"))]
{
let pid = Pid::from_raw(self.child.id() as i32).unwrap();
let status = waitid(
WaitId::Pid(pid),
WaitIdOptions::EXITED | WaitIdOptions::NOWAIT,
)
.expect("fixture completion status")
.unwrap();
assert_eq!(
status.exit_status(),
Some(0),
"fixture failed\n{}",
self.diagnostics()
);
}
#[cfg(not(any(target_os = "macos", target_os = "linux")))]
assert!(
self.child
.wait()
.expect("fixture completion status")
.success(),
"fixture failed\n{}",
self.diagnostics()
);
}
fn line(&mut self, timeout: Duration) -> Result<String, String> {
let end = Instant::now() + timeout;
loop {
let bytes = read_prefix(self.output.path(), 8193).map_err(|e| e.to_string())?;
if let Some(end) = bytes.iter().position(|&byte| byte == b'\n') {
return String::from_utf8(bytes[..end].to_vec()).map_err(|e| e.to_string());
}
if bytes.len() > 8192 {
return Err("readiness line exceeds 8192 bytes".into());
}
if self.exited()? {
return Err("exited before readiness".into());
}
if Instant::now() >= end {
return Err("readiness deadline expired".into());
}
std::thread::sleep(Duration::from_millis(10));
}
}
#[cfg(any(target_os = "macos", target_os = "linux"))]
fn exited(&mut self) -> Result<bool, String> {
let pid = Pid::from_raw(self.child.id() as i32).unwrap();
let options = WaitIdOptions::EXITED | WaitIdOptions::NOHANG | WaitIdOptions::NOWAIT;
waitid(WaitId::Pid(pid), options)
.map(|status| status.is_some())
.map_err(|e| e.to_string())
}
#[cfg(not(any(target_os = "macos", target_os = "linux")))]
fn exited(&mut self) -> Result<bool, String> {
self.child
.try_wait()
.map(|status| status.is_some())
.map_err(|e| e.to_string())
}
fn diagnostics(&self) -> String {
let bytes = read_prefix(self.diagnostics.path(), 16384).unwrap_or_default();
String::from_utf8_lossy(&bytes).into_owned()
}
#[cfg(any(target_os = "macos", target_os = "linux"))]
fn kill_descendants(&self, deadline: Instant) {
let leader = self.child.id();
let pid = Pid::from_raw(leader as i32).unwrap();
let _ = kill_process(pid, Signal::STOP);
let mut owned = vec![leader];
while Instant::now() < deadline {
let Some(listing) = process_listing(deadline) else {
eprintln!("fixture cleanup discovery timed out or failed");
break;
};
let mut added = false;
let mut running = false;
for line in listing.lines() {
if line.contains("--lifetime-guard") {
continue;
}
let mut fields = line.split_whitespace();
let child = fields.next().and_then(|value| value.parse::<u32>().ok());
let parent = fields.next().and_then(|value| value.parse::<u32>().ok());
let state = fields.next().unwrap_or("");
if child.is_some_and(|pid| owned.contains(&pid))
&& !state.starts_with(['T', 't', 'Z'])
{
running = true;
}
if let (Some(child), Some(parent)) = (child, parent)
&& owned.contains(&parent)
&& !owned.contains(&child)
{
if let Some(pid) = Pid::from_raw(child as i32) {
let _ = kill_process(pid, Signal::STOP);
}
owned.push(child);
added = true;
}
}
if !added && !running {
break;
}
std::thread::sleep(Duration::from_millis(1));
}
for child in owned[1..].iter().rev() {
if let Some(pid) = Pid::from_raw(*child as i32) {
let _ = kill_process(pid, Signal::KILL);
}
}
}
}
#[cfg(any(target_os = "macos", target_os = "linux"))]
fn process_listing(deadline: Instant) -> Option<String> {
let output = tempfile::NamedTempFile::new().ok()?;
let mut command = Command::new("ps");
command.process_group(0);
let mut child = command
.args(["-axo", "pid=,ppid=,stat=,command="])
.stdin(Stdio::null())
.stdout(output.reopen().ok()?)
.stderr(Stdio::null())
.spawn()
.ok()?;
loop {
if let Ok(Some(status)) = child.try_wait() {
return status
.success()
.then(|| std::fs::read_to_string(output.path()).ok())
.flatten();
}
if Instant::now() >= deadline {
if let Some(pid) = Pid::from_raw(child.id() as i32) {
let _ = kill_process_group(pid, Signal::KILL);
}
let _ = child.kill();
let _ = child.try_wait();
return None;
}
std::thread::sleep(Duration::from_millis(1));
}
}
fn read_prefix(path: &std::path::Path, limit: u64) -> std::io::Result<Vec<u8>> {
let mut bytes = Vec::new();
File::open(path)?.take(limit).read_to_end(&mut bytes)?;
Ok(bytes)
}
impl Drop for Process {
fn drop(&mut self) {
#[cfg(any(target_os = "macos", target_os = "linux"))]
if self.discover {
self.kill_descendants(Instant::now() + Duration::from_secs(2));
}
self.child.stdin.take();
if self.graceful {
#[cfg(any(target_os = "macos", target_os = "linux"))]
if let Some(pid) = Pid::from_raw(self.child.id() as i32) {
let _ = kill_process(pid, Signal::TERM);
let _ = kill_process(pid, Signal::CONT);
}
let end = Instant::now() + Duration::from_secs(2);
while Instant::now() < end && matches!(self.exited(), Ok(false)) {
std::thread::sleep(Duration::from_millis(10));
}
}
#[cfg(windows)]
self.job.terminate();
#[cfg(any(target_os = "macos", target_os = "linux"))]
if let Some(pid) = Pid::from_raw(self.child.id() as i32) {
let _ = kill_process_group(pid, Signal::KILL);
}
let _ = self.child.kill();
let end = Instant::now() + Duration::from_millis(200);
while matches!(self.child.try_wait(), Ok(None)) {
if Instant::now() >= end {
eprintln!("fixture leader did not reap within cleanup deadline");
break;
}
std::thread::sleep(Duration::from_millis(1));
}
}
}
#[cfg(windows)]
mod job {
use std::os::windows::io::{AsRawHandle, FromRawHandle, OwnedHandle};
use std::process::Child;
use windows_sys::Win32::Foundation::{HANDLE, INVALID_HANDLE_VALUE};
use windows_sys::Win32::System::Diagnostics::ToolHelp::{
CreateToolhelp32Snapshot, TH32CS_SNAPTHREAD, THREADENTRY32, Thread32First, Thread32Next,
};
use windows_sys::Win32::System::JobObjects::{
AssignProcessToJobObject, CreateJobObjectW, JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE,
JOBOBJECT_EXTENDED_LIMIT_INFORMATION, JobObjectExtendedLimitInformation,
SetInformationJobObject, TerminateJobObject,
};
use windows_sys::Win32::System::Threading::{
CREATE_SUSPENDED, GetCurrentProcess, OpenThread, ResumeThread, THREAD_SUSPEND_RESUME,
};
pub struct Job(OwnedHandle);
pub fn spawn_suspended(command: &mut std::process::Command) -> std::io::Result<Child> {
static OWNER: std::sync::OnceLock<Result<Job, String>> = std::sync::OnceLock::new();
let owner = OWNER.get_or_init(|| {
let job = Job::new().map_err(|error| error.to_string())?;
if unsafe { AssignProcessToJobObject(job.handle(), GetCurrentProcess()) } == 0 {
return Err(std::io::Error::last_os_error().to_string());
}
Ok(job)
});
if let Err(error) = owner {
return Err(std::io::Error::other(format!("owner job: {error}")));
}
std::os::windows::process::CommandExt::creation_flags(command, CREATE_SUSPENDED);
command.spawn()
}
impl Job {
pub fn own(mut child: Child) -> (Child, Self) {
match Self::assign(&child).and_then(|job| resume(child.id()).map(|()| job)) {
Ok(job) => (child, job),
Err(error) => {
let _ = child.kill();
let _ = child.wait();
panic!("fixture job ownership: {error}");
}
}
}
fn assign(child: &Child) -> std::io::Result<Self> {
let job = Self::new()?;
if unsafe { AssignProcessToJobObject(job.handle(), child.as_raw_handle()) } == 0 {
return Err(std::io::Error::last_os_error());
}
Ok(job)
}
fn new() -> std::io::Result<Self> {
let raw = unsafe { CreateJobObjectW(std::ptr::null(), std::ptr::null()) };
if raw.is_null() {
return Err(std::io::Error::last_os_error());
}
let job = Self(unsafe { OwnedHandle::from_raw_handle(raw) });
let mut limits = JOBOBJECT_EXTENDED_LIMIT_INFORMATION::default();
limits.BasicLimitInformation.LimitFlags = JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE;
let set = unsafe {
SetInformationJobObject(
job.handle(),
JobObjectExtendedLimitInformation,
(&raw const limits).cast(),
size_of_val(&limits) as u32,
)
};
if set == 0 {
return Err(std::io::Error::last_os_error());
}
Ok(job)
}
pub fn terminate(&self) {
unsafe { TerminateJobObject(self.handle(), 1) };
}
fn handle(&self) -> HANDLE {
self.0.as_raw_handle()
}
}
fn resume(process: u32) -> std::io::Result<()> {
let raw = unsafe { CreateToolhelp32Snapshot(TH32CS_SNAPTHREAD, 0) };
if raw == INVALID_HANDLE_VALUE {
return Err(std::io::Error::last_os_error());
}
let snapshot = unsafe { OwnedHandle::from_raw_handle(raw) };
let mut entry = THREADENTRY32 {
dwSize: size_of::<THREADENTRY32>() as u32,
..Default::default()
};
let mut resumed = false;
let mut more = unsafe { Thread32First(snapshot.as_raw_handle(), &mut entry) } != 0;
while more {
if entry.th32OwnerProcessID == process {
let thread = unsafe { OpenThread(THREAD_SUSPEND_RESUME, 0, entry.th32ThreadID) };
if thread.is_null() {
return Err(std::io::Error::last_os_error());
}
let thread = unsafe { OwnedHandle::from_raw_handle(thread) };
if unsafe { ResumeThread(thread.as_raw_handle()) } == u32::MAX {
return Err(std::io::Error::last_os_error());
}
resumed = true;
}
more = unsafe { Thread32Next(snapshot.as_raw_handle(), &mut entry) } != 0;
}
if resumed {
Ok(())
} else {
Err(std::io::Error::other("suspended fixture thread not found"))
}
}
}
#[cfg(test)]
mod owner_tests {
#[test]
#[ignore = "helper process for abrupt_owner_exit_stops_every_fixture_process"]
fn abrupt_owner_helper() {
use std::path::Path;
use std::process::Command;
use std::time::Duration;
use super::Process;
fn python() -> Command {
Command::new(if cfg!(windows) { "python" } else { "python3" })
}
fn fixture(name: &str) -> String {
format!("{}/tests/fixtures/http/{name}", env!("CARGO_MANIFEST_DIR"))
}
let Some(root) = std::env::var_os("GIRT_FIXTURE_OWNER_ROOT") else {
return;
};
let root = Path::new(&root);
#[cfg(windows)]
if std::env::var_os("GIRT_FIXTURE_OWNER_STAGE").is_some_and(|stage| stage == "suspended") {
let mut leader = python();
leader.args(["-c", "import time; time.sleep(60)"]);
#[expect(
clippy::zombie_processes,
reason = "the test kills this owner while the leader is still suspended"
)]
let leader = super::job::spawn_suspended(&mut leader).unwrap();
println!("fixture-pids {}", leader.id());
std::thread::sleep(Duration::from_secs(60));
return;
}
let owner = std::process::id().to_string();
let mut server = python();
server
.arg(fixture("server.py"))
.arg(root)
.arg("--requests")
.arg(root.join("server-requests"))
.args(["--owner", &owner]);
let mut server = Process::spawn_grouped(&mut server, false);
server.ready(|line| line.trim().parse::<u16>().map_err(|e| e.to_string()));
let mut proxy = python();
proxy
.arg(fixture("connect_proxy.py"))
.arg("1")
.arg(root.join("proxy-requests"))
.args(["--owner", &owner]);
let mut proxy = Process::spawn_grouped(&mut proxy, false);
proxy.ready(|line| line.trim().parse::<u16>().map_err(|e| e.to_string()));
let pids = Vec::from([server.child.id(), proxy.child.id()]);
#[cfg(windows)]
let (pids, _tree) = {
let mut pids = pids;
let mut tree = python();
tree.args([
"-c",
"import subprocess,sys,time\nc=subprocess.Popen([sys.executable,'-c','import time; time.sleep(60)'])\nprint(c.pid,flush=True)\ntime.sleep(60)",
]);
let mut tree = Process::spawn(&mut tree, false);
let child = tree.ready(|line| line.trim().parse::<u32>().map_err(|e| e.to_string()));
pids.extend([tree.child.id(), child]);
(pids, tree)
};
let pids = pids.iter().map(u32::to_string).collect::<Vec<_>>();
println!("fixture-pids {}", pids.join(" "));
std::thread::sleep(Duration::from_secs(60));
}
#[test]
fn abrupt_owner_exit_stops_every_fixture_process() {
use std::io::BufRead;
use std::process::{Command, Stdio};
use std::time::{Duration, Instant};
let stages: &[&str] = if cfg!(windows) {
&["ready", "suspended"]
} else {
&["ready"]
};
for stage in stages {
let root = tempfile::tempdir().unwrap();
let mut owner = Command::new(std::env::current_exe().unwrap())
.args(["abrupt_owner_helper", "--ignored", "--nocapture"])
.env("GIRT_FIXTURE_OWNER_ROOT", root.path())
.env("GIRT_FIXTURE_OWNER_STAGE", stage)
.stdin(Stdio::null())
.stdout(Stdio::piped())
.stderr(Stdio::inherit())
.spawn()
.unwrap();
let stdout = owner.stdout.take().unwrap();
let (send, receive) = std::sync::mpsc::channel();
std::thread::spawn(move || {
for line in std::io::BufReader::new(stdout).lines() {
let Ok(line) = line else { break };
if let Some(pids) = line.strip_prefix("fixture-pids ") {
let _ = send.send(pids.to_owned());
}
}
});
let Ok(pids) = receive.recv_timeout(Duration::from_secs(30)) else {
let _ = owner.kill();
let _ = owner.wait();
panic!("stand-in owner did not start its fixtures");
};
let pids: Vec<u32> = pids.split(' ').map(|pid| pid.parse().unwrap()).collect();
let watched = pids
.iter()
.map(|&pid| Watched::open(pid))
.collect::<Vec<_>>();
owner.kill().unwrap();
owner.wait().unwrap();
let end = Instant::now() + Duration::from_secs(10);
for (pid, watched) in pids.iter().zip(&watched) {
assert!(
watched.exits_by(end),
"fixture process {pid} survived its owner at stage {stage}"
);
}
}
struct Watched {
#[cfg(any(target_os = "macos", target_os = "linux"))]
pid: rustix::process::Pid,
#[cfg(windows)]
handle: std::os::windows::io::OwnedHandle,
}
#[allow(dead_code)]
impl Watched {
#[cfg(any(target_os = "macos", target_os = "linux"))]
fn open(pid: u32) -> Self {
Self {
pid: rustix::process::Pid::from_raw(pid as i32).unwrap(),
}
}
#[cfg(any(target_os = "macos", target_os = "linux"))]
fn exits_by(&self, end: Instant) -> bool {
loop {
if rustix::process::test_kill_process(self.pid).is_err() {
return true;
}
let output = Command::new("ps")
.args(["-o", "stat=", "-p", &self.pid.as_raw_nonzero().to_string()])
.output()
.unwrap();
if String::from_utf8_lossy(&output.stdout)
.trim_start()
.starts_with('Z')
{
return true;
}
if Instant::now() >= end {
return false;
}
std::thread::sleep(Duration::from_millis(20));
}
}
#[cfg(windows)]
fn open(pid: u32) -> Self {
use std::os::windows::io::FromRawHandle;
use windows_sys::Win32::System::Threading::{OpenProcess, PROCESS_SYNCHRONIZE};
let raw = unsafe { OpenProcess(PROCESS_SYNCHRONIZE, 0, pid) };
assert!(
!raw.is_null(),
"open fixture process {pid}: {}",
std::io::Error::last_os_error()
);
Self {
handle: unsafe { std::os::windows::io::OwnedHandle::from_raw_handle(raw) },
}
}
#[cfg(windows)]
fn exits_by(&self, end: Instant) -> bool {
use std::os::windows::io::AsRawHandle;
use windows_sys::Win32::Foundation::WAIT_OBJECT_0;
use windows_sys::Win32::System::Threading::WaitForSingleObject;
let remaining = end.saturating_duration_since(Instant::now()).as_millis();
let wait = unsafe {
WaitForSingleObject(self.handle.as_raw_handle(), remaining.try_into().unwrap())
};
wait == WAIT_OBJECT_0
}
}
}
}
#[cfg(all(test, any(target_os = "macos", target_os = "linux")))]
mod tests {
#[rstest::rstest]
#[case::immediate(false, false)]
#[case::graceful(true, false)]
#[case::panic(true, true)]
fn teardown_kills_descendants_in_separate_sessions(
#[case] graceful: bool,
#[case] panic: bool,
) {
use super::*;
let mut command = Command::new("python3");
command.args(["-c", "import subprocess,time\nchild=subprocess.Popen(['sleep','60'],start_new_session=True)\nprint(child.pid,flush=True)\ntime.sleep(60)"]);
let mut process = Process::spawn(&mut command, graceful);
let child = process.ready(|line| line.parse::<i32>().map_err(|e| e.to_string()));
let child = Pid::from_raw(child).unwrap();
let result = std::panic::catch_unwind(move || {
let _process = process;
assert!(!panic, "fixture owner panicked");
});
assert_eq!(result.is_err(), panic);
let end = Instant::now() + Duration::from_secs(5);
while rustix::process::test_kill_process(child).is_ok() && Instant::now() < end {
let output = Command::new("ps")
.args(["-o", "stat=", "-p", &child.as_raw_nonzero().to_string()])
.output()
.unwrap();
if String::from_utf8_lossy(&output.stdout)
.trim_start()
.starts_with('Z')
{
return;
}
std::thread::sleep(Duration::from_millis(10));
}
assert_eq!(
rustix::process::test_kill_process(child),
Err(rustix::io::Errno::SRCH)
);
}
#[test]
fn startup_failure_retains_diagnostics() {
use super::*;
let mut command = Command::new("/bin/sh");
command.args(["-c", "echo missing-prerequisite >&2; exit 17"]);
let mut process = Process::spawn(&mut command, false);
assert!(
process
.line(Duration::from_secs(1))
.unwrap_err()
.contains("exited before readiness")
);
assert!(process.diagnostics().contains("missing-prerequisite"));
}
#[test]
fn stalled_startup_has_a_finite_deadline() {
use super::*;
let mut command = Command::new("/bin/sh");
command.args(["-c", "sleep 30"]);
let mut process = Process::spawn(&mut command, false);
assert_eq!(
process.line(Duration::from_millis(30)).unwrap_err(),
"readiness deadline expired"
);
}
#[rstest::rstest]
#[case::immediate(false)]
#[case::graceful(true)]
fn malformed_readiness_reaps_the_owned_process(#[case] graceful: bool) {
use super::*;
let mut command = Command::new("/bin/sh");
command.args(["-c", "printf 'invalid\n'; sleep 30"]);
let mut process = Process::spawn(&mut command, graceful);
let pid = Pid::from_raw(process.child.id() as i32).unwrap();
let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(move || {
process.ready::<()>(|_| Err("invalid readiness".into()));
}));
assert!(result.is_err());
let options = WaitIdOptions::EXITED | WaitIdOptions::NOHANG | WaitIdOptions::NOWAIT;
assert_eq!(
waitid(WaitId::Pid(pid), options).unwrap_err(),
rustix::io::Errno::CHILD
);
}
}