use std::cell::Cell;
use std::collections::HashMap;
use std::collections::VecDeque;
use std::collections::hash_map::Entry;
use std::fs;
use std::fs::OpenOptions;
use std::future::Future;
use std::hash::Hash;
use std::hash::Hasher;
use std::io;
use std::marker::PhantomData;
use std::os::fd::AsRawFd;
use std::os::fd::FromRawFd;
use std::os::fd::OwnedFd;
use std::os::fd::RawFd;
use std::os::unix::fs::MetadataExt;
use std::os::unix::fs::OpenOptionsExt;
use std::pin::Pin;
use std::rc::Rc;
use std::sync::Arc;
#[cfg(test)]
use std::sync::Barrier;
use std::sync::LazyLock;
use std::sync::OnceLock;
use std::sync::Weak;
use std::sync::atomic::AtomicBool;
use std::sync::atomic::AtomicI32;
use std::sync::atomic::AtomicPtr;
use std::sync::atomic::AtomicU8;
use std::sync::atomic::AtomicU64;
use std::sync::atomic::AtomicUsize;
use std::sync::atomic::Ordering;
#[cfg(test)]
use std::sync::mpsc;
use std::task::Context;
use std::task::Poll;
use std::task::RawWakerVTable;
use std::task::Waker;
use std::thread;
use std::thread::JoinHandle;
#[cfg(test)]
use std::thread::ThreadId;
use std::time::Duration;
use std::time::Instant;
use nix::sys::wait::WaitPidFlag;
use parking_lot::Condvar;
use parking_lot::Mutex;
use parking_lot::MutexGuard;
use parking_lot::RwLock;
use parking_lot::RwLockReadGuard;
use reverie_process::launch_window;
use super::Errno;
use super::Error;
use super::Pid;
use super::Running;
use super::Stopped;
use super::TraceeToken;
use super::Wait;
use super::waitid;
static NOTIFIER: LazyLock<Notifier> = LazyLock::new(Notifier::new);
#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash)]
pub(super) enum WaitPolicy {
Native,
PtracerThread,
}
impl WaitPolicy {
fn check_identity(self, identity: &WorkerIdentity) -> Result<(), Errno> {
if self == Self::Native && !matches!(identity.pidfd, ThreadHandle::Pidfd(_)) {
return Err(Errno::EINVAL);
}
Ok(())
}
fn check_handle(self, handle: &EventHandle) -> Result<(), Errno> {
if let Some(error) = handle.initial_capture_error() {
return Err(error);
}
match handle.identity() {
Some(identity) => self.check_identity(identity),
None if self == Self::PtracerThread => Err(handle
.resolved()
.0
.initial_capture_refusal
.get()
.copied()
.or_else(|| *handle.event().registration_error.lock())
.unwrap_or(Errno::ENODATA)),
None => Ok(()),
}
}
}
#[cfg(test)]
static CAPTURE_ERRORS: LazyLock<Mutex<HashMap<Pid, Errno>>> =
LazyLock::new(|| Mutex::new(HashMap::new()));
#[cfg(test)]
static CAPTURE_PERSISTENT_ERRORS: LazyLock<Mutex<HashMap<Pid, Errno>>> =
LazyLock::new(|| Mutex::new(HashMap::new()));
#[cfg(test)]
static FORCE_RAW_CLAIM_REGISTRATION_RACE: LazyLock<Mutex<HashMap<Pid, ()>>> =
LazyLock::new(|| Mutex::new(HashMap::new()));
#[cfg(test)]
static SPAWN_WORKER_ERRORS: LazyLock<Mutex<HashMap<Pid, i32>>> =
LazyLock::new(|| Mutex::new(HashMap::new()));
#[cfg(test)]
static SPAWN_WORKER_COUNTS: LazyLock<Mutex<HashMap<Pid, usize>>> =
LazyLock::new(|| Mutex::new(HashMap::new()));
#[cfg(test)]
static PIDFD_OPEN_ERRORS: LazyLock<Mutex<HashMap<Pid, Errno>>> =
LazyLock::new(|| Mutex::new(HashMap::new()));
#[cfg(test)]
thread_local! {
static FORCE_LEGACY_THREAD_GROUP: Cell<Option<Pid>> = const { Cell::new(None) };
}
#[cfg(test)]
static PIDFD_LIVENESS_ERRORS: LazyLock<Mutex<HashMap<Pid, VecDeque<Errno>>>> =
LazyLock::new(|| Mutex::new(HashMap::new()));
#[cfg(test)]
static PROCFS_OBSERVATION_ERRORS: LazyLock<Mutex<HashMap<Pid, Errno>>> =
LazyLock::new(|| Mutex::new(HashMap::new()));
#[cfg(test)]
struct EventCapturePause {
captured: Arc<Barrier>,
resume: Arc<Barrier>,
}
#[cfg(test)]
#[derive(Debug)]
struct BoundedTestPause {
captured: mpsc::SyncSender<()>,
resume: mpsc::Receiver<()>,
}
#[cfg(test)]
fn pause_retirement_for_test(slot: &Mutex<Option<BoundedTestPause>>) {
pause_for_test_noting(slot, || {});
}
#[cfg(test)]
fn pause_for_test_noting(slot: &Mutex<Option<BoundedTestPause>>, note: impl FnOnce()) {
let pause = slot.lock().take();
if let Some(pause) = pause {
note();
let _ = pause.captured.send(());
let _ = pause.resume.recv_timeout(Duration::from_secs(2));
}
}
#[cfg(test)]
#[derive(Debug)]
struct WorkerDoneWaitProbe {
checked: mpsc::SyncSender<()>,
publisher_attempted: Arc<AtomicI32>,
published: Mutex<mpsc::Receiver<()>>,
observation: Mutex<Option<WorkerDoneWaitObservation>>,
}
#[cfg(test)]
#[derive(Debug)]
struct WorkerDoneWaitObservation {
check_sent: bool,
publisher_attempted: bool,
published: Result<(), mpsc::RecvTimeoutError>,
publisher_blocked_in_futex: bool,
publisher_syscall_unreadable: bool,
}
#[cfg(test)]
fn pause_worker_done_wait_for_test(slot: &Mutex<Option<Arc<WorkerDoneWaitProbe>>>) {
let probe = slot.lock().take();
if let Some(probe) = probe {
let check_sent = probe.checked.try_send(()).is_ok();
let attempt_deadline = Instant::now() + Duration::from_secs(1);
let publisher_tid = loop {
let tid = probe.publisher_attempted.load(Ordering::Acquire);
if tid != 0 || Instant::now() >= attempt_deadline {
break tid;
}
thread::sleep(Duration::from_millis(1));
};
let window_deadline = Instant::now() + Duration::from_millis(250);
let published_receiver = probe.published.lock();
let mut publisher_blocked_in_futex = false;
let mut publisher_syscall_unreadable = false;
let published = loop {
let slice = window_deadline
.saturating_duration_since(Instant::now())
.min(Duration::from_millis(1));
let received = published_receiver.recv_timeout(slice);
if !matches!(received, Err(mpsc::RecvTimeoutError::Timeout)) {
break received;
}
if publisher_tid != 0 {
match launch_window::read_to_string(format!(
"/proc/self/task/{publisher_tid}/syscall"
)) {
Ok(syscall) => {
publisher_blocked_in_futex = syscall
.split_ascii_whitespace()
.next()
.and_then(|nr| nr.parse::<libc::c_long>().ok())
== Some(libc::SYS_futex);
}
Err(_) => publisher_syscall_unreadable = true,
}
}
if publisher_blocked_in_futex || Instant::now() >= window_deadline {
break received;
}
};
drop(published_receiver);
*probe.observation.lock() = Some(WorkerDoneWaitObservation {
check_sent,
publisher_attempted: publisher_tid != 0,
published,
publisher_blocked_in_futex,
publisher_syscall_unreadable,
});
}
}
#[cfg(test)]
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
enum SyncWaitTestTransition {
WaitingForNotifier,
Returned,
}
#[cfg(test)]
#[derive(Debug)]
struct ExactTestMember {
pid: Pid,
pidfd: ThreadHandle,
phase: ExactCleanupPhase,
}
#[cfg(test)]
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
enum ExactCleanupPhase {
NeedsSignal,
NeedsReap,
}
#[cfg(test)]
#[derive(Clone, Copy, Debug)]
struct PendingExactRegistration {
pid: Pid,
error: Errno,
}
#[cfg(test)]
#[derive(Debug)]
struct NewChildDecodePause {
captured: mpsc::SyncSender<()>,
resume: mpsc::Receiver<()>,
members: Arc<Mutex<Vec<ExactTestMember>>>,
registration_error: Arc<Mutex<Option<PendingExactRegistration>>>,
pidfd_open_error: Arc<Mutex<Option<Errno>>>,
}
#[cfg(test)]
static EVENT_CAPTURE_PAUSES: LazyLock<Mutex<HashMap<Pid, EventCapturePause>>> =
LazyLock::new(|| Mutex::new(HashMap::new()));
#[cfg(test)]
static CURRENT_OR_NEW_CAPTURE_PAUSES: LazyLock<Mutex<HashMap<Pid, EventCapturePause>>> =
LazyLock::new(|| Mutex::new(HashMap::new()));
#[cfg(test)]
struct RetentionPause {
reached: mpsc::SyncSender<()>,
resume: mpsc::Receiver<()>,
}
#[cfg(test)]
static RETENTION_PAUSES: LazyLock<Mutex<HashMap<Pid, RetentionPause>>> =
LazyLock::new(|| Mutex::new(HashMap::new()));
#[cfg(test)]
static SPAWN_FAILURE_PAUSES: LazyLock<Mutex<HashMap<Pid, EventCapturePause>>> =
LazyLock::new(|| Mutex::new(HashMap::new()));
#[cfg(test)]
static PRE_REGISTRATION_REAP_PAUSES: LazyLock<Mutex<HashMap<Pid, EventCapturePause>>> =
LazyLock::new(|| Mutex::new(HashMap::new()));
#[cfg(test)]
static SYNC_WAIT_CLAIM_PAUSES: LazyLock<Mutex<HashMap<Pid, EventCapturePause>>> =
LazyLock::new(|| Mutex::new(HashMap::new()));
#[cfg(test)]
static SYNC_HANDLE_PAUSES: LazyLock<Mutex<HashMap<Pid, EventCapturePause>>> =
LazyLock::new(|| Mutex::new(HashMap::new()));
#[cfg(test)]
static CLEANUP_OWNER_CLAIM_PAUSES: LazyLock<Mutex<HashMap<Pid, EventCapturePause>>> =
LazyLock::new(|| Mutex::new(HashMap::new()));
#[cfg(test)]
static CLEANUP_CANCEL_SIGNAL_PAUSES: LazyLock<Mutex<HashMap<Pid, EventCapturePause>>> =
LazyLock::new(|| Mutex::new(HashMap::new()));
#[cfg(test)]
static ABSORBED_UNREGISTERED: LazyLock<Mutex<HashMap<Pid, usize>>> =
LazyLock::new(|| Mutex::new(HashMap::new()));
#[cfg(test)]
static ABSORB_LIVENESS_ERRORS: LazyLock<Mutex<HashMap<Pid, Errno>>> =
LazyLock::new(|| Mutex::new(HashMap::new()));
#[cfg(test)]
static STOP_CONSUMPTION_PAUSES: LazyLock<Mutex<HashMap<Pid, EventCapturePause>>> =
LazyLock::new(|| Mutex::new(HashMap::new()));
#[cfg(test)]
static SYNC_STATUS_PUBLICATION_PAUSES: LazyLock<Mutex<HashMap<Pid, EventCapturePause>>> =
LazyLock::new(|| Mutex::new(HashMap::new()));
#[cfg(test)]
static SYNC_RETURN_COMMIT_PAUSES: LazyLock<Mutex<HashMap<Pid, EventCapturePause>>> =
LazyLock::new(|| Mutex::new(HashMap::new()));
#[cfg(test)]
static CAPTURE_AFTER_FIRST_SNAPSHOT_PAUSES: LazyLock<Mutex<HashMap<Pid, EventCapturePause>>> =
LazyLock::new(|| Mutex::new(HashMap::new()));
#[cfg(test)]
static SYNC_DECODE_CAPTURE_ERRORS: LazyLock<Mutex<HashMap<Pid, Errno>>> =
LazyLock::new(|| Mutex::new(HashMap::new()));
#[cfg(test)]
static CAPTURE_THREAD_ERRORS: LazyLock<Mutex<HashMap<ThreadId, VecDeque<Errno>>>> =
LazyLock::new(|| Mutex::new(HashMap::new()));
#[cfg(test)]
static HOLD_TID_PAUSES: LazyLock<Mutex<HashMap<ThreadId, EventCapturePause>>> =
LazyLock::new(|| Mutex::new(HashMap::new()));
#[cfg(test)]
pub(super) fn inject_capture_error_for_current_thread(error: Errno) {
CAPTURE_THREAD_ERRORS
.lock()
.entry(thread::current().id())
.or_default()
.push_back(error);
}
#[cfg(all(test, not(sanitized)))]
pub(super) fn inject_sync_decode_capture_error(pid: Pid, error: Errno) {
SYNC_DECODE_CAPTURE_ERRORS.lock().insert(pid, error);
}
#[cfg(test)]
pub(super) fn pause_sync_new_child_decode(handle: &EventHandle) {
if let Some(pause) = handle.event().new_child_decode_pause.lock().take()
&& pause.captured.send(()).is_ok()
{
let _ = pause.resume.recv();
}
}
#[cfg(test)]
pub(super) fn register_new_child_for_test_cleanup(
handle: &EventHandle,
child: Pid,
) -> Result<(), Errno> {
let mut slot = handle.event().new_child_decode_pause.lock();
let Some(pause) = slot.as_mut() else {
return Ok(());
};
let injected = pause.pidfd_open_error.lock().take();
let pidfd = match injected.map_or_else(|| open_thread_pidfd(child), Err) {
Ok(pidfd) => pidfd,
Err(error) => {
*pause.registration_error.lock() = Some(PendingExactRegistration { pid: child, error });
return Err(error);
}
};
pause.members.lock().push(ExactTestMember {
pid: child,
pidfd,
phase: ExactCleanupPhase::NeedsSignal,
});
Ok(())
}
const INVALID_STATUS: i32 = -1;
const ECHILD_STATUS: i32 = -2;
const EXIT_PENDING: i32 = 0;
const EXIT_STOPPED: i32 = 1;
const EXIT_ECHILD: i32 = 2;
const EXIT_CAP_PENDING: u8 = 0;
const EXIT_CAP_AVAILABLE: u8 = 1;
const EXIT_CAP_CLAIMED: u8 = 2;
const EXIT_CAP_EXPIRED: u8 = 3;
const EXIT_CAP_FINALIZING: u8 = 4;
const WORKER_NOT_STARTED: i32 = 0;
const WORKER_STARTING: i32 = 1;
const WORKER_RUNNING: i32 = 2;
const WORKER_FINISHING: i32 = 3;
const WORKER_DONE: i32 = 4;
const WAIT_OWNER_NONE: u8 = 0;
const WAIT_OWNER_SYNC: u8 = 1;
const WAIT_OWNER_NOTIFIER: u8 = 2;
const WAIT_OWNER_SYNC_RETURNING: u8 = 3;
const WAIT_OWNER_NOTIFIER_RETURNING: u8 = 4;
const SYNC_IDENTITY_CAPTURE_RETRIES: usize = 8;
const PTRACE_EVENT_EXIT_STOP: i32 = (libc::PTRACE_EVENT_EXIT << 16) | (libc::SIGTRAP << 8) | 0x7f;
const PTRACE_EVENT_EXEC_STOP: i32 = (libc::PTRACE_EVENT_EXEC << 16) | (libc::SIGTRAP << 8) | 0x7f;
thread_local! {
static WAKER_CALLBACKS_MAY_HAVE_RUN: Cell<bool> = const { Cell::new(true) };
}
pub(super) fn note_waker_callbacks() {
let _ = WAKER_CALLBACKS_MAY_HAVE_RUN.try_with(|flag| flag.set(true));
}
#[derive(Debug, Default)]
struct WakerSlot {
waker: Mutex<Option<Arc<Waker>>>,
data: AtomicPtr<()>,
vtable: AtomicPtr<RawWakerVTable>,
}
impl Drop for WakerSlot {
fn drop(&mut self) {
if let Some(waker) = self.waker.get_mut().take() {
note_waker_callbacks();
drop(waker);
note_waker_callbacks();
}
}
}
impl WakerSlot {
fn register(&self, waker: &Waker) -> bool {
let data = waker.data().cast_mut();
let vtable = std::ptr::from_ref(waker.vtable()).cast_mut();
if self.data.load(Ordering::Acquire) == data
&& self.vtable.load(Ordering::Relaxed) == vtable
{
return false;
}
if self
.waker
.lock()
.as_ref()
.is_some_and(|registered| registered.will_wake(waker))
{
return false;
}
note_waker_callbacks();
let new = Arc::new(waker.clone());
let old = {
let mut slot = self.waker.lock();
let old = slot.replace(new);
self.vtable.store(vtable, Ordering::Relaxed);
self.data.store(data, Ordering::Release);
old
};
drop(old);
note_waker_callbacks();
true
}
fn wake(&self) -> bool {
note_waker_callbacks();
let waker = self.waker.lock().clone();
let registered = waker.is_some();
if let Some(waker) = waker {
waker.wake_by_ref();
}
note_waker_callbacks();
registered
}
}
#[derive(Debug, Default)]
struct ExitWaiter {
waker: WakerSlot,
epoch: usize,
}
#[derive(Debug, Default)]
struct ExitWaiters {
waiters: Mutex<Vec<Weak<ExitWaiter>>>,
}
impl ExitWaiters {
fn register(&self, waiter: &Arc<ExitWaiter>, waker: &Waker) {
waiter.waker.register(waker);
let weak = Arc::downgrade(waiter);
let mut waiters = self.waiters.lock();
waiters.retain(|registered| registered.strong_count() != 0);
if !waiters.iter().any(|registered| registered.ptr_eq(&weak)) {
waiters.push(weak);
}
}
fn wake_all(&self) -> usize {
let live = {
let mut waiters = self.waiters.lock();
let mut live = Vec::with_capacity(waiters.len());
waiters.retain(|waiter| {
if let Some(waiter) = waiter.upgrade() {
live.push(waiter);
true
} else {
false
}
});
live
};
let woken = live.len();
for waiter in live {
waiter.waker.wake();
}
woken
}
}
#[derive(Clone, Copy, Debug)]
struct CachedRegs {
regs: crate::Regs,
dirty: bool,
}
#[derive(Debug)]
struct Event {
exit_waiters: ExitWaiters,
exit_publication: Mutex<()>,
terminal_reaping: RwLock<bool>,
numeric_auth_epoch: AtomicUsize,
numeric_auth_checked_epoch: AtomicUsize,
#[cfg(test)]
numeric_auth_status_reads: AtomicUsize,
#[cfg(test)]
numeric_auth_force_directory: AtomicBool,
regs_cache: Mutex<Option<CachedRegs>>,
#[cfg(test)]
regs_kernel_reads: AtomicUsize,
#[cfg(test)]
regs_kernel_writes: AtomicUsize,
status_waker: WakerSlot,
status: Mutex<StatusState>,
status_changed: Condvar,
#[cfg(test)]
sync_wait_entered: Mutex<Option<mpsc::SyncSender<SyncWaitTestTransition>>>,
#[cfg(test)]
sync_wait_ineligible_pause: Mutex<Option<BoundedTestPause>>,
#[cfg(test)]
sync_wait_owner_entered: Mutex<Option<mpsc::SyncSender<()>>>,
#[cfg(test)]
notifier_registration_pause: Mutex<Option<(i32, BoundedTestPause)>>,
legacy_wait: Mutex<LegacyWaitProgress>,
legacy_wakeup: Mutex<LegacyWakeups>,
legacy_wake: Condvar,
legacy_wake_deferred: AtomicBool,
#[cfg(test)]
legacy_held_period: Mutex<Option<Duration>>,
#[cfg(test)]
legacy_held_wait_pause: Mutex<Option<BoundedTestPause>>,
#[cfg(test)]
legacy_held_wait_paused_on: Mutex<Option<(i32, u64)>>,
#[cfg(test)]
legacy_held_waits: AtomicUsize,
#[cfg(test)]
legacy_notify_wakes: AtomicUsize,
#[cfg(test)]
legacy_worker_wakes: AtomicUsize,
#[cfg(test)]
legacy_held_wait_wakes: AtomicUsize,
#[cfg(test)]
legacy_publish_pause: Mutex<Option<BoundedTestPause>>,
#[cfg(test)]
legacy_late_turns: AtomicUsize,
#[cfg(test)]
legacy_poll_period: Mutex<Option<Duration>>,
#[cfg(test)]
legacy_sleep_check_pause: Mutex<Option<BoundedTestPause>>,
#[cfg(test)]
legacy_parking_signal: Mutex<Option<mpsc::SyncSender<()>>>,
#[cfg(test)]
legacy_assist_notify_pause: Mutex<Option<BoundedTestPause>>,
legacy_wait_owner: OnceLock<LegacyWaitOwner>,
native_wait_identity: OnceLock<Arc<WorkerIdentity>>,
exit_status: AtomicI32,
exit_capability: AtomicU8,
exit_epoch: AtomicUsize,
exit_stop_follows_dead_exec: AtomicBool,
registration_error: Mutex<Option<Errno>>,
detached_waits: Mutex<Vec<(u64, Option<Arc<LegacyWaitOwner>>)>>,
detached_wait_seq: AtomicU64,
#[cfg(test)]
detached_echild_kept: AtomicUsize,
worker_state: AtomicI32,
worker_done_lock: Mutex<()>,
worker_done_changed: Condvar,
#[cfg(test)]
worker_done_wait_probe: Mutex<Option<Arc<WorkerDoneWaitProbe>>>,
#[cfg(test)]
worker_done_park_signal: Mutex<Option<mpsc::SyncSender<()>>>,
cleanup_cancel_requested: AtomicBool,
cleanup_claim_waiters: AtomicUsize,
wait_owner: AtomicU8,
wait_owner_lock: Mutex<()>,
wait_owner_changed: Condvar,
#[cfg(test)]
new_child_decode_pause: Mutex<Option<NewChildDecodePause>>,
#[cfg(test)]
cleanup_return_pause: Mutex<Option<BoundedTestPause>>,
#[cfg(test)]
terminal_publish_pause: Mutex<Option<BoundedTestPause>>,
#[cfg(test)]
registry_retirement_pause: Mutex<Option<BoundedTestPause>>,
#[cfg(test)]
worker_identity_retirement_pause: Mutex<Option<BoundedTestPause>>,
#[cfg(test)]
terminal_echild_pause: Mutex<Option<BoundedTestPause>>,
#[cfg(test)]
exit_report_pause: Mutex<Option<BoundedTestPause>>,
#[cfg(test)]
legacy_observation_pause: Mutex<Option<BoundedTestPause>>,
#[cfg(test)]
legacy_worker_cycle_pause: Mutex<Option<BoundedTestPause>>,
#[cfg(test)]
legacy_worker_turns: AtomicUsize,
#[cfg(test)]
legacy_worker_status_reads: AtomicUsize,
#[cfg(test)]
legacy_hint_turns: AtomicUsize,
#[cfg(test)]
legacy_held_expiry_finds: AtomicUsize,
#[cfg(test)]
legacy_deferred_flushes: AtomicUsize,
#[cfg(test)]
legacy_deferred_exit_wakes: AtomicUsize,
#[cfg(test)]
legacy_deferred_status_wakes: AtomicUsize,
#[cfg(all(test, not(sanitized)))]
legacy_wake_log: Mutex<Vec<LegacyWakeLogEntry>>,
#[cfg(all(test, not(sanitized)))]
legacy_deferred_receipt: Mutex<Option<LegacyCountReceipt>>,
#[cfg(all(test, not(sanitized)))]
legacy_count_serial: AtomicU64,
}
pub(super) struct TidHold<'a> {
_guard: RwLockReadGuard<'a, bool>,
}
#[derive(Debug, Default)]
struct LegacyWaitProgress {
observed: Option<i32>,
consumed: VecDeque<i32>,
error: Option<Errno>,
retiring: bool,
}
#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
struct LegacyWakeups {
targeted: u64,
probes: u64,
}
#[cfg(all(test, not(sanitized)))]
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
enum LegacyWakeLogEntry {
HoldEndedByCount(u64),
Uncounted,
Hint,
WaitBegin(u64),
CountedWake,
Notified(LegacyNotifySource),
Counted(LegacyCountReceipt),
}
#[cfg(all(test, not(sanitized)))]
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
struct LegacyCountReceipt {
serial: u64,
targeted: u64,
}
#[cfg(all(test, not(sanitized)))]
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
enum LegacyNotifySource {
Immediate(Option<LegacyCountReceipt>),
Flush(Option<LegacyCountReceipt>),
Probe(u64),
Consume(Option<LegacyCountReceipt>),
Unknown,
}
#[cfg(all(test, not(sanitized)))]
thread_local! {
static LEGACY_COUNT_RECEIPT: Cell<Option<LegacyCountReceipt>> = const { Cell::new(None) };
static LEGACY_NOTIFY_SOURCE: Cell<Option<LegacyNotifySource>> = const { Cell::new(None) };
}
#[cfg(all(test, not(sanitized)))]
fn note_legacy_notify_source(source: LegacyNotifySource) {
LEGACY_NOTIFY_SOURCE.with(|next| next.set(Some(source)));
}
#[cfg(all(test, not(sanitized)))]
fn take_legacy_count_receipt() -> Option<LegacyCountReceipt> {
LEGACY_COUNT_RECEIPT.with(Cell::take)
}
#[derive(Debug)]
pub(super) struct LegacyWaitOwner {
tid: Pid,
tgid: Pid,
lifetime: OwnedFd,
self_signal_fd: Option<OwnedFd>,
#[cfg(test)]
force_directory_proof: AtomicBool,
directory_key: ProcDirectoryKey,
proc_root: AlignedProcfs,
}
pub(super) struct AttachmentAnchor {
pid: Pid,
directory: OwnedFd,
snapshot: WorkerProcSnapshot,
inode: u64,
_root: AlignedProcfs,
}
#[cfg(test)]
type AttachmentCaptureHook = Box<dyn FnOnce(Pid, &AttachmentAnchor)>;
#[cfg(test)]
thread_local! {
static ATTACHMENT_CAPTURE_HOOK: std::cell::RefCell<Option<AttachmentCaptureHook>> = const { std::cell::RefCell::new(None) };
static PTRACER_OWNER_CAPTURES: Cell<usize> = const { Cell::new(0) };
static PTRACER_HOST_PROOFS: Cell<usize> = const { Cell::new(0) };
static WAKER_CALLBACK_RECHECKS: Cell<usize> = const { Cell::new(0) };
static PTRACER_ATTACHMENT_STATUS_READS: Cell<usize> = const { Cell::new(0) };
static RETAINED_STATUS_READS: Cell<usize> = const { Cell::new(0) };
static OTHER_STATUS_OPENS: Cell<usize> = const { Cell::new(0) };
static LEGACY_STOP_PROOFS_ACCEPTED: Cell<usize> = const { Cell::new(0) };
static LEGACY_STOP_PROOFS_REFUSED: Cell<usize> = const { Cell::new(0) };
static LEGACY_ASSIST_CONSUMPTIONS: Cell<usize> = const { Cell::new(0) };
}
#[cfg(test)]
pub(super) fn after_initial_attachment_for_test(pid: Pid, original: &AttachmentAnchor) {
let hook = ATTACHMENT_CAPTURE_HOOK.with(|slot| slot.borrow_mut().take());
if let Some(hook) = hook {
hook(pid, original);
}
}
impl AttachmentAnchor {
pub(super) fn capture(pid: Pid) -> Result<Self, Errno> {
let root = AlignedProcfs::open()?;
let directory = root.open_pid(pid, libc::O_PATH)?;
let snapshot = retained_proc_snapshot(directory.as_raw_fd()).map_err(io_errno)?;
let inode = fd_inode(&directory).map_err(io_errno)?;
Ok(Self {
pid,
directory,
snapshot,
inode,
_root: root,
})
}
fn validate_attached(&self, identity: &WorkerIdentity) -> Result<(), Errno> {
let status = retained_proc_status(self.directory.as_raw_fd())?;
let current = retained_proc_snapshot(self.directory.as_raw_fd()).map_err(io_errno)?;
if status.pid != self.pid
|| status.tracer_pid != Pid::from(nix::unistd::gettid())
|| identity.pid != self.pid
|| !self.snapshot.same_process_generation(¤t)
|| !current.same_process_generation(&identity.snapshot)
|| self.inode != identity.proc_inode
{
return Err(Errno::ESRCH);
}
Ok(())
}
}
impl LegacyWaitOwner {
fn capture_current() -> Result<Self, Errno> {
Self::capture_at(AlignedProcfs::open()?)
}
fn capture_at(root: AlignedProcfs) -> Result<Self, Errno> {
#[cfg(test)]
PTRACER_OWNER_CAPTURES.with(|captures| captures.set(captures.get() + 1));
let tid = Pid::from(nix::unistd::gettid());
let lifetime = root.open_current_thread()?;
let current = retained_proc_status(lifetime.as_raw_fd())?;
if current.pid != tid || current.tgid != Pid::from(nix::unistd::getpid()) {
return Err(Errno::EXDEV);
}
let directory_key = fd_directory_key(&lifetime).map_err(io_errno)?;
let self_signal_fd = {
let _open = launch_window::TransientOpen::begin();
pidfd_open_with_flags(tid, libc::O_EXCL)
.or_else(|_| root.open_pid(tid, libc::O_RDONLY))
.ok()
};
Ok(Self {
tid,
tgid: Pid::from(nix::unistd::getpid()),
lifetime,
self_signal_fd,
#[cfg(test)]
force_directory_proof: AtomicBool::new(false),
directory_key,
proc_root: root,
})
}
fn is_live(&self) -> Result<bool, Errno> {
if let Some(fd) = &self.self_signal_fd
&& let Ok(live) = descriptor_is_live(fd.as_raw_fd())
{
return Ok(live);
}
match retained_proc_status(self.lifetime.as_raw_fd()) {
Ok(status) => Ok(status.pid == self.tid && status.tgid == self.tgid),
Err(Errno::ENOENT | Errno::ESRCH) => Ok(false),
Err(error) => Err(error),
}
}
pub(super) fn is_current(&self) -> Result<bool, Errno> {
#[cfg(test)]
let use_descriptor = !self.force_directory_proof.load(Ordering::Relaxed);
#[cfg(not(test))]
let use_descriptor = true;
if use_descriptor
&& self
.self_signal_fd
.as_ref()
.is_some_and(|fd| descriptor_is_current_task(fd.as_raw_fd()))
{
return Ok(true);
}
if self.tid != Pid::from(nix::unistd::gettid()) {
return Ok(false);
}
Ok(self.proc_root.directory_key(c"thread-self")? == self.directory_key)
}
}
#[derive(Debug)]
struct StatusState {
pending: VecDeque<i32>,
terminal: i32,
before_exit: usize,
before_exit_epoch: usize,
retired_before_exit: u64,
retired_dead_exec: u64,
epoch_exec: Option<usize>,
}
fn stop_records_lifecycle_state(status: i32) -> bool {
!libc::WIFSTOPPED(status)
|| matches!(
(status >> 16) & 0xff,
libc::PTRACE_EVENT_FORK
| libc::PTRACE_EVENT_VFORK
| libc::PTRACE_EVENT_CLONE
| libc::PTRACE_EVENT_EXEC
)
}
fn stop_reports_new_child(status: i32) -> bool {
libc::WIFSTOPPED(status)
&& matches!(
(status >> 16) & 0xff,
libc::PTRACE_EVENT_FORK | libc::PTRACE_EVENT_VFORK | libc::PTRACE_EVENT_CLONE
)
}
impl StatusState {
fn pop_front(&mut self) -> Option<i32> {
let front = self.pending.pop_front();
if front.is_some() {
self.before_exit = self.before_exit.saturating_sub(1);
self.epoch_exec = self.epoch_exec.and_then(|position| position.checked_sub(1));
}
debug_assert!(self.before_exit <= self.pending.len());
front
}
fn retire_before_exit(&mut self) -> usize {
debug_assert!(self.before_exit <= self.pending.len());
let count = self.before_exit.min(self.pending.len());
if self
.pending
.iter()
.take(count)
.enumerate()
.any(|(position, &status)| {
stop_records_lifecycle_state(status)
&& (status != PTRACE_EVENT_EXEC_STOP || self.epoch_exec == Some(position))
})
{
return 0;
}
self.pending.drain(..count);
self.epoch_exec = self.epoch_exec.map(|position| position - count);
self.before_exit = 0;
self.retired_before_exit += count as u64;
count
}
fn retire_before_exit_of(&mut self, epoch: usize) -> usize {
if self.before_exit_epoch != epoch {
return 0;
}
self.retire_before_exit()
}
}
struct StatusReservation<'a> {
status: i32,
state: Option<MutexGuard<'a, StatusState>>,
}
enum StatusReturn<T> {
Returned(T),
Cancelled(i32),
RetiredDeadExec,
}
struct ReturnTransaction<'a> {
event: &'a Event,
returning_owner: u8,
rollback_owner: u8,
completed: bool,
}
impl ReturnTransaction<'_> {
fn commit(mut self, final_owner: u8) {
let _guard = self.event.wait_owner_lock.lock();
self.event
.wait_owner
.compare_exchange(
self.returning_owner,
final_owner,
Ordering::Release,
Ordering::Acquire,
)
.expect("status return ownership changed before commit");
self.completed = true;
self.event.wait_owner_changed.notify_all();
}
}
impl Drop for ReturnTransaction<'_> {
fn drop(&mut self) {
if self.completed {
return;
}
let _guard = self.event.wait_owner_lock.lock();
self.event
.wait_owner
.compare_exchange(
self.returning_owner,
self.rollback_owner,
Ordering::Release,
Ordering::Acquire,
)
.expect("status return ownership changed before rollback");
self.event.wait_owner_changed.notify_all();
}
}
enum ReturnTransactionStart<'a> {
Begun(ReturnTransaction<'a>),
Cancelled,
}
struct SyncWaitOwner<'a> {
event: &'a Event,
released: bool,
}
impl SyncWaitOwner<'_> {
fn decode_status_return<T>(
&mut self,
_pid: Pid,
reservation: StatusReservation<'_>,
decode: impl FnOnce(i32) -> Result<T, Error>,
) -> Result<StatusReturn<T>, Error> {
let transaction = match self.event.begin_status_return(
reservation.status,
WAIT_OWNER_SYNC,
WAIT_OWNER_SYNC_RETURNING,
) {
ReturnTransactionStart::Begun(transaction) => transaction,
ReturnTransactionStart::Cancelled => {
return Ok(StatusReturn::Cancelled(reservation.status));
}
};
#[cfg(test)]
if let Some(pause) = SYNC_RETURN_COMMIT_PAUSES.lock().remove(&_pid) {
pause.captured.wait();
pause.resume.wait();
}
let decoded = match decode(reservation.status) {
Ok(decoded) => decoded,
Err(error @ Error::Died(_)) => {
if reservation.status != PTRACE_EVENT_EXEC_STOP {
return Err(error);
}
let dead_exec = reservation.retire_dead_exec();
transaction.commit(WAIT_OWNER_NONE);
self.released = true;
if !dead_exec {
return Err(error);
}
self.event.forward_exit_stop_after_dead_exec();
return Ok(StatusReturn::RetiredDeadExec);
}
Err(error) => return Err(error),
};
reservation.commit();
transaction.commit(WAIT_OWNER_NONE);
self.released = true;
Ok(StatusReturn::Returned(decoded))
}
}
impl Drop for SyncWaitOwner<'_> {
fn drop(&mut self) {
if self.released {
return;
}
let _guard = self.event.wait_owner_lock.lock();
self.event
.wait_owner
.compare_exchange(
WAIT_OWNER_SYNC,
WAIT_OWNER_NONE,
Ordering::Release,
Ordering::Acquire,
)
.expect("synchronous wait ownership changed before release");
self.event.wait_owner_changed.notify_all();
}
}
enum SyncWaitOwnership<'a> {
Claimed(SyncWaitOwner<'a>),
Notifier,
}
struct NotifierWaitOwner<'a> {
event: &'a Event,
committed: bool,
}
impl NotifierWaitOwner<'_> {
fn commit(mut self) {
self.committed = true;
self.event.notify_wait_owner_change();
}
}
impl Drop for NotifierWaitOwner<'_> {
fn drop(&mut self) {
if self.committed {
return;
}
let _guard = self.event.wait_owner_lock.lock();
self.event
.wait_owner
.compare_exchange(
WAIT_OWNER_NOTIFIER,
WAIT_OWNER_NONE,
Ordering::Release,
Ordering::Acquire,
)
.expect("notifier wait ownership changed before rollback");
self.event.wait_owner_changed.notify_all();
}
}
enum NotifierWaitOwnership<'a> {
Claimed(NotifierWaitOwner<'a>),
Existing,
}
#[cfg(test)]
enum CancellableNotifierWaitOwnership<'a> {
Claimed(NotifierWaitOwner<'a>),
Synchronous,
Existing,
Returning,
}
impl StatusReservation<'_> {
fn commit(mut self) {
if let Some(state) = self.state.as_mut() {
let committed = state.pop_front();
debug_assert_eq!(committed, Some(self.status));
}
}
fn retire_dead_exec(mut self) -> bool {
let dead_exec = self
.state
.as_ref()
.is_some_and(|state| state.epoch_exec == Some(0));
debug_assert!(!dead_exec || self.status == PTRACE_EVENT_EXEC_STOP);
if dead_exec {
self.state.as_mut().unwrap().retired_dead_exec += 1;
}
self.commit();
dead_exec
}
}
impl Event {
pub fn new() -> Self {
Self {
exit_waiters: ExitWaiters::default(),
exit_publication: Mutex::new(()),
terminal_reaping: RwLock::new(false),
numeric_auth_epoch: AtomicUsize::new(1),
numeric_auth_checked_epoch: AtomicUsize::new(0),
#[cfg(test)]
numeric_auth_status_reads: AtomicUsize::new(0),
#[cfg(test)]
numeric_auth_force_directory: AtomicBool::new(false),
regs_cache: Mutex::new(None),
#[cfg(test)]
regs_kernel_reads: AtomicUsize::new(0),
#[cfg(test)]
regs_kernel_writes: AtomicUsize::new(0),
status_waker: WakerSlot::default(),
status: Mutex::new(StatusState {
pending: VecDeque::new(),
terminal: INVALID_STATUS,
before_exit: 0,
before_exit_epoch: 0,
retired_before_exit: 0,
retired_dead_exec: 0,
epoch_exec: None,
}),
status_changed: Condvar::new(),
#[cfg(test)]
sync_wait_entered: Mutex::new(None),
#[cfg(test)]
sync_wait_ineligible_pause: Mutex::new(None),
#[cfg(test)]
sync_wait_owner_entered: Mutex::new(None),
#[cfg(test)]
notifier_registration_pause: Mutex::new(None),
legacy_wait: Mutex::new(LegacyWaitProgress::default()),
legacy_wakeup: Mutex::new(LegacyWakeups::default()),
legacy_wake: Condvar::new(),
legacy_wake_deferred: AtomicBool::new(false),
#[cfg(test)]
legacy_held_period: Mutex::new(None),
#[cfg(test)]
legacy_held_wait_pause: Mutex::new(None),
#[cfg(test)]
legacy_held_wait_paused_on: Mutex::new(None),
#[cfg(test)]
legacy_held_waits: AtomicUsize::new(0),
#[cfg(test)]
legacy_notify_wakes: AtomicUsize::new(0),
#[cfg(test)]
legacy_worker_wakes: AtomicUsize::new(0),
#[cfg(test)]
legacy_held_wait_wakes: AtomicUsize::new(0),
#[cfg(test)]
legacy_publish_pause: Mutex::new(None),
#[cfg(test)]
legacy_late_turns: AtomicUsize::new(0),
#[cfg(test)]
legacy_poll_period: Mutex::new(None),
#[cfg(test)]
legacy_sleep_check_pause: Mutex::new(None),
#[cfg(test)]
legacy_parking_signal: Mutex::new(None),
#[cfg(test)]
legacy_assist_notify_pause: Mutex::new(None),
legacy_wait_owner: OnceLock::new(),
native_wait_identity: OnceLock::new(),
exit_status: AtomicI32::new(EXIT_PENDING),
exit_capability: AtomicU8::new(EXIT_CAP_PENDING),
exit_epoch: AtomicUsize::new(0),
exit_stop_follows_dead_exec: AtomicBool::new(false),
registration_error: Mutex::new(None),
detached_waits: Mutex::new(Vec::new()),
detached_wait_seq: AtomicU64::new(0),
#[cfg(test)]
detached_echild_kept: AtomicUsize::new(0),
worker_state: AtomicI32::new(WORKER_NOT_STARTED),
worker_done_lock: Mutex::new(()),
worker_done_changed: Condvar::new(),
#[cfg(test)]
worker_done_wait_probe: Mutex::new(None),
#[cfg(test)]
worker_done_park_signal: Mutex::new(None),
cleanup_cancel_requested: AtomicBool::new(false),
cleanup_claim_waiters: AtomicUsize::new(0),
wait_owner: AtomicU8::new(WAIT_OWNER_NONE),
wait_owner_lock: Mutex::new(()),
wait_owner_changed: Condvar::new(),
#[cfg(test)]
new_child_decode_pause: Mutex::new(None),
#[cfg(test)]
cleanup_return_pause: Mutex::new(None),
#[cfg(test)]
terminal_publish_pause: Mutex::new(None),
#[cfg(test)]
registry_retirement_pause: Mutex::new(None),
#[cfg(test)]
worker_identity_retirement_pause: Mutex::new(None),
#[cfg(test)]
terminal_echild_pause: Mutex::new(None),
#[cfg(test)]
exit_report_pause: Mutex::new(None),
#[cfg(test)]
legacy_observation_pause: Mutex::new(None),
#[cfg(test)]
legacy_worker_cycle_pause: Mutex::new(None),
#[cfg(test)]
legacy_worker_turns: AtomicUsize::new(0),
#[cfg(test)]
legacy_worker_status_reads: AtomicUsize::new(0),
#[cfg(test)]
legacy_hint_turns: AtomicUsize::new(0),
#[cfg(test)]
legacy_held_expiry_finds: AtomicUsize::new(0),
#[cfg(test)]
legacy_deferred_flushes: AtomicUsize::new(0),
#[cfg(test)]
legacy_deferred_exit_wakes: AtomicUsize::new(0),
#[cfg(test)]
legacy_deferred_status_wakes: AtomicUsize::new(0),
#[cfg(all(test, not(sanitized)))]
legacy_wake_log: Mutex::new(Vec::new()),
#[cfg(all(test, not(sanitized)))]
legacy_deferred_receipt: Mutex::new(None),
#[cfg(all(test, not(sanitized)))]
legacy_count_serial: AtomicU64::new(0),
}
}
fn mark_terminal_reaping(&self) {
*self.terminal_reaping.write() = true;
}
fn invalidate_numeric_auth(&self) {
self.numeric_auth_epoch.fetch_add(1, Ordering::AcqRel);
}
fn wake_legacy_worker(&self) {
self.count_legacy_wakeup();
#[cfg(all(test, not(sanitized)))]
let receipt = take_legacy_count_receipt();
self.legacy_wake_deferred.store(false, Ordering::Release);
#[cfg(all(test, not(sanitized)))]
note_legacy_notify_source(LegacyNotifySource::Immediate(receipt));
self.notify_legacy_worker();
}
fn defer_legacy_wake(&self) {
self.count_legacy_wakeup();
#[cfg(all(test, not(sanitized)))]
{
*self.legacy_deferred_receipt.lock() = take_legacy_count_receipt();
}
self.legacy_wake_deferred.store(true, Ordering::Release);
let _exit_woken = self.exit_waiters.wake_all();
let _status_woken = self.status_waker.wake();
#[cfg(test)]
{
if _exit_woken > 0 {
self.legacy_deferred_exit_wakes
.fetch_add(1, Ordering::Relaxed);
}
if _status_woken {
self.legacy_deferred_status_wakes
.fetch_add(1, Ordering::Relaxed);
}
}
}
fn flush_deferred_legacy_wake(&self) {
if self.legacy_wake_deferred.swap(false, Ordering::AcqRel) {
#[cfg(test)]
self.legacy_deferred_flushes.fetch_add(1, Ordering::Relaxed);
#[cfg(all(test, not(sanitized)))]
note_legacy_notify_source(LegacyNotifySource::Flush(
self.legacy_deferred_receipt.lock().take(),
));
self.notify_legacy_worker();
}
}
fn notify_legacy_worker(&self) {
let _woke = self.legacy_wake.notify_one();
#[cfg(all(test, not(sanitized)))]
{
let source = LEGACY_NOTIFY_SOURCE
.with(Cell::take)
.unwrap_or(LegacyNotifySource::Unknown);
if _woke {
self.legacy_wake_log
.lock()
.push(LegacyWakeLogEntry::Notified(source));
}
}
#[cfg(test)]
if _woke {
self.legacy_notify_wakes.fetch_add(1, Ordering::Relaxed);
}
}
fn count_legacy_wakeup(&self) {
let mut wakeups = self.legacy_wakeup.lock();
wakeups.targeted = wakeups.targeted.wrapping_add(1);
#[cfg(all(test, not(sanitized)))]
{
let receipt = LegacyCountReceipt {
serial: self.legacy_count_serial.fetch_add(1, Ordering::Relaxed) + 1,
targeted: wakeups.targeted,
};
self.legacy_wake_log
.lock()
.push(LegacyWakeLogEntry::Counted(receipt));
LEGACY_COUNT_RECEIPT.with(|slot| slot.set(Some(receipt)));
}
}
fn probe_legacy_worker(&self) {
let mut wakeups = self.legacy_wakeup.lock();
wakeups.probes = wakeups.probes.wrapping_add(1);
#[cfg(all(test, not(sanitized)))]
note_legacy_notify_source(LegacyNotifySource::Probe(wakeups.probes));
drop(wakeups);
self.notify_legacy_worker();
}
fn hold_tid(&self) -> Option<TidHold<'_>> {
let guard = self.terminal_reaping.read();
(!*guard).then_some(TidHold { _guard: guard })
}
fn expire_unclaimed_exit_capability(&self) -> bool {
loop {
let state = self.exit_capability.load(Ordering::Acquire);
match state {
EXIT_CAP_PENDING | EXIT_CAP_AVAILABLE => {
let replacement = if state == EXIT_CAP_PENDING {
EXIT_CAP_FINALIZING
} else {
EXIT_CAP_EXPIRED
};
if self
.exit_capability
.compare_exchange(state, replacement, Ordering::AcqRel, Ordering::Acquire)
.is_ok()
{
return state == EXIT_CAP_PENDING;
}
}
EXIT_CAP_CLAIMED | EXIT_CAP_EXPIRED => return false,
EXIT_CAP_FINALIZING => std::hint::spin_loop(),
state => unreachable!("invalid exit capability state {state}"),
}
}
}
fn prepare_exit_capability_for_cleanup(&self, owns_claimed: bool) -> Result<(), Errno> {
loop {
let state = self.exit_capability.load(Ordering::Acquire);
match state {
EXIT_CAP_PENDING | EXIT_CAP_AVAILABLE => {
if self
.exit_capability
.compare_exchange(
state,
EXIT_CAP_EXPIRED,
Ordering::AcqRel,
Ordering::Acquire,
)
.is_ok()
{
self.exit_waiters.wake_all();
return Ok(());
}
}
EXIT_CAP_CLAIMED if owns_claimed => {
if self
.exit_capability
.compare_exchange(
EXIT_CAP_CLAIMED,
EXIT_CAP_EXPIRED,
Ordering::AcqRel,
Ordering::Acquire,
)
.is_ok()
{
self.exit_waiters.wake_all();
return Ok(());
}
}
EXIT_CAP_CLAIMED => return Err(Errno::EALREADY),
EXIT_CAP_EXPIRED => return Ok(()),
EXIT_CAP_FINALIZING => std::hint::spin_loop(),
state => unreachable!("invalid exit capability state {state}"),
}
}
}
fn publish_exit_stop(&self, between_status_and_capability: impl FnOnce()) {
let publication = self.exit_publication.lock();
let Some(publication) = self.queue_exit_stop_after_exit(publication) else {
return;
};
#[cfg(test)]
if let Some(pause) = self.exit_report_pause.lock().take() {
let _ = pause.captured.send(());
let _ = pause.resume.recv_timeout(Duration::from_secs(2));
}
let previous = self.exit_status.compare_exchange(
EXIT_PENDING,
EXIT_STOPPED,
Ordering::Release,
Ordering::Acquire,
);
debug_assert!(matches!(
previous,
Ok(EXIT_PENDING) | Err(EXIT_STOPPED | EXIT_ECHILD)
));
if previous.is_ok() {
let mut state = self.status.lock();
state.before_exit = state.pending.len();
state.before_exit_epoch = self.exit_epoch.load(Ordering::Acquire);
drop(state);
between_status_and_capability();
let capability = self.exit_capability.compare_exchange(
EXIT_CAP_PENDING,
EXIT_CAP_AVAILABLE,
Ordering::AcqRel,
Ordering::Acquire,
);
debug_assert!(matches!(
capability,
Ok(EXIT_CAP_PENDING) | Err(EXIT_CAP_EXPIRED)
));
}
drop(publication);
self.status_changed.notify_all();
self.exit_waiters.wake_all();
}
fn publish_terminal_exit_state(&self, terminal: i32) -> i32 {
let _publication = self.exit_publication.lock();
let finalizing = self.expire_unclaimed_exit_capability();
let previous = {
let mut state = self.status.lock();
let previous = state.terminal;
if terminal == ECHILD_STATUS {
if previous == INVALID_STATUS {
state.terminal = ECHILD_STATUS;
}
} else if previous == INVALID_STATUS || previous == ECHILD_STATUS {
state.terminal = terminal;
} else {
debug_assert_eq!(previous, terminal, "terminal publication changed");
}
previous
};
#[cfg(test)]
if let Some(pause) = self.terminal_echild_pause.lock().take() {
let _ = pause.captured.send(());
let _ = pause.resume.recv_timeout(Duration::from_secs(2));
}
let _ = self.exit_status.compare_exchange(
EXIT_PENDING,
EXIT_ECHILD,
Ordering::Release,
Ordering::Acquire,
);
if finalizing {
self.exit_capability
.compare_exchange(
EXIT_CAP_FINALIZING,
EXIT_CAP_EXPIRED,
Ordering::Release,
Ordering::Acquire,
)
.expect("terminal exit-capability publication changed");
}
previous
}
fn queue_exit_stop_after_exit<'a>(
&'a self,
publication: MutexGuard<'a, ()>,
) -> Option<MutexGuard<'a, ()>> {
if self
.exit_stop_follows_dead_exec
.swap(false, Ordering::AcqRel)
{
self.status.lock().pending.push_back(PTRACE_EVENT_EXIT_STOP);
drop(publication);
self.status_changed.notify_all();
self.status_waker.wake();
return None;
}
if self.exit_status.load(Ordering::Acquire) != EXIT_STOPPED {
return Some(publication);
}
self.exit_epoch.fetch_add(1, Ordering::AcqRel);
self.exit_capability
.store(EXIT_CAP_PENDING, Ordering::Release);
self.exit_status.store(EXIT_PENDING, Ordering::Release);
self.status.lock().pending.push_back(PTRACE_EVENT_EXIT_STOP);
drop(publication);
self.status_changed.notify_all();
self.status_waker.wake();
self.exit_waiters.wake_all();
None
}
fn forward_exit_stop_after_dead_exec(&self) {
let publication = self.exit_publication.lock();
match self.exit_status.load(Ordering::Acquire) {
EXIT_PENDING => {
self.exit_stop_follows_dead_exec
.store(true, Ordering::Release);
return;
}
EXIT_STOPPED => {}
_ => return,
}
if self
.exit_capability
.compare_exchange(
EXIT_CAP_AVAILABLE,
EXIT_CAP_PENDING,
Ordering::AcqRel,
Ordering::Acquire,
)
.is_err()
{
return;
}
self.exit_epoch.fetch_add(1, Ordering::AcqRel);
self.exit_status.store(EXIT_PENDING, Ordering::Release);
self.status.lock().pending.push_back(PTRACE_EVENT_EXIT_STOP);
drop(publication);
self.status_changed.notify_all();
self.status_waker.wake();
self.exit_waiters.wake_all();
}
fn observe_exec_after_exit(&self) -> bool {
let publication = self.exit_publication.lock();
if self.exit_status.load(Ordering::Acquire) != EXIT_STOPPED {
return false;
}
self.exit_epoch.fetch_add(1, Ordering::AcqRel);
self.exit_capability
.store(EXIT_CAP_PENDING, Ordering::Release);
self.exit_status.store(EXIT_PENDING, Ordering::Release);
drop(publication);
self.exit_waiters.wake_all();
true
}
pub fn update(&self, status: i32) -> Option<i32> {
self.invalidate_numeric_auth();
if status == PTRACE_EVENT_EXIT_STOP {
self.publish_exit_stop(|| {});
return None;
}
let epoch_exec = status == PTRACE_EVENT_EXEC_STOP && self.observe_exec_after_exit();
let terminal = libc::WIFEXITED(status) || libc::WIFSIGNALED(status);
let previous = if terminal {
self.publish_terminal_exit_state(status)
} else {
let mut state = self.status.lock();
let previous = state.pending.back().copied().unwrap_or(INVALID_STATUS);
if epoch_exec {
state.epoch_exec = Some(state.pending.len());
}
state.pending.push_back(status);
previous
};
self.status_changed.notify_all();
if terminal {
self.status_waker.wake();
self.exit_waiters.wake_all();
} else {
self.status_waker.wake();
}
(previous != INVALID_STATUS).then_some(previous)
}
fn update_sync_status(&self, status: i32) {
if status != PTRACE_EVENT_EXIT_STOP {
self.update(status);
return;
}
self.invalidate_numeric_auth();
self.exit_stop_follows_dead_exec
.store(false, Ordering::Release);
let mut state = self.status.lock();
state.before_exit = state.pending.len();
state.retire_before_exit();
state.pending.push_back(status);
drop(state);
self.status_changed.notify_all();
self.status_waker.wake();
}
fn retire_statuses_before_exit_stop(&self, epoch: usize) -> usize {
self.status.lock().retire_before_exit_of(epoch)
}
fn superseded_new_child(&self, epoch: usize) -> Result<(), SupersededStopRefusal> {
let state = self.status.lock();
if state.before_exit_epoch != epoch {
return Ok(());
}
match state
.pending
.iter()
.take(state.before_exit)
.find(|&&status| stop_reports_new_child(status))
{
Some(&status) => Err(SupersededStopRefusal::NewChild(status)),
None => Ok(()),
}
}
fn mark_echild(&self) {
self.publish_terminal_exit_state(ECHILD_STATUS);
self.status_changed.notify_all();
self.status_waker.wake();
self.exit_waiters.wake_all();
}
fn note_detached_wait(&self, owner: Option<Arc<LegacyWaitOwner>>) -> u64 {
let id = self.detached_wait_seq.fetch_add(1, Ordering::AcqRel);
self.detached_waits.lock().push((id, owner));
id
}
fn end_detached_wait(&self, id: u64) {
self.detached_waits.lock().retain(|&(entry, _)| entry != id);
}
fn bind_detached_wait_owner(&self, id: u64, owner: Option<Arc<LegacyWaitOwner>>) {
let Some(owner) = owner else {
return;
};
for entry in self.detached_waits.lock().iter_mut() {
if entry.0 == id {
entry.1 = Some(Arc::clone(&owner));
break;
}
}
}
fn detach_handoff_pending(&self) -> bool {
let mut waits = self.detached_waits.lock();
waits.retain(|(_, owner)| {
owner
.as_ref()
.is_some_and(|owner| owner.is_live() != Ok(false))
});
!waits.is_empty()
}
fn mark_echild_if_unowned(&self, identity: &WorkerIdentity) -> bool {
if matches!(identity.pidfd, ThreadHandle::Procfs { .. }) {
return self
.mark_legacy_echild_if_unowned(identity)
.unwrap_or(false);
}
if identity.is_active_tracee() {
return false;
}
if self.detach_handoff_pending()
&& identity.observed_process_generation() == Some(true)
&& !identity.task_is_zombie()
{
#[cfg(test)]
self.detached_echild_kept.fetch_add(1, Ordering::Relaxed);
return false;
}
self.mark_echild();
true
}
fn mark_legacy_echild_if_unowned(&self, identity: &WorkerIdentity) -> Result<bool, Errno> {
let progress = self.legacy_wait.lock();
if !progress.consumed.is_empty() {
return Ok(false);
}
if self.detach_handoff_pending()
&& identity.observed_process_generation() == Some(true)
&& !identity.task_is_zombie()
{
#[cfg(test)]
self.detached_echild_kept.fetch_add(1, Ordering::Relaxed);
return Ok(false);
}
let closed = close_unowned_legacy_wait(identity, self)?;
drop(progress);
if closed {
self.mark_echild();
}
Ok(closed)
}
fn is_terminal(&self) -> bool {
self.status.lock().terminal != INVALID_STATUS
}
fn poll_status_reservation(&self, waker: &Waker) -> Poll<Result<StatusReservation<'_>, Errno>> {
self.poll_status_reservation_with_guard(waker, None)
}
fn poll_status_reservation_with_guard(
&self,
waker: &Waker,
guard: Option<&PtracerThreadGuard>,
) -> Poll<Result<StatusReservation<'_>, Errno>> {
self.status_waker.register(waker);
if let Some(guard) = guard
&& let Err(error) = guard.recheck_after_waker_callbacks()
{
return Poll::Ready(Err(error));
}
let state = self.status.lock();
if let Some(status) = state.pending.front().copied() {
return Poll::Ready(Ok(StatusReservation {
status,
state: Some(state),
}));
}
match state.terminal {
INVALID_STATUS => Poll::Pending,
ECHILD_STATUS => Poll::Ready(Err(Errno::ECHILD)),
status => {
Poll::Ready(Ok(StatusReservation {
status,
state: None,
}))
}
}
}
#[cfg(test)]
fn poll_status(&self, waker: &Waker) -> Poll<Result<i32, Errno>> {
match self.poll_status_reservation(waker) {
Poll::Ready(Ok(reservation)) => {
let status = reservation.status;
reservation.commit();
Poll::Ready(Ok(status))
}
Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
Poll::Pending => Poll::Pending,
}
}
fn wait_pending_status(&self, timeout: Duration) -> Option<MutexGuard<'_, StatusState>> {
let deadline = Instant::now()
.checked_add(timeout)
.unwrap_or_else(Instant::now);
let mut state = self.status.lock();
loop {
if !state.pending.is_empty() {
return Some(state);
}
if state.terminal != INVALID_STATUS {
return None;
}
let remaining = deadline.saturating_duration_since(Instant::now());
if remaining.is_zero() {
return None;
}
self.status_changed.wait_for(&mut state, remaining);
}
}
fn pending_is_empty(&self) -> bool {
self.status.lock().pending.is_empty()
}
fn notify_wait_owner_change(&self) {
let _guard = self.wait_owner_lock.lock();
self.wait_owner_changed.notify_all();
}
#[cfg(test)]
fn pause_notifier_registration_for_test(&self, phase: i32) {
let pause = {
let mut installed = self.notifier_registration_pause.lock();
if installed.as_ref().is_some_and(|(at, _)| *at == phase) {
installed.take().map(|(_, pause)| pause)
} else {
None
}
};
if let Some(pause) = pause {
let _ = pause.captured.try_send(());
let _ = pause.resume.recv_timeout(Duration::from_secs(2));
}
}
fn begin_status_return(
&self,
status: i32,
current_owner: u8,
returning_owner: u8,
) -> ReturnTransactionStart<'_> {
let _guard = self.wait_owner_lock.lock();
if !libc::WIFEXITED(status)
&& !libc::WIFSIGNALED(status)
&& (self.cleanup_cancel_requested.load(Ordering::Acquire)
|| self.cleanup_claim_waiters.load(Ordering::Acquire) != 0)
{
return ReturnTransactionStart::Cancelled;
}
self.wait_owner
.compare_exchange(
current_owner,
returning_owner,
Ordering::AcqRel,
Ordering::Acquire,
)
.expect("status return began under the wrong wait owner");
ReturnTransactionStart::Begun(ReturnTransaction {
event: self,
returning_owner,
rollback_owner: current_owner,
completed: false,
})
}
fn claim_sync_wait(&self) -> Result<SyncWaitOwnership<'_>, Errno> {
let mut guard = self.wait_owner_lock.lock();
loop {
match self.wait_owner.load(Ordering::Acquire) {
WAIT_OWNER_NONE => {
self.wait_owner
.compare_exchange(
WAIT_OWNER_NONE,
WAIT_OWNER_SYNC,
Ordering::AcqRel,
Ordering::Acquire,
)
.expect("wait owner changed while serialized");
return Ok(SyncWaitOwnership::Claimed(SyncWaitOwner {
event: self,
released: false,
}));
}
WAIT_OWNER_NOTIFIER => match self.worker_state.load(Ordering::Acquire) {
WORKER_NOT_STARTED | WORKER_STARTING => {
#[cfg(test)]
if let Some(entered) = self.sync_wait_owner_entered.lock().take() {
let _ = entered.try_send(());
}
self.wait_owner_changed.wait(&mut guard)
}
WORKER_RUNNING | WORKER_FINISHING | WORKER_DONE => {
return Ok(SyncWaitOwnership::Notifier);
}
state => unreachable!("invalid worker state {state}"),
},
WAIT_OWNER_SYNC => return Err(Errno::EBUSY),
WAIT_OWNER_SYNC_RETURNING | WAIT_OWNER_NOTIFIER_RETURNING => {
self.wait_owner_changed.wait(&mut guard)
}
owner => unreachable!("invalid wait owner {owner}"),
}
}
}
fn claim_notifier_wait(&self) -> NotifierWaitOwnership<'_> {
if !self.cleanup_cancel_requested.load(Ordering::Acquire)
&& self.cleanup_claim_waiters.load(Ordering::Acquire) == 0
&& self.wait_owner.load(Ordering::Acquire) == WAIT_OWNER_NOTIFIER
&& matches!(
self.worker_state.load(Ordering::Acquire),
WORKER_RUNNING | WORKER_FINISHING | WORKER_DONE
)
{
return NotifierWaitOwnership::Existing;
}
let mut guard = self.wait_owner_lock.lock();
loop {
match self.wait_owner.load(Ordering::Acquire) {
WAIT_OWNER_NONE => {
self.wait_owner
.compare_exchange(
WAIT_OWNER_NONE,
WAIT_OWNER_NOTIFIER,
Ordering::AcqRel,
Ordering::Acquire,
)
.expect("wait owner changed while serialized");
return NotifierWaitOwnership::Claimed(NotifierWaitOwner {
event: self,
committed: false,
});
}
WAIT_OWNER_SYNC | WAIT_OWNER_SYNC_RETURNING | WAIT_OWNER_NOTIFIER_RETURNING => {
self.wait_owner_changed.wait(&mut guard)
}
WAIT_OWNER_NOTIFIER => match self.worker_state.load(Ordering::Acquire) {
WORKER_NOT_STARTED | WORKER_STARTING => {
self.wait_owner_changed.wait(&mut guard)
}
WORKER_RUNNING | WORKER_FINISHING | WORKER_DONE => {
return NotifierWaitOwnership::Existing;
}
state => unreachable!("invalid worker state {state}"),
},
owner => unreachable!("invalid wait owner {owner}"),
}
}
}
#[cfg(test)]
fn try_claim_cancellable_notifier_wait(&self) -> CancellableNotifierWaitOwnership<'_> {
let _guard = self.wait_owner_lock.lock();
match self.wait_owner.load(Ordering::Acquire) {
WAIT_OWNER_NONE => {
self.cleanup_cancel_requested.store(true, Ordering::Release);
self.wait_owner
.compare_exchange(
WAIT_OWNER_NONE,
WAIT_OWNER_NOTIFIER,
Ordering::AcqRel,
Ordering::Acquire,
)
.expect("wait owner changed while serialized");
CancellableNotifierWaitOwnership::Claimed(NotifierWaitOwner {
event: self,
committed: false,
})
}
WAIT_OWNER_SYNC => {
self.cleanup_cancel_requested.store(true, Ordering::Release);
CancellableNotifierWaitOwnership::Synchronous
}
WAIT_OWNER_NOTIFIER => {
self.cleanup_cancel_requested.store(true, Ordering::Release);
CancellableNotifierWaitOwnership::Existing
}
WAIT_OWNER_SYNC_RETURNING | WAIT_OWNER_NOTIFIER_RETURNING => {
self.cleanup_claim_waiters.fetch_add(1, Ordering::Release);
CancellableNotifierWaitOwnership::Returning
}
owner => unreachable!("invalid wait owner {owner}"),
}
}
fn decode_status_return<T>(
&self,
reservation: StatusReservation<'_>,
decode: impl FnOnce(i32) -> Result<T, Error>,
) -> Result<StatusReturn<T>, Error> {
self.decode_status_return_with_guard(reservation, None, decode)
}
fn decode_status_return_with_guard<T>(
&self,
reservation: StatusReservation<'_>,
guard: Option<&PtracerThreadGuard>,
decode: impl FnOnce(i32) -> Result<T, Error>,
) -> Result<StatusReturn<T>, Error> {
let transaction = match self.begin_status_return(
reservation.status,
WAIT_OWNER_NOTIFIER,
WAIT_OWNER_NOTIFIER_RETURNING,
) {
ReturnTransactionStart::Begun(transaction) => transaction,
ReturnTransactionStart::Cancelled => {
return Ok(StatusReturn::Cancelled(reservation.status));
}
};
let decoded = decode(reservation.status);
if let Some(guard) = guard {
guard.recheck_after_waker_callbacks()?;
}
let decoded = match decoded {
Ok(decoded) => decoded,
Err(error @ Error::Died(_)) => {
let dead_exec = reservation.retire_dead_exec();
transaction.commit(WAIT_OWNER_NOTIFIER);
if dead_exec {
self.forward_exit_stop_after_dead_exec();
return Ok(StatusReturn::RetiredDeadExec);
}
return Err(error);
}
Err(error) => return Err(error),
};
reservation.commit();
transaction.commit(WAIT_OWNER_NOTIFIER);
Ok(StatusReturn::Returned(decoded))
}
fn wait_status_reservation_sync(
&self,
handle: &EventHandle,
) -> Result<StatusReservation<'_>, Errno> {
if handle.uses_procfs_wait() {
loop {
let assistance = handle.try_assist_legacy_wait();
let mut state = self.status.lock();
if let Some(status) = state.pending.front().copied() {
return Ok(StatusReservation {
status,
state: Some(state),
});
}
match state.terminal {
ECHILD_STATUS => return Err(Errno::ECHILD),
INVALID_STATUS => {
assistance?;
self.status_changed
.wait_for(&mut state, Duration::from_millis(1));
}
status => {
return Ok(StatusReservation {
status,
state: None,
});
}
}
}
}
let mut state = self.status.lock();
loop {
if let Some(status) = state.pending.front().copied() {
return Ok(StatusReservation {
status,
state: Some(state),
});
}
match state.terminal {
INVALID_STATUS => {
#[cfg(test)]
if let Some(entered) = self.sync_wait_entered.lock().take() {
let _ = entered.send(SyncWaitTestTransition::WaitingForNotifier);
}
self.status_changed.wait(&mut state);
}
ECHILD_STATUS => return Err(Errno::ECHILD),
status => {
return Ok(StatusReservation {
status,
state: None,
});
}
}
}
}
fn block_until_status(&self, timeout: Duration) -> Option<bool> {
if self.cleanup_cancel_requested.load(Ordering::Acquire)
|| self.cleanup_claim_waiters.load(Ordering::Acquire) != 0
|| self.wait_owner.load(Ordering::Acquire) != WAIT_OWNER_NOTIFIER
|| self.worker_state.load(Ordering::Acquire) != WORKER_RUNNING
{
return None;
}
if self.exit_status.load(Ordering::Acquire) != EXIT_PENDING {
return Some(true);
}
let deadline = Instant::now() + timeout;
let mut state = self.status.lock();
loop {
if !state.pending.is_empty()
|| state.terminal != INVALID_STATUS
|| self.exit_status.load(Ordering::Acquire) != EXIT_PENDING
{
return Some(true);
}
if self
.status_changed
.wait_until(&mut state, deadline)
.timed_out()
{
return Some(
!state.pending.is_empty()
|| state.terminal != INVALID_STATUS
|| self.exit_status.load(Ordering::Acquire) != EXIT_PENDING,
);
}
}
}
fn try_status_reservation_sync(&self) -> Option<Result<StatusReservation<'_>, Errno>> {
let state = self.status.lock();
if let Some(status) = state.pending.front().copied() {
return Some(Ok(StatusReservation {
status,
state: Some(state),
}));
}
match state.terminal {
INVALID_STATUS => None,
ECHILD_STATUS => Some(Err(Errno::ECHILD)),
status => Some(Ok(StatusReservation {
status,
state: None,
})),
}
}
fn try_terminal_reservation_sync(&self) -> Option<Result<StatusReservation<'_>, Errno>> {
let state = self.status.lock();
match state.terminal {
INVALID_STATUS => None,
ECHILD_STATUS => Some(Err(Errno::ECHILD)),
status => Some(Ok(StatusReservation {
status,
state: None,
})),
}
}
fn finish_sync_terminal(self: &Arc<Self>, pid: Pid) {
if self.try_begin_unstarted_completion() {
NOTIFIER.remove(pid, self);
self.mark_worker_done();
} else {
debug_assert_eq!(self.worker_state.load(Ordering::Acquire), WORKER_DONE);
}
}
#[cfg(test)]
fn wait_for_sync_owner_release(&self, timeout: Duration) -> bool {
let deadline = Instant::now()
.checked_add(timeout)
.unwrap_or_else(Instant::now);
let mut guard = self.wait_owner_lock.lock();
loop {
if !matches!(
self.wait_owner.load(Ordering::Acquire),
WAIT_OWNER_SYNC | WAIT_OWNER_SYNC_RETURNING
) {
return true;
}
let remaining = deadline.saturating_duration_since(Instant::now());
if remaining.is_zero() {
return false;
}
self.wait_owner_changed.wait_for(&mut guard, remaining);
}
}
#[cfg(test)]
fn wait_for_return_and_claim_cancellable_notifier(
&self,
timeout: Duration,
) -> Option<CancellableNotifierWaitOwnership<'_>> {
let deadline = Instant::now()
.checked_add(timeout)
.unwrap_or_else(Instant::now);
let mut guard = self.wait_owner_lock.lock();
loop {
match self.wait_owner.load(Ordering::Acquire) {
WAIT_OWNER_SYNC_RETURNING | WAIT_OWNER_NOTIFIER_RETURNING => {}
WAIT_OWNER_NONE => {
self.cleanup_cancel_requested.store(true, Ordering::Release);
let previous = self.cleanup_claim_waiters.fetch_sub(1, Ordering::AcqRel);
debug_assert_ne!(previous, 0);
self.wait_owner
.compare_exchange(
WAIT_OWNER_NONE,
WAIT_OWNER_NOTIFIER,
Ordering::AcqRel,
Ordering::Acquire,
)
.expect("wait owner changed while serialized");
return Some(CancellableNotifierWaitOwnership::Claimed(
NotifierWaitOwner {
event: self,
committed: false,
},
));
}
WAIT_OWNER_SYNC => {
self.cleanup_cancel_requested.store(true, Ordering::Release);
let previous = self.cleanup_claim_waiters.fetch_sub(1, Ordering::AcqRel);
debug_assert_ne!(previous, 0);
return Some(CancellableNotifierWaitOwnership::Synchronous);
}
WAIT_OWNER_NOTIFIER => {
self.cleanup_cancel_requested.store(true, Ordering::Release);
let previous = self.cleanup_claim_waiters.fetch_sub(1, Ordering::AcqRel);
debug_assert_ne!(previous, 0);
return Some(CancellableNotifierWaitOwnership::Existing);
}
owner => unreachable!("invalid wait owner {owner}"),
}
let remaining = deadline.saturating_duration_since(Instant::now());
if remaining.is_zero() {
let previous = self.cleanup_claim_waiters.fetch_sub(1, Ordering::AcqRel);
debug_assert_ne!(previous, 0);
return None;
}
self.wait_owner_changed.wait_for(&mut guard, remaining);
}
}
fn try_begin_worker_start(&self) -> bool {
let started = self
.worker_state
.compare_exchange(
WORKER_NOT_STARTED,
WORKER_STARTING,
Ordering::AcqRel,
Ordering::Acquire,
)
.is_ok();
if started {
self.notify_wait_owner_change();
}
started
}
fn mark_worker_running(&self) {
self.worker_state
.compare_exchange(
WORKER_STARTING,
WORKER_RUNNING,
Ordering::Release,
Ordering::Acquire,
)
.expect("worker start state changed before publication");
self.notify_wait_owner_change();
}
fn rollback_worker_start(&self) {
self.worker_state
.compare_exchange(
WORKER_STARTING,
WORKER_NOT_STARTED,
Ordering::AcqRel,
Ordering::Acquire,
)
.expect("failed worker start state changed before rollback");
self.notify_wait_owner_change();
}
fn worker_is_running(&self) -> bool {
self.worker_state.load(Ordering::Acquire) == WORKER_RUNNING
}
fn try_begin_unstarted_completion(&self) -> bool {
let finishing = self
.worker_state
.compare_exchange(
WORKER_NOT_STARTED,
WORKER_FINISHING,
Ordering::AcqRel,
Ordering::Acquire,
)
.is_ok();
if finishing {
self.notify_wait_owner_change();
}
finishing
}
fn mark_worker_done(&self) {
{
let _guard = self.worker_done_lock.lock();
let previous = self.worker_state.swap(WORKER_DONE, Ordering::AcqRel);
debug_assert!(matches!(previous, WORKER_RUNNING | WORKER_FINISHING));
self.worker_done_changed.notify_all();
}
self.notify_wait_owner_change();
}
fn wait_worker_done(&self, timeout: Duration) -> bool {
if self.worker_state.load(Ordering::Acquire) == WORKER_DONE {
return true;
}
let deadline = Instant::now()
.checked_add(timeout)
.unwrap_or_else(Instant::now);
let mut guard = self.worker_done_lock.lock();
loop {
if self.worker_state.load(Ordering::Acquire) == WORKER_DONE {
return true;
}
#[cfg(test)]
pause_worker_done_wait_for_test(&self.worker_done_wait_probe);
let remaining = deadline.saturating_duration_since(Instant::now());
if remaining.is_zero() {
return false;
}
#[cfg(test)]
if let Some(parking) = self.worker_done_park_signal.lock().take() {
let _ = parking.try_send(());
}
self.worker_done_changed.wait_for(&mut guard, remaining);
}
}
fn join_retired_wait_owner(&self) {
let mut guard = self.worker_done_lock.lock();
while self.worker_state.load(Ordering::Acquire) != WORKER_DONE {
self.worker_done_changed.wait(&mut guard);
}
}
pub fn poll_exit(&self, waiter: &Arc<ExitWaiter>, waker: &Waker) -> Poll<Result<(), Errno>> {
self.exit_waiters.register(waiter, waker);
let Some(publication) = self.exit_publication.try_lock() else {
waker.wake_by_ref();
return Poll::Pending;
};
let result = self.poll_exit_epoch(waiter);
drop(publication);
if result.is_ready() {
self.exit_waiters.wake_all();
}
result
}
fn poll_exit_epoch(&self, waiter: &ExitWaiter) -> Poll<Result<(), Errno>> {
if waiter.epoch != self.exit_epoch.load(Ordering::Acquire) {
return Poll::Ready(Err(Errno::EALREADY));
}
loop {
match self.exit_status.load(Ordering::Acquire) {
EXIT_STOPPED => match self.exit_capability.load(Ordering::Acquire) {
EXIT_CAP_AVAILABLE => {
if self
.exit_capability
.compare_exchange(
EXIT_CAP_AVAILABLE,
EXIT_CAP_CLAIMED,
Ordering::AcqRel,
Ordering::Acquire,
)
.is_ok()
{
return Poll::Ready(Ok(()));
}
}
EXIT_CAP_CLAIMED | EXIT_CAP_EXPIRED => {
return Poll::Ready(Err(Errno::EALREADY));
}
EXIT_CAP_PENDING | EXIT_CAP_FINALIZING => return Poll::Pending,
state => unreachable!("invalid exit capability state {state}"),
},
EXIT_ECHILD => return Poll::Ready(Err(Errno::ECHILD)),
EXIT_PENDING => match self.exit_capability.load(Ordering::Acquire) {
EXIT_CAP_PENDING | EXIT_CAP_FINALIZING => return Poll::Pending,
EXIT_CAP_AVAILABLE => {
match self.exit_status.load(Ordering::Acquire) {
EXIT_PENDING => return Poll::Pending,
EXIT_STOPPED | EXIT_ECHILD => continue,
state => unreachable!("invalid exit publication state {state}"),
}
}
EXIT_CAP_CLAIMED => return Poll::Ready(Err(Errno::EALREADY)),
EXIT_CAP_EXPIRED => {
return match self.exit_status.load(Ordering::Acquire) {
EXIT_ECHILD => Poll::Ready(Err(Errno::ECHILD)),
EXIT_PENDING | EXIT_STOPPED => Poll::Ready(Err(Errno::EALREADY)),
state => unreachable!("invalid exit publication state {state}"),
};
}
state => unreachable!("unpublished exit capability state {state}"),
},
state => unreachable!("invalid exit publication state {state}"),
}
}
}
}
#[derive(Debug)]
struct EventGeneration {
event: Arc<Event>,
identity: OnceLock<Arc<WorkerIdentity>>,
authoritative: OnceLock<EventHandle>,
initial_capture_refusal: OnceLock<Errno>,
}
#[derive(Clone, Debug)]
pub(super) struct EventHandle(Arc<EventGeneration>, Option<Arc<WorkerIdentity>>);
impl EventHandle {
pub(super) fn new() -> Self {
Self(
Arc::new(EventGeneration {
event: Arc::new(Event::new()),
identity: OnceLock::new(),
authoritative: OnceLock::new(),
initial_capture_refusal: OnceLock::new(),
}),
None,
)
}
fn with_identity(identity: Arc<WorkerIdentity>) -> Self {
let handle = Self::new();
handle
.0
.identity
.set(identity)
.expect("fresh event generation identity is unset");
if let Some(identity) = handle.identity()
&& matches!(identity.pidfd, ThreadHandle::Procfs { .. })
{
let _ = authenticate_legacy_wait_owner(identity, handle.event());
}
handle
}
pub(super) fn current_or_new(pid: Pid) -> Result<Self, Errno> {
NOTIFIER.current_or_new_for_policy(pid, WaitPolicy::Native)
}
pub(super) fn current_or_new_on_ptracer_thread(pid: Pid) -> Result<Self, Errno> {
NOTIFIER.current_or_new_for_policy(pid, WaitPolicy::PtracerThread)
}
pub(super) fn attachment_on_ptracer_thread(pid: Pid, original: &AttachmentAnchor) -> Self {
let captured = Self::current_or_new_on_ptracer_thread(pid).and_then(|handle| {
let identity = handle.identity().ok_or(Errno::ENODATA)?;
original.validate_attached(identity)?;
let connected = handle.reconnect_attachment(pid);
original.validate_attached(connected.identity().ok_or(Errno::ENODATA)?)?;
Ok(connected)
});
captured.unwrap_or_else(|error| {
let handle = Self::new();
let _ = handle.0.initial_capture_refusal.set(error);
*handle.event().registration_error.lock() = Some(error);
handle
})
}
pub(super) fn capture_current_host_owner() -> Result<Arc<LegacyWaitOwner>, Errno> {
LegacyWaitOwner::capture_current().map(Arc::new)
}
pub(super) fn current_tracer_pid(&self) -> Result<Pid, Errno> {
self.identity().ok_or(Errno::ENODATA)?.current_tracer_pid()
}
pub(super) fn check_numeric_target_lifetime(
&self,
owner: &LegacyWaitOwner,
) -> Result<(), Errno> {
let identity = self.identity().ok_or(Errno::ENODATA)?;
#[cfg(test)]
let use_descriptor = !self
.event()
.numeric_auth_force_directory
.load(Ordering::Relaxed);
#[cfg(not(test))]
let use_descriptor = true;
if use_descriptor && identity.pidfd_is_live() == Ok(true) {
return Ok(());
}
match owner.proc_root.directory_key(&identity.numeric_name) {
Ok(key) if key == identity.proc_key => {}
Ok(_) => return Err(Errno::ESRCH),
Err(Errno::ENOENT) => {
return match identity.current_tracer_pid() {
Err(Errno::ENOENT) if identity.pidfd_is_live() == Ok(false) => {
Err(Errno::ESRCH)
}
Err(error) => Err(error),
Ok(_) => Err(Errno::ENOENT),
};
}
Err(error) => return Err(error),
}
let event = self.event();
let epoch = event.numeric_auth_epoch.load(Ordering::Acquire);
if event.numeric_auth_checked_epoch.load(Ordering::Acquire) == epoch {
return Ok(());
}
#[cfg(test)]
event
.numeric_auth_status_reads
.fetch_add(1, Ordering::Relaxed);
match identity.current_tracer_pid() {
Ok(_) => {
event
.numeric_auth_checked_epoch
.store(epoch, Ordering::Release);
Ok(())
}
Err(Errno::ENOENT) if identity.pidfd_is_live() == Ok(false) => Err(Errno::ESRCH),
Err(error) => Err(error),
}
}
pub(super) fn invalidate_numeric_auth(&self) {
self.event().invalidate_numeric_auth();
}
pub(super) fn resume_proof(
&self,
pid: Pid,
owner: Option<&Arc<LegacyWaitOwner>>,
) -> ResumeProof {
ResumeProof {
owner: owner.cloned(),
event: Arc::as_ptr(self.event()).addr(),
epoch: detach_epoch(pid),
}
}
fn resume_proof_holds(
&self,
pid: Pid,
proof: Option<&ResumeProof>,
owner: Option<&Arc<LegacyWaitOwner>>,
) -> bool {
proof.is_some_and(|proof| {
Arc::as_ptr(self.event()).addr() == proof.event
&& proof.epoch.is_some()
&& detach_epoch(pid) == proof.epoch
&& owner
.zip(proof.owner.as_ref())
.is_some_and(|(owner, resumer)| Arc::ptr_eq(owner, resumer))
})
}
pub(super) fn expect_stop(&self) {
self.with_legacy_worker_event(Event::wake_legacy_worker);
}
pub(super) fn expect_stop_on_poll(&self) {
self.with_legacy_worker_event(Event::defer_legacy_wake);
}
fn with_legacy_worker_event(&self, wake: impl FnOnce(&Event)) {
let handle = self.resolved();
if handle
.identity()
.is_some_and(|identity| matches!(identity.pidfd, ThreadHandle::Procfs { .. }))
&& handle.event().native_wait_identity.get().is_none()
{
wake(handle.event());
}
}
pub(super) fn stop_may_have_ended(&self) {
self.expect_stop();
probe_legacy_workers(
self.resolved()
.identity()
.map(|identity| identity.snapshot.tgid),
);
}
pub(super) fn initial_capture_error(&self) -> Option<Errno> {
self.resolved().0.initial_capture_refusal.get().copied()
}
pub(super) fn reconnect_attachment(&self, pid: Pid) -> Self {
self.invalidate_numeric_auth();
loop {
let handle = self.resolved_handle();
let Some(identity) = handle.identity() else {
return handle;
};
if !matches!(identity.pidfd, ThreadHandle::Procfs { .. })
|| handle.event().native_wait_identity.get().is_some()
{
return handle;
}
let event = Arc::clone(handle.event());
let mut progress = event.legacy_wait.lock();
let mut gate = event.terminal_reaping.write();
if !Arc::ptr_eq(self.event(), &event) {
continue;
}
if !handle.is_current_ptracer() {
return handle;
}
let same_owner = event
.legacy_wait_owner
.get()
.is_none_or(|owner| owner.is_current() == Ok(true));
if !*gate && !progress.retiring && same_owner {
let _ = authenticate_legacy_wait_owner(identity, &event);
return handle;
}
*gate = true;
progress.retiring = true;
drop(gate);
drop(progress);
event.wake_legacy_worker();
if event.worker_state.load(Ordering::Acquire) == WORKER_NOT_STARTED {
NOTIFIER.resolve_echild(pid, &handle);
}
event.join_retired_wait_owner();
return NOTIFIER.reconnect(pid, &handle);
}
}
fn is_current_ptracer(&self) -> bool {
self.identity().is_some_and(|identity| {
identity.current_tracer_pid().ok() == Some(Pid::from(nix::unistd::gettid()))
})
}
pub(super) fn capture_current_ptracer_owner(&self) -> Result<Arc<LegacyWaitOwner>, Errno> {
let identity = self.identity().ok_or(Errno::ENODATA)?;
identity.capture_current_owner(false).map(Arc::new)
}
pub(super) fn capture_current_owner(&self) -> Result<Arc<LegacyWaitOwner>, Errno> {
let identity = self.identity().ok_or(Errno::ENODATA)?;
let owner = identity
.capture_current_owner(!matches!(identity.pidfd, ThreadHandle::Procfs { .. }))?;
if !identity.authenticates_wait_role(&owner)? {
return Err(Errno::EPERM);
}
Ok(Arc::new(owner))
}
pub(super) fn capture_current_constructor_owner(&self) -> Result<Arc<LegacyWaitOwner>, Errno> {
self.identity()
.ok_or(Errno::ENODATA)?
.capture_current_owner(true)
.map(Arc::new)
}
pub(super) fn capture_host_thread_guard(&self) -> Result<PtracerThreadGuard, Errno> {
if let Some(error) = self.initial_capture_error() {
return Err(error);
}
let identity = self.identity().ok_or_else(|| {
self.event()
.registration_error
.lock()
.unwrap_or(Errno::ENODATA)
})?;
let _open = launch_window::TransientOpen::begin();
let owner = LegacyWaitOwner::capture_at(identity.original_proc_root()?)?;
Ok(PtracerThreadGuard {
owner: Arc::new(owner),
})
}
pub(super) fn authenticates_wait_role(&self, owner: &LegacyWaitOwner) -> Result<bool, Errno> {
self.identity()
.ok_or(Errno::ENODATA)?
.authenticates_wait_role(owner)
}
pub(super) fn current_or_error(pid: Pid) -> Self {
match Self::current_or_new(pid) {
Ok(handle) => handle,
Err(error) => {
let handle = Self::new();
*handle.event().registration_error.lock() = Some(error);
handle
}
}
}
fn event(&self) -> &Arc<Event> {
&self.resolved().0.event
}
fn try_assist_legacy_wait(&self) -> Result<(), Errno> {
let result = self.assist_legacy_wait_once();
if !matches!(result, Ok(true)) {
self.event().flush_deferred_legacy_wake();
}
result.map(drop)
}
fn assist_legacy_wait_once(&self) -> Result<bool, Errno> {
let handle = self.resolved();
let Some(identity) = handle.identity() else {
return Ok(false);
};
if !matches!(identity.pidfd, ThreadHandle::Procfs { .. }) {
return Ok(false);
}
let event = handle.event();
let Some(mut progress) = event.legacy_wait.try_lock() else {
return Ok(false);
};
if event.native_wait_identity.get().is_some()
|| progress.retiring
|| !event.worker_is_running()
|| !matches!(
event.wait_owner.load(Ordering::Acquire),
WAIT_OWNER_NOTIFIER | WAIT_OWNER_NOTIFIER_RETURNING
)
{
return Ok(false);
}
if let Some(error) = progress.error {
return Err(error);
}
let mut notify = false;
#[cfg(all(test, not(sanitized)))]
let mut consume_receipt = None;
let status = match try_consume_legacy_status(identity, event, Some(handle)) {
Ok(status) => status,
Err(Errno::ECHILD) => {
event.count_legacy_wakeup();
#[cfg(all(test, not(sanitized)))]
{
consume_receipt = take_legacy_count_receipt();
}
notify = true;
None
}
Err(Errno::EINTR) => None,
Err(error) => return Err(error),
};
if let Some(status) = status {
progress.consumed.push_back(status);
#[cfg(test)]
LEGACY_ASSIST_CONSUMPTIONS.with(|consumed| consumed.set(consumed.get() + 1));
progress.observed = None;
progress.error = None;
event.count_legacy_wakeup();
#[cfg(all(test, not(sanitized)))]
{
consume_receipt = take_legacy_count_receipt();
}
notify = true;
}
drop(progress);
#[cfg(all(test, not(sanitized)))]
let consume_source = LegacyNotifySource::Consume(consume_receipt);
if notify {
event.legacy_wake_deferred.store(false, Ordering::Release);
#[cfg(test)]
pause_retirement_for_test(&event.legacy_assist_notify_pause);
#[cfg(all(test, not(sanitized)))]
note_legacy_notify_source(consume_source);
event.notify_legacy_worker();
}
Ok(notify)
}
fn uses_procfs_wait(&self) -> bool {
self.identity()
.is_some_and(|identity| matches!(identity.pidfd, ThreadHandle::Procfs { .. }))
}
pub(super) fn hold_tid(&self) -> Option<TidHold<'_>> {
loop {
let event = self.event();
#[cfg(test)]
if let Some(pause) = HOLD_TID_PAUSES.lock().remove(&thread::current().id()) {
pause.captured.wait();
pause.resume.wait();
}
let hold = event.hold_tid()?;
if Arc::ptr_eq(self.event(), event) {
return Some(hold);
}
}
}
pub(super) fn retire_statuses_before_exit_stop(&self, epoch: usize) -> usize {
self.event().retire_statuses_before_exit_stop(epoch)
}
pub(super) fn superseded_new_child(&self, epoch: usize) -> Result<(), SupersededStopRefusal> {
self.event().superseded_new_child(epoch)
}
pub(super) fn regs_cacheable(&self) -> bool {
self.identity().is_some()
}
pub(super) fn cached_regs(&self) -> Option<crate::Regs> {
self.event().regs_cache.lock().map(|cached| cached.regs)
}
pub(super) fn store_regs(&self, regs: crate::Regs) {
*self.event().regs_cache.lock() = Some(CachedRegs { regs, dirty: false });
}
pub(super) fn regs_already_current(&self, regs: &crate::Regs) -> bool {
self.event()
.regs_cache
.lock()
.is_some_and(|cached| !cached.dirty && &cached.regs == regs)
}
pub(super) fn store_regs_dirty(&self, regs: crate::Regs) {
*self.event().regs_cache.lock() = Some(CachedRegs { regs, dirty: true });
}
pub(super) fn dirty_regs(&self) -> Option<crate::Regs> {
self.event()
.regs_cache
.lock()
.filter(|cached| cached.dirty)
.map(|cached| cached.regs)
}
pub(super) fn mark_regs_clean(&self) {
if let Some(cached) = &mut *self.event().regs_cache.lock() {
cached.dirty = false;
}
}
pub(super) fn invalidate_regs(&self) {
*self.event().regs_cache.lock() = None;
}
#[cfg(test)]
pub(super) fn note_regs_kernel_read(&self) {
self.event()
.regs_kernel_reads
.fetch_add(1, Ordering::Relaxed);
}
#[cfg(test)]
pub(super) fn regs_kernel_reads(&self) -> usize {
self.event().regs_kernel_reads.load(Ordering::Relaxed)
}
#[cfg(test)]
pub(super) fn note_regs_kernel_write(&self) {
self.event()
.regs_kernel_writes
.fetch_add(1, Ordering::Relaxed);
}
#[cfg(test)]
pub(super) fn regs_kernel_writes(&self) -> usize {
self.event().regs_kernel_writes.load(Ordering::Relaxed)
}
fn identity(&self) -> Option<&Arc<WorkerIdentity>> {
self.1
.as_ref()
.or_else(|| self.0.identity.get())
.or_else(|| self.resolved().0.identity.get())
}
fn native_view(&self, current: Arc<WorkerIdentity>) -> Result<Self, Errno> {
WaitPolicy::Native.check_identity(¤t)?;
let bound = self.identity().ok_or(Errno::ENODATA)?;
if matches!(bound.pidfd, ThreadHandle::Pidfd(_)) {
return Ok(self.resolved_handle());
}
if !bound.same_generation(¤t) {
return Err(Errno::ECHILD);
}
let event = self.event();
let progress = event.legacy_wait.lock();
let gate = event.terminal_reaping.read();
if *gate
|| progress.retiring
|| matches!(
event.worker_state.load(Ordering::Acquire),
WORKER_FINISHING | WORKER_DONE
)
{
return Err(Errno::ECHILD);
}
let status = retained_proc_status(bound.proc_dir.as_raw_fd())?;
if status.pid != bound.pid || status.tgid != bound.snapshot.tgid {
return Err(Errno::ECHILD);
}
let _ = event.native_wait_identity.set(current);
let native = event.native_wait_identity.get().unwrap().clone();
let selected = Self(self.resolved().0.clone(), Some(native));
drop(gate);
drop(progress);
event.wake_legacy_worker();
event.status_changed.notify_all();
event.status_waker.wake();
Ok(selected)
}
fn worker_identity(&self, current: &Arc<WorkerIdentity>) -> Arc<WorkerIdentity> {
self.event()
.native_wait_identity
.get()
.cloned()
.or_else(|| {
self.identity()
.filter(|identity| matches!(identity.pidfd, ThreadHandle::Pidfd(_)))
.cloned()
})
.unwrap_or_else(|| current.clone())
}
fn preferred_native_view(&self) -> Option<Self> {
self.event()
.native_wait_identity
.get()
.map(|identity| Self(self.resolved().0.clone(), Some(identity.clone())))
}
fn bind_identity(&self, identity: Arc<WorkerIdentity>) -> Result<(), Arc<WorkerIdentity>> {
self.resolved().0.identity.set(identity)
}
fn resolved(&self) -> &Self {
let mut current = self;
while let Some(authoritative) = current.0.authoritative.get() {
current = authoritative;
}
current
}
fn resolved_handle(&self) -> Self {
let selected = self.resolved();
if Arc::ptr_eq(&self.0, &selected.0) {
return self.clone();
}
Self(selected.0.clone(), self.identity().cloned())
}
fn chain_contains(&self, generation: &Arc<EventGeneration>) -> bool {
let mut current = self;
loop {
if Arc::ptr_eq(¤t.0, generation) {
return true;
}
let Some(authoritative) = current.0.authoritative.get() else {
return false;
};
current = authoritative;
}
}
fn adopt_authoritative(&self, authoritative: &Self) -> Result<Self, Errno> {
static ADOPTION_LOCK: LazyLock<Mutex<()>> = LazyLock::new(|| Mutex::new(()));
let _adoption = ADOPTION_LOCK.lock();
let authoritative = authoritative.resolved_handle();
if authoritative.chain_contains(&self.0) {
if Arc::ptr_eq(&authoritative.0, &self.0) {
return Ok(authoritative);
}
return Err(Errno::ELOOP);
}
let gate = self.0.event.terminal_reaping.write();
let adopted = self.0.authoritative.set(authoritative.clone());
drop(gate);
match adopted {
Ok(()) => Ok(authoritative),
Err(_) => {
let selected = self.resolved_handle();
if Arc::ptr_eq(&selected.0, &authoritative.0) {
Ok(selected)
} else {
Err(Errno::EALREADY)
}
}
}
}
}
impl PartialEq for EventHandle {
fn eq(&self, other: &Self) -> bool {
Arc::ptr_eq(&self.0, &other.0)
}
}
impl Eq for EventHandle {}
impl Hash for EventHandle {
fn hash<H: Hasher>(&self, state: &mut H) {
Arc::as_ptr(&self.0).hash(state);
}
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
struct WorkerProcSnapshot {
tgid: Pid,
tracer_pid: Pid,
start_time: u64,
zombie: bool,
}
#[derive(Clone, Copy, Debug)]
struct RetainedProcStatus {
pid: Pid,
tgid: Pid,
tracer_pid: Pid,
}
fn retained_proc_status(directory: RawFd) -> Result<RetainedProcStatus, Errno> {
#[cfg(test)]
RETAINED_STATUS_READS.with(|reads| reads.set(reads.get() + 1));
let _open = launch_window::TransientOpen::begin();
let raw = unsafe {
libc::openat(
directory,
c"status".as_ptr(),
libc::O_RDONLY | libc::O_CLOEXEC | libc::O_NOFOLLOW,
)
};
if raw < 0 {
return Err(Errno::last());
}
let fd = unsafe { OwnedFd::from_raw_fd(raw) };
let mut bytes = [0u8; 8192];
let count = unsafe { libc::read(fd.as_raw_fd(), bytes.as_mut_ptr().cast(), bytes.len()) };
if count < 0 {
return Err(Errno::last());
}
let field = |name: &str| {
let line = bytes[..count as usize]
.split(|byte| *byte == b'\n')
.find(|line| line.starts_with(name.as_bytes()))
.ok_or(Errno::EIO)?;
let line = std::str::from_utf8(line).map_err(|_| Errno::EIO)?;
worker_status_pid(line, name).map_err(io_errno)
};
Ok(RetainedProcStatus {
pid: field("Pid:")?,
tgid: field("Tgid:")?,
tracer_pid: field("TracerPid:")?,
})
}
fn retained_proc_parent(directory: RawFd) -> Result<Pid, Errno> {
#[cfg(test)]
OTHER_STATUS_OPENS.with(|opens| opens.set(opens.get() + 1));
let _open = launch_window::TransientOpen::begin();
let raw = unsafe {
libc::openat(
directory,
c"status".as_ptr(),
libc::O_RDONLY | libc::O_CLOEXEC | libc::O_NOFOLLOW,
)
};
if raw < 0 {
return Err(Errno::last());
}
let fd = unsafe { OwnedFd::from_raw_fd(raw) };
let mut bytes = [0u8; 8192];
let count = unsafe { libc::read(fd.as_raw_fd(), bytes.as_mut_ptr().cast(), bytes.len()) };
if count < 0 {
return Err(Errno::last());
}
let line = bytes[..count as usize]
.split_inclusive(|byte| *byte == b'\n')
.find(|line| line.starts_with(b"PPid:") && line.ends_with(b"\n"))
.ok_or(Errno::EIO)?;
let line = std::str::from_utf8(line).map_err(|_| Errno::EIO)?;
worker_status_pid(line, "PPid:").map_err(io_errno)
}
impl WorkerProcSnapshot {
fn same_process_generation(&self, other: &Self) -> bool {
self.tgid == other.tgid && self.start_time == other.start_time
}
}
#[derive(Debug)]
struct WorkerIdentity {
pid: Pid,
snapshot: WorkerProcSnapshot,
pidfd: ThreadHandle,
proc_dir: OwnedFd,
proc_inode: u64,
proc_key: ProcDirectoryKey,
numeric_name: std::ffi::CString,
proc_root: Option<Arc<AlignedProcfs>>,
}
impl WorkerIdentity {
#[cfg(test)]
fn capture(pid: Pid) -> Result<Self, Errno> {
Self::capture_for_policy(pid, WaitPolicy::Native)
}
fn capture_for_policy(pid: Pid, policy: WaitPolicy) -> Result<Self, Errno> {
#[cfg(test)]
if let Some(error) = CAPTURE_PERSISTENT_ERRORS.lock().get(&pid).copied() {
return Err(error);
}
#[cfg(test)]
if let Some(error) = CAPTURE_ERRORS.lock().remove(&pid) {
return Err(error);
}
#[cfg(test)]
{
let thread = thread::current().id();
let mut errors = CAPTURE_THREAD_ERRORS.lock();
if let Some(error) = errors.get_mut(&thread).and_then(VecDeque::pop_front) {
if errors.get(&thread).is_some_and(VecDeque::is_empty) {
errors.remove(&thread);
}
return Err(error);
}
}
Self::capture_process_for_policy(pid, policy)
}
#[cfg(test)]
fn capture_process(pid: Pid) -> Result<Self, Errno> {
Self::capture_process_for_policy(pid, WaitPolicy::Native)
}
fn capture_process_for_policy(pid: Pid, policy: WaitPolicy) -> Result<Self, Errno> {
let proc_root = (policy == WaitPolicy::PtracerThread)
.then(AlignedProcfs::open)
.transpose()?
.map(Arc::new);
let proc_dir: OwnedFd = match &proc_root {
Some(root) => root.open_pid(pid, libc::O_PATH)?,
None => OpenOptions::new()
.read(true)
.custom_flags(libc::O_PATH | libc::O_CLOEXEC)
.open(format!("/proc/{pid}"))
.map_err(io_errno)?
.into(),
};
let proc_key = fd_directory_key(&proc_dir).map_err(io_errno)?;
let proc_inode = proc_key.inode;
let read_snapshot = || match &proc_root {
Some(_) => retained_proc_snapshot(proc_dir.as_raw_fd()),
None => worker_proc_snapshot(pid),
};
let before = read_snapshot().map_err(io_errno)?;
#[cfg(test)]
if let Some(pause) = CAPTURE_AFTER_FIRST_SNAPSHOT_PAUSES.lock().remove(&pid) {
pause.captured.wait();
pause.resume.wait();
}
let pidfd = open_thread_pidfd_for_policy_at(pid, policy, proc_root.as_deref())?;
let after = read_snapshot().map_err(io_errno)?;
let current_inode = match &proc_root {
Some(root) => fd_inode(&root.open_pid(pid, libc::O_PATH)?).map_err(io_errno)?,
None => fs::metadata(format!("/proc/{pid}"))
.map_err(io_errno)?
.ino(),
};
if !before.same_process_generation(&after)
|| current_inode != proc_inode
|| matches!(&pidfd, ThreadHandle::Procfs { directory, .. }
if fd_inode(directory).map_err(io_errno)? != proc_inode)
{
return Err(Errno::ESRCH);
}
Ok(Self {
pid,
snapshot: after,
pidfd,
proc_dir,
proc_inode,
proc_key,
numeric_name: std::ffi::CString::new(pid.as_raw().to_string()).expect("decimal pid"),
proc_root,
})
}
fn is_active_tracee(&self) -> bool {
self.is_same_process_generation()
&& self.read_snapshot().ok().is_some_and(|current| {
current.same_process_generation(&self.snapshot)
&& tracer_is_current(current.tracer_pid)
})
}
fn task_is_zombie(&self) -> bool {
self.read_snapshot()
.ok()
.is_some_and(|current| current.zombie)
}
fn current_tracer_pid(&self) -> Result<Pid, Errno> {
let status = retained_proc_status(self.proc_dir.as_raw_fd())?;
if status.pid != self.pid || status.tgid != self.snapshot.tgid {
return Err(Errno::ESRCH);
}
Ok(status.tracer_pid)
}
fn authenticates_wait_role(&self, owner: &LegacyWaitOwner) -> Result<bool, Errno> {
if !owner.is_current()? {
return Err(Errno::EPERM);
}
let tracer = self.current_tracer_pid()?;
if tracer == owner.tid {
return Ok(true);
}
if tracer.as_raw() != 0 || matches!(self.pidfd, ThreadHandle::Procfs { .. }) {
return Ok(false);
}
Ok(retained_proc_parent(self.proc_dir.as_raw_fd())? == owner.tgid)
}
fn capture_current_owner(&self, allow_untraced: bool) -> Result<LegacyWaitOwner, Errno> {
let _open = launch_window::TransientOpen::begin();
let root = self.original_proc_root()?;
let tracer = self.current_tracer_pid()?;
if tracer != Pid::from(nix::unistd::gettid()) && !(allow_untraced && tracer.as_raw() == 0) {
return Err(Errno::EPERM);
}
LegacyWaitOwner::capture_at(root)
}
fn original_proc_root(&self) -> Result<AlignedProcfs, Errno> {
let root = match &self.proc_root {
Some(root) => root.root.try_clone().map_err(io_errno)?,
None => {
let raw = unsafe {
libc::openat(
self.proc_dir.as_raw_fd(),
c"..".as_ptr(),
libc::O_PATH | libc::O_DIRECTORY | libc::O_CLOEXEC,
)
};
if raw < 0 {
return Err(Errno::last());
}
unsafe { OwnedFd::from_raw_fd(raw) }
}
};
AlignedProcfs::from_root(root)
}
fn is_same_process_generation(&self) -> bool {
self.procfs_generation() == Some(true)
}
fn observed_process_generation(&self) -> Option<bool> {
#[cfg(test)]
if let Some(error) = PROCFS_OBSERVATION_ERRORS.lock().remove(&self.pid) {
let error = io::Error::from_raw_os_error(error.into_raw());
return procfs_error_shows_exit(&error).then_some(false);
}
self.procfs_generation()
}
fn procfs_generation(&self) -> Option<bool> {
let current = match self.read_snapshot() {
Ok(current) => current,
Err(error) => return procfs_error_shows_exit(&error).then_some(false),
};
if !current.same_process_generation(&self.snapshot)
|| fd_inode(&self.proc_dir).ok()? != self.proc_inode
{
return Some(false);
}
if let Some(root) = &self.proc_root {
return match root.open_pid(self.pid, libc::O_PATH) {
Ok(directory) => fd_inode(&directory)
.ok()
.map(|inode| inode == self.proc_inode),
Err(Errno::ENOENT | Errno::ESRCH) => Some(false),
Err(_) => None,
};
}
match fs::metadata(format!("/proc/{}", self.pid)) {
Ok(metadata) => Some(metadata.ino() == self.proc_inode),
Err(error) => procfs_error_shows_exit(&error).then_some(false),
}
}
fn read_snapshot(&self) -> io::Result<WorkerProcSnapshot> {
match self.proc_root {
Some(_) => retained_proc_snapshot(self.proc_dir.as_raw_fd()),
None => worker_proc_snapshot(self.pid),
}
}
fn same_generation(&self, other: &Self) -> bool {
self.pid == other.pid
&& self.snapshot.same_process_generation(&other.snapshot)
&& self.proc_inode == other.proc_inode
}
fn same_live_generation(&self, other: &Self) -> Result<bool, Errno> {
if !self.same_generation(other) {
return Ok(false);
}
Ok(self.pidfd_is_live()? && other.pidfd_is_live()?)
}
fn pidfd_is_live(&self) -> Result<bool, Errno> {
#[cfg(test)]
{
let mut errors = PIDFD_LIVENESS_ERRORS.lock();
if let Some(error) = errors.get_mut(&self.pid).and_then(VecDeque::pop_front) {
if errors.get(&self.pid).is_some_and(VecDeque::is_empty) {
errors.remove(&self.pid);
}
return Err(error);
}
}
pidfd_is_live(&self.pidfd)
}
}
#[derive(Debug)]
enum ThreadHandle {
Pidfd(OwnedFd),
LegacyLeader { fd: OwnedFd },
Procfs { pid: Pid, directory: OwnedFd },
}
impl AsRawFd for ThreadHandle {
fn as_raw_fd(&self) -> RawFd {
match self {
Self::Pidfd(fd) => fd.as_raw_fd(),
Self::LegacyLeader { fd, .. } => fd.as_raw_fd(),
Self::Procfs { directory, .. } => directory.as_raw_fd(),
}
}
}
impl ThreadHandle {
#[cfg(test)]
fn try_clone(&self) -> io::Result<Self> {
Ok(match self {
Self::Pidfd(fd) => Self::Pidfd(fd.try_clone()?),
Self::LegacyLeader { fd } => Self::LegacyLeader {
fd: fd.try_clone()?,
},
Self::Procfs { pid, directory } => Self::Procfs {
pid: *pid,
directory: directory.try_clone()?,
},
})
}
fn wait_id(&self) -> Result<waitid::IdType, Errno> {
match self {
Self::Pidfd(fd) | Self::LegacyLeader { fd, .. } => {
Ok(waitid::IdType::Pidfd(fd.as_raw_fd()))
}
Self::Procfs { pid, directory } => {
match retained_proc_status(directory.as_raw_fd()) {
Ok(status) if status.pid == *pid => Ok(waitid::IdType::Pid((*pid).into())),
Ok(_) | Err(Errno::ENOENT | Errno::ESRCH) => Err(Errno::ECHILD),
Err(error) => Err(error),
}
}
}
}
fn wait_status(&self, flags: WaitPidFlag) -> Result<Option<i32>, Errno> {
match self {
Self::Pidfd(fd) | Self::LegacyLeader { fd, .. } => {
waitid::waitpidfd(fd.as_raw_fd(), flags)
}
Self::Procfs { pid, .. } => loop {
let status = self.legacy_wait_once(waitid::IdType::Pid((*pid).into()), flags)?;
if status.is_some() || flags.contains(WaitPidFlag::WNOHANG) {
return Ok(status);
}
thread::sleep(Duration::from_millis(1));
},
}
}
fn legacy_wait_once(
&self,
checked_id: waitid::IdType,
flags: WaitPidFlag,
) -> Result<Option<i32>, Errno> {
let status = observe_numeric_id(checked_id, flags)?;
self.wait_id()?;
Ok(status)
}
fn send_cancellation_signal(&self, signal: i32) -> Result<(), Errno> {
if signal == libc::SIGSTOP && !matches!(self, Self::Pidfd(_)) {
return if pidfd_is_live(self)? {
Err(Errno::EOPNOTSUPP)
} else {
Err(Errno::ESRCH)
};
}
Errno::result(unsafe {
libc::syscall(
libc::SYS_pidfd_send_signal,
self.as_raw_fd(),
signal,
std::ptr::null::<libc::siginfo_t>(),
0,
)
})
.map(|_| ())
}
fn send_process_signal(&self, signal: i32) -> Result<(), Errno> {
const PIDFD_SIGNAL_THREAD_GROUP: libc::c_uint = 1 << 1;
let flags = match self {
Self::Pidfd(_) => PIDFD_SIGNAL_THREAD_GROUP,
Self::LegacyLeader { .. } | Self::Procfs { .. } => 0,
};
Errno::result(unsafe {
libc::syscall(
libc::SYS_pidfd_send_signal,
self.as_raw_fd(),
signal,
std::ptr::null::<libc::siginfo_t>(),
flags,
)
})
.map(|_| ())
}
}
fn observe_numeric(pid: Pid, flags: WaitPidFlag) -> Result<Option<i32>, Errno> {
observe_numeric_id(waitid::IdType::Pid(pid.into()), flags)
}
fn observe_numeric_id(id: waitid::IdType, flags: WaitPidFlag) -> Result<Option<i32>, Errno> {
if !flags.contains(WaitPidFlag::WNOWAIT) {
return Err(Errno::EPERM);
}
waitid::wait_raw(id, flags | WaitPidFlag::WNOHANG)
}
#[cfg(test)]
fn open_thread_pidfd(pid: Pid) -> Result<ThreadHandle, Errno> {
open_thread_pidfd_for_policy(pid, WaitPolicy::Native)
}
#[cfg(test)]
fn open_thread_pidfd_for_policy(pid: Pid, policy: WaitPolicy) -> Result<ThreadHandle, Errno> {
open_thread_pidfd_for_policy_at(pid, policy, None)
}
fn open_thread_pidfd_for_policy_at(
pid: Pid,
policy: WaitPolicy,
proc_root: Option<&AlignedProcfs>,
) -> Result<ThreadHandle, Errno> {
#[cfg(test)]
let injected = PIDFD_OPEN_ERRORS.lock().remove(&pid);
#[cfg(test)]
let error = injected.or_else(|| {
FORCE_LEGACY_THREAD_GROUP.with(|group| {
let tgid = group.get()?;
worker_proc_snapshot(pid)
.ok()
.filter(|snapshot| snapshot.tgid == tgid)
.map(|_| Errno::EINVAL)
})
});
#[cfg(not(test))]
let error: Option<Errno> = None;
open_thread_handle_for_policy_at(pid, error, policy, proc_root)
}
#[cfg(test)]
fn open_thread_handle_with_error(pid: Pid, error: Option<Errno>) -> Result<ThreadHandle, Errno> {
open_thread_handle_for_policy(pid, error, WaitPolicy::PtracerThread)
}
#[cfg(test)]
fn open_thread_handle_for_policy(
pid: Pid,
error: Option<Errno>,
policy: WaitPolicy,
) -> Result<ThreadHandle, Errno> {
open_thread_handle_for_policy_at(pid, error, policy, None)
}
fn open_thread_handle_for_policy_at(
pid: Pid,
error: Option<Errno>,
policy: WaitPolicy,
proc_root: Option<&AlignedProcfs>,
) -> Result<ThreadHandle, Errno> {
let opened = error.map_or_else(|| pidfd_open_with_flags(pid, libc::O_EXCL), Err);
match opened {
Ok(fd) => Ok(ThreadHandle::Pidfd(fd)),
Err(error @ Errno::EINVAL) if policy == WaitPolicy::Native => {
report_native_thread_pidfd_requirement();
Err(error)
}
Err(Errno::EINVAL) if policy == WaitPolicy::PtracerThread => {
let captured_root;
let root = match proc_root {
Some(root) => root,
None => {
captured_root = AlignedProcfs::open()?;
&captured_root
}
};
let original = root.open_pid(pid, libc::O_PATH)?;
let snapshot = retained_proc_snapshot(original.as_raw_fd()).map_err(io_errno)?;
if snapshot.tgid == pid {
pidfd_open_with_flags(pid, 0).map(|fd| ThreadHandle::LegacyLeader { fd })
} else {
let directory = root.open_pid(pid, libc::O_RDONLY)?;
let original_status = retained_proc_status(original.as_raw_fd())?;
if original_status.pid != pid || original_status.tgid != snapshot.tgid {
return Err(Errno::ESRCH);
}
Ok(ThreadHandle::Procfs { pid, directory })
}
}
Err(error) => Err(error),
}
}
const NATIVE_THREAD_PIDFD_REQUIREMENT: &[u8] = b"safeptrace: native thread-pidfd waits require Linux 6.9 or newer with PIDFD_THREAD support; pidfd_open(PIDFD_THREAD) returned EINVAL. Use the explicit _on_ptracer_thread interface on older kernels.\n";
fn report_native_thread_pidfd_requirement() {
unsafe {
let fd = libc::fcntl(libc::STDERR_FILENO, libc::F_DUPFD_CLOEXEC, 3);
if fd < 0 {
return;
}
let stderr = OwnedFd::from_raw_fd(fd);
let mut metadata = std::mem::MaybeUninit::<libc::stat>::uninit();
if libc::fstat(stderr.as_raw_fd(), metadata.as_mut_ptr()) != 0 {
return;
}
let metadata = metadata.assume_init();
let kind = metadata.st_mode & libc::S_IFMT;
if kind == libc::S_IFREG {
let mut limit = std::mem::MaybeUninit::<libc::rlimit>::uninit();
if libc::getrlimit(libc::RLIMIT_FSIZE, limit.as_mut_ptr()) != 0
|| limit.assume_init().rlim_cur != libc::RLIM_INFINITY
{
return;
}
libc::write(
stderr.as_raw_fd(),
NATIVE_THREAD_PIDFD_REQUIREMENT.as_ptr().cast(),
NATIVE_THREAD_PIDFD_REQUIREMENT.len(),
);
return;
}
if kind == libc::S_IFSOCK {
libc::send(
stderr.as_raw_fd(),
NATIVE_THREAD_PIDFD_REQUIREMENT.as_ptr().cast(),
NATIVE_THREAD_PIDFD_REQUIREMENT.len(),
libc::MSG_DONTWAIT | libc::MSG_NOSIGNAL,
);
return;
}
let access = match kind {
libc::S_IFIFO => libc::O_RDWR,
libc::S_IFCHR => libc::O_WRONLY,
_ => return,
};
let flags = access | libc::O_NONBLOCK | libc::O_CLOEXEC | libc::O_NOCTTY;
let fd = libc::open(c"/proc/self/fd/2".as_ptr(), flags);
if fd < 0 {
return;
}
let output = OwnedFd::from_raw_fd(fd);
let mut reopened = std::mem::MaybeUninit::<libc::stat>::uninit();
if libc::fstat(output.as_raw_fd(), reopened.as_mut_ptr()) == 0 {
let reopened = reopened.assume_init();
if reopened.st_dev == metadata.st_dev
&& reopened.st_ino == metadata.st_ino
&& reopened.st_mode & libc::S_IFMT == kind
{
libc::write(
output.as_raw_fd(),
NATIVE_THREAD_PIDFD_REQUIREMENT.as_ptr().cast(),
NATIVE_THREAD_PIDFD_REQUIREMENT.len(),
);
}
}
}
}
#[derive(Debug)]
struct AlignedProcfs {
root: OwnedFd,
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
struct ProcDirectoryKey {
device: libc::dev_t,
inode: libc::ino_t,
}
impl ProcDirectoryKey {
fn from_stat(metadata: libc::stat) -> Self {
Self {
device: metadata.st_dev,
inode: metadata.st_ino,
}
}
}
impl AlignedProcfs {
fn open() -> Result<Self, Errno> {
let _open = launch_window::TransientOpen::begin();
let raw = unsafe {
libc::open(
c"/proc".as_ptr(),
libc::O_PATH | libc::O_DIRECTORY | libc::O_CLOEXEC,
)
};
if raw < 0 {
return Err(Errno::last());
}
let root = unsafe { OwnedFd::from_raw_fd(raw) };
Self::from_root(root)
}
fn from_root(root: OwnedFd) -> Result<Self, Errno> {
let _open = launch_window::TransientOpen::begin();
let mut filesystem = std::mem::MaybeUninit::<libc::statfs>::uninit();
if unsafe { libc::fstatfs(root.as_raw_fd(), filesystem.as_mut_ptr()) } < 0 {
return Err(Errno::last());
}
if unsafe { filesystem.assume_init() }.f_type != libc::PROC_SUPER_MAGIC {
return Err(Errno::EXDEV);
}
let root = Self { root };
#[cfg(test)]
OTHER_STATUS_OPENS.with(|opens| opens.set(opens.get() + 1));
let raw = unsafe {
libc::openat(
root.root.as_raw_fd(),
c"thread-self/status".as_ptr(),
libc::O_RDONLY | libc::O_CLOEXEC,
)
};
if raw < 0 {
return Err(Errno::last());
}
let fd = unsafe { OwnedFd::from_raw_fd(raw) };
let mut bounded = io::Read::take(fs::File::from(fd), 1024 * 1024);
let mut bytes = Vec::new();
io::Read::read_to_end(&mut bounded, &mut bytes).map_err(io_errno)?;
aligned_proc_pid_namespace(&bytes, Pid::from(nix::unistd::gettid()), || {
root.pid_namespaces()
})?;
Ok(root)
}
fn pid_namespaces(&self) -> Result<bool, Errno> {
match self.entry_key(c"thread-self/ns/pid", libc::AT_SYMLINK_NOFOLLOW) {
Ok(_) => Ok(true),
Err(Errno::ENOENT) => self
.entry_key(c"thread-self/ns/mnt", libc::AT_SYMLINK_NOFOLLOW)
.map(|_| false)
.map_err(|_| Errno::ENOENT),
Err(error) => Err(error),
}
}
fn open_pid(&self, pid: Pid, flags: i32) -> Result<OwnedFd, Errno> {
let _open = launch_window::TransientOpen::begin();
let name = std::ffi::CString::new(pid.as_raw().to_string()).expect("decimal pid");
let raw = unsafe {
libc::openat(
self.root.as_raw_fd(),
name.as_ptr(),
flags | libc::O_DIRECTORY | libc::O_CLOEXEC | libc::O_NOFOLLOW,
)
};
if raw < 0 {
Err(Errno::last())
} else {
Ok(unsafe { OwnedFd::from_raw_fd(raw) })
}
}
fn open_current_thread(&self) -> Result<OwnedFd, Errno> {
let _open = launch_window::TransientOpen::begin();
let raw = unsafe {
libc::openat(
self.root.as_raw_fd(),
c"thread-self".as_ptr(),
libc::O_PATH | libc::O_DIRECTORY | libc::O_CLOEXEC,
)
};
if raw < 0 {
return Err(Errno::last());
}
Ok(unsafe { OwnedFd::from_raw_fd(raw) })
}
#[cfg(all(test, not(sanitized)))]
fn current_thread_status(&self) -> Result<RetainedProcStatus, Errno> {
retained_proc_status(self.open_current_thread()?.as_raw_fd())
}
fn directory_key(&self, name: &std::ffi::CStr) -> Result<ProcDirectoryKey, Errno> {
self.entry_key(name, 0)
}
fn entry_key(&self, name: &std::ffi::CStr, flags: i32) -> Result<ProcDirectoryKey, Errno> {
let mut metadata = std::mem::MaybeUninit::<libc::stat>::uninit();
if unsafe {
libc::fstatat(
self.root.as_raw_fd(),
name.as_ptr(),
metadata.as_mut_ptr(),
flags,
)
} < 0
{
Err(Errno::last())
} else {
Ok(ProcDirectoryKey::from_stat(unsafe {
metadata.assume_init()
}))
}
}
}
#[cfg(test)]
fn require_aligned_proc_pid_namespace() -> Result<(), Errno> {
AlignedProcfs::open().map(|_| ())
}
fn aligned_proc_pid_namespace(
status: &[u8],
caller: Pid,
pid_namespaces: impl FnOnce() -> Result<bool, Errno>,
) -> Result<(), Errno> {
let Some(levels) = status_field(status, b"NSpid:")? else {
if pid_namespaces()? {
return Err(Errno::EOPNOTSUPP);
}
let pid = status_field(status, b"Pid:")?.ok_or(Errno::EIO)?;
return match pid.trim().parse::<i32>() {
Ok(pid) if pid == caller.as_raw() => Ok(()),
Ok(_) => Err(Errno::EXDEV),
Err(_) => Err(Errno::EIO),
};
};
let mut levels = levels.split_ascii_whitespace();
let mounted_tid = levels
.next()
.ok_or(Errno::EIO)?
.parse::<i32>()
.map_err(|_| Errno::EIO)?;
if mounted_tid != caller.as_raw() || levels.next().is_some() {
return Err(Errno::EXDEV);
}
Ok(())
}
fn status_field<'a>(status: &'a [u8], name: &[u8]) -> Result<Option<&'a str>, Errno> {
status
.split_inclusive(|byte| *byte == b'\n')
.find(|line| line.starts_with(name) && line.last() == Some(&b'\n'))
.map(|line| std::str::from_utf8(&line[name.len()..]).map_err(|_| Errno::EIO))
.transpose()
}
#[cfg(test)]
fn open_thread_pidfd_kernel(pid: Pid) -> Result<ThreadHandle, Errno> {
open_thread_handle_with_error(pid, None)
}
fn pidfd_open_with_flags(pid: Pid, flags: i32) -> Result<OwnedFd, Errno> {
let fd = unsafe { libc::syscall(libc::SYS_pidfd_open, pid.as_raw(), flags) } as i32;
if fd == -1 {
Err(io_errno(io::Error::last_os_error()))
} else {
Ok(unsafe { OwnedFd::from_raw_fd(fd) })
}
}
fn pidfd_is_live(pidfd: &ThreadHandle) -> Result<bool, Errno> {
descriptor_is_live(pidfd.as_raw_fd())
}
fn descriptor_is_live(raw_fd: RawFd) -> Result<bool, Errno> {
let result = unsafe {
libc::syscall(
libc::SYS_pidfd_send_signal,
raw_fd,
0,
std::ptr::null::<libc::siginfo_t>(),
0,
)
};
if result == 0 {
return Ok(true);
}
let error = io_errno(io::Error::last_os_error());
if error == Errno::ESRCH {
Ok(false)
} else {
Err(error)
}
}
fn descriptor_is_current_task(raw_fd: RawFd) -> bool {
let mut info = unsafe { std::mem::zeroed::<libc::siginfo_t>() };
info.si_signo = 0;
info.si_code = libc::SI_USER;
unsafe {
libc::syscall(
libc::SYS_pidfd_send_signal,
raw_fd,
0,
&info as *const libc::siginfo_t,
0,
) == 0
}
}
struct PendingWorker {
start: std::sync::mpsc::SyncSender<()>,
_handle: JoinHandle<()>,
}
impl PendingWorker {
fn start(self) {
self.start
.send(())
.expect("new notifier worker dropped its start gate");
}
}
fn spawn_worker(
pid: Pid,
event: Arc<Event>,
identity: Arc<WorkerIdentity>,
) -> io::Result<PendingWorker> {
#[cfg(test)]
event.pause_notifier_registration_for_test(WORKER_STARTING);
#[cfg(test)]
if let Some(error) = SPAWN_WORKER_ERRORS.lock().remove(&pid) {
if let Some(pause) = SPAWN_FAILURE_PAUSES.lock().remove(&pid) {
pause.captured.wait();
pause.resume.wait();
}
return Err(io::Error::from_raw_os_error(error));
}
let (start, wait_for_start) = std::sync::mpsc::sync_channel(1);
let handle = thread::Builder::new()
.name(format!("guest-{}", pid))
.spawn(move || {
if wait_for_start.recv().is_ok() {
worker_thread(pid, event, identity);
}
})?;
#[cfg(test)]
{
*SPAWN_WORKER_COUNTS.lock().entry(pid).or_default() += 1;
}
Ok(PendingWorker {
start,
_handle: handle,
})
}
fn authenticate_legacy_wait_owner(identity: &WorkerIdentity, event: &Event) -> Result<(), Errno> {
let current = identity.current_tracer_pid().map_err(|error| {
if matches!(error, Errno::ENOENT | Errno::ESRCH) {
Errno::ECHILD
} else {
error
}
})?;
if current.as_raw() == 0 {
return Err(Errno::ECHILD);
}
if current != Pid::from(nix::unistd::gettid()) {
return Err(Errno::EPERM);
}
if event.legacy_wait_owner.get().is_none() {
let owner = identity.capture_current_owner(false)?;
let _ = event.legacy_wait_owner.set(owner);
}
if !event.legacy_wait_owner.get().unwrap().is_current()? {
return Err(Errno::EPERM);
}
Ok(())
}
fn legacy_stop_proves_owner(identity: &WorkerIdentity, event: &Event) -> bool {
let Some(owner) = event.legacy_wait_owner.get() else {
return false;
};
let proved = owner.is_current() == Ok(true)
&& numeric_stop_names_retained_task(identity.pid, || identity.pidfd_is_live());
#[cfg(test)]
if proved {
LEGACY_STOP_PROOFS_ACCEPTED.with(|proofs| proofs.set(proofs.get() + 1));
} else {
LEGACY_STOP_PROOFS_REFUSED.with(|proofs| proofs.set(proofs.get() + 1));
}
proved
}
fn numeric_stop_names_retained_task(
pid: Pid,
retained_is_live: impl FnOnce() -> Result<bool, Errno>,
) -> bool {
nix::sys::ptrace::getevent(pid.into()).is_ok() && retained_is_live() == Ok(true)
}
fn try_consume_legacy_status(
identity: &WorkerIdentity,
event: &Event,
handle: Option<&EventHandle>,
) -> Result<Option<i32>, Errno> {
let Some(mut gate) = event.terminal_reaping.try_write() else {
return Ok(None);
};
if *gate || handle.is_some_and(|handle| !std::ptr::eq(handle.event().as_ref(), event)) {
return Ok(None);
}
let observe = WaitPidFlag::from_bits_retain(
WaitPidFlag::WEXITED.bits()
| WaitPidFlag::WSTOPPED.bits()
| WaitPidFlag::WNOWAIT.bits()
| WaitPidFlag::WNOHANG.bits()
| libc::__WALL
| libc::__WNOTHREAD,
);
let observed = observe_numeric(identity.pid, observe);
let stop_proved = matches!(&observed, Ok(Some(status)) if libc::WIFSTOPPED(*status))
&& legacy_stop_proves_owner(identity, event);
if !stop_proved {
authenticate_legacy_wait_owner(identity, event)?;
}
let Some(observed) = observed? else {
return Ok(None);
};
let terminal = libc::WIFEXITED(observed) || libc::WIFSIGNALED(observed);
let flags = WaitPidFlag::from_bits_retain(
if terminal {
WaitPidFlag::WEXITED.bits()
} else {
WaitPidFlag::WSTOPPED.bits()
} | WaitPidFlag::WNOHANG.bits()
| libc::__WALL
| libc::__WNOTHREAD,
);
if terminal {
*gate = true;
}
let result = waitid::wait_raw(waitid::IdType::Pid(identity.pid.into()), flags);
match result {
Ok(Some(status)) => {
if terminal {
debug_assert!(
libc::WIFEXITED(status) || libc::WIFSIGNALED(status),
"reaped a nonterminal status after a terminal observation"
);
} else {
debug_assert!(libc::WIFSTOPPED(status), "stop-only wait reaped a task");
}
Ok(Some(status))
}
Ok(None) | Err(Errno::EINTR) => {
if terminal {
*gate = false;
}
Ok(None)
}
Err(Errno::ECHILD) if !terminal => Ok(None),
Err(error) => Err(error),
}
}
fn wait_thread_consuming(pid: Pid, handle: &ThreadHandle, event: &Event) -> Result<i32, Errno> {
debug_assert!(!matches!(handle, ThreadHandle::Procfs { .. }));
let observe = WaitPidFlag::from_bits_retain(
WaitPidFlag::WEXITED.bits()
| WaitPidFlag::WSTOPPED.bits()
| WaitPidFlag::WNOWAIT.bits()
| libc::__WALL,
);
let consume_stop = WaitPidFlag::from_bits_retain(
WaitPidFlag::WSTOPPED.bits() | WaitPidFlag::WNOHANG.bits() | libc::__WALL,
);
let reap = WaitPidFlag::from_bits_retain(
WaitPidFlag::WEXITED.bits() | WaitPidFlag::WNOHANG.bits() | libc::__WALL,
);
loop {
let status = match handle.wait_status(observe) {
Ok(status) => status.expect("blocking waitid returned no status"),
Err(Errno::EINTR) => continue,
Err(error) => return Err(error),
};
if libc::WIFEXITED(status) || libc::WIFSIGNALED(status) {
event.mark_terminal_reaping();
loop {
return match handle.wait_status(reap) {
Ok(Some(reaped)) => {
debug_assert!(
libc::WIFEXITED(reaped) || libc::WIFSIGNALED(reaped),
"reaped a nonterminal status after a terminal observation"
);
Ok(reaped)
}
Ok(None) => unreachable!("observed terminal status is not reapable"),
Err(Errno::EINTR) => continue,
Err(error) => Err(error),
};
}
}
#[cfg(test)]
if let Some(pause) = STOP_CONSUMPTION_PAUSES.lock().remove(&pid) {
pause.captured.wait();
pause.resume.wait();
}
#[cfg(not(test))]
let _ = pid;
match handle.wait_status(consume_stop) {
Ok(Some(stop)) => return Ok(stop),
Ok(None) | Err(Errno::EINTR | Errno::ECHILD) => continue,
Err(error) => return Err(error),
}
}
}
fn wait_thread_status(identity: &WorkerIdentity, event: &Event) -> Option<i32> {
match wait_thread_consuming(identity.pid, &identity.pidfd, event) {
Ok(status) => Some(status),
Err(Errno::ECHILD) => None,
Err(err) => {
panic!(
"exact-thread waitid({}) failed unexpectedly: {}",
identity.pid, err
)
}
}
}
fn close_unowned_legacy_wait(identity: &WorkerIdentity, event: &Event) -> Result<bool, Errno> {
check_legacy_attachment(identity, event).map(|check| check == LegacyAttachment::Closed)
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
enum LegacyAttachment {
Closed,
Read,
Unread,
Handoff,
}
fn check_legacy_attachment(
identity: &WorkerIdentity,
event: &Event,
) -> Result<LegacyAttachment, Errno> {
let Some(mut gate) = event.terminal_reaping.try_write() else {
return Ok(LegacyAttachment::Unread);
};
if *gate {
return Ok(LegacyAttachment::Closed);
}
match identity.current_tracer_pid() {
Ok(tracer) => {
if let Some(owner) = event.legacy_wait_owner.get() {
if owner.tid == tracer && owner.is_live()? {
return Ok(LegacyAttachment::Read);
}
} else if tracer.as_raw() != 0 {
let root = identity.proc_root.as_ref().ok_or(Errno::EXDEV)?;
match root
.open_pid(tracer, libc::O_PATH)
.and_then(|directory| retained_proc_status(directory.as_raw_fd()))
{
Ok(host)
if host.pid == tracer && host.tgid == Pid::from(nix::unistd::getpid()) =>
{
return Ok(LegacyAttachment::Read);
}
Ok(_) | Err(Errno::ENOENT | Errno::ESRCH) => {}
Err(error) => return Err(error),
}
}
if tracer.as_raw() == 0
&& event.detach_handoff_pending()
&& identity.observed_process_generation() == Some(true)
&& !identity.task_is_zombie()
{
#[cfg(test)]
event.detached_echild_kept.fetch_add(1, Ordering::Relaxed);
return Ok(LegacyAttachment::Handoff);
}
}
Err(Errno::ENOENT | Errno::ESRCH) => {}
Err(error) => return Err(error),
}
*gate = true;
Ok(LegacyAttachment::Closed)
}
const LEGACY_FIRST_WAIT: Duration = Duration::from_micros(10);
const LEGACY_POLL_PERIOD: Duration = Duration::from_millis(1);
const LEGACY_WORKER_TIMER_SLACK_NS: libc::c_ulong = 1_000;
const LEGACY_CHECK_PERIOD: Duration = Duration::from_millis(10);
const LEGACY_HELD_PERIOD: Duration = Duration::from_millis(100);
const LEGACY_PROBE_WINDOW: Duration = Duration::from_millis(10);
static LEGACY_WORKERS: LazyLock<Mutex<Vec<ListedLegacyWorker>>> =
LazyLock::new(|| Mutex::new(Vec::new()));
struct ListedLegacyWorker {
tgid: Pid,
event: Weak<Event>,
}
struct LegacyWorkerListing(Weak<Event>);
impl LegacyWorkerListing {
fn new(event: &Arc<Event>, tgid: Pid) -> Self {
let event = Arc::downgrade(event);
LEGACY_WORKERS.lock().push(ListedLegacyWorker {
tgid,
event: event.clone(),
});
Self(event)
}
}
impl Drop for LegacyWorkerListing {
fn drop(&mut self) {
LEGACY_WORKERS
.lock()
.retain(|listed| !listed.event.ptr_eq(&self.0));
}
}
fn probe_legacy_workers(tgid: Option<Pid>) {
let listed: Vec<Arc<Event>> = LEGACY_WORKERS
.lock()
.iter()
.filter(|listed| tgid.is_none_or(|tgid| listed.tgid == tgid))
.filter_map(|listed| listed.event.upgrade())
.collect();
for event in listed {
event.probe_legacy_worker();
}
}
fn may_end_other_stops(status: i32) -> bool {
libc::WIFEXITED(status) || libc::WIFSIGNALED(status) || status == PTRACE_EVENT_EXIT_STOP
}
fn legacy_held_period(event: &Event) -> Duration {
#[cfg(test)]
if let Some(period) = *event.legacy_held_period.lock() {
return period;
}
let _ = event;
LEGACY_HELD_PERIOD
}
fn legacy_poll_period(event: &Event) -> Duration {
#[cfg(test)]
if let Some(period) = *event.legacy_poll_period.lock() {
return period;
}
let _ = event;
LEGACY_POLL_PERIOD
}
fn legacy_worker_thread(
_pid: Pid,
event: &Arc<Event>,
identity: &WorkerIdentity,
) -> Option<Arc<WorkerIdentity>> {
let observe = WaitPidFlag::from_bits_retain(
WaitPidFlag::WEXITED.bits()
| WaitPidFlag::WSTOPPED.bits()
| WaitPidFlag::WNOWAIT.bits()
| WaitPidFlag::WNOHANG.bits()
| libc::__WALL,
);
unsafe {
libc::prctl(
libc::PR_SET_TIMERSLACK,
LEGACY_WORKER_TIMER_SLACK_NS,
0,
0,
0,
)
};
let _listing = LegacyWorkerListing::new(event, identity.snapshot.tgid);
let mut wait = LEGACY_FIRST_WAIT;
#[cfg(test)]
let mut late = false;
let mut checked_at: Option<Instant> = None;
let mut seen = LegacyWakeups::default();
let mut held: Option<u64> = None;
#[cfg(test)]
let mut held_status = 0;
#[cfg(test)]
let mut held_wait_expired = false;
let mut probing_until: Option<Instant> = None;
loop {
#[cfg(test)]
pause_retirement_for_test(&event.legacy_worker_cycle_pause);
let mut progress = event.legacy_wait.lock();
if let Some(status) = progress.consumed.pop_front() {
drop(progress);
#[cfg(test)]
if (libc::WIFEXITED(status) || libc::WIFSIGNALED(status))
&& let Some(pause) = event.terminal_publish_pause.lock().take()
{
let _ = pause.captured.send(());
let _ = pause.resume.recv_timeout(Duration::from_secs(2));
}
#[cfg(test)]
if std::mem::take(&mut late) {
event.legacy_late_turns.fetch_add(1, Ordering::Relaxed);
}
let before_publication = event.legacy_wakeup.lock().targeted;
event.update(status);
#[cfg(test)]
pause_retirement_for_test(&event.legacy_publish_pause);
if may_end_other_stops(status) {
probe_legacy_workers(Some(identity.snapshot.tgid));
}
if libc::WIFEXITED(status) || libc::WIFSIGNALED(status) {
let mut progress = event.legacy_wait.lock();
debug_assert!(progress.consumed.is_empty());
progress.retiring = true;
progress.observed = None;
break;
}
held = libc::WIFSTOPPED(status).then_some(before_publication);
#[cfg(test)]
{
held_status = status;
}
probing_until = None;
wait = LEGACY_FIRST_WAIT;
continue;
}
if !progress.retiring
&& let Some(native) = event.native_wait_identity.get()
{
unsafe { libc::prctl(libc::PR_SET_TIMERSLACK, 0 as libc::c_ulong, 0, 0, 0) };
return Some(native.clone());
}
let observed = if progress.retiring {
Err(Errno::ECHILD)
} else {
observe_numeric(identity.pid, observe)
};
let due = !matches!(observed, Ok(None))
|| progress.error.is_some()
|| checked_at.is_none_or(|at| at.elapsed() >= LEGACY_CHECK_PERIOD);
#[cfg(test)]
event.legacy_worker_turns.fetch_add(1, Ordering::Relaxed);
let postchecked = if !due {
true
} else {
match check_legacy_attachment(identity, event) {
Ok(LegacyAttachment::Closed) => {
progress.retiring = true;
progress.observed = None;
drop(progress);
event.mark_echild();
break;
}
Ok(LegacyAttachment::Read) => {
#[cfg(test)]
event
.legacy_worker_status_reads
.fetch_add(1, Ordering::Relaxed);
checked_at = Some(Instant::now());
true
}
Ok(LegacyAttachment::Unread) => false,
Ok(LegacyAttachment::Handoff) => false,
Err(error) => {
progress.error = Some(error);
drop(progress);
event.status_changed.notify_all();
event.status_waker.wake();
event.exit_waiters.wake_all();
thread::sleep(Duration::from_millis(1));
continue;
}
}
};
let observed = match observed {
Ok(status) if !postchecked => identity.pidfd.wait_id().map(|_| status),
observed => observed,
};
#[cfg(test)]
let hinted_before = progress.observed.is_some();
match observed {
Ok(status) => {
progress.observed = status;
progress.error = None;
}
Err(Errno::ECHILD) => match close_unowned_legacy_wait(identity, event) {
Ok(true) => {
debug_assert!(progress.consumed.is_empty());
progress.retiring = true;
progress.observed = None;
drop(progress);
event.mark_echild();
break;
}
Ok(false) => {}
Err(error) => progress.error = Some(error),
},
Err(Errno::EINTR) => {}
Err(error) => progress.error = Some(error),
}
let wake = progress.observed.is_some() || progress.error.is_some();
#[cfg(test)]
if std::mem::take(&mut late) && !hinted_before && progress.observed.is_some() {
event.legacy_late_turns.fetch_add(1, Ordering::Relaxed);
}
#[cfg(test)]
if progress.observed.is_some() {
event.legacy_hint_turns.fetch_add(1, Ordering::Relaxed);
}
#[cfg(all(test, not(sanitized)))]
if !hinted_before && progress.observed.is_some() {
event.legacy_wake_log.lock().push(LegacyWakeLogEntry::Hint);
}
#[cfg(test)]
if std::mem::take(&mut held_wait_expired) && progress.observed.is_some() {
event
.legacy_held_expiry_finds
.fetch_add(1, Ordering::Relaxed);
}
#[cfg(test)]
if progress.observed.is_some() {
pause_retirement_for_test(&event.legacy_observation_pause);
}
drop(progress);
if wake {
event.status_changed.notify_all();
event.status_waker.wake();
event.exit_waiters.wake_all();
}
let held_wait =
!wake && held.is_some() && probing_until.is_none_or(|until| Instant::now() >= until);
let held_period = legacy_held_period(event);
#[cfg(test)]
if held_wait {
pause_for_test_noting(&event.legacy_held_wait_pause, || {
*event.legacy_held_wait_paused_on.lock() = held.map(|count| (held_status, count));
});
}
#[cfg(test)]
if wake {
pause_retirement_for_test(&event.legacy_sleep_check_pause);
}
#[cfg(all(test, not(sanitized)))]
let mut woke_by_notify = false;
let mut wakeups = event.legacy_wakeup.lock();
let held_timeout = held_wait && held == Some(wakeups.targeted);
let timeout = if wake {
legacy_poll_period(event)
} else if held_timeout {
held_period
} else {
wait
};
#[cfg(test)]
if wake
&& *wakeups == seen
&& let Some(parking) = event.legacy_parking_signal.lock().take()
{
let _ = parking.try_send(());
}
if *wakeups == seen {
#[cfg(test)]
if held_timeout {
let observed = event.legacy_worker_wakes.load(Ordering::Relaxed);
event
.legacy_held_wait_wakes
.store(observed, Ordering::Relaxed);
event.legacy_held_waits.fetch_add(1, Ordering::Relaxed);
}
#[cfg(all(test, not(sanitized)))]
event
.legacy_wake_log
.lock()
.push(LegacyWakeLogEntry::WaitBegin(wakeups.targeted));
let _waited = event.legacy_wake.wait_for(&mut wakeups, timeout);
#[cfg(all(test, not(sanitized)))]
{
woke_by_notify = !_waited.timed_out();
}
#[cfg(test)]
if !_waited.timed_out() {
event.legacy_worker_wakes.fetch_add(1, Ordering::Relaxed);
}
#[cfg(test)]
{
held_wait_expired = held_timeout && _waited.timed_out();
}
}
let targeted = wakeups.targeted != seen.targeted;
let probed = wakeups.probes != seen.probes;
#[cfg(all(test, not(sanitized)))]
if woke_by_notify {
event.legacy_wake_log.lock().push(if targeted || probed {
LegacyWakeLogEntry::CountedWake
} else {
LegacyWakeLogEntry::Uncounted
});
}
if held.is_some_and(|count| count != wakeups.targeted) {
#[cfg(all(test, not(sanitized)))]
event
.legacy_wake_log
.lock()
.push(LegacyWakeLogEntry::HoldEndedByCount(wakeups.targeted));
held = None;
probing_until = None;
}
seen = *wakeups;
drop(wakeups);
if targeted {
wait = LEGACY_FIRST_WAIT;
} else if probed {
if held.is_some() {
wait = LEGACY_FIRST_WAIT;
probing_until = Some(Instant::now() + LEGACY_PROBE_WINDOW);
}
} else {
#[cfg(test)]
{
late = timeout >= LEGACY_POLL_PERIOD;
}
if !wake {
wait = wait.saturating_mul(2).min(LEGACY_POLL_PERIOD);
}
}
}
None
}
fn worker_thread(pid: Pid, event: Arc<Event>, identity: Arc<WorkerIdentity>) {
let native = if matches!(identity.pidfd, ThreadHandle::Procfs { .. }) {
legacy_worker_thread(pid, &event, &identity)
} else {
Some(identity.clone())
};
if let Some(identity) = native {
let mut retrying_echild = false;
loop {
if retrying_echild && event.mark_echild_if_unowned(&identity) {
break;
}
let Some(status) = wait_thread_status(&identity, &event) else {
if !event.mark_echild_if_unowned(&identity) {
retrying_echild = true;
thread::sleep(Duration::from_millis(1));
continue;
}
break;
};
retrying_echild = false;
#[cfg(test)]
if (libc::WIFEXITED(status) || libc::WIFSIGNALED(status))
&& let Some(pause) = event.terminal_publish_pause.lock().take()
{
let _ = pause.captured.send(());
let _ = pause.resume.recv_timeout(Duration::from_secs(2));
}
event.update(status);
if may_end_other_stops(status) {
probe_legacy_workers(Some(identity.snapshot.tgid));
}
if libc::WIFEXITED(status) || libc::WIFSIGNALED(status) {
break;
}
}
}
NOTIFIER.remove(pid, &event);
#[cfg(test)]
pause_retirement_for_test(&event.worker_identity_retirement_pause);
drop(identity);
event.mark_worker_done();
}
fn check_sync_wait_authority(handle: &EventHandle, token: &TraceeToken) -> Result<(), Errno> {
token.policy.check_handle(handle)?;
if token.policy == WaitPolicy::PtracerThread {
let owner = token.ptracer_owner.clone().ok_or(Errno::EPERM)?;
if !token.ptracer_wait_role {
return Err(Errno::EPERM);
}
let mut affinity = PtracerAffinity {
owner: Some(owner),
wait_role: true,
};
affinity.check_host(handle)?;
if !handle.event().is_terminal() && !*handle.event().terminal_reaping.read() {
affinity.check_attachment(handle)?;
}
}
Ok(())
}
fn try_replay_sync_terminal(
pid: Pid,
handle: &EventHandle,
token: &TraceeToken,
) -> Option<Result<Wait, Error>> {
let handle = handle.resolved_handle();
if let Err(error) = check_sync_wait_authority(&handle, token) {
return Some(Err(error.into()));
}
let event = handle.event();
let reservation = event.try_terminal_reservation_sync()?;
Some(match reservation {
Err(error) => Err(error.into()),
Ok(reservation) => Wait::from_raw_with_token(
pid,
reservation.status,
sync_decode_token(token, handle.clone()),
),
})
}
fn try_wait_retained_notifier_terminal(
pid: Pid,
handle: &EventHandle,
token: &TraceeToken,
) -> Option<Result<Wait, Error>> {
let handle = handle.resolved_handle();
if let Err(error) = check_sync_wait_authority(&handle, token) {
return Some(Err(error.into()));
}
let identity = handle.identity()?;
let event = handle.event();
if identity.pid != pid || handle.uses_procfs_wait() || !event.worker_is_running() {
#[cfg(test)]
{
let pause = event.sync_wait_ineligible_pause.lock().take();
if let Some(pause) = pause {
let _ = pause.captured.try_send(());
let _ = pause.resume.recv_timeout(Duration::from_secs(2));
}
}
return None;
}
match event.claim_sync_wait() {
Ok(SyncWaitOwnership::Notifier) => {}
Ok(SyncWaitOwnership::Claimed(_)) => return None,
Err(error) => return Some(Err(error.into())),
}
wait_retained_notifier_terminal_status(pid, &handle, token)
}
fn wait_retained_notifier_terminal_status(
pid: Pid,
handle: &EventHandle,
token: &TraceeToken,
) -> Option<Result<Wait, Error>> {
if let Err(error) = check_sync_wait_authority(handle, token) {
return Some(Err(error.into()));
}
let event = handle.event();
let reservation = match event.wait_status_reservation_sync(handle) {
Ok(reservation) => reservation,
Err(error) => return Some(Err(error.into())),
};
if !libc::WIFEXITED(reservation.status) && !libc::WIFSIGNALED(reservation.status) {
return None;
}
match event.decode_status_return(reservation, |status| {
Wait::from_raw_with_token(pid, status, sync_decode_token(token, handle.clone()))
}) {
Ok(StatusReturn::Returned(wait)) => Some(Ok(wait)),
Ok(StatusReturn::Cancelled(_)) => Some(Err(Errno::ECANCELED.into())),
Ok(StatusReturn::RetiredDeadExec) => None,
Err(error) => Some(Err(error)),
}
}
fn reconcile_failed_sync_capture(
pid: Pid,
requested: &EventHandle,
token: &TraceeToken,
capture_error: Errno,
) -> Result<Wait, Error> {
loop {
let handle = requested.resolved_handle();
check_sync_wait_authority(&handle, token)?;
let event = Arc::clone(handle.event());
let ownership = event.claim_sync_wait()?;
if !Arc::ptr_eq(handle.event(), &event) {
drop(ownership);
continue;
}
if !handle
.identity()
.is_some_and(|identity| identity.pid == pid)
{
return Err(capture_error.into());
}
return match ownership {
SyncWaitOwnership::Notifier => {
wait_retained_notifier_terminal_status(pid, &handle, token)
.unwrap_or_else(|| Err(capture_error.into()))
}
SyncWaitOwnership::Claimed(_owner) => {
try_replay_sync_terminal(pid, &handle, token)
.unwrap_or_else(|| Err(capture_error.into()))
}
};
}
}
fn sync_decode_token(token: &TraceeToken, event: EventHandle) -> TraceeToken {
TraceeToken {
event,
policy: token.policy,
ptracer_owner: token.ptracer_owner.clone(),
ptracer_wait_role: token.ptracer_wait_role,
}
}
fn resume_cancelled_sync_status(pid: Pid, status: i32, handle: &EventHandle) -> Result<(), Error> {
if !libc::WIFSTOPPED(status) {
return Ok(());
}
match nix::sys::ptrace::cont(pid.into(), None) {
Ok(()) => {
handle.invalidate_numeric_auth();
handle.expect_stop();
Ok(())
}
Err(nix::errno::Errno::ESRCH) => {
handle.expect_stop();
Ok(())
}
Err(nix::errno::Errno::EIO) if status != PTRACE_EVENT_EXIT_STOP => Ok(()),
Err(error) => Err(Errno::new(error as i32).into()),
}
}
pub(super) fn wait_sync(pid: Pid, token: TraceeToken) -> Result<Wait, Error> {
wait_sync_for_policy(pid, token, WaitPolicy::Native)
}
fn wait_sync_for_policy(
pid: Pid,
mut token: TraceeToken,
policy: WaitPolicy,
) -> Result<Wait, Error> {
token.policy = policy;
let requested = token.event().resolved_handle();
policy.check_handle(&requested)?;
if let Some(replayed) = try_replay_sync_terminal(pid, &requested, &token) {
return replayed;
}
if let Some(terminal) = try_wait_retained_notifier_terminal(pid, &requested, &token) {
return terminal;
}
let handle = match NOTIFIER.sync_handle_for_policy(pid, &requested, policy) {
Ok(handle) => handle,
Err(error) => {
if let Some(replayed) = try_replay_sync_terminal(pid, &requested, &token) {
return replayed;
}
return reconcile_failed_sync_capture(pid, &requested, &token, error);
}
};
token.event = handle;
#[cfg(test)]
if let Some(pause) = SYNC_HANDLE_PAUSES.lock().remove(&pid) {
pause.captured.wait();
pause.resume.wait();
}
'claim: loop {
let handle = token.event().resolved_handle();
let event = Arc::clone(handle.event());
match event.claim_sync_wait()? {
SyncWaitOwnership::Notifier => {
let reservation = event.wait_status_reservation_sync(&handle)?;
match event.decode_status_return(reservation, |status| {
Wait::from_raw_with_token(pid, status, sync_decode_token(&token, handle))
})? {
StatusReturn::Returned(decoded) => return Ok(decoded),
StatusReturn::Cancelled(_) => return Err(Errno::ECANCELED.into()),
StatusReturn::RetiredDeadExec => continue,
}
}
SyncWaitOwnership::Claimed(mut owner) => {
if !Arc::ptr_eq(token.event().event(), &event) {
drop(owner);
continue;
}
#[cfg(test)]
let pause = SYNC_WAIT_CLAIM_PAUSES.lock().remove(&pid);
#[cfg(test)]
if let Some(pause) = pause {
pause.captured.wait();
pause.resume.wait();
}
let mut cancelling = false;
if let Some(reservation) = event.try_status_reservation_sync() {
let reservation = reservation?;
match owner.decode_status_return(pid, reservation, |status| {
Wait::from_raw_with_token(
pid,
status,
sync_decode_token(&token, handle.clone()),
)
})? {
StatusReturn::Returned(decoded) => return Ok(decoded),
StatusReturn::Cancelled(status) => {
resume_cancelled_sync_status(pid, status, &handle)?;
cancelling = true;
}
StatusReturn::RetiredDeadExec => continue,
}
}
loop {
let Some(identity) = handle.identity() else {
NOTIFIER.remove(pid, &event);
return Err(Errno::EIO.into());
};
if identity.pid != pid {
NOTIFIER.remove(pid, &event);
return Err(Errno::ESRCH.into());
}
let result = if matches!(identity.pidfd, ThreadHandle::Procfs { .. }) {
if !Arc::ptr_eq(handle.event(), &event) {
drop(owner);
continue 'claim;
}
let progress = event.legacy_wait.lock();
if progress.retiring {
Err(Errno::ECHILD)
} else {
match try_consume_legacy_status(identity, &event, Some(&handle)) {
Ok(Some(status)) => Ok(status),
Ok(None) | Err(Errno::EINTR) => {
drop(progress);
thread::sleep(Duration::from_millis(1));
continue;
}
Err(error) => Err(error),
}
}
} else {
wait_thread_consuming(identity.pid, &identity.pidfd, &event)
};
let status = match result {
Ok(status) => status,
Err(error) => {
if error == Errno::ECHILD {
if matches!(identity.pidfd, ThreadHandle::Procfs { .. }) {
if !event.mark_legacy_echild_if_unowned(identity)? {
thread::sleep(Duration::from_millis(1));
continue;
}
} else {
event.mark_echild();
}
event.finish_sync_terminal(pid);
} else {
NOTIFIER.remove(pid, &event);
}
return Err(error.into());
}
};
event.update_sync_status(status);
if may_end_other_stops(status) {
probe_legacy_workers(Some(identity.snapshot.tgid));
}
#[cfg(test)]
let pause = SYNC_STATUS_PUBLICATION_PAUSES.lock().remove(&pid);
#[cfg(test)]
if let Some(pause) = pause {
pause.captured.wait();
pause.resume.wait();
}
if libc::WIFEXITED(status) || libc::WIFSIGNALED(status) {
event.finish_sync_terminal(pid);
if cancelling {
drop(owner);
return Wait::from_raw_with_token(
pid,
status,
sync_decode_token(&token, handle),
);
}
}
#[cfg(test)]
if let Some(error) = SYNC_DECODE_CAPTURE_ERRORS.lock().remove(&pid) {
inject_capture_error_for_current_thread(error);
}
let reservation = event
.try_status_reservation_sync()
.expect("published synchronous status is immediately reservable")?;
match owner.decode_status_return(pid, reservation, |reserved_status| {
Wait::from_raw_with_token(
pid,
reserved_status,
sync_decode_token(&token, handle.clone()),
)
})? {
StatusReturn::Returned(decoded) => return Ok(decoded),
StatusReturn::Cancelled(reserved_status) => {
resume_cancelled_sync_status(pid, reserved_status, &handle)?;
cancelling = true;
}
StatusReturn::RetiredDeadExec => continue 'claim,
}
}
}
}
}
}
fn worker_process_start_time(pid: Pid) -> std::io::Result<u64> {
let stat = launch_window::read_to_string(format!("/proc/{pid}/stat"))?;
worker_stat_start_time(&stat)
}
fn worker_stat_start_time(stat: &str) -> std::io::Result<u64> {
let fields = stat
.rsplit_once(") ")
.ok_or_else(|| std::io::Error::new(std::io::ErrorKind::InvalidData, "malformed stat"))?
.1;
fields
.split_ascii_whitespace()
.nth(19)
.ok_or_else(|| std::io::Error::new(std::io::ErrorKind::InvalidData, "missing starttime"))?
.parse()
.map_err(|error| std::io::Error::new(std::io::ErrorKind::InvalidData, error))
}
fn retained_proc_snapshot(directory: RawFd) -> io::Result<WorkerProcSnapshot> {
fn read(directory: RawFd, name: &std::ffi::CStr) -> io::Result<String> {
let _open = launch_window::TransientOpen::begin();
let raw = unsafe {
libc::openat(
directory,
name.as_ptr(),
libc::O_RDONLY | libc::O_CLOEXEC | libc::O_NOFOLLOW,
)
};
if raw < 0 {
return Err(io::Error::last_os_error());
}
let fd = unsafe { OwnedFd::from_raw_fd(raw) };
let mut bounded = io::Read::take(fs::File::from(fd), 1024 * 1024);
let mut text = String::new();
io::Read::read_to_string(&mut bounded, &mut text)?;
Ok(text)
}
let start_time = worker_stat_start_time(&read(directory, c"stat")?)?;
#[cfg(test)]
OTHER_STATUS_OPENS.with(|opens| opens.set(opens.get() + 1));
let status = read(directory, c"status")?;
let snapshot = WorkerProcSnapshot {
tgid: worker_status_pid(&status, "Tgid:")?,
tracer_pid: worker_status_pid(&status, "TracerPid:")?,
start_time,
zombie: worker_status_is_zombie(&status),
};
if worker_stat_start_time(&read(directory, c"stat")?)? != start_time {
return Err(io::Error::new(
io::ErrorKind::NotFound,
"tracee identity changed while reading retained procfs",
));
}
Ok(snapshot)
}
fn worker_status_is_zombie(status: &str) -> bool {
status
.lines()
.find_map(|line| line.strip_prefix("State:"))
.is_some_and(|value| matches!(value.trim_start().chars().next(), Some('Z' | 'X' | 'x')))
}
fn worker_status_pid(status: &str, name: &str) -> std::io::Result<Pid> {
status
.lines()
.find_map(|line| line.strip_prefix(name))
.and_then(|value| value.trim().parse::<i32>().ok())
.map(Pid::from_raw)
.ok_or_else(|| {
std::io::Error::new(
std::io::ErrorKind::InvalidData,
format!("missing or malformed {name}"),
)
})
}
fn worker_proc_snapshot(pid: Pid) -> std::io::Result<WorkerProcSnapshot> {
let start_time = worker_process_start_time(pid)?;
#[cfg(test)]
OTHER_STATUS_OPENS.with(|opens| opens.set(opens.get() + 1));
let status = launch_window::read_to_string(format!("/proc/{pid}/status"))?;
let snapshot = WorkerProcSnapshot {
tgid: worker_status_pid(&status, "Tgid:")?,
tracer_pid: worker_status_pid(&status, "TracerPid:")?,
start_time,
zombie: worker_status_is_zombie(&status),
};
if worker_process_start_time(pid)? != start_time {
return Err(std::io::Error::new(
std::io::ErrorKind::NotFound,
"tracee identity changed while reading procfs",
));
}
Ok(snapshot)
}
fn tracer_is_current(tracer_pid: Pid) -> bool {
tracer_pid.as_raw() > 0
&& std::path::Path::new(&format!("/proc/self/task/{tracer_pid}")).exists()
}
fn procfs_error_shows_exit(error: &std::io::Error) -> bool {
match error.raw_os_error() {
Some(errno) => errno == libc::ENOENT || errno == libc::ESRCH,
None => error.kind() == std::io::ErrorKind::NotFound,
}
}
fn fd_inode(fd: &OwnedFd) -> std::io::Result<u64> {
fd_directory_key(fd).map(|key| key.inode)
}
fn fd_directory_key(fd: &OwnedFd) -> std::io::Result<ProcDirectoryKey> {
let mut metadata = std::mem::MaybeUninit::<libc::stat>::uninit();
if unsafe { libc::fstat(fd.as_raw_fd(), metadata.as_mut_ptr()) } < 0 {
Err(io::Error::last_os_error())
} else {
Ok(ProcDirectoryKey::from_stat(unsafe {
metadata.assume_init()
}))
}
}
fn io_errno(error: std::io::Error) -> Errno {
Errno::new(error.raw_os_error().unwrap_or(libc::EIO))
}
#[derive(Debug)]
struct NotifierEntry {
handle: EventHandle,
identity: Arc<WorkerIdentity>,
}
#[cfg(test)]
enum RawCleanupClaim {
Won,
Existing(EventHandle),
Lost,
}
enum EventRegistration {
Registered(EventHandle),
Adopted,
}
#[cfg(test)]
enum CancellableEventRegistration {
Registered,
Synchronous(Arc<Event>),
ReturningTimedOut,
Busy,
}
struct Notifier {
pids: Mutex<HashMap<Pid, NotifierEntry>>,
unregistered: Mutex<HashMap<Pid, Vec<Weak<EventGeneration>>>>,
}
impl Notifier {
pub fn new() -> Self {
let pids = Mutex::new(HashMap::new());
let unregistered = Mutex::new(HashMap::new());
Notifier { pids, unregistered }
}
#[cfg(test)]
fn capture_identity(&self, pid: Pid) -> Result<Arc<WorkerIdentity>, Errno> {
WorkerIdentity::capture(pid).map(Arc::new)
}
fn capture_identity_for_policy(
&self,
pid: Pid,
policy: WaitPolicy,
) -> Result<Arc<WorkerIdentity>, Errno> {
WorkerIdentity::capture_for_policy(pid, policy).map(Arc::new)
}
fn reconnect(&self, pid: Pid, prior: &EventHandle) -> EventHandle {
let Some(identity) = prior.identity() else {
return prior.clone();
};
debug_assert_eq!(
prior.event().worker_state.load(Ordering::Acquire),
WORKER_DONE
);
let pids = self.pids.lock();
if !prior.is_current_ptracer() {
return prior.clone();
}
if let Some(occupied) = pids.get(&pid)
&& occupied.identity.same_live_generation(identity) == Ok(true)
&& !*occupied.handle.event().terminal_reaping.read()
{
return occupied.handle.clone();
}
let mut unregistered = self.unregistered.lock();
let generations = unregistered.entry(pid).or_default();
generations.retain(|generation| {
generation.upgrade().is_some_and(|generation| {
!*EventHandle(generation, None)
.event()
.terminal_reaping
.read()
})
});
for generation in generations.iter().filter_map(Weak::upgrade) {
let handle = EventHandle(generation, None);
if handle
.identity()
.is_some_and(|bound| bound.same_live_generation(identity) == Ok(true))
{
return handle.resolved_handle();
}
}
let handle = EventHandle::with_identity(Arc::clone(identity));
generations.push(Arc::downgrade(&handle.0));
handle
}
fn absorb_unregistered(
&self,
pid: Pid,
current: &WorkerIdentity,
handle: &EventHandle,
) -> Result<(), Errno> {
let earlier: Vec<EventHandle> = self
.unregistered
.lock()
.get(&pid)
.map(|generations| {
generations
.iter()
.filter_map(Weak::upgrade)
.map(|generation| EventHandle(generation, None))
.collect()
})
.unwrap_or_default();
for earlier in earlier {
loop {
let resolved = earlier.resolved_handle();
if handle.chain_contains(&resolved.0) {
break;
}
#[cfg(test)]
if let Some(error) = ABSORB_LIVENESS_ERRORS.lock().remove(&pid) {
return Err(error);
}
match resolved.identity() {
Some(bound) if bound.same_live_generation(current)? => {}
_ => break,
}
match resolved.adopt_authoritative(handle) {
Ok(_) => {
#[cfg(test)]
{
*ABSORBED_UNREGISTERED.lock().entry(pid).or_default() += 1;
}
break;
}
Err(Errno::EALREADY) => continue,
Err(error) => return Err(error),
}
}
}
Ok(())
}
fn keep_unregistered(&self, pid: Pid, handle: &EventHandle) {
let resolved = handle.resolved_handle();
let mut unregistered = self.unregistered.lock();
let generations = unregistered.entry(pid).or_default();
generations.retain(|generation| generation.strong_count() > 0);
let cached = generations
.iter()
.any(|generation| std::ptr::eq(generation.as_ptr(), Arc::as_ptr(&resolved.0)));
if !cached {
generations.push(Arc::downgrade(&resolved.0));
}
}
fn retain_failed_registrant(
&self,
pids: &HashMap<Pid, NotifierEntry>,
pid: Pid,
handle: &EventHandle,
) {
let Some(identity) = handle.identity().cloned() else {
return;
};
if let Some(occupied) = pids.get(&pid)
&& occupied.identity.same_generation(&identity)
{
loop {
let resolved = handle.resolved_handle();
if occupied.handle.chain_contains(&resolved.0) {
return;
}
match resolved.adopt_authoritative(&occupied.handle) {
Ok(_) => return,
Err(Errno::EALREADY) => continue,
Err(_) => break,
}
}
}
self.keep_unregistered(pid, handle);
let reaped = match identity.pidfd_is_live() {
Ok(live) => !live,
Err(_) => identity.observed_process_generation() == Some(false),
};
if reaped {
handle.event().mark_terminal_reaping();
}
}
fn matches_current_for_policy(
bound: &WorkerIdentity,
current: &WorkerIdentity,
policy: WaitPolicy,
) -> Result<bool, Errno> {
if policy == WaitPolicy::Native && !matches!(bound.pidfd, ThreadHandle::Pidfd(_)) {
if !bound.same_generation(current) {
return Ok(false);
}
return match retained_proc_status(bound.proc_dir.as_raw_fd()) {
Ok(status) => Ok(status.pid == bound.pid && status.tgid == bound.snapshot.tgid),
Err(Errno::ENOENT | Errno::ESRCH) => Ok(false),
Err(error) => Err(error),
};
}
bound.same_live_generation(current)
}
fn view_for_policy(
handle: &EventHandle,
current: &Arc<WorkerIdentity>,
policy: WaitPolicy,
) -> Result<EventHandle, Errno> {
if policy == WaitPolicy::Native
&& !matches!(handle.identity().unwrap().pidfd, ThreadHandle::Pidfd(_))
{
return handle.native_view(current.clone());
}
policy.check_handle(handle)?;
Ok(handle.resolved_handle())
}
fn current_or_new_for_policy(
&self,
pid: Pid,
policy: WaitPolicy,
) -> Result<EventHandle, Errno> {
let cached = {
let pids = self.pids.lock();
pids.get(&pid)
.map(|occupied| (occupied.handle.clone(), Arc::clone(&occupied.identity)))
};
if let Some((handle, identity)) = cached
&& matches!(identity.pidfd_is_live(), Ok(true))
{
if policy == WaitPolicy::PtracerThread
|| handle
.identity()
.is_some_and(|identity| matches!(identity.pidfd, ThreadHandle::Pidfd(_)))
{
return Ok(handle);
}
if let Some(native) = handle.preferred_native_view() {
return Ok(native);
}
}
loop {
let current = self.capture_identity_for_policy(pid, policy)?;
if !current.is_same_process_generation() {
continue;
}
#[cfg(test)]
if let Some(pause) = CURRENT_OR_NEW_CAPTURE_PAUSES.lock().remove(&pid) {
pause.captured.wait();
pause.resume.wait();
}
let mut pids = self.pids.lock();
if !current.pidfd_is_live()? {
drop(pids);
continue;
}
if let Some(occupied) = pids.get(&pid)
&& Self::matches_current_for_policy(&occupied.identity, ¤t, policy)?
{
return Self::view_for_policy(&occupied.handle, ¤t, policy);
}
pids.remove(&pid);
let mut unregistered = self.unregistered.lock();
let cached = unregistered.get(&pid).into_iter().flatten();
for generation in cached.filter_map(Weak::upgrade) {
let handle = EventHandle(generation, None);
if let Some(bound) = handle.identity()
&& Self::matches_current_for_policy(bound, ¤t, policy)?
{
return Self::view_for_policy(&handle, ¤t, policy);
}
}
unregistered.retain(|_, generations| {
generations.retain(|generation| generation.strong_count() > 0);
!generations.is_empty()
});
let handle = EventHandle::with_identity(Arc::clone(¤t));
unregistered
.entry(pid)
.or_default()
.push(Arc::downgrade(&handle.0));
return Ok(handle);
}
}
#[cfg(test)]
fn sync_handle(&self, pid: Pid, requested: &EventHandle) -> Result<EventHandle, Errno> {
self.sync_handle_for_policy(pid, requested, WaitPolicy::Native)
}
fn sync_handle_for_policy(
&self,
pid: Pid,
requested: &EventHandle,
policy: WaitPolicy,
) -> Result<EventHandle, Errno> {
policy.check_handle(requested)?;
let result = self.resolve_sync_handle_for_policy(pid, requested, policy);
if result.is_err() {
#[cfg(test)]
if let Some(pause) = RETENTION_PAUSES.lock().remove(&pid) {
let _ = pause.reached.send(());
let _ = pause.resume.recv();
}
self.retain_failed_registrant(&self.pids.lock(), pid, requested);
}
result
}
fn resolve_sync_handle_for_policy(
&self,
pid: Pid,
requested: &EventHandle,
policy: WaitPolicy,
) -> Result<EventHandle, Errno> {
let mut capture_retries = 0;
loop {
let current = match self.capture_identity_for_policy(pid, policy) {
Err(Errno::ENOENT | Errno::ESRCH)
if capture_retries < SYNC_IDENTITY_CAPTURE_RETRIES =>
{
capture_retries += 1;
thread::yield_now();
continue;
}
result => result?,
};
if !current.is_same_process_generation() {
continue;
}
match requested.identity() {
Some(bound) if !bound.same_live_generation(¤t)? => {
return Err(Errno::ECHILD);
}
Some(_) => {}
None => {
if let Err(bound) = requested.bind_identity(Arc::clone(¤t))
&& !bound.same_live_generation(¤t)?
{
return Err(Errno::ECHILD);
}
}
}
let mut pids = self.pids.lock();
if !current.pidfd_is_live()? {
drop(pids);
continue;
}
if let Some(occupied) = pids.get(&pid)
&& Self::matches_current_for_policy(&occupied.identity, ¤t, policy)?
{
let authoritative = Self::view_for_policy(&occupied.handle, ¤t, policy)?;
return requested.adopt_authoritative(&authoritative);
}
if let Err(error) = self.absorb_unregistered(pid, ¤t, requested) {
self.keep_unregistered(pid, requested);
return Err(error);
}
pids.insert(
pid,
NotifierEntry {
handle: requested.clone(),
identity: current,
},
);
return Ok(requested.resolved_handle());
}
}
#[cfg(test)]
fn current_registered(
&self,
pid: Pid,
policy: WaitPolicy,
) -> Result<Option<EventHandle>, Errno> {
loop {
let current = match self.capture_identity_for_policy(pid, policy) {
Ok(current) => current,
Err(Errno::ENOENT | Errno::ESRCH) => return Ok(None),
Err(error) => return Err(error),
};
if !current.is_same_process_generation() {
continue;
}
let pids = self.pids.lock();
if !current.pidfd_is_live()? {
drop(pids);
continue;
}
return match pids.get(&pid) {
Some(occupied) if occupied.identity.same_live_generation(¤t)? => {
Ok(Some(occupied.handle.clone()))
}
_ => Ok(None),
};
}
}
#[cfg(test)]
fn registered_sync_handle(
&self,
pid: Pid,
exact_pidfd: &ThreadHandle,
) -> Result<Option<EventHandle>, Errno> {
if !pidfd_is_live(exact_pidfd)? {
return Ok(None);
}
let pids = self.pids.lock();
let Some(occupied) = pids.get(&pid) else {
return Ok(None);
};
if !matches!(
occupied.handle.event().wait_owner.load(Ordering::Acquire),
WAIT_OWNER_SYNC | WAIT_OWNER_SYNC_RETURNING
) || !pidfd_is_live(&occupied.identity.pidfd)?
{
return Ok(None);
}
Ok(Some(occupied.handle.clone()))
}
fn resolve_echild(&self, pid: Pid, handle: &EventHandle) -> EventHandle {
let handle = handle.resolved_handle();
let event = Arc::clone(handle.event());
let mut pids = self.pids.lock();
if event.worker_is_running() {
return handle;
}
if event.try_begin_unstarted_completion() {
if handle
.identity()
.is_some_and(|identity| matches!(identity.pidfd, ThreadHandle::Procfs { .. }))
{
event.mark_terminal_reaping();
}
event.mark_echild();
#[cfg(test)]
pause_retirement_for_test(&event.registry_retirement_pause);
if pids
.get(&pid)
.is_some_and(|current| Arc::ptr_eq(current.handle.event(), &event))
{
pids.remove(&pid);
}
event.mark_worker_done();
}
handle
}
fn record_registration_error(handle: &EventHandle, error: Errno) {
*handle.event().registration_error.lock() = Some(error);
}
#[cfg(test)]
fn event(&self, pid: Pid, handle: &EventHandle) -> Result<EventHandle, Errno> {
self.event_for_policy(pid, handle, WaitPolicy::Native)
}
fn event_for_policy(
&self,
pid: Pid,
handle: &EventHandle,
policy: WaitPolicy,
) -> Result<EventHandle, Errno> {
loop {
policy.check_handle(handle)?;
let requested = Arc::clone(handle.event());
let owner = match requested.claim_notifier_wait() {
NotifierWaitOwnership::Existing => return Ok(handle.resolved_handle()),
NotifierWaitOwnership::Claimed(owner) => owner,
};
if !Arc::ptr_eq(handle.event(), &requested) {
drop(owner);
continue;
}
#[cfg(test)]
requested.pause_notifier_registration_for_test(WORKER_NOT_STARTED);
match self.event_with_owner_for_policy(pid, handle, &requested, owner, policy)? {
EventRegistration::Registered(handle) => return Ok(handle),
EventRegistration::Adopted => {}
}
}
}
#[cfg(test)]
fn try_event_for_cleanup(
&self,
pid: Pid,
handle: &EventHandle,
return_deadline: Instant,
) -> Result<CancellableEventRegistration, Errno> {
loop {
let requested = Arc::clone(handle.event());
let mut ownership = requested.try_claim_cancellable_notifier_wait();
if matches!(ownership, CancellableNotifierWaitOwnership::Returning) {
if let Some(pause) = requested.cleanup_return_pause.lock().take()
&& pause.captured.send(()).is_ok()
{
let _ = pause.resume.recv();
}
let return_timeout = return_deadline.saturating_duration_since(Instant::now());
ownership = match requested
.wait_for_return_and_claim_cancellable_notifier(return_timeout)
{
Some(ownership) => ownership,
None => return Ok(CancellableEventRegistration::ReturningTimedOut),
};
}
let owner = match ownership {
CancellableNotifierWaitOwnership::Synchronous => {
return Ok(CancellableEventRegistration::Synchronous(Arc::clone(
&requested,
)));
}
CancellableNotifierWaitOwnership::Existing => {
return match requested.worker_state.load(Ordering::Acquire) {
WORKER_RUNNING | WORKER_FINISHING | WORKER_DONE => {
Ok(CancellableEventRegistration::Registered)
}
WORKER_NOT_STARTED | WORKER_STARTING => {
Ok(CancellableEventRegistration::Busy)
}
state => unreachable!("invalid worker state {state}"),
};
}
CancellableNotifierWaitOwnership::Returning => {
unreachable!("return wait completed without a stable ownership state")
}
CancellableNotifierWaitOwnership::Claimed(owner) => owner,
};
if !Arc::ptr_eq(handle.event(), &requested) {
drop(owner);
continue;
}
match self.event_with_owner(pid, handle, &requested, owner)? {
EventRegistration::Registered(_) => {
return Ok(CancellableEventRegistration::Registered);
}
EventRegistration::Adopted => {}
}
}
}
#[cfg(test)]
fn event_with_owner(
&self,
pid: Pid,
handle: &EventHandle,
requested: &Arc<Event>,
owner: NotifierWaitOwner<'_>,
) -> Result<EventRegistration, Errno> {
self.event_with_owner_for_policy(pid, handle, requested, owner, WaitPolicy::Native)
}
fn event_with_owner_for_policy(
&self,
pid: Pid,
handle: &EventHandle,
requested: &Arc<Event>,
owner: NotifierWaitOwner<'_>,
policy: WaitPolicy,
) -> Result<EventRegistration, Errno> {
policy.check_handle(handle)?;
let result = self.register_with_owner(pid, handle, requested, owner, policy);
if result.is_err() {
#[cfg(test)]
if let Some(pause) = RETENTION_PAUSES.lock().remove(&pid) {
let _ = pause.reached.send(());
let _ = pause.resume.recv();
}
self.retain_failed_registrant(&self.pids.lock(), pid, handle);
}
result
}
fn register_with_owner(
&self,
pid: Pid,
handle: &EventHandle,
requested: &Arc<Event>,
owner: NotifierWaitOwner<'_>,
policy: WaitPolicy,
) -> Result<EventRegistration, Errno> {
if requested.is_terminal() {
owner.commit();
return Ok(EventRegistration::Registered(handle.resolved_handle()));
}
if requested.worker_is_running() {
owner.commit();
return Ok(EventRegistration::Registered(handle.resolved_handle()));
}
loop {
let current = match self.capture_identity_for_policy(pid, policy) {
Ok(identity) => identity,
Err(Errno::ENOENT | Errno::ESRCH) => {
let resolved = self.resolve_echild(pid, handle);
owner.commit();
return Ok(EventRegistration::Registered(resolved));
}
Err(error) => {
Self::record_registration_error(handle, error);
return Err(error);
}
};
if !current.is_same_process_generation() {
continue;
}
match handle.identity() {
Some(bound) => match bound.same_live_generation(¤t) {
Ok(true) => {}
Ok(false) => {
let resolved = self.resolve_echild(pid, handle);
owner.commit();
return Ok(EventRegistration::Registered(resolved));
}
Err(error) => {
Self::record_registration_error(handle, error);
return Err(error);
}
},
None => {
if let Err(bound) = handle.bind_identity(Arc::clone(¤t)) {
match bound.same_live_generation(¤t) {
Ok(true) => {}
Ok(false) => {
let resolved = self.resolve_echild(pid, handle);
owner.commit();
return Ok(EventRegistration::Registered(resolved));
}
Err(error) => {
Self::record_registration_error(handle, error);
return Err(error);
}
}
}
}
}
#[cfg(test)]
if let Some(pause) = EVENT_CAPTURE_PAUSES.lock().remove(&pid) {
pause.captured.wait();
pause.resume.wait();
}
let mut pids = self.pids.lock();
match current.pidfd_is_live() {
Ok(true) => {}
Ok(false) => {
drop(pids);
let resolved = self.resolve_echild(pid, handle);
owner.commit();
return Ok(EventRegistration::Registered(resolved));
}
Err(error) => {
Self::record_registration_error(handle, error);
return Err(error);
}
}
match requested.worker_state.load(Ordering::Acquire) {
WORKER_FINISHING | WORKER_DONE => {
owner.commit();
return Ok(EventRegistration::Registered(handle.resolved_handle()));
}
WORKER_RUNNING => {
owner.commit();
return Ok(EventRegistration::Registered(handle.resolved_handle()));
}
WORKER_STARTING => {
unreachable!("worker STARTING publication escaped the registry lock")
}
WORKER_NOT_STARTED => {}
state => unreachable!("invalid worker state {state}"),
}
if matches!(current.pidfd, ThreadHandle::Pidfd(_))
&& handle
.identity()
.is_some_and(|identity| !matches!(identity.pidfd, ThreadHandle::Pidfd(_)))
{
let _native_view = handle.native_view(current.clone())?;
}
let admitted_worker = handle.worker_identity(¤t);
let mut worker_identity = None;
let event_handle = match pids.entry(pid) {
Entry::Occupied(occupied) if occupied.get().handle == *handle => {
policy.check_handle(handle)?;
match Self::matches_current_for_policy(
&occupied.get().identity,
¤t,
policy,
) {
Ok(true) => {}
Ok(false) => {
drop(pids);
let resolved = self.resolve_echild(pid, handle);
owner.commit();
return Ok(EventRegistration::Registered(resolved));
}
Err(error) => {
Self::record_registration_error(handle, error);
return Err(error);
}
}
if requested.try_begin_worker_start() {
worker_identity = Some(handle.worker_identity(&occupied.get().identity));
}
handle.clone()
}
Entry::Occupied(mut occupied) => {
match occupied.get().identity.same_live_generation(¤t) {
Ok(true) => {
let authoritative =
Self::view_for_policy(&occupied.get().handle, ¤t, policy)?;
drop(pids);
handle.adopt_authoritative(&authoritative)?;
drop(owner);
return Ok(EventRegistration::Adopted);
}
Ok(false) => {}
Err(error) => {
Self::record_registration_error(handle, error);
return Err(error);
}
}
if let Err(error) = self.absorb_unregistered(pid, ¤t, handle) {
self.keep_unregistered(pid, handle);
Self::record_registration_error(handle, error);
return Err(error);
}
occupied.insert(NotifierEntry {
handle: handle.clone(),
identity: admitted_worker.clone(),
});
if requested.try_begin_worker_start() {
worker_identity = Some(admitted_worker.clone());
}
handle.clone()
}
Entry::Vacant(vacant) => {
if let Err(error) = self.absorb_unregistered(pid, ¤t, handle) {
self.keep_unregistered(pid, handle);
Self::record_registration_error(handle, error);
return Err(error);
}
vacant.insert(NotifierEntry {
handle: handle.clone(),
identity: admitted_worker.clone(),
});
if requested.try_begin_worker_start() {
worker_identity = Some(admitted_worker.clone());
}
handle.clone()
}
};
let pending_worker = if let Some(identity) = worker_identity {
match spawn_worker(pid, Arc::clone(event_handle.event()), Arc::clone(&identity)) {
Ok(worker) => {
requested.mark_worker_running();
Some(worker)
}
Err(error) => {
requested.rollback_worker_start();
if pids.get(&pid).is_some_and(|entry| {
entry.handle == *handle && entry.identity.same_generation(&identity)
}) {
pids.remove(&pid);
self.keep_unregistered(pid, handle);
}
let error = io_errno(error);
Self::record_registration_error(handle, error);
return Err(error);
}
}
} else {
None
};
*requested.registration_error.lock() = None;
drop(pids);
if let Some(worker) = pending_worker {
worker.start();
}
owner.commit();
return Ok(EventRegistration::Registered(event_handle));
}
}
fn remove(&self, pid: Pid, event: &Arc<Event>) {
#[cfg(test)]
pause_retirement_for_test(&event.registry_retirement_pause);
let mut pids = self.pids.lock();
if pids
.get(&pid)
.is_some_and(|current| Arc::ptr_eq(current.handle.event(), event))
{
pids.remove(&pid);
}
}
#[cfg(test)]
fn try_claim_unstarted_raw_cleanup(
&self,
pid: Pid,
handle: &EventHandle,
) -> Result<RawCleanupClaim, Errno> {
loop {
let mut pids = self.pids.lock();
if let Some(current) = pids.get(&pid)
&& current.handle != *handle
{
let current_handle = current.handle.clone();
let current_identity = Arc::clone(¤t.identity);
drop(pids);
let live = current_identity.pidfd_is_live()?;
pids = self.pids.lock();
if !pids.get(&pid).is_some_and(|entry| {
entry.handle == current_handle
&& entry.identity.same_generation(¤t_identity)
}) {
drop(pids);
continue;
}
if live {
return Ok(RawCleanupClaim::Existing(current_handle));
}
pids.remove(&pid);
}
drop(pids);
let event = Arc::clone(handle.event());
let owner = match event.claim_notifier_wait() {
NotifierWaitOwnership::Existing => return Ok(RawCleanupClaim::Lost),
NotifierWaitOwnership::Claimed(owner) => owner,
};
if !Arc::ptr_eq(handle.event(), &event) {
drop(owner);
continue;
}
pids = self.pids.lock();
if pids
.get(&pid)
.is_some_and(|current| current.handle != *handle)
{
drop(pids);
drop(owner);
continue;
}
if !event.try_begin_unstarted_completion() {
drop(owner);
return Ok(RawCleanupClaim::Lost);
}
if pids
.get(&pid)
.is_some_and(|current| current.handle == *handle)
{
pids.remove(&pid);
}
owner.commit();
return Ok(RawCleanupClaim::Won);
}
}
}
impl Drop for Notifier {
fn drop(&mut self) {
let pids = self.pids.lock();
assert_eq!(
pids.len(),
0,
"Some tracees have not exited yet:\n{:#?}",
pids
);
}
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub struct StopSiginfo {
pub signo: i32,
pub code: i32,
pub sender_pid: i32,
pub sender_uid: u32,
}
impl StopSiginfo {
pub fn has_exit_signature(&self) -> bool {
self.signo == libc::SIGTRAP && self.code == (libc::PTRACE_EVENT_EXIT << 8) | libc::SIGTRAP
}
}
#[derive(Clone, Debug, Eq, PartialEq, thiserror::Error)]
pub enum ProcStatError {
#[error("retained proc stat I/O: {0}")]
Io(Errno),
#[error("retained proc stat: {0}")]
Format(&'static str),
#[error("retained proc stat names a different PID")]
PidMismatch,
}
#[derive(Clone, Copy, Debug, Eq, PartialEq, thiserror::Error)]
pub enum StopObservationError {
#[error("stopped observation used from another thread")]
WrongThread,
#[error("stopped observation has no registered identity: {0}")]
Identity(Errno),
#[error("stopped observation Event generation changed")]
GenerationMismatch,
#[error("stopped observation exec epoch changed")]
ExecEpochMismatch,
#[error("stopped observation PID binding changed")]
PidMismatch,
}
#[derive(Clone, Debug, Default)]
pub struct StopObservationSample {
siginfo: Option<Result<StopSiginfo, Errno>>,
flags: Option<Result<u32, ProcStatError>>,
pidfd_live: Option<Result<bool, Errno>>,
refusal: Option<StopObservationError>,
}
impl StopObservationSample {
pub fn siginfo(&self) -> Option<Result<StopSiginfo, Errno>> {
self.siginfo
}
pub fn flags(&self) -> Option<&Result<u32, ProcStatError>> {
self.flags.as_ref()
}
pub fn pidfd_live(&self) -> Option<Result<bool, Errno>> {
self.pidfd_live
}
pub fn refusal(&self) -> Option<StopObservationError> {
self.refusal
}
}
pub struct StoppedObservation {
pid: Pid,
event: EventHandle,
generation: Arc<Event>,
epoch: usize,
thread: thread::ThreadId,
ptracer_token: Option<TraceeToken>,
}
impl StoppedObservation {
pub(super) fn new(pid: Pid, token: &TraceeToken) -> Self {
let event = token.event().clone();
let generation = event.event().clone();
let epoch = generation.exit_epoch.load(Ordering::Acquire);
Self {
pid,
event,
generation,
epoch,
thread: thread::current().id(),
ptracer_token: (token.policy == WaitPolicy::PtracerThread).then(|| token.clone()),
}
}
pub fn same_generation(&self, terminal: &TerminalCleanup) -> bool {
self.pid == terminal.pid && Arc::ptr_eq(&self.generation, terminal.event.event())
}
fn binding_error(&self) -> Option<StopObservationError> {
if self.thread != thread::current().id() {
Some(StopObservationError::WrongThread)
} else if !Arc::ptr_eq(&self.generation, self.event.event()) {
Some(StopObservationError::GenerationMismatch)
} else if self.epoch != self.generation.exit_epoch.load(Ordering::Acquire) {
Some(StopObservationError::ExecEpochMismatch)
} else {
None
}
}
pub fn sample(&self, flags_for_exit_siginfo: bool) -> StopObservationSample {
let mut result = StopObservationSample::default();
if let Some(error) = self.binding_error() {
result.refusal = Some(error);
return result;
}
let Some(identity) = self.event.identity() else {
result.refusal = Some(StopObservationError::Identity(
self.generation
.registration_error
.lock()
.unwrap_or(Errno::ENODATA),
));
return result;
};
if identity.pid != self.pid {
result.refusal = Some(StopObservationError::PidMismatch);
return result;
}
let query = || {
nix::sys::ptrace::getsiginfo(self.pid.into())
.map(|info| StopSiginfo {
signo: info.si_signo,
code: info.si_code,
sender_pid: unsafe { info.si_pid() },
sender_uid: unsafe { info.si_uid() },
})
.map_err(|error| Errno::new(error as i32))
};
result.siginfo = Some(match &self.ptracer_token {
Some(token) => token.on_held_tid(query),
None => match self.generation.hold_tid() {
Some(_held) => query(),
None => Err(Errno::ESRCH),
},
});
if flags_for_exit_siginfo
&& result
.siginfo
.is_some_and(|info| info.is_ok_and(|info| info.has_exit_signature()))
{
result.flags = Some(read_bound_stat_flags(identity));
}
result.pidfd_live = Some(identity.pidfd_is_live());
result.refusal = self.binding_error();
result
}
}
impl EventHandle {
pub(super) fn bound_stat_flags(&self, pid: Pid) -> Option<Result<u32, ProcStatError>> {
#[cfg(test)]
if let Some(error) = BOUND_STAT_FLAGS_ERROR_FOR_TEST.with(|slot| slot.borrow().clone()) {
return Some(Err(error));
}
self.identity()
.filter(|identity| identity.pid == pid)
.map(|identity| read_bound_stat_flags(identity))
}
pub(super) fn bound_pidfd_live(&self, pid: Pid) -> Option<Result<bool, Errno>> {
#[cfg(test)]
if let Some(live) = BOUND_PIDFD_LIVE_FOR_TEST.with(std::cell::Cell::get) {
return Some(live);
}
self.identity()
.filter(|identity| identity.pid == pid)
.map(|identity| identity.pidfd_is_live())
}
}
#[cfg(test)]
thread_local! {
pub(super) static BOUND_STAT_FLAGS_ERROR_FOR_TEST: std::cell::RefCell<Option<ProcStatError>> = const { std::cell::RefCell::new(None) };
pub(super) static BOUND_PIDFD_LIVE_FOR_TEST: std::cell::Cell<Option<Result<bool, Errno>>> = const { std::cell::Cell::new(None) };
}
fn read_bound_stat_flags(identity: &WorkerIdentity) -> Result<u32, ProcStatError> {
read_bound_stat(identity, |bytes| {
parse_bound_stat_flags(bytes, identity.pid)
})
}
fn read_bound_stat<T>(
identity: &WorkerIdentity,
parse: impl FnOnce(&[u8]) -> Result<T, ProcStatError>,
) -> Result<T, ProcStatError> {
let _open = launch_window::TransientOpen::begin();
let raw = unsafe {
libc::openat(
identity.proc_dir.as_raw_fd(),
c"stat".as_ptr(),
libc::O_RDONLY | libc::O_CLOEXEC | libc::O_NOFOLLOW,
)
};
if raw < 0 {
return Err(ProcStatError::Io(Errno::last()));
}
let fd = unsafe { OwnedFd::from_raw_fd(raw) };
let mut bytes = [0u8; 4097];
let count = unsafe { libc::read(fd.as_raw_fd(), bytes.as_mut_ptr().cast(), bytes.len()) };
if count < 0 {
return Err(ProcStatError::Io(Errno::last()));
}
if count as usize > 4096 {
return Err(ProcStatError::Format("record exceeds 4096 bytes"));
}
let mut extra = 0u8;
let eof = unsafe { libc::read(fd.as_raw_fd(), (&mut extra as *mut u8).cast(), 1) };
if eof < 0 {
return Err(ProcStatError::Io(Errno::last()));
}
if eof != 0 {
return Err(ProcStatError::Format("record continued after bounded read"));
}
parse(&bytes[..count as usize])
}
fn parse_bound_stat_flags(bytes: &[u8], pid: Pid) -> Result<u32, ProcStatError> {
if bytes.len() > 4096 {
return Err(ProcStatError::Format("record exceeds 4096 bytes"));
}
let open = bytes
.windows(2)
.position(|part| part == b" (")
.ok_or(ProcStatError::Format("missing comm opening delimiter"))?;
let actual = std::str::from_utf8(&bytes[..open])
.ok()
.and_then(|value| value.parse::<i32>().ok())
.ok_or(ProcStatError::Format("invalid PID"))?;
if actual != pid.as_raw() {
return Err(ProcStatError::PidMismatch);
}
let close = bytes
.iter()
.rposition(|byte| *byte == b')')
.filter(|close| *close > open)
.ok_or(ProcStatError::Format("missing comm closing delimiter"))?;
if bytes.get(close + 1) != Some(&b' ') {
return Err(ProcStatError::Format("invalid comm separator"));
}
let mut fields = bytes[close + 2..]
.split(|byte| byte.is_ascii_whitespace())
.filter(|field| !field.is_empty());
let state = fields
.next()
.ok_or(ProcStatError::Format("missing state"))?;
if state.len() != 1 || !state[0].is_ascii_alphabetic() {
return Err(ProcStatError::Format("invalid state"));
}
let flags = fields
.nth(5)
.ok_or(ProcStatError::Format("missing flags"))?;
if flags.is_empty() || !flags.iter().all(u8::is_ascii_digit) {
return Err(ProcStatError::Format("flags are not unsigned decimal"));
}
std::str::from_utf8(flags)
.ok()
.and_then(|value| value.parse().ok())
.ok_or(ProcStatError::Format("flags overflow u32"))
}
pub struct TerminalCleanup {
pid: Pid,
event: EventHandle,
}
impl TerminalCleanup {
pub(super) fn new(pid: Pid, token: &TraceeToken) -> Self {
let cleanup = Self::new_unregistered(pid, token);
let _ = cleanup.ensure_registered();
cleanup
}
pub(super) fn new_unregistered(pid: Pid, token: &TraceeToken) -> Self {
let event = token.event().clone();
Self { pid, event }
}
pub fn ensure_registered(&self) -> Result<(), Errno> {
if let Some(error) = self.event.resolved().0.initial_capture_refusal.get() {
return Err(*error);
}
let policy = if self
.event
.identity()
.is_some_and(|identity| !matches!(identity.pidfd, ThreadHandle::Pidfd(_)))
{
WaitPolicy::PtracerThread
} else {
WaitPolicy::Native
};
NOTIFIER
.event_for_policy(self.pid, &self.event, policy)
.map(drop)
}
#[cfg(test)]
fn try_ensure_registered_for_cleanup(
&self,
return_deadline: Instant,
) -> Result<CancellableEventRegistration, Errno> {
NOTIFIER.try_event_for_cleanup(self.pid, &self.event, return_deadline)
}
pub fn registration_error(&self) -> Option<Errno> {
*self.event.event().registration_error.lock()
}
pub fn request_sigkill(&self) -> Result<(), Errno> {
self.request_cancellation_signal(libc::SIGKILL)
}
pub fn request_process_sigkill(&self) -> Result<(), Errno> {
if let Some(error) = self.registration_error() {
return Err(error);
}
let identity = self
.event
.identity()
.ok_or_else(|| self.registration_error().unwrap_or(Errno::ENODATA))?;
let sent = identity.pidfd.send_process_signal(libc::SIGKILL);
self.event.stop_may_have_ended();
sent
}
pub fn request_sigstop(&self) -> Result<(), Errno> {
self.request_cancellation_signal(libc::SIGSTOP)
}
pub fn has_thread_pidfd(&self) -> Result<bool, Errno> {
self.event
.identity()
.map(|identity| matches!(identity.pidfd, ThreadHandle::Pidfd(_)))
.ok_or_else(|| self.registration_error().unwrap_or(Errno::ENODATA))
}
fn request_cancellation_signal(&self, signal: i32) -> Result<(), Errno> {
if let Some(error) = self.registration_error() {
return Err(error);
}
let identity = self
.event
.identity()
.ok_or_else(|| self.registration_error().unwrap_or(Errno::ENODATA))?;
let sent = identity.pidfd.send_cancellation_signal(signal);
self.event.stop_may_have_ended();
sent
}
#[cfg(test)]
fn try_claim_unstarted_raw_cleanup(&self) -> Result<RawCleanupClaim, Errno> {
NOTIFIER.try_claim_unstarted_raw_cleanup(self.pid, &self.event)
}
#[cfg(test)]
fn finish_unstarted_raw_cleanup(&self) {
let event = self.event.event();
if self
.event
.identity()
.is_some_and(|identity| matches!(identity.pidfd, ThreadHandle::Procfs { .. }))
{
event.mark_terminal_reaping();
}
event.mark_echild();
NOTIFIER.remove(self.pid, event);
event.mark_worker_done();
}
pub fn same_generation(&self, other: &Self) -> bool {
Arc::ptr_eq(self.event.event(), other.event.event())
}
pub fn reaping_started(&self) -> bool {
self.event.hold_tid().is_none()
}
pub fn wait(&self, timeout: Duration) -> bool {
self.event.event().wait_worker_done(timeout)
}
pub fn observed_exit_status(&self) -> Result<Option<crate::ExitStatus>, Errno> {
let status = self.event.event().status.lock().terminal;
match status {
INVALID_STATUS => Ok(None),
ECHILD_STATUS => Err(Errno::ECHILD),
status => Ok(Some(crate::ExitStatus::from_raw(status))),
}
}
pub fn exit_effects_complete(&self) -> Result<bool, ProcStatError> {
if self
.observed_exit_status()
.is_ok_and(|status| status.is_some())
{
return Ok(true);
}
let identity = self
.event
.identity()
.ok_or(ProcStatError::Io(Errno::ENODATA))?;
if identity.pid != self.pid {
return Err(ProcStatError::PidMismatch);
}
match read_bound_stat(identity, |bytes| {
parse_bound_stat_flags(bytes, self.pid)?;
let close = bytes.iter().rposition(|byte| *byte == b')').unwrap();
let state = bytes[close + 2..]
.split(|byte| byte.is_ascii_whitespace())
.find(|field| !field.is_empty())
.unwrap();
Ok(matches!(state, b"Z" | b"X" | b"x"))
}) {
Err(ProcStatError::Io(Errno::ENOENT | Errno::ESRCH)) => Ok(true),
result => result,
}
}
pub fn reserve_pending_for_cleanup(
&self,
timeout: Duration,
) -> Option<PendingStatusReservation<'_>> {
WaitPolicy::Native.check_handle(&self.event).ok()?;
self.reserve_pending_for_policy(timeout, WaitPolicy::Native)
}
pub fn reserve_pending_for_cleanup_with_guard(
&self,
timeout: Duration,
guard: &PtracerThreadGuard,
) -> Result<Option<GuardedPendingStatusReservation<'_>>, Errno> {
WaitPolicy::Native.check_handle(&self.event)?;
guard.check_current()?;
self.ensure_registered()?;
let inner = self.reserve_pending_for_policy(timeout, WaitPolicy::Native);
guard.check_current()?;
Ok(inner.map(|inner| GuardedPendingStatusReservation {
inner: Some(inner),
guard: guard.clone(),
dead_exec_consumed: false,
local: PhantomData,
}))
}
fn reserve_pending_for_policy(
&self,
timeout: Duration,
policy: WaitPolicy,
) -> Option<PendingStatusReservation<'_>> {
let state = self.event.event().wait_pending_status(timeout)?;
let status = *state
.pending
.front()
.expect("pending cleanup reservation requires a FIFO front");
Some(PendingStatusReservation {
pid: self.pid,
status,
event: self.event.resolved(),
policy,
state,
})
}
pub fn wait_after_sigkill(&self) -> Result<(), Error> {
WaitPolicy::Native.check_handle(&self.event)?;
let deadline = Instant::now() + Duration::from_secs(2);
loop {
if self.wait(Duration::ZERO) {
return Ok(());
}
let remaining = deadline.saturating_duration_since(Instant::now());
let Some(reservation) = self.reserve_pending_for_cleanup(remaining) else {
if self.wait(deadline.saturating_duration_since(Instant::now())) {
return Ok(());
}
return Err(Errno::ETIMEDOUT.into());
};
let state = match reservation.decode() {
Ok(state) => state,
Err(error) => match reservation.consume_dead_exec() {
Ok(()) => return Ok(()),
Err(_) => return Err(error),
},
};
let Wait::Stopped(stopped, _) = state else {
reservation.commit();
continue;
};
reservation.commit();
match stopped.resume(None) {
Ok(_) | Err(Error::Died(_)) | Err(Error::Errno(Errno::ESRCH)) => {
return Ok(());
}
Err(error) => return Err(error),
}
}
}
pub fn pending_is_empty(&self) -> bool {
self.event.event().pending_is_empty()
}
pub fn retired_stops_before_exit(&self) -> u64 {
self.event.event().status.lock().retired_before_exit
}
pub fn retired_dead_exec_stops(&self) -> u64 {
self.event.event().status.lock().retired_dead_exec
}
pub fn queued_raw_statuses(&self) -> Vec<i32> {
self.event
.event()
.status
.lock()
.pending
.iter()
.copied()
.collect()
}
pub fn exit_stop_observed(&self) -> bool {
self.event.event().exit_status.load(Ordering::Acquire) == EXIT_STOPPED
}
pub fn revoke_unclaimed_exit_stop(&self) -> Result<(), Errno> {
self.event
.event()
.prepare_exit_capability_for_cleanup(false)
}
pub unsafe fn revoke_owned_exit_stop(&self) -> Result<(), Errno> {
let event = self.event.event();
event.prepare_exit_capability_for_cleanup(true)?;
event.status.lock().retire_before_exit();
Ok(())
}
}
#[must_use = "drop rolls the reservation back; call commit after ownership is stored"]
pub struct PendingStatusReservation<'a> {
pid: Pid,
status: i32,
event: &'a EventHandle,
policy: WaitPolicy,
state: MutexGuard<'a, StatusState>,
}
impl PendingStatusReservation<'_> {
pub fn decode(&self) -> Result<Wait, Error> {
Wait::from_raw_with_token(
self.pid,
self.status,
TraceeToken::from_event_for_policy(self.event.clone(), self.policy),
)
}
pub fn commit(mut self) {
let committed = self.state.pop_front();
debug_assert_eq!(committed, Some(self.status));
}
pub fn consume_dead_exec(self) -> Result<(), Self> {
let token = TraceeToken::from_event_for_policy(self.event.clone(), self.policy);
self.consume_dead_exec_with_token(token)
}
fn consume_dead_exec_with_token(mut self, token: TraceeToken) -> Result<(), Self> {
if self.status != PTRACE_EVENT_EXEC_STOP
|| !matches!(
Wait::from_raw_with_token(self.pid, self.status, token),
Err(Error::Died(_))
)
{
return Err(self);
}
let epoch_exec = self.state.epoch_exec == Some(0);
if epoch_exec {
self.state.retired_dead_exec += 1;
}
let committed = self.state.pop_front();
debug_assert_eq!(committed, Some(self.status));
let Self { event, state, .. } = self;
drop(state);
if epoch_exec {
event.event().forward_exit_stop_after_dead_exec();
}
Ok(())
}
}
#[must_use = "retain this reservation on refusal; commit only after ownership is stored"]
pub struct GuardedPendingStatusReservation<'a> {
inner: Option<PendingStatusReservation<'a>>,
guard: PtracerThreadGuard,
dead_exec_consumed: bool,
local: PhantomData<Rc<()>>,
}
impl GuardedPendingStatusReservation<'_> {
pub fn decode(&self) -> Result<Wait, Error> {
self.guard.check_current()?;
let result = self.inner.as_ref().ok_or(Errno::EALREADY)?.decode();
self.guard.check_current()?;
result
}
pub fn commit(&mut self) -> Result<(), Errno> {
self.guard.check_current()?;
self.inner.take().ok_or(Errno::EALREADY)?.commit();
Ok(())
}
pub fn consume_dead_exec(&mut self) -> Result<bool, Errno> {
self.guard.check_current()?;
if self.dead_exec_consumed {
return Ok(true);
}
let inner = self.inner.as_ref().ok_or(Errno::EALREADY)?;
if inner.status != PTRACE_EVENT_EXEC_STOP {
return Ok(false);
}
let dead = matches!(inner.decode(), Err(Error::Died(_)));
self.guard.check_current()?;
if !dead {
return Ok(false);
}
let mut inner = self.inner.take().unwrap();
let epoch_exec = inner.state.epoch_exec == Some(0);
if epoch_exec {
inner.state.retired_dead_exec += 1;
}
let committed = inner.state.pop_front();
debug_assert_eq!(committed, Some(inner.status));
let PendingStatusReservation { event, state, .. } = inner;
drop(state);
self.dead_exec_consumed = true;
if epoch_exec {
event.event().forward_exit_stop_after_dead_exec();
}
self.guard.check_current()?;
Ok(true)
}
}
pub struct WaitFuture {
pid: Pid,
token: TraceeToken,
}
impl WaitFuture {
pub(super) fn new(Running(pid, token, _): Running) -> Self {
Self { pid, token }
}
fn from_stopped(Stopped(pid, token, _): Stopped) -> Self {
Self { pid, token }
}
fn poll_for_policy(
&mut self,
cx: &mut Context<'_>,
policy: WaitPolicy,
) -> Poll<Result<Wait, Error>> {
self.poll_with_guard(cx, policy, None)
}
fn poll_with_guard(
&mut self,
cx: &mut Context<'_>,
policy: WaitPolicy,
guard: Option<&PtracerThreadGuard>,
) -> Poll<Result<Wait, Error>> {
let pid = self.pid;
let event_handle = match NOTIFIER.event_for_policy(pid, self.token.event(), policy) {
Ok(event) => event,
Err(error) => return Poll::Ready(Err(error.into())),
};
let event = event_handle.event();
let ptracer_owner = self.token.ptracer_owner.clone();
let ptracer_wait_role = self.token.ptracer_wait_role;
loop {
let assistance = if policy == WaitPolicy::PtracerThread {
event_handle.try_assist_legacy_wait()
} else {
Ok(())
};
let readiness = match guard {
Some(guard) => event.poll_status_reservation_with_guard(cx.waker(), Some(guard)),
None => event.poll_status_reservation(cx.waker()),
};
let reservation = match readiness {
Poll::Ready(Ok(reservation)) => reservation,
Poll::Ready(Err(errno)) => return Poll::Ready(Err(errno.into())),
Poll::Pending => {
return match assistance {
Ok(()) => Poll::Pending,
Err(error) => Poll::Ready(Err(error.into())),
};
}
};
return match event.decode_status_return_with_guard(reservation, guard, |status| {
let token = TraceeToken {
event: event_handle.clone(),
policy,
ptracer_owner: (policy == WaitPolicy::PtracerThread)
.then(|| ptracer_owner.clone())
.flatten(),
ptracer_wait_role: policy == WaitPolicy::PtracerThread && ptracer_wait_role,
};
Wait::from_raw_with_token(pid, status, token)
}) {
Ok(StatusReturn::Returned(decoded)) => Poll::Ready(Ok(decoded)),
Ok(StatusReturn::Cancelled(_)) => Poll::Ready(Err(Errno::ECANCELED.into())),
Ok(StatusReturn::RetiredDeadExec) => continue,
Err(error) => Poll::Ready(Err(error)),
};
}
}
}
impl Future for WaitFuture {
type Output = Result<Wait, Error>;
fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
self.get_mut().poll_for_policy(cx, WaitPolicy::Native)
}
}
#[derive(Clone, Copy, Debug, Eq, PartialEq, thiserror::Error)]
pub enum SupersededStopRefusal {
#[error(
"queued wait status {0:#x} is a fork, vfork or clone stop superseded by the exit stop; its child PID is no longer readable"
)]
NewChild(i32),
}
#[derive(Clone, Copy, Debug, Eq, PartialEq, thiserror::Error)]
pub enum OwnedWaitError {
#[error("owned ptrace wait refused: {0}")]
Errno(Errno),
#[error("owned ptrace wait observed death during decoding; original owner retained")]
Died,
#[error("owned ptrace wait already transferred its successful result")]
Completed,
}
#[cfg(test)]
thread_local! {
static TRACER_PID_READS_FOR_TEST: Cell<usize> = const { Cell::new(0) };
}
const DETACH_EPOCH_TID_LIMIT: usize = 1 << 22;
const DETACH_EPOCH_BLOCK: usize = 4096;
const DETACH_EPOCH_BLOCK_COUNT: usize = DETACH_EPOCH_TID_LIMIT / DETACH_EPOCH_BLOCK;
static DETACH_EPOCH_BLOCKS: [OnceLock<Box<[AtomicU64]>>; DETACH_EPOCH_BLOCK_COUNT] =
[const { OnceLock::new() }; DETACH_EPOCH_BLOCK_COUNT];
fn detach_epoch_slot(pid: Pid) -> Option<(usize, usize)> {
let tid = usize::try_from(pid.as_raw())
.ok()
.filter(|tid| *tid < DETACH_EPOCH_TID_LIMIT)?;
Some((tid / DETACH_EPOCH_BLOCK, tid % DETACH_EPOCH_BLOCK))
}
fn detach_epoch(pid: Pid) -> Option<u64> {
let (block, index) = detach_epoch_slot(pid)?;
Some(
DETACH_EPOCH_BLOCKS[block]
.get()
.map_or(0, |block| block[index].load(Ordering::SeqCst)),
)
}
pub(super) fn note_detach(pid: Pid) {
if let Some((block, index)) = detach_epoch_slot(pid) {
DETACH_EPOCH_BLOCKS[block]
.get_or_init(|| (0..DETACH_EPOCH_BLOCK).map(|_| AtomicU64::new(0)).collect())[index]
.fetch_add(1, Ordering::SeqCst);
}
}
#[derive(Clone, Debug)]
pub(super) struct ResumeProof {
owner: Option<Arc<LegacyWaitOwner>>,
event: usize,
epoch: Option<u64>,
}
fn note_detached_wait_if_untraced(
running: &Running,
affinity_owner: Option<Arc<LegacyWaitOwner>>,
) -> Option<(Arc<Event>, u64)> {
let token = &running.1;
let handle = token.event();
let owner = affinity_owner.or_else(|| token.ptracer_owner.clone());
if handle.resume_proof_holds(running.0, running.2.0.as_ref(), owner.as_ref()) {
return None;
}
let identity = handle.identity()?;
#[cfg(test)]
TRACER_PID_READS_FOR_TEST.with(|reads| reads.set(reads.get() + 1));
if identity.current_tracer_pid() != Ok(Pid::from_raw(0)) {
return None;
}
let event = Arc::clone(handle.event());
let id = event.note_detached_wait(owner);
Some((event, id))
}
fn end_detached_wait(pending: &mut Option<(Arc<Event>, u64)>) {
if let Some((event, id)) = pending.take() {
event.end_detached_wait(id);
}
}
fn bind_detached_wait_owner(
pending: &Option<(Arc<Event>, u64)>,
owner: Option<Arc<LegacyWaitOwner>>,
) {
if let Some((event, id)) = pending {
event.bind_detached_wait_owner(*id, owner);
}
}
#[must_use = "this future owns an unfinished generation-bound wait"]
pub struct OwnedWaitFuture {
inner: Option<WaitFuture>,
observed_death: bool,
managed_guard: Option<PtracerThreadGuard>,
detached_wait: Option<(Arc<Event>, u64)>,
}
impl OwnedWaitFuture {
pub(super) fn new(running: Running) -> Self {
let detached_wait = if running.1.ptracer_owner.is_some() {
note_detached_wait_if_untraced(&running, None)
} else {
None
};
Self::with_detached_wait(running, detached_wait)
}
fn new_for_ptracer_driver(
running: Running,
affinity_owner: Option<Arc<LegacyWaitOwner>>,
) -> Self {
let detached_wait = note_detached_wait_if_untraced(&running, affinity_owner);
Self::with_detached_wait(running, detached_wait)
}
fn with_detached_wait(running: Running, detached_wait: Option<(Arc<Event>, u64)>) -> Self {
Self {
inner: Some(WaitFuture::new(running)),
observed_death: false,
managed_guard: None,
detached_wait,
}
}
pub(super) fn from_stopped(stopped: Stopped) -> Self {
Self {
inner: Some(WaitFuture::from_stopped(stopped)),
observed_death: false,
managed_guard: None,
detached_wait: None,
}
}
fn bind_detached_wait_owner(&mut self, owner: Option<Arc<LegacyWaitOwner>>) {
bind_detached_wait_owner(&self.detached_wait, owner);
}
fn poll_for_policy(
&mut self,
cx: &mut Context<'_>,
policy: WaitPolicy,
) -> Poll<Result<Wait, OwnedWaitError>> {
self.poll_with_guard(cx, policy, None)
}
fn poll_with_guard(
&mut self,
cx: &mut Context<'_>,
policy: WaitPolicy,
guard: Option<&PtracerThreadGuard>,
) -> Poll<Result<Wait, OwnedWaitError>> {
let Some(inner) = self.inner.as_mut() else {
return Poll::Ready(Err(OwnedWaitError::Completed));
};
match inner.poll_with_guard(cx, policy, guard) {
Poll::Pending => Poll::Pending,
Poll::Ready(Ok(wait)) => {
self.inner.take();
end_detached_wait(&mut self.detached_wait);
Poll::Ready(Ok(wait))
}
Poll::Ready(Err(Error::Errno(errno))) => Poll::Ready(Err(OwnedWaitError::Errno(errno))),
Poll::Ready(Err(Error::Died(zombie))) => {
drop(zombie);
self.observed_death = true;
Poll::Ready(Err(OwnedWaitError::Died))
}
}
}
pub fn block_until_ready(&self, timeout: Duration) -> Option<bool> {
let inner = self.inner.as_ref()?;
let handle = inner.token.event().resolved_handle();
if handle.uses_procfs_wait() {
return None;
}
handle.event().block_until_status(timeout)
}
pub fn poll_with_ptracer_guard(
&mut self,
cx: &mut Context<'_>,
guard: &PtracerThreadGuard,
) -> Poll<Result<Wait, OwnedWaitError>> {
if let Some(error) = self
.inner
.as_ref()
.and_then(|inner| WaitPolicy::Native.check_handle(inner.token.event()).err())
{
return Poll::Ready(Err(OwnedWaitError::Errno(error)));
}
let guard = match PtracerThreadGuard::retain_for_poll(&mut self.managed_guard, guard) {
Ok(guard) => guard,
Err(error) => return Poll::Ready(Err(OwnedWaitError::Errno(error))),
};
self.poll_with_guard(cx, WaitPolicy::Native, Some(&guard))
}
pub fn generation(&self) -> Option<super::TraceeGeneration> {
self.inner
.as_ref()
.map(|inner| super::TraceeGeneration(inner.pid, inner.token.clone()))
}
pub fn terminal_cleanup(&self) -> Option<TerminalCleanup> {
self.inner
.as_ref()
.map(|inner| TerminalCleanup::new_unregistered(inner.pid, &inner.token))
}
pub fn into_zombie_after_observed_death(mut self) -> Result<super::Zombie, Self> {
if !self.observed_death || self.inner.is_none() {
return Err(self);
}
let inner = self.inner.take().unwrap();
Ok(super::Zombie(Running::from_token(inner.pid, inner.token)))
}
}
impl Drop for OwnedWaitFuture {
fn drop(&mut self) {
end_detached_wait(&mut self.detached_wait);
}
}
impl Future for OwnedWaitFuture {
type Output = Result<Wait, OwnedWaitError>;
fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
self.get_mut().poll_for_policy(cx, WaitPolicy::Native)
}
}
pub struct ExitFuture {
pid: Pid,
event: EventHandle,
waiter: Arc<ExitWaiter>,
managed_guard: Option<PtracerThreadGuard>,
managed_delivery: Option<Stopped>,
}
impl ExitFuture {
pub(super) fn new(pid: Pid, token: &TraceeToken) -> Self {
Self {
pid,
event: token.event().clone(),
waiter: Arc::new(ExitWaiter {
waker: WakerSlot::default(),
epoch: token.event().event().exit_epoch.load(Ordering::Acquire),
}),
managed_guard: None,
managed_delivery: None,
}
}
fn poll_for_policy(
&mut self,
cx: &mut Context<'_>,
policy: WaitPolicy,
) -> Poll<Result<Stopped, Error>> {
let event_handle = match NOTIFIER.event_for_policy(self.pid, &self.event, policy) {
Ok(event) => event,
Err(error) => return Poll::Ready(Err(error.into())),
};
let event = event_handle.event();
let assistance = if policy == WaitPolicy::PtracerThread {
event_handle.try_assist_legacy_wait()
} else {
Ok(())
};
match event.poll_exit(&self.waiter, cx.waker()) {
Poll::Ready(Ok(())) => Poll::Ready(Ok(Stopped::from_exit_token(
self.pid,
TraceeToken::from_event_for_policy(event_handle, policy),
self.waiter.epoch,
))),
Poll::Ready(Err(errno)) => Poll::Ready(Err(errno.into())),
Poll::Pending => match assistance {
Ok(()) => Poll::Pending,
Err(error) => Poll::Ready(Err(error.into())),
},
}
}
pub fn poll_with_ptracer_guard(
&mut self,
cx: &mut Context<'_>,
guard: &PtracerThreadGuard,
) -> Poll<Result<Stopped, Error>> {
if let Err(error) = WaitPolicy::Native.check_handle(&self.event) {
return Poll::Ready(Err(error.into()));
}
let guard = match PtracerThreadGuard::retain_for_poll(&mut self.managed_guard, guard) {
Ok(guard) => guard,
Err(error) => return Poll::Ready(Err(error.into())),
};
if let Some(stopped) = self.managed_delivery.take() {
return Poll::Ready(Ok(stopped));
}
let event_handle =
match NOTIFIER.event_for_policy(self.pid, &self.event, WaitPolicy::Native) {
Ok(event) => event,
Err(error) => return Poll::Ready(Err(error.into())),
};
let event = event_handle.event();
event.exit_waiters.register(&self.waiter, cx.waker());
if let Err(error) = guard.check_current() {
return Poll::Ready(Err(error.into()));
}
let Some(publication) = event.exit_publication.try_lock() else {
cx.waker().wake_by_ref();
return Poll::Pending;
};
let result = event.poll_exit_epoch(&self.waiter);
drop(publication);
if result.is_ready() {
if matches!(result, Poll::Ready(Ok(()))) {
self.managed_delivery = Some(Stopped::from_exit_token(
self.pid,
TraceeToken::from_event_for_policy(event_handle.clone(), WaitPolicy::Native),
self.waiter.epoch,
));
}
event.exit_waiters.wake_all();
if let Err(error) = guard.check_current() {
return Poll::Ready(Err(error.into()));
}
}
match result {
Poll::Ready(Ok(())) => Poll::Ready(Ok(self.managed_delivery.take().unwrap())),
Poll::Ready(Err(error)) => Poll::Ready(Err(error.into())),
Poll::Pending => Poll::Pending,
}
}
}
impl Future for ExitFuture {
type Output = Result<Stopped, Error>;
fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
self.get_mut().poll_for_policy(cx, WaitPolicy::Native)
}
}
#[derive(Clone)]
pub struct PtracerThreadGuard {
owner: Arc<LegacyWaitOwner>,
}
impl PtracerThreadGuard {
fn retain_for_poll(retained: &mut Option<Self>, supplied: &Self) -> Result<Self, Errno> {
if let Some(guard) = retained.as_ref() {
if !Arc::ptr_eq(&guard.owner, &supplied.owner) {
return Err(Errno::EPERM);
}
guard.check_current()?;
return Ok(guard.clone());
}
supplied.check_current()?;
*retained = Some(supplied.clone());
Ok(supplied.clone())
}
pub(super) fn from_owner(owner: Arc<LegacyWaitOwner>) -> Self {
Self { owner }
}
pub fn check_current(&self) -> Result<(), Errno> {
if self.owner.is_current()? {
let _ = WAKER_CALLBACKS_MAY_HAVE_RUN.try_with(|flag| flag.set(false));
Ok(())
} else {
Err(Errno::EPERM)
}
}
fn recheck_after_waker_callbacks(&self) -> Result<(), Errno> {
if WAKER_CALLBACKS_MAY_HAVE_RUN
.try_with(Cell::get)
.unwrap_or(true)
{
#[cfg(test)]
WAKER_CALLBACK_RECHECKS.with(|rechecks| rechecks.set(rechecks.get() + 1));
self.check_current()
} else {
Ok(())
}
}
pub(super) fn check_wait_role(&self, event: &EventHandle) -> Result<(), Errno> {
if event.authenticates_wait_role(&self.owner)? {
Ok(())
} else {
Err(Errno::EPERM)
}
}
}
struct PtracerAffinity {
owner: Option<Arc<LegacyWaitOwner>>,
wait_role: bool,
}
#[must_use = "retain this original synchronous wait across refusals"]
pub struct PtracerSyncDriver {
input: Option<(Pid, TraceeToken)>,
affinity: PtracerAffinity,
detached_wait: Option<(Arc<Event>, u64)>,
}
impl Drop for PtracerSyncDriver {
fn drop(&mut self) {
end_detached_wait(&mut self.detached_wait);
}
}
impl PtracerSyncDriver {
pub(super) fn new(running: Running) -> Self {
let affinity = PtracerAffinity::new(
running.1.event(),
running.1.ptracer_owner.clone(),
running.1.ptracer_wait_role,
);
let detached_wait = note_detached_wait_if_untraced(&running, affinity.owner.clone());
Self {
input: Some((running.0, running.1)),
affinity,
detached_wait,
}
}
pub fn wait_on_ptracer_thread(&mut self) -> Result<Wait, OwnedWaitError> {
let Some((pid, token)) = self.input.as_mut() else {
return Err(OwnedWaitError::Completed);
};
let handle = token.event();
self.affinity
.check_host(handle)
.map_err(OwnedWaitError::Errno)?;
if !handle.event().is_terminal() && !*handle.event().terminal_reaping.read() {
self.affinity
.check_attachment(handle)
.map_err(OwnedWaitError::Errno)?;
}
token.ptracer_owner = self.affinity.owner.clone();
token.ptracer_wait_role = self.affinity.wait_role;
bind_detached_wait_owner(&self.detached_wait, self.affinity.owner.clone());
match wait_sync_for_policy(*pid, token.clone(), WaitPolicy::PtracerThread) {
Ok(state) => {
self.input.take();
end_detached_wait(&mut self.detached_wait);
Ok(state)
}
Err(Error::Errno(error)) => Err(OwnedWaitError::Errno(error)),
Err(Error::Died(_)) => Err(OwnedWaitError::Died),
}
}
}
#[must_use = "this interface owns its unfinished synchronous wait"]
pub struct PtracerSyncWait {
driver: PtracerSyncDriver,
local: PhantomData<Rc<()>>,
}
impl PtracerSyncWait {
pub(super) fn new(running: Running) -> Self {
Self {
driver: PtracerSyncDriver::new(running),
local: PhantomData,
}
}
pub fn wait(&mut self) -> Result<Wait, OwnedWaitError> {
self.driver.wait_on_ptracer_thread()
}
pub fn into_driver(self) -> PtracerSyncDriver {
self.driver
}
}
impl PtracerAffinity {
fn new(handle: &EventHandle, owner: Option<Arc<LegacyWaitOwner>>, wait_role: bool) -> Self {
let mut affinity = Self { owner, wait_role };
if affinity.owner.is_none() || !affinity.wait_role {
let _ = affinity.check_host(handle);
}
affinity
}
fn check_host(&mut self, handle: &EventHandle) -> Result<(), Errno> {
#[cfg(test)]
PTRACER_HOST_PROOFS.with(|proofs| proofs.set(proofs.get() + 1));
WaitPolicy::PtracerThread.check_handle(handle)?;
if self.owner.is_none() {
self.owner = Some(handle.capture_current_owner()?);
self.wait_role = true;
}
let owner = self.owner.as_ref().unwrap();
if !owner.is_current()? {
return Err(Errno::EPERM);
}
if !self.wait_role {
if !handle.authenticates_wait_role(owner)? {
return Err(Errno::EPERM);
}
self.wait_role = true;
}
Ok(())
}
fn check_attachment(&self, handle: &EventHandle) -> Result<(), Errno> {
let identity = handle.identity().ok_or(Errno::ENODATA)?;
let owner = self.owner.as_ref().ok_or(Errno::EPERM)?;
if let Some(_held) = handle.hold_tid()
&& nix::sys::ptrace::getevent(identity.pid.into()).is_ok()
&& identity.pidfd_is_live() == Ok(true)
{
return Ok(());
}
#[cfg(test)]
PTRACER_ATTACHMENT_STATUS_READS.with(|reads| reads.set(reads.get() + 1));
let tracer = match identity.current_tracer_pid() {
Ok(tracer) => tracer,
Err(Errno::ENOENT | Errno::ESRCH) => return Ok(()),
Err(error) => return Err(error),
};
if tracer != owner.tid
&& !(tracer.as_raw() == 0 && !matches!(identity.pidfd, ThreadHandle::Procfs { .. }))
{
return Err(Errno::EPERM);
}
Ok(())
}
}
#[must_use = "retain this original wait driver until it returns its state"]
pub struct PtracerWaitDriver {
inner: OwnedWaitFuture,
affinity: PtracerAffinity,
observed_death: bool,
}
impl PtracerWaitDriver {
pub(super) fn new(running: Running) -> Self {
let affinity = PtracerAffinity::new(
running.1.event(),
running.1.ptracer_owner.clone(),
running.1.ptracer_wait_role,
);
let inner = OwnedWaitFuture::new_for_ptracer_driver(running, affinity.owner.clone());
Self {
inner,
affinity,
observed_death: false,
}
}
pub(super) fn from_stopped(stopped: Stopped) -> Self {
let affinity = PtracerAffinity::new(
stopped.1.event(),
stopped.1.ptracer_owner.clone(),
stopped.1.ptracer_wait_role,
);
Self {
inner: OwnedWaitFuture::from_stopped(stopped),
affinity,
observed_death: false,
}
}
pub fn block_until_ready(&self, timeout: Duration) -> Option<bool> {
self.inner.block_until_ready(timeout)
}
pub fn poll_on_ptracer_thread(
&mut self,
cx: &mut Context<'_>,
) -> Poll<Result<Wait, OwnedWaitError>> {
let Some(inner) = self.inner.inner.as_ref() else {
return Poll::Ready(Err(OwnedWaitError::Completed));
};
let handle = inner.token.event();
if let Err(error) = self.affinity.check_host(handle) {
return Poll::Ready(Err(OwnedWaitError::Errno(error)));
}
if !handle.event().is_terminal()
&& !*handle.event().terminal_reaping.read()
&& let Err(error) = self.affinity.check_attachment(handle)
{
if !handle.event().is_terminal() && !*handle.event().terminal_reaping.read() {
return Poll::Ready(Err(OwnedWaitError::Errno(error)));
}
}
self.inner.inner.as_mut().unwrap().token.ptracer_owner = self.affinity.owner.clone();
self.inner.inner.as_mut().unwrap().token.ptracer_wait_role = self.affinity.wait_role;
self.inner
.bind_detached_wait_owner(self.affinity.owner.clone());
let result = self.inner.poll_for_policy(cx, WaitPolicy::PtracerThread);
if matches!(result, Poll::Ready(Err(OwnedWaitError::Died))) {
self.observed_death = true;
}
result
}
pub fn terminal_cleanup(&self) -> Option<TerminalCleanup> {
self.inner
.inner
.as_ref()
.map(|inner| TerminalCleanup::new(inner.pid, &inner.token))
}
pub fn generation(&self) -> Option<super::TraceeGeneration> {
self.inner
.inner
.as_ref()
.map(|inner| super::TraceeGeneration(inner.pid, inner.token.clone()))
}
pub fn cleanup_on_ptracer_thread(&self) -> Option<PtracerCleanupDriver> {
self.inner
.inner
.as_ref()
.map(|inner| PtracerTerminalCleanup::new(inner.pid, &inner.token).into_driver())
}
pub fn into_zombie_after_observed_death(mut self) -> Result<super::Zombie, Self> {
if !self.observed_death || self.inner.inner.is_none() {
return Err(self);
}
let inner = self.inner.inner.take().unwrap();
Ok(super::Zombie(Running::from_token(inner.pid, inner.token)))
}
}
#[must_use = "this future owns an unfinished generation-bound wait"]
pub struct PtracerOwnedWaitFuture {
driver: PtracerWaitDriver,
local: PhantomData<Rc<()>>,
}
impl PtracerOwnedWaitFuture {
pub(super) fn new(running: Running) -> Self {
Self {
driver: PtracerWaitDriver::new(running),
local: PhantomData,
}
}
pub(super) fn from_stopped(stopped: Stopped) -> Self {
Self {
driver: PtracerWaitDriver::from_stopped(stopped),
local: PhantomData,
}
}
pub fn into_driver(self) -> PtracerWaitDriver {
self.driver
}
}
impl Future for PtracerOwnedWaitFuture {
type Output = Result<Wait, OwnedWaitError>;
fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
self.get_mut().driver.poll_on_ptracer_thread(cx)
}
}
pub type PtracerWaitFuture = PtracerOwnedWaitFuture;
#[must_use = "retain this original exit driver through non-owning refusals"]
pub struct PtracerExitDriver {
inner: ExitFuture,
affinity: PtracerAffinity,
}
impl PtracerExitDriver {
pub(super) fn new(pid: Pid, token: &TraceeToken) -> Self {
Self {
inner: ExitFuture::new(pid, token),
affinity: PtracerAffinity::new(
token.event(),
token.ptracer_owner.clone(),
token.ptracer_wait_role,
),
}
}
pub fn poll_on_ptracer_thread(&mut self, cx: &mut Context<'_>) -> Poll<Result<Stopped, Error>> {
let handle = &self.inner.event;
if let Err(error) = self.affinity.check_host(handle) {
return Poll::Ready(Err(error.into()));
}
let event = handle.event();
let settled = event.is_terminal()
|| self.inner.waiter.epoch != event.exit_epoch.load(Ordering::Acquire)
|| matches!(
event.exit_capability.load(Ordering::Acquire),
EXIT_CAP_CLAIMED | EXIT_CAP_EXPIRED
);
if !settled
&& let Err(error) = self.affinity.check_attachment(handle)
&& !event.is_terminal()
&& !*event.terminal_reaping.read()
{
return Poll::Ready(Err(error.into()));
}
match self.inner.poll_for_policy(cx, WaitPolicy::PtracerThread) {
Poll::Ready(Ok(mut stopped)) => {
stopped.1.ptracer_owner = self.affinity.owner.clone();
stopped.1.ptracer_wait_role = self.affinity.wait_role;
Poll::Ready(Ok(stopped))
}
result => result,
}
}
}
#[must_use = "this future retains its original exit epoch"]
pub struct PtracerExitFuture {
driver: PtracerExitDriver,
local: PhantomData<Rc<()>>,
}
impl PtracerExitFuture {
pub(super) fn new(pid: Pid, token: &TraceeToken) -> Self {
Self {
driver: PtracerExitDriver::new(pid, token),
local: PhantomData,
}
}
pub fn into_driver(self) -> PtracerExitDriver {
self.driver
}
}
impl Future for PtracerExitFuture {
type Output = Result<Stopped, Error>;
fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
self.get_mut().driver.poll_on_ptracer_thread(cx)
}
}
pub struct PtracerCleanupDriver {
shared: TerminalCleanup,
affinity: PtracerAffinity,
}
impl PtracerCleanupDriver {
fn new(shared: TerminalCleanup) -> Self {
let affinity = PtracerAffinity::new(&shared.event, None, false);
Self { shared, affinity }
}
pub fn shared(&self) -> &TerminalCleanup {
&self.shared
}
pub fn progress_on_ptracer_thread(&mut self) -> Result<(), Errno> {
self.affinity.check_host(&self.shared.event)?;
self.progress_after_host_check()
}
fn progress_after_host_check(&self) -> Result<(), Errno> {
self.shared.ensure_registered()?;
let unsettled = !self.shared.event.event().is_terminal()
&& !*self.shared.event.event().terminal_reaping.read();
if unsettled
&& let Err(error) = self.affinity.check_attachment(&self.shared.event)
&& !self.shared.event.event().is_terminal()
&& !*self.shared.event.event().terminal_reaping.read()
{
return Err(error);
}
self.shared.event.try_assist_legacy_wait()
}
pub fn wait_on_ptracer_thread(&mut self, timeout: Duration) -> Result<bool, Errno> {
let deadline = Instant::now()
.checked_add(timeout)
.unwrap_or_else(Instant::now);
loop {
self.progress_on_ptracer_thread()?;
let remaining = deadline.saturating_duration_since(Instant::now());
if self.shared.wait(remaining.min(Duration::from_millis(1))) {
return Ok(true);
}
if remaining.is_zero() || Instant::now() >= deadline {
return Ok(false);
}
}
}
pub fn reserve_pending_on_ptracer_thread(
&mut self,
timeout: Duration,
) -> Result<Option<PtracerPendingStatusReservation<'_>>, Errno> {
self.affinity.check_host(&self.shared.event)?;
let deadline = Instant::now()
.checked_add(timeout)
.unwrap_or_else(Instant::now);
loop {
self.progress_after_host_check()?;
let remaining = deadline.saturating_duration_since(Instant::now());
if let Some(reservation) = self.shared.reserve_pending_for_policy(
remaining.min(Duration::from_millis(1)),
WaitPolicy::PtracerThread,
) {
return Ok(Some(PtracerPendingStatusReservation {
inner: reservation,
owner: self.affinity.owner.clone(),
wait_role: self.affinity.wait_role,
local: PhantomData,
}));
}
if self.shared.event.event().is_terminal()
|| remaining.is_zero()
|| Instant::now() >= deadline
{
return Ok(None);
}
}
}
}
pub struct PtracerTerminalCleanup {
driver: PtracerCleanupDriver,
local: PhantomData<Rc<()>>,
}
impl PtracerTerminalCleanup {
pub(super) fn new(pid: Pid, token: &TraceeToken) -> Self {
if let Some(owner) = &token.ptracer_owner {
Self {
driver: PtracerCleanupDriver {
shared: TerminalCleanup::new_unregistered(pid, token),
affinity: PtracerAffinity {
owner: Some(Arc::clone(owner)),
wait_role: token.ptracer_wait_role,
},
},
local: PhantomData,
}
} else {
Self::from_shared(TerminalCleanup::new_unregistered(pid, token))
}
}
fn from_shared(shared: TerminalCleanup) -> Self {
Self {
driver: PtracerCleanupDriver::new(shared),
local: PhantomData,
}
}
pub fn shared(&self) -> &TerminalCleanup {
self.driver.shared()
}
pub fn into_driver(self) -> PtracerCleanupDriver {
self.driver
}
pub fn progress(&mut self) -> Result<(), Errno> {
self.driver.progress_on_ptracer_thread()
}
pub fn wait(&mut self, timeout: Duration) -> Result<bool, Errno> {
self.driver.wait_on_ptracer_thread(timeout)
}
pub fn reserve_pending_for_cleanup(
&mut self,
timeout: Duration,
) -> Result<Option<PtracerPendingStatusReservation<'_>>, Errno> {
self.driver.reserve_pending_on_ptracer_thread(timeout)
}
}
#[must_use = "drop rolls back; commit after original ownership is stored"]
pub struct PtracerPendingStatusReservation<'a> {
inner: PendingStatusReservation<'a>,
owner: Option<Arc<LegacyWaitOwner>>,
wait_role: bool,
local: PhantomData<Rc<()>>,
}
impl PtracerPendingStatusReservation<'_> {
pub fn decode(&self) -> Result<Wait, Error> {
Wait::from_raw_with_token(self.inner.pid, self.inner.status, self.token())
}
pub fn commit(self) {
self.inner.commit();
}
pub fn consume_dead_exec(self) -> Result<(), Self> {
let token = self.token();
match self.inner.consume_dead_exec_with_token(token) {
Ok(()) => Ok(()),
Err(inner) => Err(Self {
inner,
owner: self.owner,
wait_role: self.wait_role,
local: PhantomData,
}),
}
}
fn token(&self) -> TraceeToken {
TraceeToken {
event: self.inner.event.clone(),
policy: WaitPolicy::PtracerThread,
ptracer_owner: self.owner.clone(),
ptracer_wait_role: self.wait_role,
}
}
}
impl TerminalCleanup {
pub fn on_ptracer_thread(&self) -> PtracerTerminalCleanup {
PtracerTerminalCleanup::from_shared(Self {
pid: self.pid,
event: self.event.clone(),
})
}
}
#[cfg(test)]
mod test {
#![allow(
unexpected_cfgs,
reason = "`fbcode_build` is supplied by the internal Buck build"
)]
include!("adaptive_api_tests.rs");
include!("managed_native_tests.rs");
include!("exit_probe_tests.rs");
include!("stop_auth_cache_tests.rs");
include!("retired_tid_tests.rs");
include!("legacy_thread_tests.rs");
include!("legacy_owner_tests.rs");
include!("ptracer_thread_tests.rs");
include!("attachment_generation_tests.rs");
include!("stop_observation_tests.rs");
include!("group_exit_status_tests.rs");
include!("retirement_ack_tests.rs");
include!("completion_wakeup_tests.rs");
include!("native_final_wait_tests.rs");
use std::collections::hash_map::DefaultHasher;
use std::env;
use std::io;
use std::io::Read;
use std::io::Write;
use std::mem;
use std::os::fd::AsRawFd;
use std::os::fd::FromRawFd;
use std::os::unix::process::CommandExt;
use std::os::unix::process::ExitStatusExt;
use std::process::Command;
use std::process::Output;
use std::process::Stdio;
use std::sync::atomic::AtomicBool;
use std::sync::atomic::AtomicUsize;
use std::sync::mpsc;
use std::task::Wake;
use std::thread::JoinHandle;
use std::time::Duration;
use nix::sys::signal::Signal;
use nix::sys::wait::WaitStatus;
use nix::unistd::ForkResult;
use nix::unistd::Pid;
use super::*;
use crate::Options;
use crate::test_fork::fork;
#[cfg(not(sanitized))]
fn emit_completion_marker(marker: &str) {
let line = format!("\n{marker}\n");
assert!(line.len() <= libc::PIPE_BUF);
assert_eq!(
unsafe { libc::write(libc::STDOUT_FILENO, line.as_ptr().cast(), line.len()) },
line.len() as isize,
"could not write actual completion marker"
);
}
#[derive(Default)]
struct WakeCounter(AtomicUsize);
impl Wake for WakeCounter {
fn wake(self: Arc<Self>) {
self.0.fetch_add(1, Ordering::SeqCst);
}
fn wake_by_ref(self: &Arc<Self>) {
self.0.fetch_add(1, Ordering::SeqCst);
}
}
fn handle_hash(handle: &EventHandle) -> u64 {
let mut hasher = DefaultHasher::new();
handle.hash(&mut hasher);
hasher.finish()
}
const SUBPROCESS_POLL_INTERVAL: Duration = Duration::from_millis(5);
const SUBPROCESS_DRAIN_TIMEOUT: Duration = Duration::from_secs(1);
const PID_NAMESPACE_TEST_TIMEOUT: Duration = Duration::from_secs(5);
const TRACEE_WAIT_TIMEOUT: Duration = Duration::from_secs(2);
#[cfg(not(sanitized))]
const EXACT_TEST_SUBPROCESS_TIMEOUT: Duration = Duration::from_secs(60);
const REGISTRATION_RETRY_TIMEOUT: Duration = Duration::from_millis(100);
#[derive(Debug)]
struct BoundedSubprocess {
output: Output,
timed_out: bool,
}
fn kill_process_group(process_group: i32) -> io::Result<()> {
let result = unsafe { libc::kill(-process_group, libc::SIGKILL) };
if result == 0 {
return Ok(());
}
let error = io::Error::last_os_error();
if error.raw_os_error() == Some(libc::ESRCH) {
Ok(())
} else {
Err(error)
}
}
fn pidfd_open(pid: i32) -> io::Result<ThreadHandle> {
let fd = unsafe { libc::syscall(libc::SYS_pidfd_open, pid, 0) } as i32;
if fd == -1 {
Err(io::Error::last_os_error())
} else {
Ok(ThreadHandle::Pidfd(unsafe { OwnedFd::from_raw_fd(fd) }))
}
}
fn pidfd_send_signal(pidfd: &ThreadHandle, signal: i32) -> io::Result<()> {
let result = unsafe {
libc::syscall(
libc::SYS_pidfd_send_signal,
pidfd.as_raw_fd(),
signal,
std::ptr::null::<libc::siginfo_t>(),
0,
)
};
if result == 0 {
Ok(())
} else {
Err(io::Error::last_os_error())
}
}
fn pidfd_exited(pidfd: &ThreadHandle) -> io::Result<bool> {
if matches!(pidfd, ThreadHandle::Procfs { .. }) {
return pidfd_is_live(pidfd)
.map(|live| !live)
.map_err(|error| io::Error::from_raw_os_error(error.into_raw()));
}
let mut pollfd = libc::pollfd {
fd: pidfd.as_raw_fd(),
events: libc::POLLIN,
revents: 0,
};
let result = unsafe { libc::poll(&mut pollfd, 1, 0) };
if result == -1 {
Err(io::Error::last_os_error())
} else {
Ok(result == 1 && pollfd.revents & libc::POLLIN != 0)
}
}
struct PipeDrain {
stop: Arc<AtomicBool>,
thread: Option<JoinHandle<io::Result<Vec<u8>>>>,
}
impl PipeDrain {
fn is_finished(&self) -> bool {
self.thread.as_ref().is_none_or(JoinHandle::is_finished)
}
fn stop_and_join(&mut self) -> io::Result<Vec<u8>> {
self.stop.store(true, Ordering::Release);
self.join()
}
fn join(&mut self) -> io::Result<Vec<u8>> {
self.thread
.take()
.expect("pipe reader joined twice")
.join()
.map_err(|_| io::Error::other("exact-test pipe reader panicked"))?
}
}
impl Drop for PipeDrain {
fn drop(&mut self) {
self.stop.store(true, Ordering::Release);
if let Some(thread) = self.thread.take() {
let _ = thread.join();
}
}
}
fn drain_pipe<R>(mut pipe: R) -> io::Result<PipeDrain>
where
R: Read + AsRawFd + Send + 'static,
{
let flags = unsafe { libc::fcntl(pipe.as_raw_fd(), libc::F_GETFL) };
if flags == -1
|| unsafe { libc::fcntl(pipe.as_raw_fd(), libc::F_SETFL, flags | libc::O_NONBLOCK) }
== -1
{
return Err(io::Error::last_os_error());
}
let stop = Arc::new(AtomicBool::new(false));
let reader_stop = Arc::clone(&stop);
let thread = thread::spawn(move || {
let mut bytes = Vec::new();
let mut buffer = [0_u8; 8192];
loop {
match pipe.read(&mut buffer) {
Ok(0) => return Ok(bytes),
Ok(read) => bytes.extend_from_slice(&buffer[..read]),
Err(error) if error.kind() == io::ErrorKind::Interrupted => {}
Err(error) if error.kind() == io::ErrorKind::WouldBlock => {
if reader_stop.load(Ordering::Acquire) {
return Ok(bytes);
}
thread::sleep(Duration::from_millis(1));
}
Err(error) => return Err(error),
}
}
});
Ok(PipeDrain {
stop,
thread: Some(thread),
})
}
struct ChildProcessGroup {
child: std::process::Child,
pidfd: ThreadHandle,
process_group: i32,
reaped: bool,
}
impl ChildProcessGroup {
fn new(mut child: std::process::Child) -> io::Result<Self> {
let process_group = child.id() as i32;
let pidfd = match pidfd_open(process_group) {
Ok(pidfd) => pidfd,
Err(error) => {
let _ = child.kill();
let _ = child.wait();
return Err(error);
}
};
Ok(Self {
child,
pidfd,
process_group,
reaped: false,
})
}
fn wait(&mut self) -> io::Result<std::process::ExitStatus> {
let status = self.child.wait()?;
self.reaped = true;
Ok(status)
}
fn terminate_group(&self) -> io::Result<()> {
kill_process_group(self.process_group)?;
Ok(())
}
}
impl Drop for ChildProcessGroup {
fn drop(&mut self) {
if !self.reaped {
let _ = kill_process_group(self.process_group);
let _ = self.child.wait();
self.reaped = true;
}
}
}
fn run_exact_test_bounded(
inner: &str,
environment: &[(&str, &str)],
pid_namespace: bool,
timeout: Duration,
) -> io::Result<BoundedSubprocess> {
const PROJECT_REAPED_PGID: &str = "SAFEPTRACE_PROJECT_REAPED_PGID";
let projected_reaped_pgid = environment
.iter()
.find_map(|(name, value)| (*name == PROJECT_REAPED_PGID).then_some(*value))
.map(str::parse::<i32>)
.transpose()
.map_err(|error| io::Error::new(io::ErrorKind::InvalidInput, error))?;
let test_binary = env::current_exe()?;
let mut command = if pid_namespace {
let mut command = Command::new("unshare");
command.args([
"--user",
"--map-root-user",
"--pid",
"--fork",
"--mount-proc",
]);
command.arg(test_binary);
command
} else {
Command::new(test_binary)
};
command
.args(["--exact", inner, "--nocapture"])
.envs(environment.iter().copied())
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.process_group(0);
let mut group = ChildProcessGroup::new(crate::test_fork::spawn(&mut command)?)?;
let mut stdout = drain_pipe(
group
.child
.stdout
.take()
.expect("bounded child stdout pipe"),
)?;
let mut stderr = match drain_pipe(
group
.child
.stderr
.take()
.expect("bounded child stderr pipe"),
) {
Ok(stderr) => stderr,
Err(error) => {
let _ = group.terminate_group();
let _ = stdout.stop_and_join();
let _ = group.wait();
return Err(error);
}
};
let deadline = Instant::now() + timeout;
let timed_out = loop {
if pidfd_exited(&group.pidfd)? {
break false;
}
if Instant::now() >= deadline {
group.terminate_group()?;
break true;
}
thread::sleep(SUBPROCESS_POLL_INTERVAL);
};
let drain_deadline = Instant::now() + SUBPROCESS_DRAIN_TIMEOUT;
while !stdout.is_finished() || !stderr.is_finished() {
if Instant::now() >= drain_deadline {
break;
}
thread::sleep(SUBPROCESS_POLL_INTERVAL);
}
let drain_timed_out = !stdout.is_finished() || !stderr.is_finished();
if drain_timed_out {
group.terminate_group()?;
}
let stdout_result = if timed_out || drain_timed_out {
stdout.stop_and_join()
} else {
stdout.join()
};
let stderr_result = if timed_out || drain_timed_out {
stderr.stop_and_join()
} else {
stderr.join()
};
let status = group.wait()?;
if let Some(projected_reaped_pgid) = projected_reaped_pgid {
group.process_group = projected_reaped_pgid;
}
let stdout = stdout_result?;
let stderr = stderr_result?;
if projected_reaped_pgid.is_some() {
return Err(io::Error::other(
"injected exact-test reader failure after leader reap",
));
}
if drain_timed_out {
return Err(io::Error::new(
io::ErrorKind::TimedOut,
format!(
"exact-test pipes stayed open after leader exit; status={status}; stdout={}; stderr={}",
String::from_utf8_lossy(&stdout),
String::from_utf8_lossy(&stderr)
),
));
}
Ok(BoundedSubprocess {
output: Output {
status,
stdout,
stderr,
},
timed_out,
})
}
fn run_exact_in_pid_namespace_bounded(
inner: &str,
environment: &[(&str, &str)],
) -> Option<Output> {
match run_exact_test_bounded(inner, environment, true, PID_NAMESPACE_TEST_TIMEOUT) {
Ok(result) if result.timed_out => panic!(
"PID-namespace exact test exceeded {:?}:\nstdout:\n{}\nstderr:\n{}",
PID_NAMESPACE_TEST_TIMEOUT,
String::from_utf8_lossy(&result.output.stdout),
String::from_utf8_lossy(&result.output.stderr),
),
Ok(result) => Some(result.output),
Err(error) if error.kind() == io::ErrorKind::NotFound => None,
Err(error) => panic!("failed to run bounded PID-namespace exact test: {error}"),
}
}
#[derive(Debug, Eq, PartialEq)]
enum ExactReuseOutcome {
Exercised,
Unavailable,
}
fn classify_exact_reuse_output(
output: Option<&Output>,
exercised_marker: &str,
unavailable_marker: &str,
) -> Result<ExactReuseOutcome, String> {
let Some(output) = output else {
return Ok(ExactReuseOutcome::Unavailable);
};
let stdout = String::from_utf8_lossy(&output.stdout);
let stderr = String::from_utf8_lossy(&output.stderr);
let exercised = stdout.contains(exercised_marker);
let unavailable = stdout.contains(unavailable_marker);
if output.status.success() {
return match (exercised, unavailable) {
(true, false) => Ok(ExactReuseOutcome::Exercised),
(false, true) => Ok(ExactReuseOutcome::Unavailable),
_ => Err(format!(
"successful exact test emitted ambiguous/missing outcome marker; status={}; stdout:\n{stdout}\nstderr:\n{stderr}",
output.status
)),
};
}
let known_unshare_denial = stderr.lines().any(|line| {
matches!(
line,
"unshare: unshare failed: Operation not permitted"
| "unshare: write failed /proc/self/uid_map: Operation not permitted"
| "unshare: mount /proc failed: Operation not permitted"
)
});
if known_unshare_denial && !exercised && !unavailable {
Ok(ExactReuseOutcome::Unavailable)
} else {
Err(format!(
"exact-test subprocess failed; status={}; stdout:\n{stdout}\nstderr:\n{stderr}",
output.status
))
}
}
fn waitpid_status_bounded(pid: Pid, flags: i32, timeout: Duration) -> io::Result<i32> {
let deadline = Instant::now() + timeout;
loop {
let mut status = 0;
let waited = unsafe { libc::waitpid(pid.as_raw(), &mut status, flags | libc::WNOHANG) };
if waited == pid.as_raw() {
return Ok(status);
}
if waited == -1 {
let error = io::Error::last_os_error();
if error.raw_os_error() == Some(libc::EINTR) {
continue;
}
return Err(error);
}
if Instant::now() >= deadline {
return Err(io::Error::new(
io::ErrorKind::TimedOut,
format!("waitpid({pid}) exceeded {timeout:?}"),
));
}
thread::sleep(SUBPROCESS_POLL_INTERVAL);
}
}
fn stopped_tracee_bounded(
pid: Pid,
capability: impl FnOnce(crate::Pid) -> Stopped,
) -> io::Result<Stopped> {
let status = waitpid_status_bounded(pid, libc::WUNTRACED, TRACEE_WAIT_TIMEOUT)?;
if !libc::WIFSTOPPED(status) {
return Err(io::Error::other(format!(
"tracee {pid} reached non-stop status {status:#x}"
)));
}
if libc::WSTOPSIG(status) != libc::SIGSTOP {
return Err(io::Error::other(format!(
"tracee {pid} stopped with signal {} instead of SIGSTOP",
libc::WSTOPSIG(status)
)));
}
Ok(capability(pid.into()))
}
fn reap_tracee_bounded(pid: Pid) -> io::Result<()> {
let deadline = Instant::now() + TRACEE_WAIT_TIMEOUT;
loop {
let remaining = deadline.saturating_duration_since(Instant::now());
match waitpid_status_bounded(pid, libc::__WALL, remaining) {
Ok(status) if libc::WIFEXITED(status) || libc::WIFSIGNALED(status) => return Ok(()),
Ok(status) if libc::WIFSTOPPED(status) => {
let _ = nix::sys::ptrace::cont(pid, None);
}
Ok(status) => {
return Err(io::Error::other(format!(
"tracee {pid} cleanup reached unexpected status {status:#x}"
)));
}
Err(error) if error.raw_os_error() == Some(libc::ECHILD) => {
if !std::path::Path::new(&format!("/proc/{pid}")).exists() {
return Ok(());
}
}
Err(error) => return Err(error),
}
if Instant::now() >= deadline {
return Err(io::Error::new(
io::ErrorKind::TimedOut,
format!("tracee {pid} survived bounded cleanup"),
));
}
thread::sleep(SUBPROCESS_POLL_INTERVAL);
}
}
fn reap_tracee_pidfd_bounded(pid: Pid, pidfd: &ThreadHandle) -> io::Result<()> {
let flags = WaitPidFlag::from_bits_retain(
WaitPidFlag::WEXITED.bits()
| WaitPidFlag::WSTOPPED.bits()
| WaitPidFlag::WCONTINUED.bits()
| WaitPidFlag::WNOHANG.bits()
| libc::__WALL,
);
let deadline = Instant::now() + TRACEE_WAIT_TIMEOUT;
loop {
match pidfd.wait_id().and_then(|id| waitid::waitid(id, flags)) {
Ok(WaitStatus::Exited(waited, _) | WaitStatus::Signaled(waited, _, _)) => {
assert_eq!(waited, pid, "pidfd reaped a different tracee");
return Ok(());
}
Ok(
WaitStatus::Stopped(waited, _)
| WaitStatus::PtraceEvent(waited, _, _)
| WaitStatus::PtraceSyscall(waited),
) => {
assert_eq!(waited, pid, "pidfd observed a different tracee stop");
nix::sys::ptrace::cont(pid, None).map_err(|error| {
io::Error::other(format!("resume pidfd-bound cleanup stop: {error}"))
})?;
}
Ok(WaitStatus::StillAlive | WaitStatus::Continued(_)) => {}
Err(Errno::EINTR) => continue,
Err(Errno::ECHILD) => return Ok(()),
Err(error) => {
return Err(io::Error::other(format!(
"waitid(P_PIDFD) cleanup failed: {error}"
)));
}
}
if Instant::now() >= deadline {
return Err(io::Error::new(
io::ErrorKind::TimedOut,
format!("pidfd cleanup for tracee {pid} timed out"),
));
}
thread::sleep(SUBPROCESS_POLL_INTERVAL);
}
}
enum TraceeCleanupOwnership {
PreRegistration,
NotifierOwned {
terminal: TerminalCleanup,
owns_claimed_exit: bool,
raw_cleanup_claimed: bool,
raw_cleanup_finishes_event: bool,
cleanup_signal_sent: bool,
},
}
enum CleanupWaitOwner {
Notifier,
Raw { finish_event: bool },
TerminalAck,
Authoritative(TerminalCleanup),
}
struct TraceeCleanupGuard {
pid: Pid,
pidfd: ThreadHandle,
ownership: TraceeCleanupOwnership,
armed: bool,
}
impl TraceeCleanupGuard {
fn new(pid: Pid) -> io::Result<Self> {
Ok(Self {
pid,
pidfd: pidfd_open(pid.as_raw())?,
ownership: TraceeCleanupOwnership::PreRegistration,
armed: true,
})
}
fn store_terminal(&mut self, candidate: TerminalCleanup) -> io::Result<()> {
match &self.ownership {
TraceeCleanupOwnership::PreRegistration => {
self.ownership = TraceeCleanupOwnership::NotifierOwned {
terminal: candidate,
owns_claimed_exit: false,
raw_cleanup_claimed: false,
raw_cleanup_finishes_event: false,
cleanup_signal_sent: false,
};
}
TraceeCleanupOwnership::NotifierOwned { terminal, .. } => {
if terminal.pid != candidate.pid || !terminal.same_generation(&candidate) {
return Err(io::Error::new(
io::ErrorKind::InvalidInput,
"cleanup guard cannot bind a different notifier generation",
));
}
}
}
Ok(())
}
fn bind_terminal(&mut self, candidate: TerminalCleanup) -> io::Result<()> {
self.store_terminal(candidate)?;
self.terminal()
.expect("notifier ownership transition stores terminal cleanup")
.ensure_registered()
.map_err(|error| io::Error::other(format!("register cleanup notifier: {error}")))
}
fn bind_notifier(&mut self, stopped: &Stopped) -> io::Result<()> {
self.bind_terminal(TerminalCleanup::new_unregistered(stopped.0, &stopped.1))
}
fn bind_running_notifier(&mut self, running: &Running) -> io::Result<()> {
self.bind_terminal(TerminalCleanup::new_unregistered(running.0, &running.1))
}
fn terminal(&self) -> Option<&TerminalCleanup> {
match &self.ownership {
TraceeCleanupOwnership::PreRegistration => None,
TraceeCleanupOwnership::NotifierOwned { terminal, .. } => Some(terminal),
}
}
fn exit_event(&mut self, stopped: &Stopped) -> io::Result<ExitFuture> {
self.bind_notifier(stopped)?;
Ok(stopped.exit_event())
}
#[cfg(not(sanitized))]
fn exit_event_on_ptracer_thread(
&mut self,
stopped: &Stopped,
) -> io::Result<PtracerExitFuture> {
self.bind_notifier(stopped)?;
Ok(stopped.exit_event_on_ptracer_thread())
}
fn mark_claimed_exit(&mut self) {
let TraceeCleanupOwnership::NotifierOwned {
owns_claimed_exit, ..
} = &mut self.ownership
else {
panic!("exit capability claimed before notifier ownership transition");
};
*owns_claimed_exit = true;
}
fn disarm(&mut self) {
self.armed = false;
}
fn cleanup_before_registration(pid: Pid, pidfd: &ThreadHandle) -> io::Result<()> {
match pidfd_send_signal(pidfd, libc::SIGKILL) {
Ok(()) => {
if let Some(pause) = PRE_REGISTRATION_REAP_PAUSES.lock().remove(&pid.into()) {
pause.captured.wait();
pause.resume.wait();
}
reap_tracee_pidfd_bounded(pid, pidfd)
}
Err(error) if error.raw_os_error() == Some(libc::ESRCH) => Ok(()),
Err(error) => Err(error),
}
}
fn acquire_cleanup_wait_owner(
pid: Pid,
pidfd: &ThreadHandle,
terminal: &TerminalCleanup,
signal_sent: &mut bool,
) -> io::Result<CleanupWaitOwner> {
let policy = match terminal.event.identity().map(|identity| &identity.pidfd) {
Some(ThreadHandle::Pidfd(_)) | None => WaitPolicy::Native,
Some(_) => WaitPolicy::PtracerThread,
};
let authoritative =
match NOTIFIER.current_registered(pid.into(), policy) {
Ok(Some(authoritative)) => Some(authoritative),
Ok(None) | Err(_) => NOTIFIER
.registered_sync_handle(pid.into(), pidfd)
.map_err(|error| {
io::Error::other(format!(
"resolve exact-pidfd synchronous cleanup authority: {error}"
))
})?,
};
if let Some(authoritative) = authoritative {
terminal
.event
.adopt_authoritative(&authoritative)
.map_err(|error| {
io::Error::other(format!("adopt cleanup wait authority: {error}"))
})?;
}
let pause = CLEANUP_OWNER_CLAIM_PAUSES.lock().remove(&pid.into());
if let Some(pause) = pause {
pause.captured.wait();
pause.resume.wait();
}
let deadline = Instant::now() + REGISTRATION_RETRY_TIMEOUT;
let return_deadline = Instant::now() + TRACEE_WAIT_TIMEOUT;
let last_error = loop {
match terminal.try_ensure_registered_for_cleanup(return_deadline) {
Ok(CancellableEventRegistration::Registered) => {
return match terminal.event.event().worker_state.load(Ordering::Acquire) {
WORKER_RUNNING => Ok(CleanupWaitOwner::Notifier),
WORKER_FINISHING | WORKER_DONE => Ok(CleanupWaitOwner::TerminalAck),
state => Err(io::Error::other(format!(
"successful cleanup registration retained worker state {state}"
))),
};
}
Ok(CancellableEventRegistration::Synchronous(event)) => {
if !*signal_sent {
match pidfd_send_signal(pidfd, libc::SIGKILL) {
Ok(()) => *signal_sent = true,
Err(error) if error.raw_os_error() == Some(libc::ESRCH) => {
*signal_sent = true;
}
Err(error) => return Err(error),
}
}
if let Some(pause) = CLEANUP_CANCEL_SIGNAL_PAUSES.lock().remove(&pid.into())
{
pause.captured.wait();
pause.resume.wait();
}
if !event.wait_for_sync_owner_release(TRACEE_WAIT_TIMEOUT) {
return Err(io::Error::new(
io::ErrorKind::TimedOut,
"synchronous wait owner did not release after exact pidfd cancellation",
));
}
}
Ok(CancellableEventRegistration::ReturningTimedOut) => {
return Err(io::Error::new(
io::ErrorKind::TimedOut,
"typed status return transactions starved cleanup past its deadline",
));
}
Ok(CancellableEventRegistration::Busy) if Instant::now() < deadline => {
thread::sleep(SUBPROCESS_POLL_INTERVAL);
}
Ok(CancellableEventRegistration::Busy) => break Errno::EBUSY,
Err(_) if Instant::now() < deadline => {
thread::sleep(SUBPROCESS_POLL_INTERVAL);
}
Err(error) => break error,
}
};
if FORCE_RAW_CLAIM_REGISTRATION_RACE
.lock()
.remove(&pid.into())
.is_some()
{
CAPTURE_PERSISTENT_ERRORS.lock().remove(&pid.into());
match terminal
.try_ensure_registered_for_cleanup(Instant::now() + TRACEE_WAIT_TIMEOUT)
{
Ok(CancellableEventRegistration::Registered) => {}
Ok(CancellableEventRegistration::Synchronous(_)) => {
return Err(io::Error::other(
"forced cleanup registration race found a synchronous owner",
));
}
Ok(CancellableEventRegistration::ReturningTimedOut) => {
return Err(io::Error::other(
"forced cleanup registration race timed out behind a returning owner",
));
}
Ok(CancellableEventRegistration::Busy) => {
return Err(io::Error::other(
"forced cleanup registration race remained unpublished",
));
}
Err(error) => {
return Err(io::Error::other(format!(
"force cleanup registration race: {error}"
)));
}
}
}
match terminal
.try_claim_unstarted_raw_cleanup()
.map_err(|error| {
io::Error::other(format!("claim exact raw cleanup owner: {error}"))
})? {
RawCleanupClaim::Won => {
return Ok(CleanupWaitOwner::Raw { finish_event: true });
}
RawCleanupClaim::Existing(event) => {
return Ok(CleanupWaitOwner::Authoritative(TerminalCleanup {
pid: terminal.pid,
event,
}));
}
RawCleanupClaim::Lost => {}
}
let event = terminal.event.event();
match event.worker_state.load(Ordering::Acquire) {
WORKER_RUNNING => Ok(CleanupWaitOwner::Notifier),
WORKER_FINISHING | WORKER_DONE => Ok(CleanupWaitOwner::TerminalAck),
state => Err(io::Error::other(format!(
"cleanup registration remained {last_error} and raw-owner CAS lost in state {state}"
))),
}
}
fn notifier_retired(terminal: &TerminalCleanup, timeout: Duration) -> bool {
let procfs = terminal
.event
.identity()
.is_some_and(|identity| matches!(identity.pidfd, ThreadHandle::Procfs { .. }));
if procfs && let Ok(retired) = terminal.on_ptracer_thread().wait(timeout) {
return retired;
}
terminal.wait(timeout)
}
fn cleanup_with_notifier(
pid: Pid,
pidfd: &ThreadHandle,
terminal: &TerminalCleanup,
owns_claimed_exit: bool,
signal_sent: &mut bool,
) -> io::Result<()> {
if !*signal_sent {
match pidfd_send_signal(pidfd, libc::SIGKILL) {
Ok(()) => *signal_sent = true,
Err(error) if error.raw_os_error() == Some(libc::ESRCH) => {
*signal_sent = true;
if Self::notifier_retired(terminal, TRACEE_WAIT_TIMEOUT) {
return Ok(());
}
return Err(io::Error::new(
io::ErrorKind::TimedOut,
format!("notifier for dead tracee {pid} did not finish"),
));
}
Err(error) => return Err(error),
}
}
if owns_claimed_exit {
unsafe { terminal.revoke_owned_exit_stop() }
} else {
terminal.revoke_unclaimed_exit_stop()
}
.map_err(|error| io::Error::other(format!("revoke exit capability: {error}")))?;
let deadline = Instant::now() + TRACEE_WAIT_TIMEOUT;
let mut exit_stop_resumed = false;
loop {
if Self::notifier_retired(terminal, Duration::ZERO) {
return Ok(());
}
if terminal.exit_stop_observed() && !exit_stop_resumed {
nix::sys::ptrace::cont(pid, None).map_err(|error| {
io::Error::other(format!("resume cleanup exit stop: {error}"))
})?;
exit_stop_resumed = true;
}
if Instant::now() >= deadline {
return Err(io::Error::new(
io::ErrorKind::TimedOut,
format!("notifier cleanup for tracee {pid} timed out"),
));
}
thread::sleep(SUBPROCESS_POLL_INTERVAL);
}
}
fn cleanup_with_raw_wait(
pid: Pid,
pidfd: &ThreadHandle,
terminal: &TerminalCleanup,
finish_event: bool,
signal_sent: &mut bool,
) -> io::Result<()> {
if !*signal_sent {
match pidfd_send_signal(pidfd, libc::SIGKILL) {
Ok(()) => *signal_sent = true,
Err(error) if error.raw_os_error() == Some(libc::ESRCH) => {
*signal_sent = true;
}
Err(error) => return Err(error),
}
}
reap_tracee_pidfd_bounded(pid, pidfd)?;
if finish_event {
terminal.finish_unstarted_raw_cleanup();
} else if !terminal.wait(TRACEE_WAIT_TIMEOUT) {
return Err(io::Error::new(
io::ErrorKind::TimedOut,
"synthetic notifier completion did not finish after exact raw reap",
));
}
Ok(())
}
fn cleanup_with_terminal_ack(
pidfd: &ThreadHandle,
terminal: &TerminalCleanup,
signal_sent: &mut bool,
) -> io::Result<()> {
if terminal.wait(TRACEE_WAIT_TIMEOUT) {
return Ok(());
}
if !*signal_sent {
match pidfd_send_signal(pidfd, libc::SIGKILL) {
Ok(()) => *signal_sent = true,
Err(error) if error.raw_os_error() == Some(libc::ESRCH) => {
*signal_sent = true;
}
Err(error) => return Err(error),
}
}
let deadline = Instant::now() + TRACEE_WAIT_TIMEOUT;
loop {
if terminal.wait(Duration::ZERO) {
return Ok(());
}
if pidfd_exited(pidfd)? {
return Err(io::Error::new(
io::ErrorKind::TimedOut,
"cleanup owner omitted terminal acknowledgment after tracee exit",
));
}
if Instant::now() >= deadline {
return Err(io::Error::new(
io::ErrorKind::TimedOut,
"cleanup owner left the exact pidfd target live",
));
}
thread::sleep(SUBPROCESS_POLL_INTERVAL);
}
}
fn cleanup(&mut self) -> io::Result<()> {
if !self.armed {
return Ok(());
}
if matches!(self.ownership, TraceeCleanupOwnership::PreRegistration) {
Self::cleanup_before_registration(self.pid, &self.pidfd)?;
self.armed = false;
return Ok(());
}
let needs_owner = matches!(
self.ownership,
TraceeCleanupOwnership::NotifierOwned {
raw_cleanup_claimed: false,
..
}
);
if needs_owner {
let TraceeCleanupOwnership::NotifierOwned {
terminal,
cleanup_signal_sent,
..
} = &mut self.ownership
else {
unreachable!()
};
match Self::acquire_cleanup_wait_owner(
self.pid,
&self.pidfd,
terminal,
cleanup_signal_sent,
)? {
CleanupWaitOwner::Notifier => {
let TraceeCleanupOwnership::NotifierOwned {
terminal,
owns_claimed_exit,
cleanup_signal_sent,
..
} = &mut self.ownership
else {
unreachable!()
};
Self::cleanup_with_notifier(
self.pid,
&self.pidfd,
terminal,
*owns_claimed_exit,
cleanup_signal_sent,
)?;
self.armed = false;
return Ok(());
}
CleanupWaitOwner::Raw { finish_event } => {
let TraceeCleanupOwnership::NotifierOwned {
raw_cleanup_claimed,
raw_cleanup_finishes_event,
..
} = &mut self.ownership
else {
unreachable!()
};
*raw_cleanup_claimed = true;
*raw_cleanup_finishes_event = finish_event;
}
CleanupWaitOwner::TerminalAck => {
let TraceeCleanupOwnership::NotifierOwned {
terminal,
cleanup_signal_sent,
..
} = &mut self.ownership
else {
unreachable!()
};
Self::cleanup_with_terminal_ack(
&self.pidfd,
terminal,
cleanup_signal_sent,
)?;
self.armed = false;
return Ok(());
}
CleanupWaitOwner::Authoritative(authoritative) => {
let TraceeCleanupOwnership::NotifierOwned {
terminal,
owns_claimed_exit,
..
} = &mut self.ownership
else {
unreachable!()
};
*terminal = authoritative;
*owns_claimed_exit = false;
return self.cleanup();
}
}
}
let TraceeCleanupOwnership::NotifierOwned {
terminal,
raw_cleanup_claimed: true,
raw_cleanup_finishes_event,
cleanup_signal_sent,
..
} = &mut self.ownership
else {
unreachable!("cleanup wait ownership must be resolved")
};
Self::cleanup_with_raw_wait(
self.pid,
&self.pidfd,
terminal,
*raw_cleanup_finishes_event,
cleanup_signal_sent,
)?;
self.armed = false;
Ok(())
}
}
impl Drop for TraceeCleanupGuard {
fn drop(&mut self) {
let _ = self.cleanup();
}
}
#[test]
fn nspid_alignment_needs_the_line_only_where_pid_namespaces_exist() {
let caller = super::Pid::from_raw(4242);
let unasked = || -> Result<bool, Errno> { panic!("NSpid is present") };
let aligned = b"Name:\tx\nPid:\t4242\nNSpid:\t4242\n";
let outer_mount = b"Name:\tx\nPid:\t17\nNSpid:\t17\t4242\n";
let no_nspid = b"Name:\tx\nPid:\t4242\nPPid:\t1\n";
let other_pid = b"Name:\tx\nPid:\t17\nPPid:\t1\n";
assert_eq!(aligned_proc_pid_namespace(aligned, caller, unasked), Ok(()));
assert_eq!(
aligned_proc_pid_namespace(outer_mount, caller, unasked),
Err(Errno::EXDEV)
);
assert_eq!(
aligned_proc_pid_namespace(no_nspid, caller, || Ok(true)),
Err(Errno::EOPNOTSUPP)
);
assert_eq!(
aligned_proc_pid_namespace(no_nspid, caller, || Ok(false)),
Ok(())
);
assert_eq!(
aligned_proc_pid_namespace(other_pid, caller, || Ok(false)),
Err(Errno::EXDEV)
);
assert_eq!(
aligned_proc_pid_namespace(no_nspid, caller, || Err(Errno::EACCES)),
Err(Errno::EACCES)
);
}
#[test]
fn procfs_reports_whether_the_kernel_has_pid_namespaces() {
let listed = fs::symlink_metadata("/proc/thread-self/ns/pid").is_ok();
assert_eq!(AlignedProcfs::open().unwrap().pid_namespaces(), Ok(listed));
}
#[test]
fn stale_cleanup_guard_pid_projection_cannot_signal_replacement() {
let (old_pid, mut old_cleanup) =
spawn_stopped_process(None).expect("spawn old cleanup generation");
pidfd_send_signal(&old_cleanup.pidfd, libc::SIGKILL)
.expect("signal old generation through pidfd");
reap_tracee_bounded(old_pid).expect("reap old cleanup generation");
let (replacement_pid, replacement_cleanup) =
spawn_stopped_process(None).expect("spawn replacement cleanup generation");
old_cleanup.pid = replacement_pid;
drop(old_cleanup);
assert_eq!(
unsafe { libc::kill(replacement_pid.as_raw(), 0) },
0,
"stale cleanup guard signaled a live replacement"
);
reap_stopped_process(replacement_cleanup);
}
#[test]
fn fallback_guard_unwind_does_not_orphan_stopped_child() {
let identity = Arc::new(Mutex::new(None));
let captured = Arc::clone(&identity);
let unwind = std::panic::catch_unwind(std::panic::AssertUnwindSafe(move || {
let (pid, _cleanup) =
spawn_stopped_process(None).expect("spawn guarded fallback child");
*captured.lock() = Some(
WorkerIdentity::capture_process(pid.into())
.expect("capture guarded fallback child"),
);
panic!("exercise fallback unwind cleanup");
}));
assert!(unwind.is_err());
let identity = identity
.lock()
.take()
.expect("fallback panic captured child identity");
assert!(
!identity.is_same_process_generation(),
"fallback panic orphaned its stopped child"
);
}
#[tokio::test(flavor = "current_thread")]
async fn cleanup_guard_transitions_before_first_exit_future_poll() {
let (_, stopped, mut cleanup) =
spawn_traced_process(None).expect("spawn notifier-owned cleanup tracee");
stopped
.setoptions(Options::PTRACE_O_TRACEEXIT)
.expect("enable notifier-owned cleanup exit stop");
let mut exit = Box::pin(
cleanup
.exit_event(&stopped)
.expect("bind cleanup guard before ExitFuture poll"),
);
assert!(matches!(
cleanup.ownership,
TraceeCleanupOwnership::NotifierOwned { .. }
));
let waker = futures::task::noop_waker();
let mut context = Context::from_waker(&waker);
assert_eq!(exit.as_mut().poll(&mut context), Poll::Pending);
drop(exit);
cleanup
.cleanup()
.expect("notifier-owned cleanup after canceled ExitFuture");
assert!(!cleanup.armed);
}
#[test]
fn registration_is_reused_across_status_events() {
let counter = Arc::new(WakeCounter::default());
let waker = Waker::from(Arc::clone(&counter));
let event = Event::new();
assert_eq!(event.poll_status(&waker), Poll::Pending);
assert_eq!(event.poll_status(&waker), Poll::Pending);
let stopped = (libc::SIGSTOP << 8) | 0x7f;
assert_eq!(event.update(stopped), None);
assert_eq!(counter.0.load(Ordering::SeqCst), 1);
assert_eq!(event.poll_status(&waker), Poll::Ready(Ok(stopped)));
assert!(!event.status_waker.register(&waker));
}
#[cfg(not(sanitized))]
struct ReentrantWaker {
reenter: Box<dyn Fn() + Send + Sync>,
armed: AtomicBool,
reentries: AtomicUsize,
wakes: AtomicUsize,
}
#[cfg(not(sanitized))]
static REENTRANT_WAKER_VTABLE: RawWakerVTable = RawWakerVTable::new(
reentrant_waker_clone,
reentrant_waker_wake,
reentrant_waker_wake,
reentrant_waker_drop,
);
#[cfg(not(sanitized))]
fn reentrant_waker(data: *const ()) -> &'static ReentrantWaker {
unsafe { &*data.cast::<ReentrantWaker>() }
}
#[cfg(not(sanitized))]
fn reentrant_waker_clone(data: *const ()) -> std::task::RawWaker {
reentrant_waker(data).reenter_once();
std::task::RawWaker::new(data, &REENTRANT_WAKER_VTABLE)
}
#[cfg(not(sanitized))]
fn reentrant_waker_wake(data: *const ()) {
let waker = reentrant_waker(data);
waker.reenter_once();
waker.wakes.fetch_add(1, Ordering::SeqCst);
}
#[cfg(not(sanitized))]
fn reentrant_waker_drop(data: *const ()) {
reentrant_waker(data).reenter_once();
}
#[cfg(not(sanitized))]
impl ReentrantWaker {
fn leak(reenter: impl Fn() + Send + Sync + 'static) -> &'static Self {
Box::leak(Box::new(Self {
reenter: Box::new(reenter),
armed: AtomicBool::new(false),
reentries: AtomicUsize::new(0),
wakes: AtomicUsize::new(0),
}))
}
fn waker(&'static self) -> Waker {
let data = std::ptr::from_ref(self).cast::<()>();
unsafe { Waker::from_raw(std::task::RawWaker::new(data, &REENTRANT_WAKER_VTABLE)) }
}
fn arm(&self) {
self.armed.store(true, Ordering::SeqCst);
}
fn reenter_once(&self) {
if self.armed.swap(false, Ordering::SeqCst) {
self.reentries.fetch_add(1, Ordering::SeqCst);
(self.reenter)();
}
}
fn counts(&self) -> (usize, usize) {
(
self.reentries.load(Ordering::SeqCst),
self.wakes.load(Ordering::SeqCst),
)
}
}
#[cfg(not(sanitized))]
fn returns_within_bound(what: &str, operation: impl FnOnce() + Send + 'static) {
let (returned, receiver) = mpsc::channel();
thread::spawn(move || {
operation();
let _ = returned.send(());
});
match receiver.recv_timeout(Duration::from_secs(10)) {
Ok(()) => {}
Err(mpsc::RecvTimeoutError::Timeout) => panic!(
"{what}: did not return within 10 s: a waker callback that re-entered the \
slot blocked on its lock"
),
Err(mpsc::RecvTimeoutError::Disconnected) => panic!("{what}: panicked"),
}
}
#[test]
#[cfg(not(sanitized))]
fn waker_slots_run_no_waker_callback_under_their_lock() {
let event = Arc::new(Event::new());
let waiter = Arc::new(ExitWaiter::default());
let reentrant = {
let (event, waiter) = (Arc::clone(&event), Arc::clone(&waiter));
ReentrantWaker::leak(move || event.exit_waiters.register(&waiter, Waker::noop()))
};
event.exit_waiters.register(&waiter, &reentrant.waker());
reentrant.arm();
returns_within_bound("a deferral's wake of an exit waiter", {
let event = Arc::clone(&event);
move || event.defer_legacy_wake()
});
assert_eq!(
reentrant.counts(),
(1, 1),
"the exit waiter's waker was not woken once, re-entering its slot once"
);
let event = Arc::new(Event::new());
let reentrant = {
let event = Arc::clone(&event);
ReentrantWaker::leak(move || {
let _ = event.poll_status(Waker::noop());
})
};
assert_eq!(event.poll_status(&reentrant.waker()), Poll::Pending);
reentrant.arm();
returns_within_bound("a deferral's wake of a status waiter", {
let event = Arc::clone(&event);
move || event.defer_legacy_wake()
});
assert_eq!(
reentrant.counts(),
(1, 1),
"the status waiter's waker was not woken once, re-entering its slot once"
);
let event = Arc::new(Event::new());
let counter = Arc::new(WakeCounter::default());
assert_eq!(
event.poll_status(&Waker::from(Arc::clone(&counter))),
Poll::Pending
);
let reentrant = {
let event = Arc::clone(&event);
ReentrantWaker::leak(move || {
let _ = event.poll_status(Waker::noop());
})
};
reentrant.arm();
returns_within_bound("registering a waker whose clone re-enters the slot", {
let event = Arc::clone(&event);
move || {
let _ = event.poll_status(&reentrant.waker());
}
});
assert_eq!(
reentrant.counts(),
(1, 0),
"the registration's clone did not re-enter the slot once"
);
assert!(
!event.status_waker.register(&reentrant.waker()),
"the slot does not hold the waker registered last"
);
let event = Arc::new(Event::new());
let reentrant = {
let event = Arc::clone(&event);
ReentrantWaker::leak(move || {
let _ = event.poll_status(Waker::noop());
})
};
assert_eq!(event.poll_status(&reentrant.waker()), Poll::Pending);
reentrant.arm();
returns_within_bound("replacing a waker whose drop re-enters the slot", {
let event = Arc::clone(&event);
move || {
let _ = event.poll_status(&Waker::from(counter));
}
});
assert_eq!(
reentrant.counts(),
(1, 0),
"dropping the replaced waker did not re-enter the slot once"
);
}
#[test]
fn bounded_test_subprocess_kills_and_reaps_hung_inner() {
const HANG: &str = "SAFEPTRACE_BOUNDED_INNER_HANG";
if env::var_os(HANG).is_some() {
println!("BOUNDED_INNER_HANG_READY");
io::stdout().flush().expect("flush hang marker");
loop {
thread::park();
}
}
let inner = "notifier::test::bounded_test_subprocess_kills_and_reaps_hung_inner";
let started = Instant::now();
let result =
run_exact_test_bounded(inner, &[(HANG, "1")], false, Duration::from_millis(100))
.expect("run deliberately hung exact-test subprocess");
assert!(result.timed_out, "hung subprocess unexpectedly completed");
assert!(
String::from_utf8_lossy(&result.output.stdout).contains("BOUNDED_INNER_HANG_READY"),
"hung subprocess output was not drained concurrently: {result:?}"
);
assert!(
started.elapsed() < Duration::from_secs(2),
"hung subprocess cleanup exceeded its external bound"
);
}
#[test]
fn drain_failure_after_reap_never_signals_projected_reused_pgid() {
const SENTINEL: &str = "SAFEPTRACE_PGID_SENTINEL_INNER";
const PROJECT_REAPED_PGID: &str = "SAFEPTRACE_PROJECT_REAPED_PGID";
if env::var_os(SENTINEL).is_some() {
loop {
thread::park();
}
}
if env::var_os(PROJECT_REAPED_PGID).is_some() {
println!("PROJECTED_REAPED_PGID_INNER_COMPLETE");
return;
}
let inner = "notifier::test::drain_failure_after_reap_never_signals_projected_reused_pgid";
let mut sentinel_command = Command::new(env::current_exe().expect("current test binary"));
sentinel_command
.args(["--exact", inner, "--nocapture"])
.env(SENTINEL, "1")
.stdout(Stdio::null())
.stderr(Stdio::null())
.process_group(0);
let mut sentinel = ChildProcessGroup::new(
crate::test_fork::spawn(&mut sentinel_command).expect("spawn projected-PGID sentinel"),
)
.expect("open projected-PGID sentinel pidfd");
let projected_pgid = sentinel.process_group.to_string();
let error = run_exact_test_bounded(
inner,
&[(PROJECT_REAPED_PGID, projected_pgid.as_str())],
false,
PID_NAMESPACE_TEST_TIMEOUT,
)
.expect_err("injected post-reap reader failure must surface");
assert!(
error
.to_string()
.contains("injected exact-test reader failure"),
"{error}"
);
assert!(
!pidfd_exited(&sentinel.pidfd).expect("poll projected-PGID sentinel"),
"post-reap error cleanup signaled a recycled process group"
);
sentinel
.terminate_group()
.expect("terminate projected-PGID sentinel");
sentinel.wait().expect("reap projected-PGID sentinel");
}
#[test]
fn exact_new_child_cleanup_never_signals_projected_reused_pgid() {
const SENTINEL: &str = "SAFEPTRACE_NEW_CHILD_PGID_SENTINEL";
if env::var_os(SENTINEL).is_some() {
loop {
thread::park();
}
}
let mut old_command = Command::new("/bin/true");
old_command.process_group(0);
let mut old = ChildProcessGroup::new(
crate::test_fork::spawn(&mut old_command)
.expect("spawn old exact-cleanup group leader"),
)
.expect("open old exact-cleanup leader pidfd");
let old_pid = old.process_group;
let stale_pidfd = old
.pidfd
.try_clone()
.expect("clone stale exact-cleanup pidfd");
old.wait().expect("reap old exact-cleanup leader");
let inner = "notifier::test::exact_new_child_cleanup_never_signals_projected_reused_pgid";
let mut sentinel_command = Command::new(env::current_exe().expect("current test binary"));
sentinel_command
.args(["--exact", inner, "--nocapture"])
.env(SENTINEL, "1")
.stdout(Stdio::null())
.stderr(Stdio::null())
.process_group(0);
let mut sentinel = ChildProcessGroup::new(
crate::test_fork::spawn(&mut sentinel_command)
.expect("spawn exact-cleanup projected-PGID sentinel"),
)
.expect("open exact-cleanup sentinel pidfd");
let mut stale_guard = ExactNewChildCleanupGuard {
projected_pgid: Pid::from_raw(sentinel.process_group),
event: EventHandle::new(),
members: Arc::new(Mutex::new(vec![ExactTestMember {
pid: crate::Pid::from_raw(old_pid),
pidfd: stale_pidfd,
phase: ExactCleanupPhase::NeedsSignal,
}])),
registration_error: Arc::new(Mutex::new(None)),
pidfd_open_error: Arc::new(Mutex::new(None)),
signal_errors: HashMap::new(),
reap_errors: HashMap::new(),
armed: true,
};
stale_guard
.cleanup()
.expect("stale exact cleanup ignores projected numeric PGID");
assert!(
!pidfd_exited(&sentinel.pidfd).expect("poll exact-cleanup sentinel"),
"stale exact cleanup signaled the projected replacement process group"
);
sentinel
.terminate_group()
.expect("terminate exact-cleanup sentinel");
sentinel.wait().expect("reap exact-cleanup sentinel");
}
#[test]
fn exact_reuse_nonzero_inner_is_a_hard_failure() {
const FAIL: &str = "SAFEPTRACE_EXACT_REUSE_FAIL_INNER";
if env::var_os(FAIL).is_some() {
println!("ACTUAL_PID_REUSE_UNAVAILABLE");
io::stdout().flush().expect("flush misleading marker");
std::process::exit(23);
}
let inner = "notifier::test::exact_reuse_nonzero_inner_is_a_hard_failure";
let result =
run_exact_test_bounded(inner, &[(FAIL, "1")], false, PID_NAMESPACE_TEST_TIMEOUT)
.expect("run nonzero exact-test subprocess");
assert!(!result.timed_out);
let error = classify_exact_reuse_output(
Some(&result.output),
"ACTUAL_PID_REUSE_EXERCISED",
"ACTUAL_PID_REUSE_UNAVAILABLE",
)
.expect_err("nonzero inner test must not become an unavailable fallback");
assert!(error.contains("status=exit status: 23"), "{error}");
}
#[test]
fn exact_reuse_classifier_accepts_only_known_uid_map_denial() {
let output = Output {
status: std::process::ExitStatus::from_raw(1 << 8),
stdout: Vec::new(),
stderr: b"unshare: write failed /proc/self/uid_map: Operation not permitted\n".to_vec(),
};
assert_eq!(
classify_exact_reuse_output(
Some(&output),
"ACTUAL_PID_REUSE_EXERCISED",
"ACTUAL_PID_REUSE_UNAVAILABLE",
),
Ok(ExactReuseOutcome::Unavailable)
);
let proc_mount = Output {
status: std::process::ExitStatus::from_raw(1 << 8),
stdout: Vec::new(),
stderr: b"unshare: mount /proc failed: Operation not permitted\n".to_vec(),
};
assert_eq!(
classify_exact_reuse_output(
Some(&proc_mount),
"ACTUAL_PID_REUSE_EXERCISED",
"ACTUAL_PID_REUSE_UNAVAILABLE",
),
Ok(ExactReuseOutcome::Unavailable)
);
let nearby = Output {
status: std::process::ExitStatus::from_raw(1 << 8),
stdout: Vec::new(),
stderr: b"unshare: write failed /proc/self/uid_map: Input/output error\n".to_vec(),
};
assert!(
classify_exact_reuse_output(
Some(&nearby),
"ACTUAL_PID_REUSE_EXERCISED",
"ACTUAL_PID_REUSE_UNAVAILABLE",
)
.is_err(),
"nearby non-permission uid_map failure became an unavailable fallback"
);
}
#[test]
fn registration_updates_when_the_executor_changes_wakers() {
let slot = WakerSlot::default();
let first = Waker::from(Arc::new(WakeCounter::default()));
let second_counter = Arc::new(WakeCounter::default());
let second = Waker::from(Arc::clone(&second_counter));
assert!(slot.register(&first));
assert!(!slot.register(&first));
assert!(slot.register(&second));
slot.wake();
assert_eq!(second_counter.0.load(Ordering::SeqCst), 1);
}
#[test]
fn exit_event_code() {
assert_eq!(
WaitStatus::from_raw(Pid::from_raw(42), PTRACE_EVENT_EXIT_STOP),
Ok(WaitStatus::PtraceEvent(
Pid::from_raw(42),
Signal::SIGTRAP,
libc::PTRACE_EVENT_EXIT
))
);
}
fn notifier_owned_event() -> Event {
let event = Event::new();
event
.wait_owner
.store(WAIT_OWNER_NOTIFIER, Ordering::Release);
event.worker_state.store(WORKER_RUNNING, Ordering::Release);
event
}
#[test]
fn block_until_status_refuses_an_event_the_worker_does_not_own() {
let event = Event::new();
event.update((libc::SIGTRAP << 8) | 0x7f);
assert_eq!(event.block_until_status(Duration::from_secs(5)), None);
let event = notifier_owned_event();
event.cleanup_claim_waiters.fetch_add(1, Ordering::AcqRel);
assert_eq!(event.block_until_status(Duration::from_secs(5)), None);
let event = notifier_owned_event();
event
.cleanup_cancel_requested
.store(true, Ordering::Release);
assert_eq!(event.block_until_status(Duration::from_secs(5)), None);
}
#[test]
fn block_until_status_returns_at_once_for_a_queued_stop_without_taking_it() {
let event = notifier_owned_event();
let stop = (libc::SIGTRAP << 8) | 0x7f;
event.update(stop);
assert_eq!(event.block_until_status(Duration::from_secs(5)), Some(true));
assert_eq!(event.status.lock().pending.front().copied(), Some(stop));
assert_eq!(
event.wait_owner.load(Ordering::Acquire),
WAIT_OWNER_NOTIFIER
);
}
#[test]
fn block_until_status_returns_for_an_exit_stop_and_a_terminal_status() {
let event = notifier_owned_event();
event.update(PTRACE_EVENT_EXIT_STOP);
assert_eq!(event.block_until_status(Duration::from_secs(5)), Some(true));
let event = notifier_owned_event();
event.update(42 << 8);
assert_eq!(event.block_until_status(Duration::from_secs(5)), Some(true));
}
#[test]
fn block_until_status_times_out_on_an_empty_fifo() {
let event = notifier_owned_event();
let start = Instant::now();
assert_eq!(
event.block_until_status(Duration::from_millis(20)),
Some(false)
);
assert!(start.elapsed() >= Duration::from_millis(20));
}
#[test]
fn block_until_status_wakes_on_a_later_update() {
let event = Arc::new(notifier_owned_event());
let publisher = {
let event = Arc::clone(&event);
thread::spawn(move || {
thread::sleep(Duration::from_millis(20));
event.update((libc::SIGTRAP << 8) | 0x7f);
})
};
let start = Instant::now();
assert_eq!(
event.block_until_status(Duration::from_secs(10)),
Some(true)
);
assert!(start.elapsed() < Duration::from_secs(10));
publisher.join().unwrap();
}
#[test]
fn block_until_status_wakes_when_an_exit_stop_is_published_during_the_wait() {
let event = Arc::new(notifier_owned_event());
let publisher = {
let event = Arc::clone(&event);
thread::spawn(move || {
thread::sleep(Duration::from_millis(20));
event.update(PTRACE_EVENT_EXIT_STOP);
})
};
let start = Instant::now();
assert_eq!(
event.block_until_status(Duration::from_secs(10)),
Some(true)
);
assert!(start.elapsed() < Duration::from_secs(5));
publisher.join().unwrap();
assert!(event.status.lock().pending.is_empty());
assert_eq!(event.exit_status.load(Ordering::Acquire), EXIT_STOPPED);
assert_eq!(
event.exit_capability.load(Ordering::Acquire),
EXIT_CAP_AVAILABLE
);
}
#[test]
fn block_until_status_returns_at_entry_after_a_published_exit_stop() {
let event = notifier_owned_event();
event.update(PTRACE_EVENT_EXIT_STOP);
let start = Instant::now();
assert_eq!(
event.block_until_status(Duration::from_secs(10)),
Some(true)
);
assert!(start.elapsed() < Duration::from_secs(5));
assert!(event.status.lock().pending.is_empty());
assert_eq!(
event.exit_capability.load(Ordering::Acquire),
EXIT_CAP_AVAILABLE
);
assert_eq!(
event.wait_owner.load(Ordering::Acquire),
WAIT_OWNER_NOTIFIER
);
}
#[test]
fn a_contended_exit_poll_wakes_its_waiter() {
let counter = Arc::new(WakeCounter::default());
let waker = Waker::from(Arc::clone(&counter));
let event = Event::new();
let waiter = Arc::new(ExitWaiter::default());
event.update(PTRACE_EVENT_EXIT_STOP);
let contender = event.exit_publication.lock();
assert_eq!(event.poll_exit(&waiter, &waker), Poll::Pending);
assert_eq!(counter.0.load(Ordering::SeqCst), 1);
assert_eq!(
event.exit_capability.load(Ordering::Acquire),
EXIT_CAP_AVAILABLE
);
drop(contender);
assert_eq!(event.poll_exit(&waiter, &waker), Poll::Ready(Ok(())));
}
#[test]
fn a_stale_exit_waiter_contending_with_a_current_poller_is_woken() {
let counter = Arc::new(WakeCounter::default());
let waker = Waker::from(Arc::clone(&counter));
let event = Event::new();
let stale = Arc::new(ExitWaiter::default());
event.update(PTRACE_EVENT_EXIT_STOP);
event.update(PTRACE_EVENT_EXEC_STOP);
assert_eq!(event.exit_status.load(Ordering::Acquire), EXIT_PENDING);
let current_poller = event.exit_publication.lock();
assert_eq!(event.poll_exit(&stale, &waker), Poll::Pending);
assert_eq!(counter.0.load(Ordering::SeqCst), 1);
drop(current_poller);
assert_eq!(
event.poll_exit(&stale, &waker),
Poll::Ready(Err(Errno::EALREADY))
);
}
#[test]
fn held_unclaimed_exit_waiter_expires_before_terminal_publication() {
let counter = Arc::new(WakeCounter::default());
let waker = Waker::from(Arc::clone(&counter));
let event = Event::new();
let waiter = Arc::new(ExitWaiter::default());
assert_eq!(event.poll_exit(&waiter, &waker), Poll::Pending);
event.update(PTRACE_EVENT_EXIT_STOP);
event.update(42 << 8);
assert_eq!(
event.poll_exit(&waiter, &waker),
Poll::Ready(Err(Errno::EALREADY))
);
assert_eq!(event.poll_status(&waker), Poll::Ready(Ok(42 << 8)));
}
#[test]
fn exit_stop_capability_is_single_claim_while_terminal_status_fans_out() {
let waker = Waker::from(Arc::new(WakeCounter::default()));
let event = Event::new();
let winner = Arc::new(ExitWaiter::default());
let duplicate = Arc::new(ExitWaiter::default());
event.update(PTRACE_EVENT_EXIT_STOP);
assert_eq!(event.poll_exit(&winner, &waker), Poll::Ready(Ok(())));
assert_eq!(
event.poll_exit(&duplicate, &waker),
Poll::Ready(Err(Errno::EALREADY)),
"one Event generation minted two exit-stop capabilities"
);
event.update(42 << 8);
assert_eq!(event.poll_status(&waker), Poll::Ready(Ok(42 << 8)));
assert_eq!(event.poll_status(&waker), Poll::Ready(Ok(42 << 8)));
assert_eq!(
event.poll_exit(&winner, &waker),
Poll::Ready(Err(Errno::EALREADY))
);
}
#[test]
fn exec_rearms_exit_only_after_a_real_exec_status_and_rejects_old_waiters() {
let waker = Waker::from(Arc::new(WakeCounter::default()));
let event = Event::new();
let winner = Arc::new(ExitWaiter::default());
let duplicate = Arc::new(ExitWaiter::default());
event.update(PTRACE_EVENT_EXIT_STOP);
assert_eq!(event.poll_exit(&winner, &waker), Poll::Ready(Ok(())));
let signal = (libc::SIGSTOP << 8) | 0x7f;
event.update(signal);
assert_eq!(
event.poll_exit(&duplicate, &waker),
Poll::Ready(Err(Errno::EALREADY))
);
let exec = (libc::PTRACE_EVENT_EXEC << 16) | (libc::SIGTRAP << 8) | 0x7f;
event.update(exec);
let current = Arc::new(ExitWaiter {
epoch: 1,
..ExitWaiter::default()
});
let competitor = Arc::new(ExitWaiter {
epoch: 1,
..ExitWaiter::default()
});
assert_eq!(event.poll_exit(¤t, &waker), Poll::Pending);
event.update(PTRACE_EVENT_EXIT_STOP);
for old in [&winner, &duplicate] {
assert_eq!(
event.poll_exit(old, &waker),
Poll::Ready(Err(Errno::EALREADY))
);
}
assert_eq!(event.poll_exit(¤t, &waker), Poll::Ready(Ok(())));
assert_eq!(
event.poll_exit(&competitor, &waker),
Poll::Ready(Err(Errno::EALREADY))
);
event.update(0);
assert_eq!(event.poll_status(&waker), Poll::Ready(Ok(signal)));
assert_eq!(event.poll_status(&waker), Poll::Ready(Ok(exec)));
assert_eq!(event.poll_status(&waker), Poll::Ready(Ok(0)));
}
const PRE_EXIT_STEP_STOP: i32 = (libc::SIGTRAP << 8) | 0x7f;
const POST_EXIT_EXEC_STOP: i32 = (libc::PTRACE_EVENT_EXEC << 16) | (libc::SIGTRAP << 8) | 0x7f;
fn queued(event: &Event) -> Vec<i32> {
event.status.lock().pending.iter().copied().collect()
}
#[test]
fn exit_capability_resume_retires_only_stops_queued_before_exit() {
let waker = Waker::from(Arc::new(WakeCounter::default()));
let handle = EventHandle::new();
let event = handle.event();
let winner = Arc::new(ExitWaiter::default());
event.update(PRE_EXIT_STEP_STOP);
event.update(PTRACE_EVENT_EXIT_STOP);
assert_eq!(event.poll_exit(&winner, &waker), Poll::Ready(Ok(())));
event.update(POST_EXIT_EXEC_STOP);
assert_eq!(queued(event), [PRE_EXIT_STEP_STOP, POST_EXIT_EXEC_STOP]);
let pid = crate::Pid::from_raw(4242);
let token = TraceeToken::from_event(handle.clone());
let plain = Stopped::from_token(pid, token.clone());
let exit = Stopped::from_exit_token(pid, token.clone(), 0);
assert_eq!(plain, exit, "the exit-stop mark changed Stopped identity");
assert_eq!(format!("{plain:?}"), format!("{exit:?}"));
let _ = plain.into_running();
assert_eq!(queued(event), [PRE_EXIT_STEP_STOP, POST_EXIT_EXEC_STOP]);
let _ = exit.into_running();
assert_eq!(queued(event), [POST_EXIT_EXEC_STOP]);
assert_eq!(
TerminalCleanup::new_unregistered(pid, &token).retired_stops_before_exit(),
1
);
assert_eq!(
event.poll_status(&waker),
Poll::Ready(Ok(POST_EXIT_EXEC_STOP))
);
assert_eq!(event.retire_statuses_before_exit_stop(0), 0);
assert_eq!(event.status.lock().retired_before_exit, 1);
}
const ABSENT_PID: i32 = i32::MAX;
fn claimed_exit_after_stale_step() -> (EventHandle, TraceeToken) {
let waker = Waker::from(Arc::new(WakeCounter::default()));
let handle = EventHandle::new();
let event = handle.event();
event.update(PRE_EXIT_STEP_STOP);
event.update(PTRACE_EVENT_EXIT_STOP);
let winner = Arc::new(ExitWaiter::default());
assert_eq!(event.poll_exit(&winner, &waker), Poll::Ready(Ok(())));
event.update(POST_EXIT_EXEC_STOP);
assert_eq!(queued(event), [PRE_EXIT_STEP_STOP, POST_EXIT_EXEC_STOP]);
let token = TraceeToken::from_event(handle.clone());
(handle, token)
}
#[test]
fn exit_capability_esrch_retires_only_stops_queued_before_exit() {
let pid = crate::Pid::from_raw(ABSENT_PID);
let retired = |token: &TraceeToken| {
TerminalCleanup::new_unregistered(pid, token).retired_stops_before_exit()
};
let (handle, token) = claimed_exit_after_stale_step();
let plain = Stopped::from_token(pid, token.clone());
let (plain, errno) = plain
.resume_retaining(None)
.expect_err("absent PID resumed");
assert_eq!(errno, Errno::ESRCH);
drop(plain);
assert_eq!(
queued(handle.event()),
[PRE_EXIT_STEP_STOP, POST_EXIT_EXEC_STOP]
);
let exit = Stopped::from_exit_token(pid, token.clone(), 0);
let (retained, errno) = exit.resume_retaining(None).expect_err("absent PID resumed");
assert_eq!(errno, Errno::ESRCH);
assert_eq!(queued(handle.event()), [POST_EXIT_EXEC_STOP]);
assert_eq!(retired(&token), 1);
drop(retained);
let (handle, token) = claimed_exit_after_stale_step();
let exit = Stopped::from_exit_token(pid, token.clone(), 0);
assert!(matches!(exit.resume(None), Err(Error::Died(_))));
assert_eq!(queued(handle.event()), [POST_EXIT_EXEC_STOP]);
assert_eq!(retired(&token), 1);
let (handle, token) = claimed_exit_after_stale_step();
let exit = Stopped::from_exit_token(pid, token.clone(), 0);
assert!(matches!(exit.getevent(), Err(Error::Died(_))));
assert_eq!(queued(handle.event()), [POST_EXIT_EXEC_STOP]);
assert_eq!(retired(&token), 1);
assert!(matches!(exit.getevent(), Err(Error::Died(_))));
assert_eq!(queued(handle.event()), [POST_EXIT_EXEC_STOP]);
assert_eq!(retired(&token), 1);
let (handle, token) = claimed_exit_after_stale_step();
let exit = Stopped::from_exit_token(pid, token.clone(), 0);
let (retained, errno) = exit
.finish_retained_resume(Err(Errno::EIO))
.expect_err("EIO is a refusal");
assert_eq!(errno, Errno::EIO);
assert_eq!(
queued(handle.event()),
[PRE_EXIT_STEP_STOP, POST_EXIT_EXEC_STOP]
);
assert_eq!(retired(&token), 0);
drop(retained);
}
#[test]
fn exit_capability_from_an_earlier_epoch_retires_nothing() {
let waker = Waker::from(Arc::new(WakeCounter::default()));
let pid = crate::Pid::from_raw(ABSENT_PID);
let handle = EventHandle::new();
let event = handle.event();
let token = TraceeToken::from_event(handle.clone());
event.update(PRE_EXIT_STEP_STOP);
event.update(PTRACE_EVENT_EXIT_STOP);
let first = Arc::new(ExitWaiter::default());
assert_eq!(event.poll_exit(&first, &waker), Poll::Ready(Ok(())));
assert_eq!(event.status.lock().before_exit_epoch, 0);
assert_eq!(
event.poll_status(&waker),
Poll::Ready(Ok(PRE_EXIT_STEP_STOP))
);
event.update(POST_EXIT_EXEC_STOP);
assert_eq!(
event.poll_status(&waker),
Poll::Ready(Ok(POST_EXIT_EXEC_STOP))
);
let later = (libc::SIGUSR1 << 8) | 0x7f;
event.update(later);
event.update(PTRACE_EVENT_EXIT_STOP);
assert_eq!(event.exit_epoch.load(Ordering::Acquire), 1);
assert_eq!(event.status.lock().before_exit_epoch, 1);
assert_eq!(event.status.lock().before_exit, 1);
let stale = Stopped::from_exit_token(pid, token.clone(), 0);
assert!(matches!(stale.resume(None), Err(Error::Died(_))));
let _ = Stopped::from_exit_token(pid, token.clone(), 0).into_running();
assert_eq!(queued(event), [later]);
assert_eq!(event.status.lock().retired_before_exit, 0);
let current = Stopped::from_exit_token(pid, token.clone(), 1);
assert!(matches!(current.resume(None), Err(Error::Died(_))));
assert!(queued(event).is_empty());
assert_eq!(event.status.lock().retired_before_exit, 1);
}
#[test]
fn second_exit_stop_without_exec_is_queued_and_expires_the_first() {
let waker = Waker::from(Arc::new(WakeCounter::default()));
let handle = EventHandle::new();
let event = handle.event();
event.update(PRE_EXIT_STEP_STOP);
event.update(PTRACE_EVENT_EXIT_STOP);
let first = Arc::new(ExitWaiter::default());
assert_eq!(event.poll_exit(&first, &waker), Poll::Ready(Ok(())));
assert_eq!(event.exit_status.load(Ordering::Acquire), EXIT_STOPPED);
assert_eq!(queued(event), [PRE_EXIT_STEP_STOP]);
event.update(PTRACE_EVENT_EXIT_STOP);
assert_eq!(event.exit_epoch.load(Ordering::Acquire), 1);
assert_eq!(event.exit_status.load(Ordering::Acquire), EXIT_PENDING);
assert_eq!(
event.exit_capability.load(Ordering::Acquire),
EXIT_CAP_PENDING
);
assert_eq!(queued(event), [PRE_EXIT_STEP_STOP, PTRACE_EVENT_EXIT_STOP]);
let stale = Arc::new(ExitWaiter::default());
assert_eq!(
event.poll_exit(&stale, &waker),
Poll::Ready(Err(Errno::EALREADY))
);
let current = Arc::new(ExitWaiter {
epoch: 1,
..ExitWaiter::default()
});
assert_eq!(event.poll_exit(¤t, &waker), Poll::Pending);
assert_eq!(
event.poll_status(&waker),
Poll::Ready(Ok(PRE_EXIT_STEP_STOP))
);
assert_eq!(
event.poll_status(&waker),
Poll::Ready(Ok(PTRACE_EVENT_EXIT_STOP))
);
assert!(queued(event).is_empty());
}
#[test]
fn owned_exit_revocation_retires_only_stops_queued_before_exit() {
let waker = Waker::from(Arc::new(WakeCounter::default()));
let pid = crate::Pid::from_raw(ABSENT_PID);
let handle = EventHandle::new();
let event = handle.event();
let token = TraceeToken::from_event(handle.clone());
let cleanup = TerminalCleanup::new_unregistered(pid, &token);
event.update(PRE_EXIT_STEP_STOP);
event.update(PTRACE_EVENT_EXIT_STOP);
let winner = Arc::new(ExitWaiter::default());
assert_eq!(event.poll_exit(&winner, &waker), Poll::Ready(Ok(())));
assert_eq!(cleanup.revoke_unclaimed_exit_stop(), Err(Errno::EALREADY));
assert_eq!(queued(event), [PRE_EXIT_STEP_STOP]);
unsafe { cleanup.revoke_owned_exit_stop() }.expect("owned revocation");
assert_eq!(queued(event), []);
assert_eq!(cleanup.retired_stops_before_exit(), 1);
let (handle, token) = claimed_exit_after_stale_step();
let cleanup = TerminalCleanup::new_unregistered(pid, &token);
unsafe { cleanup.revoke_owned_exit_stop() }.expect("owned revocation");
assert_eq!(queued(handle.event()), [POST_EXIT_EXEC_STOP]);
assert_eq!(cleanup.retired_stops_before_exit(), 1);
assert_eq!(cleanup.queued_raw_statuses(), [POST_EXIT_EXEC_STOP]);
let handle = EventHandle::new();
let event = handle.event();
event.update(PRE_EXIT_STEP_STOP);
event.update(PTRACE_EVENT_EXIT_STOP);
let token = TraceeToken::from_event(handle.clone());
let cleanup = TerminalCleanup::new_unregistered(pid, &token);
cleanup
.revoke_unclaimed_exit_stop()
.expect("unclaimed revocation");
assert_eq!(queued(event), [PRE_EXIT_STEP_STOP]);
assert_eq!(cleanup.retired_stops_before_exit(), 0);
}
#[test]
fn exit_retirement_follows_consumers_that_pop_the_prefix_first() {
let waker = Waker::from(Arc::new(WakeCounter::default()));
let event = Event::new();
let second = (libc::SIGUSR1 << 8) | 0x7f;
event.update(PRE_EXIT_STEP_STOP);
event.update(second);
event.update(PTRACE_EVENT_EXIT_STOP);
event.update(POST_EXIT_EXEC_STOP);
assert_eq!(event.status.lock().before_exit, 2);
assert_eq!(
event.poll_status(&waker),
Poll::Ready(Ok(PRE_EXIT_STEP_STOP))
);
assert_eq!(event.status.lock().before_exit, 1);
assert_eq!(event.retire_statuses_before_exit_stop(0), 1);
assert_eq!(queued(&event), [POST_EXIT_EXEC_STOP]);
}
fn dead_epoch_exec_at_front() -> (EventHandle, TraceeToken) {
let (handle, token) = claimed_exit_after_stale_step();
let event = handle.event();
assert_eq!(event.exit_epoch.load(Ordering::Acquire), 1);
assert_eq!(event.status.lock().epoch_exec, Some(1));
let pid = crate::Pid::from_raw(ABSENT_PID);
let _ = Stopped::from_exit_token(pid, token.clone(), 0).into_running();
assert_eq!(queued(event), [POST_EXIT_EXEC_STOP]);
assert_eq!(event.status.lock().epoch_exec, Some(0));
(handle, token)
}
fn notifier_decode_front(
handle: &EventHandle,
token: &TraceeToken,
) -> Result<StatusReturn<Wait>, Error> {
let waker = Waker::from(Arc::new(WakeCounter::default()));
let event = handle.event();
let pid = crate::Pid::from_raw(ABSENT_PID);
event
.wait_owner
.store(WAIT_OWNER_NOTIFIER, Ordering::Release);
let reservation = match event.poll_status_reservation(&waker) {
Poll::Ready(Ok(reservation)) => reservation,
_ => panic!("no status to decode"),
};
let result = event.decode_status_return(reservation, |status| {
Wait::from_raw_with_token(pid, status, token.clone())
});
assert_eq!(
event.wait_owner.load(Ordering::Acquire),
WAIT_OWNER_NOTIFIER,
"the async decode left the wait owner changed"
);
result
}
#[test]
fn dead_epoch_exec_forwards_its_published_exit_stop_to_the_fifo() {
let waker = Waker::from(Arc::new(WakeCounter::default()));
let (handle, token) = dead_epoch_exec_at_front();
let event = handle.event();
event.update(PTRACE_EVENT_EXIT_STOP);
assert_eq!(event.exit_status.load(Ordering::Acquire), EXIT_STOPPED);
assert_eq!(
event.exit_capability.load(Ordering::Acquire),
EXIT_CAP_AVAILABLE
);
assert_eq!(queued(event), [POST_EXIT_EXEC_STOP]);
assert!(matches!(
notifier_decode_front(&handle, &token),
Ok(StatusReturn::RetiredDeadExec)
));
assert_eq!(queued(event), [PTRACE_EVENT_EXIT_STOP]);
assert_eq!(event.exit_epoch.load(Ordering::Acquire), 2);
assert_eq!(event.exit_status.load(Ordering::Acquire), EXIT_PENDING);
assert_eq!(
event.exit_capability.load(Ordering::Acquire),
EXIT_CAP_PENDING
);
assert!(!event.exit_stop_follows_dead_exec.load(Ordering::Acquire));
assert_eq!(event.status.lock().epoch_exec, None);
let pid = crate::Pid::from_raw(ABSENT_PID);
let cleanup = TerminalCleanup::new_unregistered(pid, &token);
assert_eq!(cleanup.retired_dead_exec_stops(), 1);
assert_eq!(cleanup.retired_stops_before_exit(), 1);
let epoch_one = Arc::new(ExitWaiter {
epoch: 1,
..ExitWaiter::default()
});
assert_eq!(
event.poll_exit(&epoch_one, &waker),
Poll::Ready(Err(Errno::EALREADY))
);
assert_eq!(
event.poll_status(&waker),
Poll::Ready(Ok(PTRACE_EVENT_EXIT_STOP))
);
event.update(libc::SIGKILL);
assert_eq!(event.poll_status(&waker), Poll::Ready(Ok(libc::SIGKILL)));
}
#[test]
fn dead_epoch_exec_leaves_a_claimed_exit_stop_with_its_claimant() {
let waker = Waker::from(Arc::new(WakeCounter::default()));
let (handle, token) = dead_epoch_exec_at_front();
let event = handle.event();
event.update(PTRACE_EVENT_EXIT_STOP);
let claimant = Arc::new(ExitWaiter {
epoch: 1,
..ExitWaiter::default()
});
assert_eq!(event.poll_exit(&claimant, &waker), Poll::Ready(Ok(())));
assert_eq!(
event.exit_capability.load(Ordering::Acquire),
EXIT_CAP_CLAIMED
);
assert!(matches!(
notifier_decode_front(&handle, &token),
Ok(StatusReturn::RetiredDeadExec)
));
assert!(queued(event).is_empty());
assert_eq!(event.exit_epoch.load(Ordering::Acquire), 1);
assert_eq!(event.exit_status.load(Ordering::Acquire), EXIT_STOPPED);
assert_eq!(
event.exit_capability.load(Ordering::Acquire),
EXIT_CAP_CLAIMED
);
assert!(!event.exit_stop_follows_dead_exec.load(Ordering::Acquire));
assert_eq!(event.status.lock().retired_dead_exec, 1);
}
#[test]
fn dead_epoch_exec_queues_the_next_reported_exit_stop() {
let waker = Waker::from(Arc::new(WakeCounter::default()));
let (handle, token) = dead_epoch_exec_at_front();
let event = handle.event();
assert_eq!(event.exit_status.load(Ordering::Acquire), EXIT_PENDING);
assert!(matches!(
notifier_decode_front(&handle, &token),
Ok(StatusReturn::RetiredDeadExec)
));
assert!(queued(event).is_empty());
assert!(event.exit_stop_follows_dead_exec.load(Ordering::Acquire));
assert_eq!(event.exit_epoch.load(Ordering::Acquire), 1);
assert_eq!(event.status.lock().retired_dead_exec, 1);
assert_eq!(event.poll_status(&waker), Poll::Pending);
event.update(PTRACE_EVENT_EXIT_STOP);
assert_eq!(queued(event), [PTRACE_EVENT_EXIT_STOP]);
assert!(!event.exit_stop_follows_dead_exec.load(Ordering::Acquire));
assert_eq!(event.exit_epoch.load(Ordering::Acquire), 1);
assert_eq!(event.exit_status.load(Ordering::Acquire), EXIT_PENDING);
assert_eq!(
event.exit_capability.load(Ordering::Acquire),
EXIT_CAP_PENDING
);
let epoch_one = Arc::new(ExitWaiter {
epoch: 1,
..ExitWaiter::default()
});
assert_eq!(event.poll_exit(&epoch_one, &waker), Poll::Pending);
assert_eq!(
event.poll_status(&waker),
Poll::Ready(Ok(PTRACE_EVENT_EXIT_STOP))
);
event.update(libc::SIGKILL);
assert_eq!(event.poll_status(&waker), Poll::Ready(Ok(libc::SIGKILL)));
}
#[test]
fn dead_epoch_exec_forwards_an_exit_stop_published_during_its_retirement() {
let waker = Waker::from(Arc::new(WakeCounter::default()));
let (handle, token) = dead_epoch_exec_at_front();
let event = handle.event();
assert_eq!(event.exit_status.load(Ordering::Acquire), EXIT_PENDING);
let (captured, paused_rx) = mpsc::sync_channel(1);
let (resume_tx, resume) = mpsc::sync_channel(1);
*event.exit_report_pause.lock() = Some(BoundedTestPause { captured, resume });
let publisher_handle = handle.clone();
let publisher = thread::spawn(move || {
publisher_handle.event().update(PTRACE_EVENT_EXIT_STOP);
});
paused_rx
.recv_timeout(Duration::from_secs(1))
.expect("the exit stop report did not reach its publication");
let (retired_tx, retired_rx) = mpsc::sync_channel(1);
let retirer_handle = handle.clone();
let retirer_token = token.clone();
let retirer = thread::spawn(move || {
let retired = matches!(
notifier_decode_front(&retirer_handle, &retirer_token),
Ok(StatusReturn::RetiredDeadExec)
);
let _ = retired_tx.send(retired);
});
let deadline = Instant::now() + Duration::from_secs(1);
while event.status.lock().retired_dead_exec == 0 {
assert!(Instant::now() < deadline, "the dead Exec was not retired");
thread::sleep(Duration::from_millis(1));
}
let forwarded_before_publication = retired_rx.recv_timeout(Duration::from_millis(50)).ok();
resume_tx
.send(())
.expect("release the exit stop publication");
publisher.join().expect("join the exit stop publisher");
let retired = match forwarded_before_publication {
Some(retired) => retired,
None => retired_rx
.recv_timeout(Duration::from_secs(1))
.expect("the retirement did not finish after the publication"),
};
retirer.join().expect("join the retirer");
assert!(
retired,
"the dead Exec decode did not return RetiredDeadExec"
);
assert_eq!(queued(event), [PTRACE_EVENT_EXIT_STOP]);
assert!(!event.exit_stop_follows_dead_exec.load(Ordering::Acquire));
assert_eq!(event.exit_epoch.load(Ordering::Acquire), 2);
assert_eq!(event.exit_status.load(Ordering::Acquire), EXIT_PENDING);
assert_eq!(
event.exit_capability.load(Ordering::Acquire),
EXIT_CAP_PENDING
);
assert_eq!(event.status.lock().retired_dead_exec, 1);
let epoch_one = Arc::new(ExitWaiter {
epoch: 1,
..ExitWaiter::default()
});
assert_eq!(
event.poll_exit(&epoch_one, &waker),
Poll::Ready(Err(Errno::EALREADY))
);
assert_eq!(
event.poll_status(&waker),
Poll::Ready(Ok(PTRACE_EVENT_EXIT_STOP))
);
event.update(libc::SIGKILL);
assert_eq!(event.poll_status(&waker), Poll::Ready(Ok(libc::SIGKILL)));
assert!(
forwarded_before_publication.is_none(),
"the forward finished while the exit stop publication was pending"
);
}
#[test]
fn dead_exec_that_did_not_advance_the_epoch_returns_its_death() {
let waker = Waker::from(Arc::new(WakeCounter::default()));
let handle = EventHandle::new();
let event = handle.event();
let token = TraceeToken::from_event(handle.clone());
event.update(POST_EXIT_EXEC_STOP);
assert_eq!(event.exit_epoch.load(Ordering::Acquire), 0);
assert_eq!(event.status.lock().epoch_exec, None);
assert!(matches!(
notifier_decode_front(&handle, &token),
Err(Error::Died(_))
));
assert!(queued(event).is_empty());
assert!(!event.exit_stop_follows_dead_exec.load(Ordering::Acquire));
assert_eq!(event.status.lock().retired_dead_exec, 0);
event.update(PTRACE_EVENT_EXIT_STOP);
assert!(queued(event).is_empty());
assert_eq!(event.exit_status.load(Ordering::Acquire), EXIT_STOPPED);
let waiter = Arc::new(ExitWaiter::default());
assert_eq!(event.poll_exit(&waiter, &waker), Poll::Ready(Ok(())));
}
#[test]
fn sync_owner_consumes_a_dead_exec_and_rolls_back_any_other_status() {
let pid = crate::Pid::from_raw(ABSENT_PID);
let decode_front = |handle: &EventHandle, token: &TraceeToken| {
let event = handle.event();
let mut owner = match event.claim_sync_wait().expect("claim sync owner") {
SyncWaitOwnership::Claimed(owner) => owner,
SyncWaitOwnership::Notifier => panic!("sync decode found the notifier"),
};
let reservation = event
.try_status_reservation_sync()
.expect("a status is queued")
.expect("the status is valid");
let result = owner.decode_status_return(pid, reservation, |status| {
Wait::from_raw_with_token(pid, status, token.clone())
});
drop(owner);
result
};
let (handle, token) = dead_epoch_exec_at_front();
let event = handle.event();
event.update(PTRACE_EVENT_EXIT_STOP);
assert!(matches!(
decode_front(&handle, &token),
Ok(StatusReturn::RetiredDeadExec)
));
assert_eq!(event.wait_owner.load(Ordering::Acquire), WAIT_OWNER_NONE);
assert_eq!(queued(event), [PTRACE_EVENT_EXIT_STOP]);
assert_eq!(event.exit_epoch.load(Ordering::Acquire), 2);
assert_eq!(event.status.lock().retired_dead_exec, 1);
let handle = EventHandle::new();
let event = handle.event();
let token = TraceeToken::from_event(handle.clone());
event.update(POST_EXIT_EXEC_STOP);
event.update(PRE_EXIT_STEP_STOP);
assert!(matches!(decode_front(&handle, &token), Err(Error::Died(_))));
assert_eq!(event.wait_owner.load(Ordering::Acquire), WAIT_OWNER_NONE);
assert_eq!(queued(event), [PRE_EXIT_STEP_STOP]);
assert_eq!(event.status.lock().retired_dead_exec, 0);
assert_eq!(event.exit_epoch.load(Ordering::Acquire), 0);
let fork = (libc::PTRACE_EVENT_FORK << 16) | (libc::SIGTRAP << 8) | 0x7f;
let handle = EventHandle::new();
let event = handle.event();
let token = TraceeToken::from_event(handle.clone());
event.update(fork);
assert!(matches!(decode_front(&handle, &token), Err(Error::Died(_))));
assert_eq!(event.wait_owner.load(Ordering::Acquire), WAIT_OWNER_NONE);
assert_eq!(queued(event), [fork]);
}
#[test]
fn synchronous_exit_stop_retires_only_stops_queued_before_it() {
let event = Event::new();
event.update(PRE_EXIT_STEP_STOP);
event.update_sync_status(PTRACE_EVENT_EXIT_STOP);
event.update(POST_EXIT_EXEC_STOP);
assert_eq!(
queued(&event),
[PTRACE_EVENT_EXIT_STOP, POST_EXIT_EXEC_STOP]
);
assert_eq!(event.status.lock().retired_before_exit, 1);
assert_eq!(event.status.lock().before_exit, 0);
let reservation = event
.try_status_reservation_sync()
.expect("queued synchronous exit stop is reservable")
.expect("queued synchronous exit stop is not ECHILD");
assert_eq!(reservation.status, PTRACE_EVENT_EXIT_STOP);
reservation.commit();
assert_eq!(queued(&event), [POST_EXIT_EXEC_STOP]);
}
#[test]
fn exit_retirement_keeps_a_prefix_that_records_lifecycle_state() {
let waker = Waker::from(Arc::new(WakeCounter::default()));
let clone = (libc::PTRACE_EVENT_CLONE << 16) | (libc::SIGTRAP << 8) | 0x7f;
let event = Event::new();
event.update(PRE_EXIT_STEP_STOP);
event.update(clone);
event.update(PTRACE_EVENT_EXIT_STOP);
assert_eq!(event.retire_statuses_before_exit_stop(0), 0);
assert_eq!(queued(&event), [PRE_EXIT_STEP_STOP, clone]);
assert_eq!(event.status.lock().retired_before_exit, 0);
assert_eq!(
event.poll_status(&waker),
Poll::Ready(Ok(PRE_EXIT_STEP_STOP))
);
assert_eq!(event.poll_status(&waker), Poll::Ready(Ok(clone)));
assert_eq!(event.status.lock().before_exit, 0);
let event = Event::new();
event.update(clone);
event.update_sync_status(PTRACE_EVENT_EXIT_STOP);
assert_eq!(queued(&event), [clone, PTRACE_EVENT_EXIT_STOP]);
assert_eq!(event.status.lock().retired_before_exit, 0);
}
#[test]
fn exit_retirement_retires_a_prefix_exec_except_the_epoch_exec() {
let fork = (libc::PTRACE_EVENT_FORK << 16) | (libc::SIGTRAP << 8) | 0x7f;
let later = (libc::SIGUSR1 << 8) | 0x7f;
let event = Event::new();
event.update(PRE_EXIT_STEP_STOP);
event.update(PTRACE_EVENT_EXEC_STOP);
event.update(PTRACE_EVENT_EXIT_STOP);
event.update(later);
assert_eq!(event.status.lock().epoch_exec, None);
assert_eq!(event.retire_statuses_before_exit_stop(0), 2);
assert_eq!(queued(&event), [later]);
assert_eq!(event.status.lock().retired_before_exit, 2);
assert_eq!(event.status.lock().retired_dead_exec, 0);
let event = Event::new();
event.update(PTRACE_EVENT_EXEC_STOP);
event.update_sync_status(PTRACE_EVENT_EXIT_STOP);
assert_eq!(queued(&event), [PTRACE_EVENT_EXIT_STOP]);
assert_eq!(event.status.lock().retired_before_exit, 1);
let event = Event::new();
event.update(PTRACE_EVENT_EXEC_STOP);
event.update(fork);
event.update(PTRACE_EVENT_EXIT_STOP);
assert_eq!(event.retire_statuses_before_exit_stop(0), 0);
assert_eq!(queued(&event), [PTRACE_EVENT_EXEC_STOP, fork]);
assert_eq!(event.status.lock().retired_before_exit, 0);
let (handle, _token) = dead_epoch_exec_at_front();
let event = handle.event();
event.update(PTRACE_EVENT_EXIT_STOP);
assert_eq!(event.exit_status.load(Ordering::Acquire), EXIT_STOPPED);
assert_eq!(event.status.lock().before_exit_epoch, 1);
assert_eq!(event.status.lock().before_exit, 1);
assert_eq!(event.retire_statuses_before_exit_stop(1), 0);
assert_eq!(queued(event), [POST_EXIT_EXEC_STOP]);
assert_eq!(event.status.lock().epoch_exec, Some(0));
assert_eq!(event.status.lock().retired_before_exit, 1);
}
#[test]
fn exit_retirement_retires_a_prefix_vfork_done_stop() {
let vfork_done = (libc::PTRACE_EVENT_VFORK_DONE << 16) | (libc::SIGTRAP << 8) | 0x7f;
let seccomp = (libc::PTRACE_EVENT_SECCOMP << 16) | (libc::SIGTRAP << 8) | 0x7f;
let syscall = ((libc::SIGTRAP | 0x80) << 8) | 0x7f;
let group_stop = (libc::PTRACE_EVENT_STOP << 16) | (libc::SIGSTOP << 8) | 0x7f;
let fork = (libc::PTRACE_EVENT_FORK << 16) | (libc::SIGTRAP << 8) | 0x7f;
let later = (libc::SIGUSR1 << 8) | 0x7f;
let event = Event::new();
event.update(vfork_done);
event.update(PTRACE_EVENT_EXIT_STOP);
event.update(later);
assert_eq!(event.retire_statuses_before_exit_stop(0), 1);
assert_eq!(queued(&event), [later]);
assert_eq!(event.status.lock().retired_before_exit, 1);
let prefix = [seccomp, syscall, PRE_EXIT_STEP_STOP, group_stop, vfork_done];
let event = Event::new();
for status in prefix {
event.update(status);
}
event.update(PTRACE_EVENT_EXIT_STOP);
assert_eq!(event.retire_statuses_before_exit_stop(0), prefix.len());
assert!(queued(&event).is_empty());
assert_eq!(event.status.lock().retired_before_exit, prefix.len() as u64);
let event = Event::new();
event.update(vfork_done);
event.update_sync_status(PTRACE_EVENT_EXIT_STOP);
assert_eq!(queued(&event), [PTRACE_EVENT_EXIT_STOP]);
assert_eq!(event.status.lock().retired_before_exit, 1);
let event = Event::new();
event.update(vfork_done);
event.update(fork);
event.update(PTRACE_EVENT_EXIT_STOP);
assert_eq!(event.retire_statuses_before_exit_stop(0), 0);
assert_eq!(queued(&event), [vfork_done, fork]);
assert_eq!(event.status.lock().retired_before_exit, 0);
}
#[test]
fn exit_capability_refuses_a_new_child_stop_queued_before_it() {
let pid = crate::Pid::from_raw(ABSENT_PID);
let fork = (libc::PTRACE_EVENT_FORK << 16) | (libc::SIGTRAP << 8) | 0x7f;
let vfork = (libc::PTRACE_EVENT_VFORK << 16) | (libc::SIGTRAP << 8) | 0x7f;
let clone = (libc::PTRACE_EVENT_CLONE << 16) | (libc::SIGTRAP << 8) | 0x7f;
let vfork_done = (libc::PTRACE_EVENT_VFORK_DONE << 16) | (libc::SIGTRAP << 8) | 0x7f;
for child in [fork, vfork, clone] {
let handle = EventHandle::new();
let event = handle.event();
let token = TraceeToken::from_event(handle.clone());
event.update(PRE_EXIT_STEP_STOP);
event.update(child);
event.update(PTRACE_EVENT_EXIT_STOP);
assert_eq!(
Stopped::from_exit_token(pid, token.clone(), 0).superseded_new_child(),
Err(SupersededStopRefusal::NewChild(child))
);
assert_eq!(
Stopped::from_exit_token(pid, token.clone(), 1).superseded_new_child(),
Ok(())
);
assert_eq!(
Stopped::from_token(pid, token).superseded_new_child(),
Ok(())
);
assert_eq!(queued(event), [PRE_EXIT_STEP_STOP, child]);
assert_eq!(event.status.lock().before_exit, 2);
}
for (before, after) in [
(
vec![PRE_EXIT_STEP_STOP, PTRACE_EVENT_EXEC_STOP, vfork_done],
vec![],
),
(vec![PRE_EXIT_STEP_STOP], vec![fork]),
] {
let handle = EventHandle::new();
let event = handle.event();
let token = TraceeToken::from_event(handle.clone());
for &status in &before {
event.update(status);
}
event.update(PTRACE_EVENT_EXIT_STOP);
for &status in &after {
event.update(status);
}
assert_eq!(
Stopped::from_exit_token(pid, token, 0).superseded_new_child(),
Ok(()),
"refused with prefix {before:x?} and suffix {after:x?}"
);
}
}
#[test]
fn exit_retirement_races_consumers_and_later_pushes_without_losing_a_later_status() {
const ITERATIONS: usize = 4000;
const PRE: usize = 6;
const POST: i32 = 6;
let pre = (libc::SIGUSR1 << 8) | 0x7f;
let post: Vec<i32> = (0..POST)
.map(|index| ((0x40 + index) << 16) | (libc::SIGUSR2 << 8) | 0x7f)
.collect();
let mut retired_total = 0;
let mut consumed_total = 0;
for _ in 0..ITERATIONS {
let event = Event::new();
let barrier = std::sync::Barrier::new(3);
let producer_done = AtomicBool::new(false);
let resumer_done = AtomicBool::new(false);
let (consumed, retired) = std::thread::scope(|scope| {
let producer = scope.spawn(|| {
barrier.wait();
for _ in 0..PRE {
event.update(pre);
}
event.update(PTRACE_EVENT_EXIT_STOP);
for &status in &post {
event.update(status);
}
producer_done.store(true, Ordering::Release);
});
let resumer = scope.spawn(|| {
barrier.wait();
while event.exit_capability.load(Ordering::Acquire) != EXIT_CAP_AVAILABLE {
std::hint::spin_loop();
}
let retired = event.retire_statuses_before_exit_stop(0);
resumer_done.store(true, Ordering::Release);
retired
});
let consumer = scope.spawn(|| {
barrier.wait();
let mut consumed = 0;
loop {
match event.try_status_reservation_sync() {
Some(Ok(reservation)) if reservation.status == pre => {
reservation.commit();
consumed += 1;
}
Some(_) => break,
None => {
if producer_done.load(Ordering::Acquire)
&& resumer_done.load(Ordering::Acquire)
{
break;
}
std::hint::spin_loop();
}
}
}
consumed
});
producer.join().unwrap();
(consumer.join().unwrap(), resumer.join().unwrap())
});
assert_eq!(
queued(&event),
post,
"a status queued after the exit stop was lost"
);
assert_eq!(
consumed + retired,
PRE,
"a pre-exit status was retired twice or kept"
);
assert_eq!(event.status.lock().retired_before_exit, retired as u64);
assert_eq!(event.status.lock().before_exit, 0);
retired_total += retired;
consumed_total += consumed;
}
eprintln!(
"exit retirement race: iterations={ITERATIONS} consumed={consumed_total} retired={retired_total}"
);
assert!(
retired_total > 0 && consumed_total > 0,
"the race never interleaved consumption and retirement"
);
}
fn spawn_exec_tracee(deadline: Instant) -> (Pid, TraceeCleanupGuard, Stopped) {
let root = match unsafe { fork() }.unwrap() {
ForkResult::Parent { child } => child,
ForkResult::Child => {
crate::traceme_and_stop().unwrap();
let args = [c"/bin/sleep".as_ptr(), c"30".as_ptr(), std::ptr::null()];
unsafe {
libc::execv(args[0], args.as_ptr());
libc::_exit(127);
}
}
};
let root_cleanup = TraceeCleanupGuard::new(root).unwrap();
let status = waitpid_status_bounded(
root,
libc::WUNTRACED,
deadline.saturating_duration_since(Instant::now()),
)
.unwrap();
assert!(libc::WIFSTOPPED(status));
assert_eq!(libc::WSTOPSIG(status), libc::SIGSTOP);
let stopped = Stopped::new_unchecked(root.into());
stopped
.setoptions(
Options::PTRACE_O_TRACEEXEC
| Options::PTRACE_O_TRACEEXIT
| Options::PTRACE_O_EXITKILL,
)
.unwrap();
(root, root_cleanup, stopped)
}
#[test]
fn cleanup_reservation_consumes_only_a_dead_exec() {
let pid = crate::Pid::from_raw(ABSENT_PID);
let cleanup_of = |handle: &EventHandle| TerminalCleanup {
pid,
event: handle.clone(),
};
let handle = EventHandle::new();
let event = handle.event();
event.update(PRE_EXIT_STEP_STOP);
let cleanup = cleanup_of(&handle);
let pending = cleanup
.reserve_pending_for_cleanup(Duration::ZERO)
.expect("a status is queued");
assert!(pending.decode().is_ok());
let Err(pending) = pending.consume_dead_exec() else {
panic!("a decodable non-Exec status was consumed");
};
drop(pending);
assert_eq!(queued(event), [PRE_EXIT_STEP_STOP]);
let handle = EventHandle::new();
let event = handle.event();
event.update(POST_EXIT_EXEC_STOP);
event.update(PRE_EXIT_STEP_STOP);
let cleanup = cleanup_of(&handle);
let pending = cleanup
.reserve_pending_for_cleanup(Duration::ZERO)
.expect("a status is queued");
assert!(matches!(pending.decode(), Err(Error::Died(_))));
assert!(pending.consume_dead_exec().is_ok());
assert_eq!(queued(event), [PRE_EXIT_STEP_STOP]);
assert_eq!(event.status.lock().retired_dead_exec, 0);
assert_eq!(event.exit_epoch.load(Ordering::Acquire), 0);
let (handle, _token) = dead_epoch_exec_at_front();
let event = handle.event();
event.update(PTRACE_EVENT_EXIT_STOP);
let cleanup = cleanup_of(&handle);
let pending = cleanup
.reserve_pending_for_cleanup(Duration::ZERO)
.expect("the epoch Exec is queued");
assert!(pending.consume_dead_exec().is_ok());
assert_eq!(queued(event), [PTRACE_EVENT_EXIT_STOP]);
assert_eq!(event.status.lock().retired_dead_exec, 1);
assert_eq!(event.status.lock().epoch_exec, None);
assert_eq!(event.exit_epoch.load(Ordering::Acquire), 2);
}
#[tokio::test(flavor = "current_thread")]
async fn cleanup_reservation_consumes_a_killed_exec_and_reaches_its_exit() {
let deadline = Instant::now() + Duration::from_secs(3);
let (root, mut root_cleanup, stopped) = spawn_exec_tracee(deadline);
let mut exit = Box::pin(root_cleanup.exit_event(&stopped).unwrap());
let terminal = stopped.terminal_cleanup();
let running = stopped.resume(None).unwrap();
while terminal.pending_is_empty() {
assert!(Instant::now() < deadline, "no exec stop queued");
thread::sleep(Duration::from_millis(1));
}
assert_eq!(queued(terminal.event.event()), [PTRACE_EVENT_EXEC_STOP]);
let live = terminal
.reserve_pending_for_cleanup(Duration::ZERO)
.expect("the Exec is queued");
let decoded = live.decode();
assert!(
matches!(
decoded,
Ok(crate::Wait::Stopped(_, crate::Event::Exec(former))) if former == root.into()
),
"a live exec decoded as {decoded:?}"
);
let Err(live) = live.consume_dead_exec() else {
panic!("a live Exec was consumed");
};
drop(live);
assert_eq!(queued(terminal.event.event()), [PTRACE_EVENT_EXEC_STOP]);
assert_eq!(unsafe { libc::kill(root.as_raw(), libc::SIGKILL) }, 0);
while !terminal.exit_stop_observed() {
assert!(Instant::now() < deadline, "no exit stop observed");
thread::sleep(Duration::from_millis(1));
}
let pending = terminal
.reserve_pending_for_cleanup(Duration::ZERO)
.expect("the Exec stays queued");
let decoded = pending.decode();
assert!(
matches!(decoded, Err(Error::Died(_))),
"a killed exec decoded as {decoded:?}"
);
assert!(
pending.consume_dead_exec().is_ok(),
"the dead Exec was not consumed"
);
assert!(terminal.pending_is_empty());
drop(running);
let exit_stop = tokio::time::timeout(
deadline.saturating_duration_since(Instant::now()),
exit.as_mut(),
)
.await
.unwrap()
.unwrap();
root_cleanup.mark_claimed_exit();
let final_wait = tokio::time::timeout(
deadline.saturating_duration_since(Instant::now()),
exit_stop.resume(None).unwrap().wait_owned(),
)
.await
.unwrap()
.unwrap();
assert_eq!(
final_wait.assume_exited(),
(
root.into(),
crate::ExitStatus::Signaled(Signal::SIGKILL, false)
)
);
assert!(terminal.wait(deadline.saturating_duration_since(Instant::now())));
root_cleanup.disarm();
}
#[test]
fn synchronous_wait_consumes_a_killed_exec_and_reaches_its_exit() {
let deadline = Instant::now() + Duration::from_secs(3);
let (root, mut root_cleanup, stopped) = spawn_exec_tracee(deadline);
let running = stopped.resume(None).unwrap();
let retry = Running::from_token(running.0, running.1.clone());
let retained = running.1.event().clone();
let published = Arc::new(Barrier::new(2));
let resume = Arc::new(Barrier::new(2));
SYNC_STATUS_PUBLICATION_PAUSES.lock().insert(
root.into(),
EventCapturePause {
captured: Arc::clone(&published),
resume: Arc::clone(&resume),
},
);
let killer = thread::spawn(move || {
published.wait();
let killed = unsafe { libc::kill(root.as_raw(), libc::SIGKILL) };
resume.wait();
killed
});
let first = running.wait();
assert_eq!(killer.join().unwrap(), 0);
assert!(
matches!(first, Err(Error::Died(_))),
"a killed exec's synchronous wait returned {first:?}"
);
assert!(
retained.event().status.lock().pending.is_empty(),
"the dead Exec stayed queued"
);
let second = retry.wait().unwrap();
let (exit_stop, crate::Event::Exit) = second.assume_stopped() else {
panic!("the next synchronous wait did not reach the exit stop")
};
let final_wait = exit_stop.resume(None).unwrap().wait().unwrap();
assert_eq!(
final_wait.assume_exited(),
(
root.into(),
crate::ExitStatus::Signaled(Signal::SIGKILL, false)
)
);
assert!(Instant::now() < deadline);
root_cleanup.disarm();
}
#[tokio::test(flavor = "current_thread")]
async fn exit_resume_retires_a_killed_exec_and_the_owned_wait_reaches_its_exit() {
let deadline = Instant::now() + Duration::from_secs(3);
let (root, mut root_cleanup, stopped) = spawn_exec_tracee(deadline);
let mut exit = Box::pin(root_cleanup.exit_event(&stopped).unwrap());
let terminal = stopped.terminal_cleanup();
let running = stopped.resume(None).unwrap();
while terminal.pending_is_empty() {
assert!(Instant::now() < deadline, "no exec stop queued");
thread::sleep(Duration::from_millis(1));
}
assert_eq!(queued(terminal.event.event()), [PTRACE_EVENT_EXEC_STOP]);
assert_eq!(unsafe { libc::kill(root.as_raw(), libc::SIGKILL) }, 0);
while !terminal.exit_stop_observed() {
assert!(Instant::now() < deadline, "no exit stop observed");
thread::sleep(Duration::from_millis(1));
}
drop(running);
let exit_stop = tokio::time::timeout(
deadline.saturating_duration_since(Instant::now()),
exit.as_mut(),
)
.await
.unwrap()
.unwrap();
root_cleanup.mark_claimed_exit();
assert_eq!(exit_stop.superseded_new_child(), Ok(()));
assert_eq!(queued(terminal.event.event()), [PTRACE_EVENT_EXEC_STOP]);
let Ok(running) = exit_stop.resume_retaining(None) else {
panic!("resuming the exit stop failed");
};
assert!(terminal.pending_is_empty(), "the dead Exec stayed queued");
assert_eq!(terminal.retired_stops_before_exit(), 1);
assert_eq!(terminal.retired_dead_exec_stops(), 0);
let final_wait = tokio::time::timeout(
deadline.saturating_duration_since(Instant::now()),
running.wait_owned(),
)
.await
.unwrap();
let Ok(final_wait) = final_wait else {
panic!("the owned wait after the exit resume returned {final_wait:?}");
};
assert_eq!(
final_wait.assume_exited(),
(
root.into(),
crate::ExitStatus::Signaled(Signal::SIGKILL, false)
)
);
assert!(terminal.wait(deadline.saturating_duration_since(Instant::now())));
root_cleanup.disarm();
}
#[tokio::test(flavor = "current_thread")]
async fn exit_capability_refuses_a_killed_fork_stop_queued_before_it() {
let deadline = Instant::now() + Duration::from_secs(3);
let fork_stop = (libc::PTRACE_EVENT_FORK << 16) | (libc::SIGTRAP << 8) | 0x7f;
let root = match unsafe { fork() }.unwrap() {
ForkResult::Parent { child } => child,
ForkResult::Child => {
crate::traceme_and_stop().unwrap();
unsafe {
if libc::fork() == 0 {
libc::pause();
libc::_exit(0);
}
libc::pause();
libc::_exit(0);
}
}
};
let mut root_cleanup = TraceeCleanupGuard::new(root).unwrap();
let status = waitpid_status_bounded(
root,
libc::WUNTRACED,
deadline.saturating_duration_since(Instant::now()),
)
.unwrap();
assert!(libc::WIFSTOPPED(status));
assert_eq!(libc::WSTOPSIG(status), libc::SIGSTOP);
let stopped = Stopped::new_unchecked(root.into());
stopped
.setoptions(
Options::PTRACE_O_TRACEFORK
| Options::PTRACE_O_TRACEEXIT
| Options::PTRACE_O_EXITKILL,
)
.unwrap();
let mut exit = Box::pin(root_cleanup.exit_event(&stopped).unwrap());
let terminal = stopped.terminal_cleanup();
let running = stopped.resume(None).unwrap();
while terminal.pending_is_empty() {
assert!(Instant::now() < deadline, "no fork stop queued");
thread::sleep(Duration::from_millis(1));
}
assert_eq!(queued(terminal.event.event()), [fork_stop]);
let child = Pid::from_raw(nix::sys::ptrace::getevent(root).unwrap() as i32);
let _child_cleanup = TraceeCleanupGuard::new(child).unwrap();
assert_eq!(unsafe { libc::kill(root.as_raw(), libc::SIGKILL) }, 0);
while !terminal.exit_stop_observed() {
assert!(Instant::now() < deadline, "no exit stop observed");
thread::sleep(Duration::from_millis(1));
}
drop(running);
let exit_stop = tokio::time::timeout(
deadline.saturating_duration_since(Instant::now()),
exit.as_mut(),
)
.await
.unwrap()
.unwrap();
root_cleanup.mark_claimed_exit();
let refusal = exit_stop.superseded_new_child();
assert_eq!(refusal, Err(SupersededStopRefusal::NewChild(fork_stop)));
assert_eq!(
refusal.unwrap_err().to_string(),
"queued wait status 0x1057f is a fork, vfork or clone stop superseded by the exit stop; its child PID is no longer readable"
);
assert_eq!(queued(terminal.event.event()), [fork_stop]);
let Ok(running) = exit_stop.resume_retaining(None) else {
panic!("resuming the exit stop failed");
};
assert_eq!(queued(terminal.event.event()), [fork_stop]);
assert_eq!(terminal.retired_stops_before_exit(), 0);
let mut wait = running.wait_owned();
let first = tokio::time::timeout(
deadline.saturating_duration_since(Instant::now()),
&mut wait,
)
.await
.unwrap();
assert!(
matches!(first, Err(OwnedWaitError::Died)),
"the killed fork stop decoded as {first:?}"
);
let final_wait = tokio::time::timeout(
deadline.saturating_duration_since(Instant::now()),
&mut wait,
)
.await
.unwrap()
.unwrap();
assert_eq!(
final_wait.assume_exited(),
(
root.into(),
crate::ExitStatus::Signaled(Signal::SIGKILL, false)
)
);
assert!(terminal.wait(deadline.saturating_duration_since(Instant::now())));
root_cleanup.disarm();
}
#[tokio::test(flavor = "current_thread")]
async fn actual_nonleader_exec_rearms_one_exit_and_refuses_old_and_competing_owners() {
actual_nonleader_exec_owned_case(false).await;
}
fn spawn_exec_true_thread_in_fork_child() {
extern "C" fn exec_true(_: *mut libc::c_void) -> *mut libc::c_void {
let args = [c"/bin/true".as_ptr(), std::ptr::null()];
unsafe {
libc::execv(args[0], args.as_ptr());
libc::_exit(127)
}
}
let mut thread = std::mem::MaybeUninit::<libc::pthread_t>::uninit();
let created = unsafe {
libc::pthread_create(
thread.as_mut_ptr(),
std::ptr::null(),
exec_true,
std::ptr::null_mut(),
)
};
if created != 0 {
unsafe { libc::_exit(126) };
}
}
async fn actual_nonleader_exec_owned_case(force_legacy: bool) {
let deadline = Instant::now() + Duration::from_secs(3);
let root = match unsafe { fork() }.unwrap() {
ForkResult::Parent { child } => child,
ForkResult::Child => {
crate::traceme_and_stop().unwrap();
spawn_exec_true_thread_in_fork_child();
loop {
unsafe {
libc::pause();
}
}
}
};
let _legacy = force_legacy.then(|| LegacyThreadGroup::new(root));
let mut root_cleanup = TraceeCleanupGuard::new(root).unwrap();
let status = waitpid_status_bounded(
root,
libc::WUNTRACED,
deadline.saturating_duration_since(Instant::now()),
)
.unwrap();
assert!(libc::WIFSTOPPED(status));
assert_eq!(libc::WSTOPSIG(status), libc::SIGSTOP);
let stopped = if force_legacy {
Stopped::new_unchecked_on_ptracer_thread(root.into()).unwrap()
} else {
Stopped::new_unchecked(root.into())
};
stopped
.setoptions(
Options::PTRACE_O_TRACECLONE
| Options::PTRACE_O_TRACEEXEC
| Options::PTRACE_O_TRACEEXIT
| Options::PTRACE_O_EXITKILL,
)
.unwrap();
root_cleanup.bind_notifier(&stopped).unwrap();
let terminal = stopped.terminal_cleanup();
let mut old_exit = Box::pin(if force_legacy {
futures::future::Either::Left(stopped.exit_event_on_ptracer_thread())
} else {
futures::future::Either::Right(stopped.exit_event())
});
let mut old_duplicate = Box::pin(if force_legacy {
futures::future::Either::Left(stopped.exit_event_on_ptracer_thread())
} else {
futures::future::Either::Right(stopped.exit_event())
});
let wait = tokio::time::timeout(
deadline.saturating_duration_since(Instant::now()),
if force_legacy {
futures::future::Either::Left(
stopped.resume(None).unwrap().wait_owned_on_ptracer_thread(),
)
} else {
futures::future::Either::Right(stopped.resume(None).unwrap().wait_owned())
},
)
.await
.unwrap()
.unwrap();
let (parent, crate::Event::NewChild(crate::ChildOp::Clone, child)) = wait.assume_stopped()
else {
panic!("missing actual clone")
};
let former = child.pid();
let child_terminal = child.terminal_cleanup();
let child_identity = child_terminal.event.identity().unwrap();
if force_legacy {
assert!(matches!(child_identity.pidfd, ThreadHandle::Procfs { .. }));
}
let pre_exec_wait_id = force_legacy.then(|| child_identity.pidfd.wait_id().unwrap());
let pidfd = child_identity.pidfd.try_clone().unwrap();
let mut child_cleanup = TraceeCleanupGuard {
pid: former.into(),
pidfd,
ownership: TraceeCleanupOwnership::PreRegistration,
armed: true,
};
child_cleanup.bind_running_notifier(&child).unwrap();
let initial = tokio::time::timeout(
deadline.saturating_duration_since(Instant::now()),
if force_legacy {
futures::future::Either::Left(child.wait_owned_on_ptracer_thread())
} else {
futures::future::Either::Right(child.wait_owned())
},
)
.await
.unwrap()
.unwrap();
let (child, event) = initial.assume_stopped();
assert_eq!(event, crate::Event::Signal(Signal::SIGSTOP));
let old_running = parent.resume(None).unwrap();
let mut former_wait = Box::pin(if force_legacy {
futures::future::Either::Left(
child.resume(None).unwrap().wait_owned_on_ptracer_thread(),
)
} else {
futures::future::Either::Right(child.resume(None).unwrap().wait_owned())
});
let first = tokio::time::timeout(
deadline.saturating_duration_since(Instant::now()),
old_exit.as_mut(),
)
.await
.unwrap()
.unwrap();
root_cleanup.mark_claimed_exit();
drop(old_running); let old_status = first.getevent().unwrap();
let next = tokio::time::timeout(
deadline.saturating_duration_since(Instant::now()),
if force_legacy {
futures::future::Either::Left(
first
.resume_retaining(None)
.unwrap()
.wait_owned_on_ptracer_thread(),
)
} else {
futures::future::Either::Right(first.resume_retaining(None).unwrap().wait_owned())
},
)
.await
.unwrap()
.unwrap();
let (replacement, crate::Event::Exec(actual_former)) = next.assume_stopped() else {
panic!("missing actual replacement exec")
};
assert_eq!(actual_former, former);
assert_eq!(replacement.pid(), root.into());
assert!(replacement.terminal_cleanup().same_generation(&terminal));
if let Some(checked_id) = pre_exec_wait_id {
assert!(matches!(checked_id, waitid::IdType::Pid(pid) if pid == former.into()));
assert!(!pidfd_is_live(&child_identity.pidfd).unwrap());
let (projected_pid, mut projected_cleanup) = spawn_stopped_process(None).unwrap();
let observation = WaitPidFlag::from_bits_retain(
WaitPidFlag::WEXITED.bits()
| WaitPidFlag::WSTOPPED.bits()
| WaitPidFlag::WNOWAIT.bits()
| libc::__WALL,
);
assert_eq!(
child_identity
.pidfd
.legacy_wait_once(waitid::IdType::Pid(projected_pid), observation,),
Err(Errno::ECHILD),
);
assert!(!pidfd_exited(&projected_cleanup.pidfd).unwrap());
projected_cleanup.cleanup().unwrap();
}
let mut next_exit = Box::pin(if force_legacy {
futures::future::Either::Left(replacement.exit_event_on_ptracer_thread())
} else {
futures::future::Either::Right(replacement.exit_event())
});
let mut next_duplicate = Box::pin(if force_legacy {
futures::future::Either::Left(replacement.exit_event_on_ptracer_thread())
} else {
futures::future::Either::Right(replacement.exit_event())
});
for old in [&mut old_exit, &mut old_duplicate] {
assert!(matches!(
old.as_mut().await,
Err(Error::Errno(Errno::EALREADY))
));
}
let replacement_running = replacement.resume(None).unwrap();
let second = tokio::time::timeout(
deadline.saturating_duration_since(Instant::now()),
next_exit.as_mut(),
)
.await
.unwrap()
.unwrap();
drop(replacement_running);
assert!(matches!(
next_duplicate.as_mut().await,
Err(Error::Errno(Errno::EALREADY))
));
assert!(matches!(
old_exit.as_mut().await,
Err(Error::Errno(Errno::EALREADY))
));
let second_status = second.getevent().unwrap();
let final_wait = tokio::time::timeout(
deadline.saturating_duration_since(Instant::now()),
if force_legacy {
futures::future::Either::Left(
second
.resume_retaining(None)
.unwrap()
.wait_owned_on_ptracer_thread(),
)
} else {
futures::future::Either::Right(second.resume_retaining(None).unwrap().wait_owned())
},
)
.await
.unwrap()
.unwrap();
assert_eq!(
final_wait.assume_exited(),
(root.into(), crate::ExitStatus::Exited(0))
);
assert!(matches!(
tokio::time::timeout(
deadline.saturating_duration_since(Instant::now()),
former_wait.as_mut()
)
.await
.unwrap(),
Err(OwnedWaitError::Errno(Errno::ECHILD))
));
assert!(terminal.wait(deadline.saturating_duration_since(Instant::now())));
assert!(child_terminal.wait(deadline.saturating_duration_since(Instant::now())));
eprintln!(
"actual owned exec: leader={root}, former={former}, first_exit_event={old_status}, actual_exec_former={actual_former}, second_exit_event={second_status}, final=Exited(0), former=ECHILD (no fabricated status)"
);
root_cleanup.disarm();
child_cleanup.disarm(); }
#[test]
fn cleanup_requires_exclusive_transfer_of_claimed_exit_stop() {
let waker = Waker::from(Arc::new(WakeCounter::default()));
let event = Event::new();
let waiter = Arc::new(ExitWaiter::default());
event.update(PTRACE_EVENT_EXIT_STOP);
assert_eq!(event.poll_exit(&waiter, &waker), Poll::Ready(Ok(())));
assert_eq!(
event.prepare_exit_capability_for_cleanup(false),
Err(Errno::EALREADY)
);
event
.prepare_exit_capability_for_cleanup(true)
.expect("exclusive cleanup transfer rejected claimed exit stop");
assert_eq!(
event.poll_exit(&waiter, &waker),
Poll::Ready(Err(Errno::EALREADY))
);
}
#[test]
fn simultaneous_exit_waiters_are_all_woken_for_one_claim() {
for claim_a_first in [true, false] {
let counter_a = Arc::new(WakeCounter::default());
let counter_b = Arc::new(WakeCounter::default());
let waker_a = Waker::from(Arc::clone(&counter_a));
let waker_b = Waker::from(Arc::clone(&counter_b));
let event = Event::new();
let waiter_a = Arc::new(ExitWaiter::default());
let waiter_b = Arc::new(ExitWaiter::default());
assert_eq!(event.poll_exit(&waiter_a, &waker_a), Poll::Pending);
assert_eq!(event.poll_exit(&waiter_b, &waker_b), Poll::Pending);
event.update(PTRACE_EVENT_EXIT_STOP);
assert_eq!(counter_a.0.load(Ordering::SeqCst), 1);
assert_eq!(counter_b.0.load(Ordering::SeqCst), 1);
let (winner, winner_waker, duplicate, duplicate_waker) = if claim_a_first {
(&waiter_a, &waker_a, &waiter_b, &waker_b)
} else {
(&waiter_b, &waker_b, &waiter_a, &waker_a)
};
assert_eq!(event.poll_exit(winner, winner_waker), Poll::Ready(Ok(())));
assert_eq!(
event.poll_exit(duplicate, duplicate_waker),
Poll::Ready(Err(Errno::EALREADY))
);
}
}
#[test]
fn exit_waiter_retries_capability_before_status_publication() {
let counter = Arc::new(WakeCounter::default());
let waker = Waker::from(Arc::clone(&counter));
let event = Event::new();
let waiter = Arc::new(ExitWaiter::default());
event
.exit_capability
.store(EXIT_CAP_AVAILABLE, Ordering::Release);
assert_eq!(event.poll_exit(&waiter, &waker), Poll::Pending);
event.exit_status.store(EXIT_STOPPED, Ordering::Release);
event.exit_waiters.wake_all();
assert_eq!(counter.0.load(Ordering::SeqCst), 1);
assert_eq!(event.poll_exit(&waiter, &waker), Poll::Ready(Ok(())));
}
#[test]
fn exit_waiter_retries_forced_status_before_capability_publication() {
let counter = Arc::new(WakeCounter::default());
let waker = Waker::from(Arc::clone(&counter));
let event = Arc::new(Event::new());
let waiter = Arc::new(ExitWaiter::default());
let (paused_tx, paused_rx) = mpsc::channel();
let (resume_tx, resume_rx) = mpsc::channel();
let publisher_event = Arc::clone(&event);
let publisher = thread::spawn(move || {
publisher_event.publish_exit_stop(|| {
paused_tx.send(()).expect("report exit publication gap");
resume_rx.recv().expect("resume exit publication");
});
});
paused_rx
.recv_timeout(Duration::from_secs(1))
.expect("exit publication did not reach forced gap");
assert_eq!(event.exit_status.load(Ordering::Acquire), EXIT_STOPPED);
assert_eq!(
event.exit_capability.load(Ordering::Acquire),
EXIT_CAP_PENDING
);
assert_eq!(event.poll_exit(&waiter, &waker), Poll::Pending);
assert_eq!(counter.0.load(Ordering::SeqCst), 1);
resume_tx.send(()).expect("release exit publication");
publisher.join().expect("join exit publisher");
assert_eq!(counter.0.load(Ordering::SeqCst), 2);
assert_eq!(event.poll_exit(&waiter, &waker), Poll::Ready(Ok(())));
}
#[test]
fn terminal_finalization_waits_for_exit_capability_publication() {
let event = Arc::new(Event::new());
let waiter = Arc::new(ExitWaiter::default());
let waker = Waker::from(Arc::new(WakeCounter::default()));
let (paused_tx, paused_rx) = mpsc::channel();
let (resume_tx, resume_rx) = mpsc::channel();
let publisher_event = Arc::clone(&event);
let publisher = thread::spawn(move || {
publisher_event.publish_exit_stop(|| {
paused_tx.send(()).expect("report exit publication gap");
resume_rx.recv().expect("resume exit publication");
});
});
paused_rx
.recv_timeout(Duration::from_secs(1))
.expect("exit publication did not reach forced gap");
let (started_tx, started_rx) = mpsc::channel();
let (done_tx, done_rx) = mpsc::channel();
let finalizer_event = Arc::clone(&event);
let finalizer = thread::spawn(move || {
started_tx.send(()).expect("report terminal attempt");
finalizer_event.update(42 << 8);
done_tx.send(()).expect("report terminal completion");
});
started_rx
.recv_timeout(Duration::from_secs(1))
.expect("terminal finalizer did not start");
assert_eq!(
done_rx.recv_timeout(Duration::from_millis(20)),
Err(mpsc::RecvTimeoutError::Timeout),
"terminal publication interleaved the exit-stop atomic gap"
);
resume_tx.send(()).expect("release exit publication");
publisher.join().expect("join exit publisher");
done_rx
.recv_timeout(Duration::from_secs(1))
.expect("terminal finalizer stayed blocked");
finalizer.join().expect("join terminal finalizer");
assert_eq!(event.exit_status.load(Ordering::Acquire), EXIT_STOPPED);
assert_eq!(
event.exit_capability.load(Ordering::Acquire),
EXIT_CAP_EXPIRED
);
assert_eq!(
event.poll_exit(&waiter, &waker),
Poll::Ready(Err(Errno::EALREADY))
);
assert_eq!(event.poll_status(&waker), Poll::Ready(Ok(42 << 8)));
}
#[test]
fn terminal_status_is_observable_before_exit_echild() {
let counter = Arc::new(WakeCounter::default());
let waker = Waker::from(Arc::clone(&counter));
let event = Arc::new(Event::new());
let waiter = Arc::new(ExitWaiter::default());
assert_eq!(event.poll_exit(&waiter, &waker), Poll::Pending);
let status = libc::SIGKILL;
assert!(libc::WIFSIGNALED(status));
let (captured, paused_rx) = mpsc::sync_channel(1);
let (resume_tx, resume) = mpsc::sync_channel(1);
*event.terminal_echild_pause.lock() = Some(BoundedTestPause { captured, resume });
let publisher_event = Arc::clone(&event);
let publisher = thread::spawn(move || publisher_event.update(status));
paused_rx
.recv_timeout(Duration::from_secs(1))
.expect("terminal publication did not reach the gap before ECHILD");
assert_eq!(event.exit_status.load(Ordering::Acquire), EXIT_PENDING);
assert_eq!(
event.exit_capability.load(Ordering::Acquire),
EXIT_CAP_FINALIZING
);
assert_eq!(event.status.lock().terminal, status);
assert_eq!(event.poll_exit(&waiter, &waker), Poll::Pending);
resume_tx.send(()).expect("release terminal publication");
assert_eq!(publisher.join().expect("join terminal publisher"), None);
assert_eq!(event.exit_status.load(Ordering::Acquire), EXIT_ECHILD);
assert_eq!(
event.exit_capability.load(Ordering::Acquire),
EXIT_CAP_EXPIRED
);
assert_eq!(
event.poll_exit(&waiter, &waker),
Poll::Ready(Err(Errno::ECHILD))
);
assert_eq!(event.status.lock().terminal, status);
}
#[test]
fn terminal_publication_keeps_actual_status_over_echild_marker() {
let event = Event::new();
event.mark_echild();
assert_eq!(event.status.lock().terminal, ECHILD_STATUS);
assert_eq!(event.update(42 << 8), Some(ECHILD_STATUS));
assert_eq!(event.status.lock().terminal, 42 << 8);
event.mark_echild();
assert_eq!(event.status.lock().terminal, 42 << 8);
assert_eq!(event.exit_status.load(Ordering::Acquire), EXIT_ECHILD);
}
#[test]
fn cancelled_last_exit_waiter_does_not_orphan_first_waiter() {
let counter_a = Arc::new(WakeCounter::default());
let counter_b = Arc::new(WakeCounter::default());
let waker_a = Waker::from(Arc::clone(&counter_a));
let waker_b = Waker::from(Arc::clone(&counter_b));
let event = Event::new();
let waiter_a = Arc::new(ExitWaiter::default());
let waiter_b = Arc::new(ExitWaiter::default());
assert_eq!(event.poll_exit(&waiter_a, &waker_a), Poll::Pending);
assert_eq!(event.poll_exit(&waiter_b, &waker_b), Poll::Pending);
drop(waiter_b);
event.update(PTRACE_EVENT_EXIT_STOP);
assert_eq!(counter_a.0.load(Ordering::SeqCst), 1);
assert_eq!(counter_b.0.load(Ordering::SeqCst), 0);
assert_eq!(event.poll_exit(&waiter_a, &waker_a), Poll::Ready(Ok(())));
event.update(42 << 8);
assert_eq!(event.poll_status(&waker_a), Poll::Ready(Ok(42 << 8)));
assert_eq!(event.poll_status(&waker_a), Poll::Ready(Ok(42 << 8)));
}
#[test]
fn terminal_status_remains_available_to_late_waiters() {
let waker = Waker::from(Arc::new(WakeCounter::default()));
for terminal in [42 << 8, Signal::SIGILL as i32] {
let event = Event::new();
let waiter = Arc::new(ExitWaiter::default());
event.update(PTRACE_EVENT_EXIT_STOP);
event.update(terminal);
assert_eq!(event.poll_status(&waker), Poll::Ready(Ok(terminal)));
assert_eq!(event.poll_status(&waker), Poll::Ready(Ok(terminal)));
assert_eq!(
event.poll_exit(&waiter, &waker),
Poll::Ready(Err(Errno::EALREADY))
);
}
}
#[test]
fn queued_stop_precedes_monotonic_terminal_status() {
let waker = Waker::from(Arc::new(WakeCounter::default()));
let event = Event::new();
let stopped = (Signal::SIGSTOP as i32) << 8 | 0x7f;
let terminal = 42 << 8;
event.update(stopped);
event.update(terminal);
assert_eq!(event.poll_status(&waker), Poll::Ready(Ok(stopped)));
assert_eq!(event.poll_status(&waker), Poll::Ready(Ok(terminal)));
assert_eq!(event.poll_status(&waker), Poll::Ready(Ok(terminal)));
}
#[test]
fn failed_cleanup_decode_preserves_fifo_front_for_retry() {
let pid = Pid::from_raw(i32::MAX - 31);
let handle = EventHandle::new();
let child_event = (libc::PTRACE_EVENT_FORK << 16) | (libc::SIGTRAP << 8) | 0x7f;
handle.event().update(child_event);
let cleanup = TerminalCleanup {
pid: pid.into(),
event: handle,
};
let reservation = cleanup
.reserve_pending_for_cleanup(Duration::ZERO)
.expect("reserve child-event FIFO front");
assert!(reservation.decode().is_err(), "fake child event decoded");
drop(reservation);
let retry = cleanup
.reserve_pending_for_cleanup(Duration::ZERO)
.expect("decode failure removed FIFO front");
assert_eq!(retry.status, child_event);
}
#[test]
fn worker_completion_does_not_hide_pending_cleanup_status() {
let pid = Pid::from_raw(i32::MAX - 32);
let handle = EventHandle::new();
let stopped = (Signal::SIGSTOP as i32) << 8 | 0x7f;
handle.event().update(stopped);
assert!(handle.event().try_begin_worker_start());
handle.event().mark_worker_running();
handle.event().mark_worker_done();
let cleanup = TerminalCleanup {
pid: pid.into(),
event: handle,
};
assert!(cleanup.wait(Duration::ZERO));
assert!(!cleanup.pending_is_empty());
}
#[test]
fn authoritative_adoption_is_cycle_free_and_preserves_handle_hash() {
let first = EventHandle::new();
let second = EventHandle::new();
let first_hash = handle_hash(&first);
let second_hash = handle_hash(&second);
let start = Arc::new(Barrier::new(3));
let first_racer = first.clone();
let second_target = second.clone();
let first_start = Arc::clone(&start);
let forward = thread::spawn(move || {
first_start.wait();
first_racer.adopt_authoritative(&second_target)
});
let second_racer = second.clone();
let first_target = first.clone();
let second_start = Arc::clone(&start);
let reverse = thread::spawn(move || {
second_start.wait();
second_racer.adopt_authoritative(&first_target)
});
start.wait();
forward.join().expect("join forward adoption").unwrap();
reverse.join().expect("join reverse adoption").unwrap();
assert!(Arc::ptr_eq(first.event(), second.event()));
assert_ne!(first, second, "stable token identity must not be rewritten");
assert_eq!(handle_hash(&first), first_hash);
assert_eq!(handle_hash(&second), second_hash);
let rejected = EventHandle::new();
let adopted = if first.0.authoritative.get().is_some() {
&first
} else {
&second
};
assert_eq!(
adopted.adopt_authoritative(&rejected),
Err(Errno::EALREADY),
"an adopted handle cannot be redirected to a second authority"
);
assert!(Arc::ptr_eq(first.event(), second.event()));
}
#[test]
fn adoption_waits_for_a_request_holding_the_old_gate() {
let requested = EventHandle::new();
let authoritative = EventHandle::new();
let hold = requested.hold_tid().expect("hold the unmarked gate");
let (adopted_tx, adopted_rx) = mpsc::channel();
let adopter = {
let requested = requested.clone();
let authoritative = authoritative.clone();
thread::spawn(move || {
let result = requested.adopt_authoritative(&authoritative);
adopted_tx.send(()).expect("report adoption");
result
})
};
let deadline = Instant::now() + TRACEE_WAIT_TIMEOUT;
while requested.0.event.terminal_reaping.try_read().is_some() {
assert!(
Instant::now() < deadline,
"adoption never waited for the request holding the old gate"
);
thread::yield_now();
}
assert!(
adopted_rx.try_recv().is_err() && requested.0.authoritative.get().is_none(),
"adoption redirected a handle while a request held its old gate"
);
drop(hold);
adopter
.join()
.expect("join adopter")
.expect("adopt the authoritative generation");
authoritative.event().mark_terminal_reaping();
assert!(
requested.hold_tid().is_none(),
"a request through the adopted handle escaped the authoritative reap"
);
}
#[test]
fn request_racing_adoption_takes_the_authoritative_gate() {
let requested = EventHandle::new();
let authoritative = EventHandle::new();
let read_event = Arc::new(Barrier::new(2));
let resume = Arc::new(Barrier::new(2));
let request = {
let requested = requested.clone();
let read_event = Arc::clone(&read_event);
let resume = Arc::clone(&resume);
thread::spawn(move || {
HOLD_TID_PAUSES.lock().insert(
thread::current().id(),
EventCapturePause {
captured: read_event,
resume,
},
);
requested.hold_tid().is_some()
})
};
read_event.wait();
requested
.adopt_authoritative(&authoritative)
.expect("adopt while the request is paused");
authoritative.event().mark_terminal_reaping();
resume.wait();
assert!(
!request.join().expect("join the paused request"),
"a request that read the old Event before the adoption held the old gate after the authoritative reap"
);
}
#[test]
fn capabilities_created_before_registration_share_one_tid_gate() {
let (pid, cleanup) = spawn_stopped_process(None).expect("spawn unregistered child");
let first = EventHandle::current_or_new(pid.into()).expect("first capability");
let second = EventHandle::current_or_new(pid.into()).expect("second capability");
assert!(
!NOTIFIER.pids.lock().contains_key(&pid.into()),
"a lookup registered the generation"
);
assert!(
Arc::ptr_eq(first.event(), second.event()),
"two capabilities for one live generation carry different TID gates"
);
first.event().mark_terminal_reaping();
assert!(
second.hold_tid().is_none(),
"a reap marked through one capability left the other's TID gate open"
);
drop((first, second));
reap_stopped_process(cleanup);
}
#[tokio::test(flavor = "current_thread")]
async fn retaining_resume_refuses_wrong_ptracer_then_transfers_the_same_owner_once() {
let deadline = Instant::now() + Duration::from_secs(3);
let (pid, stopped, mut cleanup) = spawn_traced_process(None).unwrap();
stopped.setoptions(Options::PTRACE_O_TRACEEXIT).unwrap();
let terminal = stopped.terminal_cleanup();
let original = stopped.1.clone();
let mut exit = Box::pin(cleanup.exit_event(&stopped).unwrap());
let duplicate = stopped.exit_event();
let (retained, errno) = thread::spawn(move || stopped.resume_retaining(None))
.join()
.unwrap()
.expect_err("wrong ptracer thread resumed the child");
assert_eq!(errno, Errno::ESRCH);
assert_eq!(retained.1, original);
assert!(retained.terminal_cleanup().same_generation(&terminal));
let running = retained.resume_retaining(None).unwrap();
let stopped = tokio::time::timeout(
deadline.saturating_duration_since(Instant::now()),
&mut exit,
)
.await
.unwrap()
.unwrap();
cleanup.mark_claimed_exit();
drop(running); assert!(matches!(
duplicate.await,
Err(Error::Errno(Errno::EALREADY))
));
let mut terminal_wait = stopped.resume_retaining(None).unwrap().wait_owned();
let result = tokio::time::timeout(
deadline.saturating_duration_since(Instant::now()),
&mut terminal_wait,
)
.await
.unwrap()
.unwrap();
assert_eq!(
result.assume_exited(),
(pid.into(), crate::ExitStatus::Exited(42))
);
assert!(matches!(
(&mut terminal_wait).await,
Err(OwnedWaitError::Completed)
));
assert!(terminal.wait(deadline.saturating_duration_since(Instant::now())));
cleanup.disarm();
}
fn regs_kernel_reads(stopped: &Stopped) -> usize {
stopped.1.event().regs_kernel_reads()
}
fn regs_kernel_writes(stopped: &Stopped) -> usize {
stopped.1.event().regs_kernel_writes()
}
fn regs_ip(regs: &crate::Regs) -> u64 {
#[cfg(target_arch = "x86_64")]
{
regs.rip
}
#[cfg(target_arch = "aarch64")]
{
regs.pc
}
}
fn regs_scratch(regs: &crate::Regs) -> u64 {
#[cfg(target_arch = "x86_64")]
{
regs.rbx
}
#[cfg(target_arch = "aarch64")]
{
regs.regs[19]
}
}
fn regs_scratch_set(regs: &mut crate::Regs, value: u64) {
#[cfg(target_arch = "x86_64")]
{
regs.rbx = value;
}
#[cfg(target_arch = "aarch64")]
{
regs.regs[19] = value;
}
}
async fn next_stopped(running: crate::Running, deadline: Instant) -> Stopped {
let wait = tokio::time::timeout(
deadline.saturating_duration_since(Instant::now()),
running.next_state(),
)
.await
.unwrap()
.unwrap();
wait.assume_stopped().0
}
#[tokio::test(flavor = "current_thread")]
async fn regs_cache_reads_kernel_once_per_stop() {
let (_pid, stopped, cleanup) = spawn_traced_process(None).unwrap();
let first = stopped.getregs().unwrap();
let second = stopped.getregs().unwrap();
assert_eq!(regs_ip(&first), regs_ip(&second));
assert_eq!(regs_kernel_reads(&stopped), 1);
drop(stopped);
reap_stopped_process(cleanup);
}
#[tokio::test(flavor = "current_thread")]
async fn regs_cache_setregs_updates_without_kernel_read() {
let (_pid, stopped, cleanup) = spawn_traced_process(None).unwrap();
let mut regs = stopped.getregs().unwrap();
assert_eq!(regs_kernel_reads(&stopped), 1);
regs_scratch_set(&mut regs, 0x5eed);
stopped.setregs(®s).unwrap();
assert_eq!(regs_scratch(&stopped.getregs().unwrap()), 0x5eed);
assert_eq!(regs_kernel_reads(&stopped), 1);
drop(stopped);
reap_stopped_process(cleanup);
}
#[tokio::test(flavor = "current_thread")]
async fn regs_cache_step_invalidates() {
let deadline = Instant::now() + Duration::from_secs(3);
let (_pid, stopped, cleanup) = spawn_traced_process(None).unwrap();
let before = stopped.getregs().unwrap();
assert_eq!(regs_kernel_reads(&stopped), 1);
let stopped = next_stopped(stopped.step(None).unwrap(), deadline).await;
let after = stopped.getregs().unwrap();
assert_eq!(regs_kernel_reads(&stopped), 2);
assert_ne!(regs_ip(&before), regs_ip(&after));
drop(stopped);
reap_stopped_process(cleanup);
}
#[tokio::test(flavor = "current_thread")]
async fn regs_cache_syscall_invalidates() {
let deadline = Instant::now() + Duration::from_secs(3);
let (_pid, stopped, cleanup) = spawn_traced_process(None).unwrap();
stopped.getregs().unwrap();
assert_eq!(regs_kernel_reads(&stopped), 1);
let stopped = next_stopped(stopped.syscall(None).unwrap(), deadline).await;
stopped.getregs().unwrap();
assert_eq!(regs_kernel_reads(&stopped), 2);
drop(stopped);
reap_stopped_process(cleanup);
}
#[tokio::test(flavor = "current_thread")]
async fn regs_cache_signal_delivery_invalidates() {
let deadline = Instant::now() + Duration::from_secs(3);
let (_pid, stopped, cleanup) = spawn_traced_process(None).unwrap();
stopped.getregs().unwrap();
assert_eq!(regs_kernel_reads(&stopped), 1);
let stopped = next_stopped(stopped.step(Some(Signal::SIGWINCH)).unwrap(), deadline).await;
stopped.getregs().unwrap();
assert_eq!(regs_kernel_reads(&stopped), 2);
drop(stopped);
reap_stopped_process(cleanup);
}
#[tokio::test(flavor = "current_thread")]
async fn regs_cache_exec_invalidates() {
let deadline = Instant::now() + Duration::from_secs(3);
let (root, mut cleanup, stopped) = spawn_exec_tracee(deadline);
let mut exit = Box::pin(cleanup.exit_event(&stopped).unwrap());
stopped.getregs().unwrap();
assert_eq!(regs_kernel_reads(&stopped), 1);
let stopped = next_stopped(stopped.resume(None).unwrap(), deadline).await;
stopped.getregs().unwrap();
assert_eq!(regs_kernel_reads(&stopped), 2);
assert_eq!(unsafe { libc::kill(root.as_raw(), libc::SIGKILL) }, 0);
drop(stopped);
let exit_stop = tokio::time::timeout(
deadline.saturating_duration_since(Instant::now()),
exit.as_mut(),
)
.await
.unwrap()
.unwrap();
cleanup.mark_claimed_exit();
let final_wait = tokio::time::timeout(
deadline.saturating_duration_since(Instant::now()),
exit_stop.resume(None).unwrap().wait_owned(),
)
.await
.unwrap()
.unwrap();
assert_eq!(
final_wait.assume_exited(),
(
root.into(),
crate::ExitStatus::Signaled(Signal::SIGKILL, false)
)
);
cleanup.disarm();
}
#[tokio::test(flavor = "current_thread")]
async fn regs_cache_hit_keeps_ptracer_thread_refusal() {
let (_pid, stopped, cleanup) = spawn_traced_process_as(None, |pid| {
Stopped::new_unchecked_on_ptracer_thread(pid).unwrap()
})
.unwrap();
stopped.getregs().unwrap();
assert_eq!(regs_kernel_reads(&stopped), 1);
let generation = stopped.generation();
let refused = thread::spawn(move || generation.assume_stopped().getregs())
.join()
.unwrap();
assert!(
matches!(refused, Err(Error::Errno(Errno::EPERM))),
"{refused:?}"
);
stopped.getregs().unwrap();
assert_eq!(regs_kernel_reads(&stopped), 1);
drop(stopped);
reap_stopped_process(cleanup);
}
#[tokio::test(flavor = "current_thread")]
async fn regs_writeback_collapses_setregs_until_flush() {
let (_pid, stopped, cleanup) = spawn_traced_process(None).unwrap();
let mut regs = stopped.getregs().unwrap();
regs_scratch_set(&mut regs, 0x1111);
stopped.setregs(®s).unwrap();
regs_scratch_set(&mut regs, 0x2222);
stopped.setregs(®s).unwrap();
assert_eq!(regs_kernel_writes(&stopped), 0);
assert_eq!(regs_scratch(&stopped.getregs().unwrap()), 0x2222);
assert_eq!(regs_kernel_reads(&stopped), 1);
stopped.peekuser(0).unwrap();
assert_eq!(regs_kernel_writes(&stopped), 1);
stopped.1.event().invalidate_regs();
assert_eq!(regs_scratch(&stopped.getregs().unwrap()), 0x2222);
assert_eq!(regs_kernel_reads(&stopped), 2);
let mut regs = stopped.getregs().unwrap();
regs_scratch_set(&mut regs, 0x3333);
stopped.setregs(®s).unwrap();
assert_eq!(regs_kernel_writes(&stopped), 1);
stopped.peekuser(0).unwrap();
assert_eq!(regs_kernel_writes(&stopped), 2);
drop(stopped);
reap_stopped_process(cleanup);
}
#[cfg(target_arch = "aarch64")]
#[tokio::test(flavor = "current_thread")]
async fn regs_writeback_set_syscall_flushes_before_invalidating() {
let (_pid, stopped, cleanup) = spawn_traced_process(None).unwrap();
let mut regs = stopped.getregs().unwrap();
regs_scratch_set(&mut regs, 0x7777);
stopped.setregs(®s).unwrap();
assert_eq!(regs_kernel_writes(&stopped), 0);
stopped.set_syscall(-1).unwrap();
assert_eq!(regs_kernel_writes(&stopped), 1);
assert_eq!(regs_scratch(&stopped.getregs().unwrap()), 0x7777);
assert_eq!(regs_kernel_reads(&stopped), 2);
drop(stopped);
reap_stopped_process(cleanup);
}
#[tokio::test(flavor = "current_thread")]
async fn regs_writeback_elides_a_no_op_setregs() {
let (_pid, stopped, cleanup) = spawn_traced_process(None).unwrap();
let regs = stopped.getregs().unwrap();
stopped.setregs(®s).unwrap();
stopped.peekuser(0).unwrap();
assert_eq!(regs_kernel_writes(&stopped), 0);
assert_eq!(regs_kernel_reads(&stopped), 1);
drop(stopped);
reap_stopped_process(cleanup);
}
#[tokio::test(flavor = "current_thread")]
async fn regs_writeback_flushes_at_syscall_resume() {
let deadline = Instant::now() + Duration::from_secs(3);
let (_pid, stopped, cleanup) = spawn_traced_process(None).unwrap();
let mut regs = stopped.getregs().unwrap();
regs_scratch_set(&mut regs, 0x4444);
stopped.setregs(®s).unwrap();
assert_eq!(regs_kernel_writes(&stopped), 0);
let stopped = next_stopped(stopped.syscall(None).unwrap(), deadline).await;
assert_eq!(regs_kernel_writes(&stopped), 1);
stopped.getregs().unwrap();
assert_eq!(regs_kernel_reads(&stopped), 2);
drop(stopped);
reap_stopped_process(cleanup);
}
#[tokio::test(flavor = "current_thread")]
async fn regs_writeback_flushes_at_detach() {
let (_pid, stopped, cleanup) = spawn_traced_process(None).unwrap();
let mut regs = stopped.getregs().unwrap();
regs_scratch_set(&mut regs, 0x5555);
stopped.setregs(®s).unwrap();
assert_eq!(regs_kernel_writes(&stopped), 0);
let generation = stopped.generation();
drop(stopped.detach(None::<Signal>).unwrap());
let detached = generation.assume_stopped();
assert_eq!(regs_kernel_writes(&detached), 1);
drop(detached);
reap_stopped_process(cleanup);
}
#[tokio::test(flavor = "current_thread")]
async fn regs_writeback_foreign_thread_setregs_refused() {
let (_pid, stopped, cleanup) = spawn_traced_process_as(None, |pid| {
Stopped::new_unchecked_on_ptracer_thread(pid).unwrap()
})
.unwrap();
let regs = stopped.getregs().unwrap();
let original = regs_scratch(®s);
let generation = stopped.generation();
let refused = thread::spawn(move || {
let foreign = generation.assume_stopped();
let mut regs = regs;
regs_scratch_set(&mut regs, 0x6666);
foreign.setregs(®s)
})
.join()
.unwrap();
assert!(
matches!(refused, Err(Error::Errno(Errno::EPERM))),
"{refused:?}"
);
stopped.peekuser(0).unwrap();
assert_eq!(regs_kernel_writes(&stopped), 0);
stopped.1.event().invalidate_regs();
assert_eq!(regs_scratch(&stopped.getregs().unwrap()), original);
drop(stopped);
reap_stopped_process(cleanup);
}
#[tokio::test(flavor = "current_thread")]
async fn regs_cache_is_per_generation() {
let (_p, first, first_cleanup) = spawn_traced_process(None).unwrap();
let (_p, second, second_cleanup) = spawn_traced_process(None).unwrap();
first.getregs().unwrap();
first.getregs().unwrap();
second.getregs().unwrap();
assert_eq!(regs_kernel_reads(&first), 1);
assert_eq!(regs_kernel_reads(&second), 1);
drop((first, second));
reap_stopped_process(first_cleanup);
reap_stopped_process(second_cleanup);
}
#[tokio::test(flavor = "current_thread")]
async fn owned_wait_registration_refusal_retains_original_and_then_reaps() {
let deadline = Instant::now() + Duration::from_secs(3);
let (pid, mut cleanup) = spawn_stopped_process(None).unwrap();
let running = Running::new(pid.into());
let original = running.1.clone();
let terminal = TerminalCleanup::new_unregistered(pid.into(), &original);
CAPTURE_ERRORS.lock().insert(pid.into(), Errno::EMFILE);
let mut wait = running.wait_owned();
assert!(matches!(
(&mut wait).await,
Err(OwnedWaitError::Errno(Errno::EMFILE))
));
assert_eq!(wait.inner.as_ref().unwrap().token, original);
assert!(!terminal.wait(Duration::ZERO));
cleanup.store_terminal(terminal).unwrap();
cleanup.terminal().unwrap().ensure_registered().unwrap();
pidfd_send_signal(&cleanup.pidfd, libc::SIGKILL).unwrap();
let result = tokio::time::timeout(
deadline.saturating_duration_since(Instant::now()),
&mut wait,
)
.await
.unwrap()
.unwrap();
assert_eq!(
result.assume_exited(),
(
pid.into(),
crate::ExitStatus::Signaled(Signal::SIGKILL, false)
)
);
assert!(wait.inner.is_none());
assert!(matches!((&mut wait).await, Err(OwnedWaitError::Completed)));
assert!(
cleanup
.terminal()
.unwrap()
.wait(deadline.saturating_duration_since(Instant::now()))
);
cleanup.disarm();
}
#[test]
fn terminal_cleanup_removes_stale_pid_registration() {
let pid = Pid::from_raw(i32::MAX - 17);
let running = Running::new(pid.into());
let cleanup = running.terminal_cleanup();
let old_event = cleanup.event.clone();
assert!(cleanup.wait(Duration::from_secs(1)));
let replacement_handle = EventHandle::new();
let replacement = NOTIFIER
.event(pid.into(), &replacement_handle)
.expect("resolve absent replacement");
assert!(
!Arc::ptr_eq(old_event.event(), replacement.event()),
"terminal cleanup retained a stale PID registry entry"
);
}
#[test]
fn exit_effects_observer_is_bound_to_the_original_task_not_a_reused_number() {
const INNER: &str = "SAFEPTRACE_EXIT_EFFECTS_REUSE_INNER";
if env::var_os(INNER).is_none() {
let output = run_exact_in_pid_namespace_bounded(
"notifier::test::exit_effects_observer_is_bound_to_the_original_task_not_a_reused_number",
&[(INNER, "1")],
).expect("the actual reuse control requires the owned PID namespace launcher");
assert!(
output.status.success(),
"actual reuse control failed: {output:?}"
);
assert!(String::from_utf8_lossy(&output.stdout).contains("EXIT_EFFECTS_ACTUAL_REUSE"));
return;
}
let (pid, mut cleanup) =
spawn_stopped_process(Some(100)).expect("create original fixed-ID task");
let running = Running::new(pid.into());
let terminal = running.terminal_cleanup();
terminal.ensure_registered().unwrap();
assert!(!terminal.exit_effects_complete().unwrap());
let mismatched = TerminalCleanup {
pid: crate::Pid::from_raw(101),
event: terminal.event.clone(),
};
assert!(matches!(
mismatched.exit_effects_complete(),
Err(ProcStatError::PidMismatch)
));
cleanup
.store_terminal(TerminalCleanup::new_unregistered(pid.into(), &running.1))
.unwrap();
cleanup.cleanup().unwrap();
assert!(terminal.exit_effects_complete().unwrap());
let (replacement, replacement_cleanup) =
spawn_stopped_process(Some(100)).expect("reuse the exact old number");
assert_eq!(replacement, pid);
let replacement_running = Running::new(replacement.into());
let replacement_terminal = replacement_running.terminal_cleanup();
replacement_terminal.ensure_registered().unwrap();
assert!(!terminal.same_generation(&replacement_terminal));
assert!(
terminal.exit_effects_complete().unwrap(),
"old completion remains about the old task"
);
assert!(
!replacement_terminal.exit_effects_complete().unwrap(),
"old completion cannot certify the live replacement"
);
reap_stopped_process(replacement_cleanup);
println!("EXIT_EFFECTS_ACTUAL_REUSE");
}
#[test]
fn terminal_cleanup_surfaces_and_retries_typed_capture_error() {
let (pid, mut child_cleanup) =
spawn_stopped_process(None).expect("spawn capture-error child");
CAPTURE_ERRORS.lock().insert(pid.into(), Errno::EMFILE);
let running = distinct_running(pid.into());
let error = child_cleanup
.bind_running_notifier(&running)
.expect_err("first notifier registration must surface EMFILE");
assert!(error.to_string().contains("EMFILE"), "{error}");
assert!(matches!(
child_cleanup.ownership,
TraceeCleanupOwnership::NotifierOwned { .. }
));
let cleanup = child_cleanup
.terminal()
.expect("failed registration retains notifier ownership");
assert_eq!(cleanup.registration_error(), Some(Errno::EMFILE));
assert!(!cleanup.wait(Duration::ZERO));
cleanup
.ensure_registered()
.expect("retry notifier registration after EMFILE");
assert_eq!(cleanup.registration_error(), None);
pidfd_send_signal(&child_cleanup.pidfd, libc::SIGKILL)
.expect("signal capture-error child through pidfd");
assert!(cleanup.wait(Duration::from_secs(1)));
child_cleanup.disarm();
assert!(!std::path::Path::new(&format!("/proc/{pid}")).exists());
}
#[test]
fn unsupported_pidfd_thread_fails_closed_before_registration() {
for error in [Errno::EINVAL, Errno::ENOSYS, Errno::EMFILE, Errno::EACCES] {
let (pid, cleanup) =
spawn_stopped_process(None).expect("spawn unsupported-pidfd child");
let running = Running::new(pid.into());
let terminal = TerminalCleanup::new_unregistered(pid.into(), &running.1);
PIDFD_OPEN_ERRORS.lock().insert(pid.into(), error);
assert_eq!(terminal.ensure_registered(), Err(error));
assert_eq!(terminal.registration_error(), Some(error));
assert_eq!(
terminal.event.event().worker_state.load(Ordering::Acquire),
WORKER_NOT_STARTED
);
assert!(!NOTIFIER.pids.lock().contains_key(&pid.into()));
assert_eq!(
terminal.event.event().wait_owner.load(Ordering::Acquire),
WAIT_OWNER_NONE,
"failed registration must release wait ownership"
);
reap_stopped_process(cleanup);
}
}
#[test]
fn native_thread_pidfd_refusal_skips_finite_file_size_limit() {
let result = run_exact_test_bounded(
"notifier::test::native_thread_pidfd_refusal_reports_kernel_requirement",
&[("SAFEPTRACE_NATIVE_PIDFD_DIAGNOSTIC_INNER", "file-limit")],
false,
Duration::from_secs(5),
)
.unwrap();
assert!(!result.timed_out, "{result:?}");
assert!(result.output.status.success(), "{result:?}");
assert!(result.output.stderr.is_empty(), "{result:?}");
}
#[test]
fn native_thread_pidfd_refusal_reports_kernel_requirement() {
const INNER: &str = "SAFEPTRACE_NATIVE_PIDFD_DIAGNOSTIC_INNER";
let Some(mode) = env::var_os(INNER) else {
for mode in ["pipe", "full", "closed", "file"] {
let result = run_exact_test_bounded(
"notifier::test::native_thread_pidfd_refusal_reports_kernel_requirement",
&[(INNER, mode)],
false,
Duration::from_secs(5),
)
.unwrap();
assert!(!result.timed_out, "{mode}: {result:?}");
assert!(result.output.status.success(), "{mode}: {result:?}");
if mode == "pipe" {
assert_eq!(result.output.stderr, NATIVE_THREAD_PIDFD_REQUIREMENT);
io::stderr().write_all(&result.output.stderr).unwrap();
} else {
assert!(result.output.stderr.is_empty(), "{mode}: {result:?}");
}
}
return;
};
let mode = mode.to_str().unwrap();
let (pid, cleanup) = spawn_stopped_process(None).unwrap();
let instruction = |code, jt, jf, k| libc::sock_filter { code, jt, jf, k };
let filter = [
instruction(0x20, 0, 0, 0),
instruction(0x15, 0, 3, libc::SYS_pidfd_open as u32),
instruction(0x20, 0, 0, 24),
instruction(0x15, 0, 1, libc::O_EXCL as u32),
instruction(0x06, 0, 0, libc::SECCOMP_RET_ERRNO | libc::EINVAL as u32),
instruction(0x06, 0, 0, libc::SECCOMP_RET_ALLOW),
];
let program = libc::sock_fprog {
len: filter.len() as u16,
filter: filter.as_ptr().cast_mut(),
};
assert_eq!(
unsafe { libc::prctl(libc::PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0) },
0
);
assert_eq!(
unsafe {
libc::syscall(
libc::SYS_seccomp,
libc::SECCOMP_SET_MODE_FILTER,
0,
&program,
)
},
0
);
assert_eq!(
pidfd_open_with_flags(pid.into(), libc::O_EXCL).unwrap_err(),
Errno::EINVAL
);
let ordinary = pidfd_open_with_flags(pid.into(), 0).unwrap();
assert_eq!(descriptor_is_live(ordinary.as_raw_fd()), Ok(true));
let saved = unsafe { libc::fcntl(libc::STDERR_FILENO, libc::F_DUPFD_CLOEXEC, 3) };
assert!(saved >= 0);
let saved = unsafe { OwnedFd::from_raw_fd(saved) };
let mut full_reader = None;
let mut regular_output = None;
let mut previous_sigpipe = None;
let mut previous_file_limit = None;
let mut previous_sigxfsz = None;
if mode == "file" || mode == "file-limit" {
let fd = unsafe {
libc::memfd_create(c"native-pidfd-diagnostic".as_ptr(), libc::MFD_CLOEXEC)
};
assert!(fd >= 0);
let fd = unsafe { OwnedFd::from_raw_fd(fd) };
assert_eq!(
unsafe { libc::write(fd.as_raw_fd(), b"before\n".as_ptr().cast(), 7) },
7
);
assert_eq!(
unsafe { libc::dup2(fd.as_raw_fd(), libc::STDERR_FILENO) },
libc::STDERR_FILENO
);
regular_output = Some(fd);
if mode == "file-limit" {
let mut limit = std::mem::MaybeUninit::<libc::rlimit>::uninit();
assert_eq!(
unsafe { libc::getrlimit(libc::RLIMIT_FSIZE, limit.as_mut_ptr()) },
0
);
let limit = unsafe { limit.assume_init() };
let finite = libc::rlimit {
rlim_cur: 7,
rlim_max: limit.rlim_max,
};
assert_eq!(unsafe { libc::setrlimit(libc::RLIMIT_FSIZE, &finite) }, 0);
previous_file_limit = Some(limit);
previous_sigxfsz = Some(unsafe { libc::signal(libc::SIGXFSZ, libc::SIG_DFL) });
assert_ne!(previous_sigxfsz.unwrap(), libc::SIG_ERR);
let mut mask = std::mem::MaybeUninit::<libc::sigset_t>::uninit();
assert_eq!(
unsafe {
libc::pthread_sigmask(
libc::SIG_SETMASK,
std::ptr::null(),
mask.as_mut_ptr(),
)
},
0
);
assert_eq!(
unsafe { libc::sigismember(mask.as_ptr(), libc::SIGXFSZ) },
0
);
}
} else if mode != "pipe" {
let mut descriptors = [-1; 2];
assert_eq!(
unsafe { libc::pipe2(descriptors.as_mut_ptr(), libc::O_CLOEXEC) },
0
);
let reader = unsafe { OwnedFd::from_raw_fd(descriptors[0]) };
let writer = unsafe { OwnedFd::from_raw_fd(descriptors[1]) };
if mode == "full" {
assert_eq!(
unsafe { libc::fcntl(writer.as_raw_fd(), libc::F_SETPIPE_SZ, 4096) },
4096
);
assert_eq!(
unsafe { libc::write(writer.as_raw_fd(), [b'x'; 4096].as_ptr().cast(), 4096) },
4096
);
full_reader = Some(reader);
} else {
assert_eq!(mode, "closed");
drop(reader);
previous_sigpipe = Some(unsafe { libc::signal(libc::SIGPIPE, libc::SIG_DFL) });
assert_ne!(previous_sigpipe.unwrap(), libc::SIG_ERR);
}
assert_eq!(
unsafe { libc::dup2(writer.as_raw_fd(), libc::STDERR_FILENO) },
libc::STDERR_FILENO
);
}
let flags = unsafe { libc::fcntl(libc::STDERR_FILENO, libc::F_GETFL) };
assert!(flags >= 0);
assert_eq!(flags & libc::O_NONBLOCK, 0);
assert_eq!(Running::try_new(pid.into()).unwrap_err(), Errno::EINVAL);
assert!(!NOTIFIER.pids.lock().contains_key(&pid.into()));
let explicit = Running::new_on_ptracer_thread(pid.into()).unwrap();
assert!(matches!(
explicit.1.event().identity().unwrap().pidfd,
ThreadHandle::LegacyLeader { .. }
));
assert_eq!(
unsafe { libc::fcntl(libc::STDERR_FILENO, libc::F_GETFL) },
flags,
"the diagnostic changed the caller's stderr flags"
);
if let Some(previous) = previous_file_limit {
let mut limit = std::mem::MaybeUninit::<libc::rlimit>::uninit();
assert_eq!(
unsafe { libc::getrlimit(libc::RLIMIT_FSIZE, limit.as_mut_ptr()) },
0
);
let limit = unsafe { limit.assume_init() };
assert_eq!(limit.rlim_cur, 7);
assert_eq!(limit.rlim_max, previous.rlim_max);
let mut action = std::mem::MaybeUninit::<libc::sigaction>::uninit();
assert_eq!(
unsafe { libc::sigaction(libc::SIGXFSZ, std::ptr::null(), action.as_mut_ptr()) },
0
);
assert_eq!(unsafe { action.assume_init() }.sa_sigaction, libc::SIG_DFL);
let mut mask = std::mem::MaybeUninit::<libc::sigset_t>::uninit();
assert_eq!(
unsafe {
libc::pthread_sigmask(libc::SIG_SETMASK, std::ptr::null(), mask.as_mut_ptr())
},
0
);
assert_eq!(
unsafe { libc::sigismember(mask.as_ptr(), libc::SIGXFSZ) },
0
);
assert_eq!(unsafe { libc::setrlimit(libc::RLIMIT_FSIZE, &previous) }, 0);
assert_ne!(
unsafe { libc::signal(libc::SIGXFSZ, previous_sigxfsz.unwrap()) },
libc::SIG_ERR
);
}
if mode == "file" || mode == "file-limit" {
assert_eq!(
unsafe { libc::write(libc::STDERR_FILENO, b"after\n".as_ptr().cast(), 6) },
6
);
}
assert_eq!(
unsafe { libc::dup2(saved.as_raw_fd(), libc::STDERR_FILENO) },
libc::STDERR_FILENO
);
if let Some(previous) = previous_sigpipe {
assert_ne!(
unsafe { libc::signal(libc::SIGPIPE, previous) },
libc::SIG_ERR
);
}
if mode == "full" {
let mut bytes = [0; 4096];
assert_eq!(
unsafe {
libc::read(
full_reader.as_ref().unwrap().as_raw_fd(),
bytes.as_mut_ptr().cast(),
bytes.len(),
)
},
bytes.len() as isize
);
assert_eq!(bytes, [b'x'; 4096]);
} else if let Some(fd) = regular_output {
let expected = if mode == "file-limit" {
b"before\nafter\n".to_vec()
} else {
[
b"before\n".as_slice(),
NATIVE_THREAD_PIDFD_REQUIREMENT,
b"after\n".as_slice(),
]
.concat()
};
assert_eq!(unsafe { libc::lseek(fd.as_raw_fd(), 0, libc::SEEK_SET) }, 0);
let mut actual = vec![0; expected.len() + 1];
let read =
unsafe { libc::read(fd.as_raw_fd(), actual.as_mut_ptr().cast(), actual.len()) };
assert_eq!(read, expected.len() as isize);
actual.truncate(read as usize);
assert_eq!(
actual, expected,
"stderr logs or their shared file offset were overwritten"
);
}
drop(explicit);
reap_stopped_process(cleanup);
}
#[test]
fn fatal_signal_preserves_registration_refusal() {
let (pid, mut cleanup) = spawn_stopped_process(None).expect("spawn registration control");
let running = distinct_running(pid.into());
let terminal = TerminalCleanup::new_unregistered(pid.into(), &running.1);
CAPTURE_ERRORS.lock().insert(pid.into(), Errno::EMFILE);
assert_eq!(terminal.ensure_registered(), Err(Errno::EMFILE));
assert_eq!(terminal.request_sigkill(), Err(Errno::EMFILE));
assert!(
terminal.event.identity().is_none(),
"signal recaptured identity"
);
assert!(!terminal.wait(Duration::ZERO));
assert!(
!pidfd_exited(&cleanup.pidfd).unwrap(),
"refusal signalled the child"
);
cleanup
.cleanup()
.expect("reap refused control through its test-owned pidfd");
}
#[test]
fn fatal_signal_preserves_worker_spawn_refusal() {
let (pid, mut cleanup) = spawn_stopped_process(None).unwrap();
let running = Running::new(pid.into());
SPAWN_WORKER_ERRORS.lock().insert(pid.into(), libc::EAGAIN);
assert!(cleanup.bind_running_notifier(&running).is_err());
let terminal = cleanup.terminal().unwrap();
assert_eq!(terminal.registration_error(), Some(Errno::EAGAIN));
assert_eq!(terminal.request_sigkill(), Err(Errno::EAGAIN));
assert!(!terminal.wait(Duration::ZERO));
assert!(!pidfd_exited(&cleanup.pidfd).unwrap());
cleanup.cleanup().unwrap();
}
#[tokio::test(flavor = "current_thread")]
async fn fatal_signal_requires_real_exit_and_exclusive_exit_capability() {
let (pid, stopped, mut cleanup) = spawn_traced_process(None).unwrap();
stopped.setoptions(Options::PTRACE_O_TRACEEXIT).unwrap();
let exit = cleanup.exit_event(&stopped).unwrap();
let duplicate = stopped.exit_event();
let terminal = stopped.terminal_cleanup();
terminal.request_sigkill().unwrap();
drop(stopped);
let exit_stopped = tokio::time::timeout(TRACEE_WAIT_TIMEOUT, exit)
.await
.unwrap()
.unwrap();
cleanup.mark_claimed_exit();
assert!(
!terminal.wait(Duration::ZERO),
"signal fabricated terminal acknowledgment"
);
assert!(matches!(
tokio::time::timeout(TRACEE_WAIT_TIMEOUT, duplicate)
.await
.unwrap(),
Err(Error::Errno(Errno::EALREADY))
));
let waited = tokio::time::timeout(
TRACEE_WAIT_TIMEOUT,
exit_stopped.resume(None).unwrap().next_state(),
)
.await
.unwrap()
.unwrap();
assert_eq!(
waited.assume_exited(),
(
pid.into(),
crate::ExitStatus::Signaled(Signal::SIGKILL, false)
)
);
assert!(terminal.wait(TRACEE_WAIT_TIMEOUT));
assert!(!std::path::Path::new(&format!("/proc/{pid}")).exists());
assert_eq!(terminal.request_sigkill(), Err(Errno::ESRCH));
cleanup.disarm();
}
#[test]
fn fatal_signal_does_not_follow_a_replaced_numeric_pid_label() {
let (pid, mut cleanup) = spawn_stopped_process(None).unwrap();
let running = Running::new(pid.into());
cleanup.bind_running_notifier(&running).unwrap();
let mut stale = running.terminal_cleanup();
stale.request_sigkill().unwrap();
assert!(stale.wait(TRACEE_WAIT_TIMEOUT));
cleanup.disarm();
let (other_pid, other_cleanup) = spawn_stopped_process(None).unwrap();
stale.pid = other_pid.into();
assert_eq!(stale.request_sigkill(), Err(Errno::ESRCH));
assert!(!pidfd_exited(&other_cleanup.pidfd).unwrap());
reap_stopped_process(other_cleanup);
}
#[test]
fn pidfd_liveness_errors_are_typed_across_owner_paths() {
let (lookup_pid, lookup_cleanup) =
spawn_stopped_process(None).expect("spawn liveness lookup child");
PIDFD_LIVENESS_ERRORS
.lock()
.entry(lookup_pid.into())
.or_default()
.push_back(Errno::EPERM);
assert!(matches!(
EventHandle::current_or_new(lookup_pid.into()),
Err(Errno::EPERM)
));
reap_stopped_process(lookup_cleanup);
let (event_pid, event_cleanup) =
spawn_stopped_process(None).expect("spawn liveness event child");
let event_running = Running::new(event_pid.into());
let event_terminal = TerminalCleanup::new_unregistered(event_pid.into(), &event_running.1);
PIDFD_LIVENESS_ERRORS
.lock()
.entry(event_pid.into())
.or_default()
.push_back(Errno::EINVAL);
assert_eq!(event_terminal.ensure_registered(), Err(Errno::EINVAL));
assert_eq!(event_terminal.registration_error(), Some(Errno::EINVAL));
reap_stopped_process(event_cleanup);
let (raw_pid, mut raw_cleanup) =
spawn_stopped_process(None).expect("spawn liveness raw-claim child");
let authoritative_running = Running::new(raw_pid.into());
let authoritative = authoritative_running.terminal_cleanup();
let competing_running = distinct_running(raw_pid.into());
let competing = TerminalCleanup::new_unregistered(raw_pid.into(), &competing_running.1);
PIDFD_LIVENESS_ERRORS
.lock()
.entry(raw_pid.into())
.or_default()
.push_back(Errno::EBADF);
assert!(matches!(
competing.try_claim_unstarted_raw_cleanup(),
Err(Errno::EBADF)
));
raw_cleanup
.store_terminal(authoritative)
.expect("bind raw-claim cleanup to authoritative worker");
raw_cleanup
.cleanup()
.expect("cleanup liveness raw-claim child");
}
#[test]
fn synchronous_wait_claim_blocks_and_wakes_registration() {
let (pid, mut cleanup) =
spawn_stopped_process(None).expect("spawn sync-first wait-owner child");
let synchronous_running = Running::new(pid.into());
let notifier_running = distinct_running(pid.into());
let terminal = TerminalCleanup::new_unregistered(pid.into(), ¬ifier_running.1);
let claimed = Arc::new(Barrier::new(2));
let resume_wait = Arc::new(Barrier::new(2));
SYNC_WAIT_CLAIM_PAUSES.lock().insert(
pid.into(),
EventCapturePause {
captured: Arc::clone(&claimed),
resume: Arc::clone(&resume_wait),
},
);
let synchronous = thread::spawn(move || synchronous_running.wait());
claimed.wait();
let (registered_tx, registered_rx) = mpsc::channel();
let registration = thread::spawn(move || {
let result = terminal.ensure_registered();
registered_tx
.send(())
.expect("report sync-first registration completion");
(result, terminal)
});
thread::sleep(Duration::from_millis(20));
assert!(
registered_rx.try_recv().is_err(),
"registration bypassed the synchronous wait-owner claim"
);
resume_wait.wait();
nix::sys::signal::kill(pid, Signal::SIGCONT).expect("resume sync-first child");
assert!(matches!(
synchronous.join().expect("join synchronous wait"),
Ok(Wait::Exited(waited, crate::ExitStatus::Exited(0))) if waited == pid.into()
));
registered_rx
.recv_timeout(TRACEE_WAIT_TIMEOUT)
.expect("registration was not woken after synchronous wait release");
let (result, terminal) = registration.join().expect("join blocked registration");
result.expect("registration retries after synchronous wait release");
assert!(terminal.wait(TRACEE_WAIT_TIMEOUT));
cleanup.disarm();
}
#[test]
fn synchronous_wait_losing_to_notifier_consumes_event_fifo() {
let (pid, mut cleanup) =
spawn_stopped_process(None).expect("spawn notifier-first wait-owner child");
let notifier_running = Running::new(pid.into());
let terminal = TerminalCleanup::new_unregistered(pid.into(), ¬ifier_running.1);
terminal
.ensure_registered()
.expect("register notifier-first worker");
let synchronous_running = distinct_running(pid.into());
let canonicalized = Arc::new(Barrier::new(2));
let resume_wait = Arc::new(Barrier::new(2));
SYNC_HANDLE_PAUSES.lock().insert(
pid.into(),
EventCapturePause {
captured: Arc::clone(&canonicalized),
resume: Arc::clone(&resume_wait),
},
);
let synchronous = thread::spawn(move || synchronous_running.wait());
canonicalized.wait();
nix::sys::signal::kill(pid, Signal::SIGCONT).expect("resume notifier-first child");
resume_wait.wait();
let observed = synchronous
.join()
.expect("join notifier-owned synchronous wait");
assert!(
matches!(
observed,
Ok(Wait::Exited(waited, crate::ExitStatus::Exited(0))) if waited == pid.into()
),
"unexpected notifier-owned synchronous wait: {observed:?}"
);
assert!(terminal.wait(TRACEE_WAIT_TIMEOUT));
cleanup.disarm();
}
#[test]
fn distinct_raw_claim_adopts_synchronous_registry_owner() {
let (pid, mut cleanup) =
spawn_stopped_process(None).expect("spawn sync-versus-raw owner child");
let synchronous_running = Running::new(pid.into());
let claimed = Arc::new(Barrier::new(2));
let resume_wait = Arc::new(Barrier::new(2));
SYNC_WAIT_CLAIM_PAUSES.lock().insert(
pid.into(),
EventCapturePause {
captured: Arc::clone(&claimed),
resume: Arc::clone(&resume_wait),
},
);
let synchronous = thread::spawn(move || synchronous_running.wait());
claimed.wait();
let raw_running = distinct_running(pid.into());
let raw = TerminalCleanup::new_unregistered(pid.into(), &raw_running.1);
let authoritative = match raw
.try_claim_unstarted_raw_cleanup()
.expect("resolve distinct raw claim")
{
RawCleanupClaim::Existing(authoritative) => authoritative,
_ => panic!("distinct raw claim did not adopt synchronous authority"),
};
assert!(Arc::ptr_eq(
authoritative.event(),
NOTIFIER
.pids
.lock()
.get(&pid.into())
.expect("synchronous registry authority")
.handle
.event()
));
resume_wait.wait();
nix::sys::signal::kill(pid, Signal::SIGCONT).expect("resume sync-versus-raw child");
assert!(
synchronous
.join()
.expect("join sync-versus-raw wait")
.is_ok()
);
cleanup.disarm();
}
#[test]
fn cleanup_cancels_distinct_synchronous_owner_before_registration() {
let (pid, mut cleanup) =
spawn_stopped_process(None).expect("spawn cleanup-versus-sync child");
let synchronous_running = Running::new(pid.into());
let claimed = Arc::new(Barrier::new(2));
let resume_wait = Arc::new(Barrier::new(2));
SYNC_WAIT_CLAIM_PAUSES.lock().insert(
pid.into(),
EventCapturePause {
captured: Arc::clone(&claimed),
resume: Arc::clone(&resume_wait),
},
);
let synchronous = thread::spawn(move || synchronous_running.wait());
claimed.wait();
let cleanup_running = distinct_running(pid.into());
cleanup
.store_terminal(TerminalCleanup::new_unregistered(
pid.into(),
&cleanup_running.1,
))
.expect("store distinct cleanup Event");
let cleaning = thread::spawn(move || {
let result = cleanup.cleanup();
(result, cleanup)
});
thread::sleep(Duration::from_millis(20));
resume_wait.wait();
assert!(matches!(
synchronous.join().expect("join cancelled synchronous wait"),
Ok(Wait::Exited(waited, crate::ExitStatus::Signaled(Signal::SIGKILL, _)))
if waited == pid.into()
));
let (result, cleanup) = cleaning.join().expect("join distinct cleanup");
result.expect("cleanup waits for synchronous owner release");
assert!(matches!(
cleanup.ownership,
TraceeCleanupOwnership::NotifierOwned {
raw_cleanup_claimed: false,
cleanup_signal_sent: true,
..
}
));
}
#[test]
fn cleanup_claim_catches_sync_started_after_authority_resolution() {
let (pid, mut cleanup) =
spawn_stopped_process(None).expect("spawn cleanup claim-race child");
let running = Running::new(pid.into());
cleanup
.store_terminal(TerminalCleanup::new_unregistered(pid.into(), &running.1))
.expect("store claim-race cleanup Event");
let cleanup_ready = Arc::new(Barrier::new(2));
let resume_cleanup = Arc::new(Barrier::new(2));
CLEANUP_OWNER_CLAIM_PAUSES.lock().insert(
pid.into(),
EventCapturePause {
captured: Arc::clone(&cleanup_ready),
resume: Arc::clone(&resume_cleanup),
},
);
let sync_claimed = Arc::new(Barrier::new(2));
let resume_sync = Arc::new(Barrier::new(2));
SYNC_WAIT_CLAIM_PAUSES.lock().insert(
pid.into(),
EventCapturePause {
captured: Arc::clone(&sync_claimed),
resume: Arc::clone(&resume_sync),
},
);
let cancel_sent = Arc::new(Barrier::new(2));
let resume_cancel = Arc::new(Barrier::new(2));
CLEANUP_CANCEL_SIGNAL_PAUSES.lock().insert(
pid.into(),
EventCapturePause {
captured: Arc::clone(&cancel_sent),
resume: Arc::clone(&resume_cancel),
},
);
let cleaning = thread::spawn(move || {
let result = cleanup.cleanup();
(result, cleanup)
});
cleanup_ready.wait();
let synchronous = thread::spawn(move || running.wait());
sync_claimed.wait();
resume_cleanup.wait();
cancel_sent.wait();
resume_cancel.wait();
resume_sync.wait();
assert!(matches!(
synchronous.join().expect("join claim-race synchronous wait"),
Ok(Wait::Exited(waited, crate::ExitStatus::Signaled(Signal::SIGKILL, _)))
if waited == pid.into()
));
let (result, cleanup) = cleaning.join().expect("join claim-race cleanup");
result.expect("serialized cleanup claim cancels late synchronous owner");
assert!(matches!(
cleanup.ownership,
TraceeCleanupOwnership::NotifierOwned {
cleanup_signal_sent: true,
..
}
));
}
#[test]
fn regular_stop_return_commit_precedes_late_cleanup_cancellation() {
let (pid, mut cleanup) =
spawn_stopped_process(None).expect("spawn regular return-race child");
let running = Running::new(pid.into());
let retained = running.1.event().clone();
cleanup
.store_terminal(TerminalCleanup::new_unregistered(pid.into(), &running.1))
.expect("store regular return-race Event");
retained.event().update((libc::SIGSTOP << 8) | 0x7f);
let return_decided = Arc::new(Barrier::new(2));
let resume_return = Arc::new(Barrier::new(2));
SYNC_RETURN_COMMIT_PAUSES.lock().insert(
pid.into(),
EventCapturePause {
captured: Arc::clone(&return_decided),
resume: Arc::clone(&resume_return),
},
);
let cleanup_ready = Arc::new(Barrier::new(2));
let resume_cleanup = Arc::new(Barrier::new(2));
CLEANUP_OWNER_CLAIM_PAUSES.lock().insert(
pid.into(),
EventCapturePause {
captured: Arc::clone(&cleanup_ready),
resume: Arc::clone(&resume_cleanup),
},
);
let synchronous = thread::spawn(move || {
matches!(
running.wait(),
Ok(Wait::Stopped(_, crate::Event::Signal(Signal::SIGSTOP)))
)
});
return_decided.wait();
let cleaning = thread::spawn(move || {
let result = cleanup.cleanup();
(result, cleanup)
});
cleanup_ready.wait();
resume_cleanup.wait();
thread::yield_now();
resume_return.wait();
assert!(
synchronous.join().expect("join regular return decision"),
"cleanup cancelled a regular stop after its return decision"
);
let (result, _cleanup) = cleaning.join().expect("join regular return cleanup");
result.expect("cleanup after regular return ownership handoff");
}
#[test]
fn exit_stop_return_commit_precedes_late_cleanup_cancellation() {
let (pid, stopped, mut cleanup) =
spawn_traced_process(None).expect("spawn exit return-race tracee");
stopped
.setoptions(Options::PTRACE_O_TRACEEXIT)
.expect("enable exit return-race stop");
let running = stopped
.resume(None)
.expect("resume exit return-race tracee");
cleanup
.store_terminal(TerminalCleanup::new_unregistered(pid.into(), &running.1))
.expect("store exit return-race Event");
let return_decided = Arc::new(Barrier::new(2));
let resume_return = Arc::new(Barrier::new(2));
SYNC_RETURN_COMMIT_PAUSES.lock().insert(
pid.into(),
EventCapturePause {
captured: Arc::clone(&return_decided),
resume: Arc::clone(&resume_return),
},
);
let cleanup_ready = Arc::new(Barrier::new(2));
let resume_cleanup = Arc::new(Barrier::new(2));
CLEANUP_OWNER_CLAIM_PAUSES.lock().insert(
pid.into(),
EventCapturePause {
captured: Arc::clone(&cleanup_ready),
resume: Arc::clone(&resume_cleanup),
},
);
let coordinator = thread::spawn(move || {
return_decided.wait();
let cleaning = thread::spawn(move || {
let result = cleanup.cleanup();
(result, cleanup)
});
cleanup_ready.wait();
resume_cleanup.wait();
thread::yield_now();
resume_return.wait();
cleaning.join().expect("join exit return cleanup")
});
let (exit_stopped, event) = running
.wait()
.expect("exit stop return decision")
.assume_stopped();
assert_eq!(event, crate::Event::Exit);
let _running = exit_stopped
.resume(None)
.expect("resume returned exit-stop capability");
let (result, _cleanup) = coordinator.join().expect("join exit return coordinator");
result.expect("cleanup after exit return ownership handoff");
}
#[test]
fn decode_error_rolls_back_return_transaction_and_wakes_cleanup() {
let pid: crate::Pid = Pid::this().into();
let event = Arc::new(Event::new());
event.update((libc::SIGSTOP << 8) | 0x7f);
let return_begun = Arc::new(Barrier::new(2));
let resume_decode = Arc::new(Barrier::new(2));
SYNC_RETURN_COMMIT_PAUSES.lock().insert(
pid,
EventCapturePause {
captured: Arc::clone(&return_begun),
resume: Arc::clone(&resume_decode),
},
);
let (first_claim_tx, first_claim_rx) = mpsc::channel();
let (retry_tx, retry_rx) = mpsc::channel();
let (decoded_tx, decoded_rx) = mpsc::channel();
let (release_owner_tx, release_owner_rx) = mpsc::channel();
thread::scope(|scope| {
let decoding_event = Arc::clone(&event);
scope.spawn(move || {
let mut owner = match decoding_event
.claim_sync_wait()
.expect("claim rollback-test synchronous owner")
{
SyncWaitOwnership::Claimed(owner) => owner,
SyncWaitOwnership::Notifier => {
panic!("rollback test unexpectedly found notifier")
}
};
let reservation = decoding_event
.try_status_reservation_sync()
.expect("rollback status is present")
.expect("rollback status is valid");
let result: Result<StatusReturn<()>, Error> =
owner.decode_status_return(pid, reservation, |_| Err(Errno::EIO.into()));
decoded_tx
.send(result)
.expect("report decode-error rollback");
release_owner_rx
.recv()
.expect("retain restored synchronous owner for cleanup retry");
});
return_begun.wait();
assert_eq!(
event.wait_owner.load(Ordering::Acquire),
WAIT_OWNER_SYNC_RETURNING
);
let cleanup_event = Arc::clone(&event);
scope.spawn(move || {
let returning = matches!(
cleanup_event.try_claim_cancellable_notifier_wait(),
CancellableNotifierWaitOwnership::Returning
);
first_claim_tx
.send(returning)
.expect("report in-flight return claim");
let synchronous = matches!(
cleanup_event
.wait_for_return_and_claim_cancellable_notifier(TRACEE_WAIT_TIMEOUT),
Some(CancellableNotifierWaitOwnership::Synchronous)
);
retry_tx
.send(synchronous)
.expect("report rolled-back synchronous claim");
});
assert!(
first_claim_rx
.recv_timeout(TRACEE_WAIT_TIMEOUT)
.expect("cleanup did not observe RETURNING")
);
assert!(!event.cleanup_cancel_requested.load(Ordering::Acquire));
resume_decode.wait();
assert!(matches!(
decoded_rx
.recv_timeout(TRACEE_WAIT_TIMEOUT)
.expect("decode error did not complete"),
Err(Error::Errno(Errno::EIO))
));
assert!(
retry_rx
.recv_timeout(TRACEE_WAIT_TIMEOUT)
.expect("cleanup was not woken by rollback")
);
assert!(event.cleanup_cancel_requested.load(Ordering::Acquire));
release_owner_tx
.send(())
.expect("release restored synchronous owner");
});
assert_eq!(event.wait_owner.load(Ordering::Acquire), WAIT_OWNER_NONE);
assert_eq!(
event.status.lock().pending.front().copied(),
Some((libc::SIGSTOP << 8) | 0x7f),
"decode error consumed the rollback-safe FIFO reservation"
);
}
#[test]
fn pending_cleanup_claim_prevents_repeated_return_starvation() {
let event = Arc::new(Event::new());
event
.wait_owner
.store(WAIT_OWNER_NOTIFIER, Ordering::Release);
let stopped = (libc::SIGSTOP << 8) | 0x7f;
let transaction = match event.begin_status_return(
stopped,
WAIT_OWNER_NOTIFIER,
WAIT_OWNER_NOTIFIER_RETURNING,
) {
ReturnTransactionStart::Begun(transaction) => transaction,
ReturnTransactionStart::Cancelled => panic!("first return was unexpectedly cancelled"),
};
let (observed_tx, observed_rx) = mpsc::channel();
let cleanup_event = Arc::clone(&event);
let cleanup = thread::spawn(move || {
assert!(matches!(
cleanup_event.try_claim_cancellable_notifier_wait(),
CancellableNotifierWaitOwnership::Returning
));
observed_tx
.send(())
.expect("report pending cleanup return claim");
matches!(
cleanup_event.wait_for_return_and_claim_cancellable_notifier(TRACEE_WAIT_TIMEOUT),
Some(CancellableNotifierWaitOwnership::Existing)
)
});
observed_rx
.recv_timeout(TRACEE_WAIT_TIMEOUT)
.expect("cleanup did not register its RETURNING claim");
assert_eq!(event.cleanup_claim_waiters.load(Ordering::Acquire), 1);
transaction.commit(WAIT_OWNER_NOTIFIER);
for _ in 0..64 {
assert!(matches!(
event.begin_status_return(
stopped,
WAIT_OWNER_NOTIFIER,
WAIT_OWNER_NOTIFIER_RETURNING,
),
ReturnTransactionStart::Cancelled
));
thread::yield_now();
}
assert!(cleanup.join().expect("join pending cleanup claimant"));
assert!(event.cleanup_cancel_requested.load(Ordering::Acquire));
assert_eq!(event.cleanup_claim_waiters.load(Ordering::Acquire), 0);
assert_eq!(
event.wait_owner.load(Ordering::Acquire),
WAIT_OWNER_NOTIFIER
);
}
#[test]
fn chained_returning_adoption_preserves_cleanup_deadline() {
let first = EventHandle::new();
let second = EventHandle::new();
first
.event()
.wait_owner
.store(WAIT_OWNER_SYNC, Ordering::Release);
second
.event()
.wait_owner
.store(WAIT_OWNER_SYNC, Ordering::Release);
let stopped = (libc::SIGSTOP << 8) | 0x7f;
let first_return = match first.event().begin_status_return(
stopped,
WAIT_OWNER_SYNC,
WAIT_OWNER_SYNC_RETURNING,
) {
ReturnTransactionStart::Begun(transaction) => transaction,
ReturnTransactionStart::Cancelled => panic!("first chained return was cancelled"),
};
let second_return = match second.event().begin_status_return(
stopped,
WAIT_OWNER_SYNC,
WAIT_OWNER_SYNC_RETURNING,
) {
ReturnTransactionStart::Begun(transaction) => transaction,
ReturnTransactionStart::Cancelled => panic!("second chained return was cancelled"),
};
let deadline = Instant::now() + Duration::from_millis(150);
let started = Instant::now();
let cleanup_handle = first.clone();
let cleanup = thread::spawn(move || {
NOTIFIER.try_event_for_cleanup(Pid::this().into(), &cleanup_handle, deadline)
});
let claim_deadline = Instant::now() + TRACEE_WAIT_TIMEOUT;
while first.event().cleanup_claim_waiters.load(Ordering::Acquire) == 0 {
assert!(
Instant::now() < claim_deadline,
"cleanup did not wait on first RETURNING generation"
);
thread::yield_now();
}
first
.adopt_authoritative(&second)
.expect("chain cleanup handle to second RETURNING generation");
thread::sleep(Duration::from_millis(100));
first_return.commit(WAIT_OWNER_NONE);
let result = cleanup.join().expect("join chained RETURNING cleanup");
assert!(matches!(
result,
Ok(CancellableEventRegistration::ReturningTimedOut)
));
assert!(
started.elapsed() < Duration::from_millis(220),
"adoption reset the absolute cleanup deadline: {:?}",
started.elapsed()
);
drop(second_return);
assert_eq!(
second.event().cleanup_claim_waiters.load(Ordering::Acquire),
0
);
}
#[tokio::test(flavor = "current_thread")]
async fn new_child_decode_return_precedes_cleanup_for_fork_vfork_and_clone() {
for repetition in 0..8 {
for child_op in [
crate::ChildOp::Fork,
crate::ChildOp::Vfork,
crate::ChildOp::Clone,
] {
let mut race = start_new_child_return_race(child_op, WAIT_OWNER_SYNC_RETURNING)
.unwrap_or_else(|error| {
panic!("sync repetition {repetition} start {child_op:?}: {error}")
});
let wait = race
.running
.take()
.expect("sync race waiter")
.wait()
.unwrap_or_else(|error| {
panic!("sync repetition {repetition} wait {child_op:?}: {error}")
});
finish_new_child_return_race(child_op, race, wait).unwrap_or_else(|error| {
panic!("sync repetition {repetition} finish {child_op:?}: {error}")
});
}
}
for repetition in 0..8 {
for child_op in [
crate::ChildOp::Fork,
crate::ChildOp::Vfork,
crate::ChildOp::Clone,
] {
let mut race = start_new_child_return_race(child_op, WAIT_OWNER_NOTIFIER_RETURNING)
.unwrap_or_else(|error| {
panic!("async repetition {repetition} start {child_op:?}: {error}")
});
let wait = race
.running
.take()
.expect("async race waiter")
.next_state()
.await
.unwrap_or_else(|error| {
panic!("async repetition {repetition} wait {child_op:?}: {error}")
});
finish_new_child_return_race(child_op, race, wait).unwrap_or_else(|error| {
panic!("async repetition {repetition} finish {child_op:?}: {error}")
});
}
}
}
#[test]
fn cleanup_cancellation_retains_pending_regular_stop_until_terminal() {
let (pid, mut cleanup) =
spawn_stopped_process(None).expect("spawn pending-stop cancellation child");
let running = Running::new(pid.into());
let retained = running.1.event().clone();
cleanup
.store_terminal(TerminalCleanup::new_unregistered(pid.into(), &running.1))
.expect("store pending-stop cleanup Event");
let sync_claimed = Arc::new(Barrier::new(2));
let resume_sync = Arc::new(Barrier::new(2));
SYNC_WAIT_CLAIM_PAUSES.lock().insert(
pid.into(),
EventCapturePause {
captured: Arc::clone(&sync_claimed),
resume: Arc::clone(&resume_sync),
},
);
let cancel_sent = Arc::new(Barrier::new(2));
let resume_cancel = Arc::new(Barrier::new(2));
CLEANUP_CANCEL_SIGNAL_PAUSES.lock().insert(
pid.into(),
EventCapturePause {
captured: Arc::clone(&cancel_sent),
resume: Arc::clone(&resume_cancel),
},
);
let synchronous = thread::spawn(move || running.wait());
sync_claimed.wait();
let stopped = (libc::SIGSTOP << 8) | 0x7f;
retained.event().update(stopped);
let cleaning = thread::spawn(move || {
let result = cleanup.cleanup();
(result, cleanup)
});
cancel_sent.wait();
resume_cancel.wait();
resume_sync.wait();
assert!(matches!(
synchronous.join().expect("join pending-stop synchronous wait"),
Ok(Wait::Exited(waited, crate::ExitStatus::Signaled(Signal::SIGKILL, _)))
if waited == pid.into()
));
let (result, cleanup) = cleaning.join().expect("join pending-stop cleanup");
result.expect("pending-stop cleanup reaches terminal state");
assert_eq!(
retained.event().status.lock().pending.front().copied(),
Some(stopped),
"cancellation consumed the pending regular stop instead of transferring it"
);
assert!(matches!(
cleanup.ownership,
TraceeCleanupOwnership::NotifierOwned {
cleanup_signal_sent: true,
..
}
));
}
#[test]
fn cleanup_cancellation_retains_exit_stop_and_sync_returns_terminal() {
let (pid, stopped, mut cleanup) =
spawn_traced_process(None).expect("spawn exit-stop cancellation tracee");
stopped
.setoptions(Options::PTRACE_O_TRACEEXIT)
.expect("enable cancellation exit stop");
let running = stopped
.resume(None)
.expect("resume cancellation exit tracee");
let retained = running.1.event().clone();
cleanup
.store_terminal(TerminalCleanup::new_unregistered(pid.into(), &running.1))
.expect("store exit-stop cleanup Event");
let status_published = Arc::new(Barrier::new(2));
let resume_status = Arc::new(Barrier::new(2));
SYNC_STATUS_PUBLICATION_PAUSES.lock().insert(
pid.into(),
EventCapturePause {
captured: Arc::clone(&status_published),
resume: Arc::clone(&resume_status),
},
);
let cancel_sent = Arc::new(Barrier::new(2));
let resume_cancel = Arc::new(Barrier::new(2));
CLEANUP_CANCEL_SIGNAL_PAUSES.lock().insert(
pid.into(),
EventCapturePause {
captured: Arc::clone(&cancel_sent),
resume: Arc::clone(&resume_cancel),
},
);
let coordinator = thread::spawn(move || {
status_published.wait();
let cleaning = thread::spawn(move || {
let result = cleanup.cleanup();
(result, cleanup)
});
cancel_sent.wait();
resume_cancel.wait();
resume_status.wait();
cleaning.join().expect("join exit-stop cleanup")
});
let observed = running.wait();
assert!(
matches!(observed, Ok(Wait::Exited(waited, _)) if waited == pid.into()),
"cancelled synchronous exit-stop returned {observed:?}"
);
let (result, _cleanup) = coordinator.join().expect("join exit-stop coordinator");
result.expect("exit-stop cancellation reaches terminal state");
assert_eq!(
retained.event().status.lock().pending.front().copied(),
Some(PTRACE_EVENT_EXIT_STOP),
"cancellation consumed or duplicated the retained exit stop"
);
assert_eq!(
retained.event().exit_capability.load(Ordering::Acquire),
EXIT_CAP_EXPIRED,
"synchronous cancellation minted an ExitFuture capability"
);
}
#[test]
fn synchronous_late_wait_replays_terminal_after_reap() {
let (pid, mut cleanup) =
spawn_stopped_process(None).expect("spawn late terminal replay child");
let running = Running::new(pid.into());
let retry = Running::from_token(pid.into(), running.1.clone());
pidfd_send_signal(&cleanup.pidfd, libc::SIGKILL)
.expect("signal late terminal replay child");
let first = running.wait().expect("consume exact terminal status");
let late = retry
.wait()
.expect("replay retained terminal after procfs disappearance");
assert!(matches!(
(&first, &late),
(
Wait::Exited(first_pid, crate::ExitStatus::Signaled(Signal::SIGKILL, _)),
Wait::Exited(late_pid, crate::ExitStatus::Signaled(Signal::SIGKILL, _))
) if *first_pid == pid.into() && *late_pid == pid.into()
));
cleanup.disarm();
}
#[test]
fn synchronous_wait_reaps_a_kill_that_ends_the_observed_stop() {
let (pid, stopped, mut cleanup) =
spawn_traced_process(None).expect("spawn stop-consumption race tracee");
let running = stopped
.resume(Some(Signal::SIGSTOP))
.expect("resume stop-consumption race tracee into a group stop");
let observed = Arc::new(Barrier::new(2));
let resume = Arc::new(Barrier::new(2));
STOP_CONSUMPTION_PAUSES.lock().insert(
pid.into(),
EventCapturePause {
captured: Arc::clone(&observed),
resume: Arc::clone(&resume),
},
);
let pidfd = cleanup.pidfd.try_clone().expect("clone race tracee pidfd");
let killer = thread::spawn(move || {
observed.wait();
pidfd_send_signal(&pidfd, libc::SIGKILL).expect("kill the observed stop");
let terminal = WaitPidFlag::from_bits_retain(
WaitPidFlag::WEXITED.bits()
| WaitPidFlag::WNOWAIT.bits()
| WaitPidFlag::WNOHANG.bits()
| libc::__WALL,
);
let deadline = Instant::now() + TRACEE_WAIT_TIMEOUT;
let status = loop {
match waitid::waitpidfd(pidfd.as_raw_fd(), terminal) {
Ok(Some(status)) => break status,
Ok(None) | Err(Errno::EINTR) => {}
Err(error) => panic!("observe the killed tracee: {error}"),
}
assert!(Instant::now() < deadline, "SIGKILL never ended the stop");
thread::sleep(Duration::from_millis(1));
};
resume.wait();
(status, pidfd)
});
let result = running.wait();
let (status, pidfd) = killer.join().expect("join stop-consumption killer");
assert!(libc::WIFSIGNALED(status) && libc::WTERMSIG(status) == libc::SIGKILL);
assert!(
matches!(
result,
Ok(Wait::Exited(waited, crate::ExitStatus::Signaled(Signal::SIGKILL, _)))
if waited == pid.into()
),
"a kill that ended the observed stop returned {result:?}"
);
let all = WaitPidFlag::from_bits_retain(
WaitPidFlag::WEXITED.bits()
| WaitPidFlag::WSTOPPED.bits()
| WaitPidFlag::WNOWAIT.bits()
| WaitPidFlag::WNOHANG.bits()
| libc::__WALL,
);
assert_eq!(
waitid::waitpidfd(pidfd.as_raw_fd(), all),
Err(Errno::ECHILD),
"the synchronous wait left the killed tracee unreaped"
);
cleanup.disarm();
}
#[test]
fn persistent_registration_failure_claims_exact_raw_cleanup() {
let (pid, mut cleanup) =
spawn_stopped_process(None).expect("spawn failed-bind cleanup child");
let identity = WorkerIdentity::capture_process(pid.into())
.expect("capture failed-bind cleanup generation");
CAPTURE_PERSISTENT_ERRORS
.lock()
.insert(pid.into(), Errno::EMFILE);
let running = Running::new(pid.into());
cleanup
.bind_running_notifier(&running)
.expect_err("inject persistent notifier registration failure");
assert!(matches!(
cleanup.ownership,
TraceeCleanupOwnership::NotifierOwned { .. }
));
cleanup
.cleanup()
.expect("persistent failure falls back to exact raw cleanup");
CAPTURE_PERSISTENT_ERRORS.lock().remove(&pid.into());
assert!(matches!(
cleanup.ownership,
TraceeCleanupOwnership::NotifierOwned {
raw_cleanup_claimed: true,
cleanup_signal_sent: true,
..
}
));
assert!(
!identity.is_same_process_generation(),
"persistent notifier registration failure orphaned its child"
);
}
#[test]
fn persistent_registration_failure_drop_reaps_child() {
let (pid, mut cleanup) =
spawn_stopped_process(None).expect("spawn persistent-drop cleanup child");
let identity = WorkerIdentity::capture_process(pid.into())
.expect("capture persistent-drop cleanup generation");
CAPTURE_PERSISTENT_ERRORS
.lock()
.insert(pid.into(), Errno::EIO);
let running = Running::new(pid.into());
cleanup
.bind_running_notifier(&running)
.expect_err("inject persistent Drop registration failure");
drop(cleanup);
CAPTURE_PERSISTENT_ERRORS.lock().remove(&pid.into());
assert!(
!identity.is_same_process_generation(),
"persistent registration failure was ignored by Drop"
);
}
#[test]
fn registration_winning_raw_claim_race_uses_notifier_cleanup() {
let (pid, mut cleanup) = spawn_stopped_process(None).expect("spawn CAS-loss cleanup child");
let identity = WorkerIdentity::capture_process(pid.into())
.expect("capture CAS-loss cleanup generation");
CAPTURE_PERSISTENT_ERRORS
.lock()
.insert(pid.into(), Errno::EMFILE);
FORCE_RAW_CLAIM_REGISTRATION_RACE
.lock()
.insert(pid.into(), ());
let running = Running::new(pid.into());
cleanup
.bind_running_notifier(&running)
.expect_err("inject CAS-loss registration failure");
cleanup
.cleanup()
.expect("CAS loss transfers cleanup to notifier worker");
CAPTURE_PERSISTENT_ERRORS.lock().remove(&pid.into());
assert!(matches!(
cleanup.ownership,
TraceeCleanupOwnership::NotifierOwned {
raw_cleanup_claimed: false,
..
}
));
assert_eq!(
cleanup
.terminal()
.expect("CAS-loss terminal cleanup")
.event
.event()
.worker_state
.load(Ordering::Acquire),
WORKER_DONE
);
assert!(
!identity.is_same_process_generation(),
"notifier-owned CAS-loss cleanup orphaned its child"
);
}
#[test]
fn second_guard_cannot_infer_raw_wait_ownership() {
let (pid, mut winner) = spawn_stopped_process(None).expect("spawn two-guard cleanup child");
let mut loser = TraceeCleanupGuard::new(pid).expect("open loser guard pidfd");
let identity = WorkerIdentity::capture_process(pid.into())
.expect("capture two-guard cleanup generation");
let running = Running::new(pid.into());
winner
.store_terminal(TerminalCleanup::new_unregistered(pid.into(), &running.1))
.expect("store winner terminal generation");
loser
.store_terminal(TerminalCleanup::new_unregistered(pid.into(), &running.1))
.expect("store loser terminal generation");
assert!(matches!(
winner
.terminal()
.expect("winner terminal")
.try_claim_unstarted_raw_cleanup(),
Ok(RawCleanupClaim::Won)
));
let TraceeCleanupOwnership::NotifierOwned {
raw_cleanup_claimed,
raw_cleanup_finishes_event,
..
} = &mut winner.ownership
else {
unreachable!()
};
*raw_cleanup_claimed = true;
*raw_cleanup_finishes_event = true;
let mut loser_signal_sent = false;
let loser_owner = TraceeCleanupGuard::acquire_cleanup_wait_owner(
pid,
&loser.pidfd,
loser.terminal().expect("loser terminal"),
&mut loser_signal_sent,
)
.expect("loser observes winner state");
assert!(matches!(loser_owner, CleanupWaitOwner::TerminalAck));
winner.cleanup().expect("winner performs exact raw cleanup");
loser
.cleanup()
.expect("loser observes terminal acknowledgment");
assert!(matches!(
loser.ownership,
TraceeCleanupOwnership::NotifierOwned {
raw_cleanup_claimed: false,
..
}
));
assert!(
!identity.is_same_process_generation(),
"two-guard cleanup left its child generation live"
);
}
#[test]
fn reaped_pidfd_esrch_never_waits_on_projected_replacement() {
let (old_pid, mut old_cleanup) =
spawn_stopped_process(None).expect("spawn ESRCH old generation");
let running = Running::new(old_pid.into());
old_cleanup
.store_terminal(TerminalCleanup::new_unregistered(
old_pid.into(),
&running.1,
))
.expect("store ESRCH old terminal");
assert!(matches!(
old_cleanup
.terminal()
.expect("ESRCH old terminal")
.try_claim_unstarted_raw_cleanup(),
Ok(RawCleanupClaim::Won)
));
let TraceeCleanupOwnership::NotifierOwned {
raw_cleanup_claimed,
raw_cleanup_finishes_event,
..
} = &mut old_cleanup.ownership
else {
unreachable!()
};
*raw_cleanup_claimed = true;
*raw_cleanup_finishes_event = true;
pidfd_send_signal(&old_cleanup.pidfd, libc::SIGKILL).expect("signal ESRCH old generation");
reap_tracee_pidfd_bounded(old_pid, &old_cleanup.pidfd)
.expect("pidfd-reap ESRCH old generation");
let (replacement_pid, replacement_cleanup) =
spawn_stopped_process(None).expect("spawn ESRCH replacement generation");
old_cleanup.pid = replacement_pid;
old_cleanup
.cleanup()
.expect("reaped pidfd cleanup avoids numeric replacement wait");
assert_eq!(
unsafe { libc::kill(replacement_pid.as_raw(), 0) },
0,
"ESRCH cleanup touched the projected replacement"
);
reap_stopped_process(replacement_cleanup);
}
fn pre_registration_cleanup_never_reaps_reused_pid(requested_pid: Option<i32>) -> bool {
let Some((old_pid, mut old_cleanup)) = spawn_stopped_process(requested_pid) else {
return false;
};
let captured = Arc::new(Barrier::new(2));
let resume = Arc::new(Barrier::new(2));
PRE_REGISTRATION_REAP_PAUSES.lock().insert(
old_pid.into(),
EventCapturePause {
captured: Arc::clone(&captured),
resume: Arc::clone(&resume),
},
);
let cleanup = thread::spawn(move || old_cleanup.cleanup());
captured.wait();
reap_tracee_bounded(old_pid).expect("competing owner reaps old pre-registration child");
thread::sleep(Duration::from_millis(20));
let Some((replacement_pid, replacement_cleanup)) = spawn_stopped_process(requested_pid)
else {
resume.wait();
let _ = cleanup.join();
return false;
};
resume.wait();
let cleanup_result = cleanup.join().expect("join pre-registration cleanup");
let replacement_live = unsafe { libc::kill(replacement_pid.as_raw(), 0) } == 0;
reap_stopped_process(replacement_cleanup);
cleanup_result.expect("exact old pidfd reap must not wait on replacement PID");
assert!(replacement_live, "old cleanup touched the replacement PID");
true
}
#[test]
fn pre_registration_cleanup_uses_exact_pidfd_after_competing_reap() {
const INNER: &str = "SAFEPTRACE_PRE_REG_REAP_REUSE_INNER";
if env::var_os(INNER).is_some() {
if pre_registration_cleanup_never_reaps_reused_pid(Some(100)) {
println!("ACTUAL_PRE_REG_REAP_REUSE_EXERCISED");
} else {
println!("ACTUAL_PRE_REG_REAP_REUSE_UNAVAILABLE");
}
return;
}
let inner =
"notifier::test::pre_registration_cleanup_uses_exact_pidfd_after_competing_reap";
let actual_reuse = run_exact_in_pid_namespace_bounded(inner, &[(INNER, "1")]);
match classify_exact_reuse_output(
actual_reuse.as_ref(),
"ACTUAL_PRE_REG_REAP_REUSE_EXERCISED",
"ACTUAL_PRE_REG_REAP_REUSE_UNAVAILABLE",
)
.unwrap_or_else(|error| panic!("actual pre-registration reuse regression failed: {error}"))
{
ExactReuseOutcome::Exercised => return,
ExactReuseOutcome::Unavailable => {}
}
assert!(pre_registration_cleanup_never_reaps_reused_pid(None));
}
#[test]
fn registration_claim_after_capture_excludes_raw_cleanup() {
let (old_pid, mut old_cleanup) =
spawn_stopped_process(None).expect("spawn delayed-registration old generation");
let running = Running::new(old_pid.into());
old_cleanup
.store_terminal(TerminalCleanup::new_unregistered(
old_pid.into(),
&running.1,
))
.expect("store delayed-registration terminal");
let delayed = TerminalCleanup::new_unregistered(old_pid.into(), &running.1);
let raw = TerminalCleanup::new_unregistered(old_pid.into(), &running.1);
let captured = Arc::new(Barrier::new(2));
let resume = Arc::new(Barrier::new(2));
EVENT_CAPTURE_PAUSES.lock().insert(
old_pid.into(),
EventCapturePause {
captured: Arc::clone(&captured),
resume: Arc::clone(&resume),
},
);
let registration = thread::spawn(move || delayed.ensure_registered());
captured.wait();
let (raw_tx, raw_rx) = mpsc::channel();
let raw_claim = thread::spawn(move || {
let claim = raw.try_claim_unstarted_raw_cleanup();
raw_tx.send(()).expect("report raw-claim completion");
claim
});
thread::sleep(Duration::from_millis(20));
assert!(matches!(raw_rx.try_recv(), Err(mpsc::TryRecvError::Empty)));
resume.wait();
registration
.join()
.expect("join delayed registration")
.expect("terminal delayed registration returns exact Event");
raw_rx
.recv_timeout(TRACEE_WAIT_TIMEOUT)
.expect("raw claim was not woken after registration publication");
assert!(matches!(
raw_claim.join().expect("join excluded raw claim"),
Ok(RawCleanupClaim::Lost)
));
old_cleanup
.cleanup()
.expect("notifier owner cleans delayed-registration child");
}
#[test]
fn worker_spawn_failure_rolls_back_registry_and_cleanup_retries() {
let (pid, mut cleanup) = spawn_stopped_process(None).expect("spawn worker-failure child");
let identity =
WorkerIdentity::capture_process(pid.into()).expect("capture worker-failure generation");
let running = Running::new(pid.into());
SPAWN_WORKER_ERRORS.lock().insert(pid.into(), libc::EAGAIN);
let error = cleanup
.bind_running_notifier(&running)
.expect_err("inject notifier worker spawn failure");
assert!(error.to_string().contains("EAGAIN"), "{error}");
let terminal = cleanup.terminal().expect("spawn-failure terminal");
assert_eq!(
terminal.event.event().worker_state.load(Ordering::Acquire),
WORKER_NOT_STARTED
);
assert!(
!NOTIFIER
.pids
.lock()
.get(&pid.into())
.is_some_and(|entry| entry.handle == terminal.event)
);
cleanup
.cleanup()
.expect("spawn-failure cleanup retries registration");
assert!(
!identity.is_same_process_generation(),
"worker spawn failure orphaned its child"
);
}
#[test]
fn echild_resolver_waits_for_starting_spawn_rollback() {
let (pid, cleanup) = spawn_stopped_process(None).expect("spawn STARTING rollback child");
let running = Running::new(pid.into());
let starter = TerminalCleanup::new_unregistered(pid.into(), &running.1);
let resolver = TerminalCleanup::new_unregistered(pid.into(), &running.1);
let spawn_paused = Arc::new(Barrier::new(2));
let resume_spawn = Arc::new(Barrier::new(2));
SPAWN_WORKER_ERRORS.lock().insert(pid.into(), libc::EAGAIN);
SPAWN_FAILURE_PAUSES.lock().insert(
pid.into(),
EventCapturePause {
captured: Arc::clone(&spawn_paused),
resume: Arc::clone(&resume_spawn),
},
);
let starting = thread::spawn(move || starter.ensure_registered());
spawn_paused.wait();
let resolver_started = Arc::new(Barrier::new(2));
let resolver_entered = Arc::clone(&resolver_started);
let resolving = thread::spawn(move || {
inject_capture_error_for_current_thread(Errno::ENOENT);
resolver_entered.wait();
resolver.ensure_registered()
});
resolver_started.wait();
thread::sleep(Duration::from_millis(20));
resume_spawn.wait();
assert_eq!(
starting.join().expect("join failed STARTING spawn"),
Err(Errno::EAGAIN)
);
resolving
.join()
.expect("join STARTING ECHILD resolver")
.expect("resolver completes rolled-back Event");
assert_eq!(
running
.1
.event()
.event()
.worker_state
.load(Ordering::Acquire),
WORKER_DONE
);
assert!(!NOTIFIER.pids.lock().contains_key(&pid.into()));
reap_stopped_process(cleanup);
}
#[test]
fn delayed_current_or_new_cannot_overwrite_replacement_entry() {
let (old_pid, old_cleanup) =
spawn_stopped_process(None).expect("spawn delayed lookup old generation");
let captured = Arc::new(Barrier::new(2));
let resume = Arc::new(Barrier::new(2));
CURRENT_OR_NEW_CAPTURE_PAUSES.lock().insert(
old_pid.into(),
EventCapturePause {
captured: Arc::clone(&captured),
resume: Arc::clone(&resume),
},
);
let lookup = thread::spawn(move || EventHandle::current_or_new(old_pid.into()));
captured.wait();
reap_stopped_process(old_cleanup);
let (replacement_pid, replacement_cleanup) =
spawn_stopped_process(None).expect("spawn delayed lookup replacement");
let mut replacement_identity = WorkerIdentity::capture_process(replacement_pid.into())
.expect("capture delayed lookup replacement");
replacement_identity.pid = old_pid.into();
let replacement_identity = Arc::new(replacement_identity);
let replacement_handle = EventHandle::with_identity(Arc::clone(&replacement_identity));
assert!(replacement_handle.event().try_begin_worker_start());
replacement_handle.event().mark_worker_running();
NOTIFIER.pids.lock().insert(
old_pid.into(),
NotifierEntry {
handle: replacement_handle.clone(),
identity: replacement_identity,
},
);
resume.wait();
assert!(lookup.join().expect("join delayed current_or_new").is_err());
assert!(
NOTIFIER
.pids
.lock()
.get(&old_pid.into())
.is_some_and(|entry| entry.handle == replacement_handle)
);
assert_eq!(
replacement_handle
.event()
.worker_state
.load(Ordering::Acquire),
WORKER_RUNNING
);
NOTIFIER.pids.lock().remove(&old_pid.into());
replacement_handle.event().mark_echild();
replacement_handle.event().mark_worker_done();
reap_stopped_process(replacement_cleanup);
}
fn delayed_current_or_new_revalidates_after_pid_reuse(requested_pid: Option<i32>) -> bool {
let Some((old_pid, old_cleanup)) = spawn_stopped_process(requested_pid) else {
return false;
};
let captured = Arc::new(Barrier::new(2));
let resume = Arc::new(Barrier::new(2));
CURRENT_OR_NEW_CAPTURE_PAUSES.lock().insert(
old_pid.into(),
EventCapturePause {
captured: Arc::clone(&captured),
resume: Arc::clone(&resume),
},
);
let lookup = thread::spawn(move || EventHandle::current_or_new(old_pid.into()));
captured.wait();
reap_stopped_process(old_cleanup);
thread::sleep(Duration::from_millis(20));
let Some((replacement_pid, replacement_cleanup)) = spawn_stopped_process(requested_pid)
else {
resume.wait();
let _ = lookup.join();
return false;
};
resume.wait();
let selected = lookup.join().expect("join post-validation lookup");
if replacement_pid == old_pid {
let selected = selected.expect("recapture exact reused PID generation");
assert!(
selected
.identity()
.is_some_and(|identity| identity.is_same_process_generation()),
"post-validation lookup returned the reaped PID generation"
);
} else {
assert_eq!(
selected.expect_err("reaped PID lookup must not return a stale handle"),
Errno::ENOENT
);
}
assert_eq!(
unsafe { libc::kill(replacement_pid.as_raw(), 0) },
0,
"stale lookup touched the replacement generation"
);
reap_stopped_process(replacement_cleanup);
true
}
#[test]
fn current_or_new_linearizes_after_validation_before_pid_reuse() {
const INNER: &str = "SAFEPTRACE_LOOKUP_REUSE_INNER";
if env::var_os(INNER).is_some() {
if delayed_current_or_new_revalidates_after_pid_reuse(Some(100)) {
println!("ACTUAL_LOOKUP_PID_REUSE_EXERCISED");
} else {
println!("ACTUAL_LOOKUP_PID_REUSE_UNAVAILABLE");
}
return;
}
let inner = "notifier::test::current_or_new_linearizes_after_validation_before_pid_reuse";
let actual_reuse = run_exact_in_pid_namespace_bounded(inner, &[(INNER, "1")]);
match classify_exact_reuse_output(
actual_reuse.as_ref(),
"ACTUAL_LOOKUP_PID_REUSE_EXERCISED",
"ACTUAL_LOOKUP_PID_REUSE_UNAVAILABLE",
)
.unwrap_or_else(|error| panic!("actual lookup PID-reuse regression failed: {error}"))
{
ExactReuseOutcome::Exercised => return,
ExactReuseOutcome::Unavailable => {}
}
assert!(delayed_current_or_new_revalidates_after_pid_reuse(None));
}
fn delayed_event_never_rebinds_after_pid_reuse(requested_pid: Option<i32>) -> bool {
let Some((old_pid, old_cleanup)) = spawn_stopped_process(requested_pid) else {
return false;
};
let running = Running::new(old_pid.into());
let terminal = TerminalCleanup::new_unregistered(old_pid.into(), &running.1);
let captured = Arc::new(Barrier::new(2));
let resume = Arc::new(Barrier::new(2));
EVENT_CAPTURE_PAUSES.lock().insert(
old_pid.into(),
EventCapturePause {
captured: Arc::clone(&captured),
resume: Arc::clone(&resume),
},
);
let registration = thread::spawn(move || {
terminal.ensure_registered()?;
Ok::<_, Errno>(terminal)
});
captured.wait();
reap_stopped_process(old_cleanup);
thread::sleep(Duration::from_millis(20));
let Some((replacement_pid, replacement_cleanup)) = spawn_stopped_process(requested_pid)
else {
resume.wait();
let _ = registration.join();
return false;
};
resume.wait();
let terminal = registration
.join()
.expect("join post-validation registration")
.expect("stale registration resolves its bound Event");
let completed_without_replacement = terminal.wait(Duration::from_millis(50));
let replacement_live = unsafe { libc::kill(replacement_pid.as_raw(), 0) } == 0;
reap_stopped_process(replacement_cleanup);
assert!(
completed_without_replacement,
"stale registration rebound its Event to the replacement generation"
);
assert!(
replacement_live,
"stale registration touched the replacement"
);
assert!(!NOTIFIER.pids.lock().contains_key(&old_pid.into()));
true
}
#[test]
fn event_linearizes_after_validation_before_pid_reuse() {
const INNER: &str = "SAFEPTRACE_EVENT_REUSE_INNER";
if env::var_os(INNER).is_some() {
if delayed_event_never_rebinds_after_pid_reuse(Some(100)) {
println!("ACTUAL_EVENT_PID_REUSE_EXERCISED");
} else {
println!("ACTUAL_EVENT_PID_REUSE_UNAVAILABLE");
}
return;
}
let inner = "notifier::test::event_linearizes_after_validation_before_pid_reuse";
let actual_reuse = run_exact_in_pid_namespace_bounded(inner, &[(INNER, "1")]);
match classify_exact_reuse_output(
actual_reuse.as_ref(),
"ACTUAL_EVENT_PID_REUSE_EXERCISED",
"ACTUAL_EVENT_PID_REUSE_UNAVAILABLE",
)
.unwrap_or_else(|error| panic!("actual event PID-reuse regression failed: {error}"))
{
ExactReuseOutcome::Exercised => return,
ExactReuseOutcome::Unavailable => {}
}
assert!(delayed_event_never_rebinds_after_pid_reuse(None));
}
#[test]
fn distinct_event_guard_uses_live_registry_worker() {
let (pid, mut cleanup) =
spawn_stopped_process(None).expect("spawn distinct-Event cleanup child");
let identity = WorkerIdentity::capture_process(pid.into())
.expect("capture distinct-Event cleanup generation");
let authoritative_running = Running::new(pid.into());
let authoritative = authoritative_running.terminal_cleanup();
let competing_running = distinct_running(pid.into());
cleanup
.store_terminal(TerminalCleanup::new_unregistered(
pid.into(),
&competing_running.1,
))
.expect("store competing cleanup Event");
assert!(
!cleanup
.terminal()
.expect("competing terminal")
.same_generation(&authoritative)
);
assert_eq!(
SPAWN_WORKER_COUNTS.lock().get(&pid.into()).copied(),
Some(1),
"authoritative registration starts exactly one worker"
);
cleanup
.cleanup()
.expect("competing guard adopts authoritative worker");
assert!(
cleanup
.terminal()
.expect("adopted authoritative terminal")
.same_generation(&authoritative)
);
assert!(matches!(
cleanup.ownership,
TraceeCleanupOwnership::NotifierOwned {
raw_cleanup_claimed: false,
..
}
));
assert_eq!(
SPAWN_WORKER_COUNTS.lock().remove(&pid.into()),
Some(1),
"competing Event must not start a second worker"
);
assert!(
!identity.is_same_process_generation(),
"competing Event raw-waited alongside the registry worker"
);
}
#[test]
fn registry_replacement_does_not_terminalize_running_old_event() {
let (pid, child_cleanup) =
spawn_stopped_process(None).expect("spawn old active registry tracee");
let identity = Arc::new(
WorkerIdentity::capture_process(pid.into()).expect("capture old active generation"),
);
let old_handle = EventHandle::with_identity(Arc::clone(&identity));
assert!(old_handle.event().try_begin_worker_start());
old_handle.event().mark_worker_running();
NOTIFIER.pids.lock().insert(
pid.into(),
NotifierEntry {
handle: old_handle.clone(),
identity: Arc::clone(&identity),
},
);
let replacement = EventHandle::with_identity(Arc::clone(&identity));
NOTIFIER.pids.lock().insert(
pid.into(),
NotifierEntry {
handle: replacement,
identity,
},
);
NOTIFIER.resolve_echild(pid.into(), &old_handle);
let terminal = old_handle.event().status.lock().terminal;
NOTIFIER.pids.lock().remove(&pid.into());
reap_stopped_process(child_cleanup);
assert_eq!(
terminal, INVALID_STATUS,
"registry replacement terminalized an Event with its own active worker"
);
}
#[test]
fn running_old_event_bypasses_replacement_capture_failure() {
let (pid, child_cleanup) =
spawn_stopped_process(None).expect("spawn old event fast-path tracee");
let identity = Arc::new(
WorkerIdentity::capture_process(pid.into()).expect("capture old event generation"),
);
let old_handle = EventHandle::with_identity(Arc::clone(&identity));
assert!(old_handle.event().try_begin_worker_start());
old_handle.event().mark_worker_running();
let replacement = EventHandle::with_identity(Arc::clone(&identity));
NOTIFIER.pids.lock().insert(
pid.into(),
NotifierEntry {
handle: replacement,
identity,
},
);
inject_capture_error_for_current_thread(Errno::EMFILE);
let selected = NOTIFIER
.event(pid.into(), &old_handle)
.expect("running old Event must not recapture replacement identity");
assert!(matches!(
NOTIFIER.capture_identity(pid.into()),
Err(Errno::EMFILE)
));
NOTIFIER.pids.lock().remove(&pid.into());
reap_stopped_process(child_cleanup);
assert!(
Arc::ptr_eq(selected.event(), old_handle.event()),
"replacement registry entry displaced the old Event worker"
);
}
#[test]
fn registered_exit_stop_survives_procfs_disappearance_until_final_status() {
let (pid, child_cleanup) =
spawn_stopped_process(None).expect("spawn exiting notifier tracee");
let identity = Arc::new(
WorkerIdentity::capture_process(pid.into()).expect("capture notifier generation"),
);
let handle = EventHandle::with_identity(Arc::clone(&identity));
handle.event().update(PTRACE_EVENT_EXIT_STOP);
assert!(handle.event().try_begin_worker_start());
handle.event().mark_worker_running();
NOTIFIER.pids.lock().insert(
pid.into(),
NotifierEntry {
handle: handle.clone(),
identity,
},
);
reap_stopped_process(child_cleanup);
NOTIFIER
.event(pid.into(), &handle)
.expect("reuse exact registered notifier generation");
let terminal = Signal::SIGKILL as i32;
handle.event().update(terminal);
let waker = Waker::from(Arc::new(WakeCounter::default()));
let observed = handle.event().poll_status(&waker);
NOTIFIER.pids.lock().remove(&pid.into());
assert_eq!(
observed,
Poll::Ready(Ok(terminal)),
"procfs disappearance replaced the old worker's final status with ECHILD"
);
}
#[tokio::test(flavor = "current_thread")]
async fn pre_traceme_snapshot_does_not_synthesize_echild_for_live_generation() {
let (pid, _stopped, mut cleanup) =
spawn_traced_process(None).expect("spawn pre-TRACEME snapshot tracee");
let mut identity =
WorkerIdentity::capture_process(pid.into()).expect("capture live traced generation");
assert!(tracer_is_current(identity.snapshot.tracer_pid));
identity.snapshot.tracer_pid = crate::Pid::from_raw(0);
assert!(
identity.is_active_tracee(),
"mutable TracerPid transition changed immutable process generation"
);
let handle = EventHandle::with_identity(Arc::new(identity));
cleanup
.store_terminal(TerminalCleanup {
pid: pid.into(),
event: handle.clone(),
})
.expect("store pre-TRACEME Event generation");
let stopped = Stopped::from_token(pid.into(), TraceeToken::from_event(handle));
let exited = stopped
.resume(None)
.expect("resume pre-TRACEME snapshot tracee")
.next_state()
.await
.expect("live generation became synthetic ECHILD");
assert_eq!(exited.assume_exited().1, crate::ExitStatus::Exited(42));
cleanup.disarm();
}
#[test]
fn capture_accepts_tracer_attachment_during_identity_snapshot() {
let (pid, mut cleanup) =
spawn_stopped_process(None).expect("spawn capture-transition child");
let first_snapshot = Arc::new(Barrier::new(2));
let resume_capture = Arc::new(Barrier::new(2));
CAPTURE_AFTER_FIRST_SNAPSHOT_PAUSES.lock().insert(
pid.into(),
EventCapturePause {
captured: Arc::clone(&first_snapshot),
resume: Arc::clone(&resume_capture),
},
);
let capturing = thread::spawn(move || WorkerIdentity::capture_process(pid.into()));
first_snapshot.wait();
nix::sys::ptrace::seize(pid, Options::empty())
.expect("attach tracer between identity snapshots");
resume_capture.wait();
let identity = capturing
.join()
.expect("join capture-transition thread")
.expect("TracerPid transition changed process generation");
assert!(
tracer_is_current(identity.snapshot.tracer_pid),
"capture did not normalize to the post-attachment TracerPid"
);
cleanup.cleanup().expect("cleanup capture-transition child");
}
fn retained_nonterminal_rejects_replacement(requested_pid: Option<i32>) -> bool {
let Some((old_pid, mut old_cleanup)) = spawn_stopped_process(requested_pid) else {
return false;
};
let old_running = distinct_running(old_pid.into());
let old_handle = old_running.1.event().clone();
old_handle
.bind_identity(Arc::new(
WorkerIdentity::capture_process(old_pid.into())
.expect("capture retained-stop old generation"),
))
.expect("bind retained-stop old identity");
old_handle.event().update((libc::SIGSTOP << 8) | 0x7f);
old_cleanup
.cleanup()
.expect("reap retained-stop old generation");
let Some((replacement_pid, replacement_cleanup)) = spawn_stopped_process(requested_pid)
else {
return false;
};
let projected = Running::from_token(replacement_pid.into(), old_running.1);
assert!(
matches!(
projected.wait(),
Err(Error::Errno(Errno::ECHILD | Errno::ESRCH))
),
"old nonterminal FIFO status replayed as a replacement-PID capability"
);
reap_stopped_process(replacement_cleanup);
true
}
#[test]
fn retained_nonterminal_is_not_replayed_for_pid_reuse_projection() {
const INNER: &str = "SAFEPTRACE_RETAINED_STOP_REUSE_INNER";
if env::var_os(INNER).is_some() {
if retained_nonterminal_rejects_replacement(Some(100)) {
println!("ACTUAL_RETAINED_STOP_REUSE_EXERCISED");
} else {
println!("ACTUAL_RETAINED_STOP_REUSE_UNAVAILABLE");
}
return;
}
let inner = "notifier::test::retained_nonterminal_is_not_replayed_for_pid_reuse_projection";
let actual_reuse = run_exact_in_pid_namespace_bounded(inner, &[(INNER, "1")]);
match classify_exact_reuse_output(
actual_reuse.as_ref(),
"ACTUAL_RETAINED_STOP_REUSE_EXERCISED",
"ACTUAL_RETAINED_STOP_REUSE_UNAVAILABLE",
)
.unwrap_or_else(|error| panic!("actual retained-stop reuse regression failed: {error}"))
{
ExactReuseOutcome::Exercised => return,
ExactReuseOutcome::Unavailable => {}
}
assert!(retained_nonterminal_rejects_replacement(None));
}
struct NewChildReturnRace {
pid: Pid,
running: Option<Running>,
parent_identity: WorkerIdentity,
coordinator: JoinHandle<(io::Result<()>, TraceeCleanupGuard)>,
exact_cleanup: ExactNewChildCleanupGuard,
}
struct ExactNewChildCleanupGuard {
projected_pgid: Pid,
event: EventHandle,
members: Arc<Mutex<Vec<ExactTestMember>>>,
registration_error: Arc<Mutex<Option<PendingExactRegistration>>>,
pidfd_open_error: Arc<Mutex<Option<Errno>>>,
signal_errors: HashMap<crate::Pid, VecDeque<Errno>>,
reap_errors: HashMap<crate::Pid, VecDeque<Errno>>,
armed: bool,
}
impl ExactNewChildCleanupGuard {
const DROP_CLEANUP_ATTEMPTS: usize = 3;
fn new(projected_pgid: Pid, event: EventHandle) -> io::Result<Self> {
let leader: crate::Pid = projected_pgid.into();
let pidfd = open_thread_pidfd(leader).map_err(|error| {
io::Error::other(format!(
"open exact fixture leader {projected_pgid} pidfd: {error}"
))
})?;
Ok(Self {
projected_pgid,
event,
members: Arc::new(Mutex::new(vec![ExactTestMember {
pid: leader,
pidfd,
phase: ExactCleanupPhase::NeedsSignal,
}])),
registration_error: Arc::new(Mutex::new(None)),
pidfd_open_error: Arc::new(Mutex::new(None)),
signal_errors: HashMap::new(),
reap_errors: HashMap::new(),
armed: true,
})
}
fn disarm_completed(&mut self) -> io::Result<()> {
if let Some(pending) = *self.registration_error.lock() {
return Err(io::Error::other(format!(
"cannot disarm with pending exact registration for {}: {}",
pending.pid, pending.error
)));
}
let members = self.members.lock();
for member in members.iter() {
if !pidfd_exited(&member.pidfd)? {
return Err(io::Error::other(format!(
"cannot disarm live exact member {} in phase {:?}",
member.pid, member.phase
)));
}
}
drop(members);
self.members.lock().clear();
self.armed = false;
Ok(())
}
fn inject_signal_error(&mut self, pid: crate::Pid, error: Errno) {
self.signal_errors.entry(pid).or_default().push_back(error);
}
fn inject_reap_error(&mut self, pid: crate::Pid, error: Errno) {
self.reap_errors.entry(pid).or_default().push_back(error);
}
fn inject_pidfd_open_error(&self, error: Errno) {
*self.pidfd_open_error.lock() = Some(error);
}
fn install_new_child_decode_pause(
&self,
captured: mpsc::SyncSender<()>,
resume: mpsc::Receiver<()>,
) {
*self.event.event().new_child_decode_pause.lock() = Some(NewChildDecodePause {
captured,
resume,
members: Arc::clone(&self.members),
registration_error: Arc::clone(&self.registration_error),
pidfd_open_error: Arc::clone(&self.pidfd_open_error),
});
}
fn take_injected_error(
errors: &mut HashMap<crate::Pid, VecDeque<Errno>>,
pid: crate::Pid,
) -> Option<Errno> {
let error = errors.get_mut(&pid).and_then(VecDeque::pop_front);
if errors.get(&pid).is_some_and(VecDeque::is_empty) {
errors.remove(&pid);
}
error
}
fn cleanup_error(&self, failures: &[String]) -> io::Error {
let retained = self
.members
.lock()
.iter()
.map(|member| format!("{}:{:?}", member.pid, member.phase))
.collect::<Vec<_>>()
.join(", ");
let pending = (*self.registration_error.lock())
.map(|pending| format!("{}:{}", pending.pid, pending.error))
.unwrap_or_else(|| "none".to_owned());
io::Error::other(format!(
"exact fixture cleanup for projected PGID {} failed; retained=[{}]; pending_registration={}; {}",
self.projected_pgid,
retained,
pending,
failures.join("; ")
))
}
fn cleanup(&mut self) -> io::Result<()> {
if !self.armed {
return Ok(());
}
self.event.event().new_child_decode_pause.lock().take();
self.event.event().cleanup_return_pause.lock().take();
let mut failures = Vec::new();
let pending_registration = *self.registration_error.lock();
if let Some(pending) = pending_registration {
match open_thread_pidfd(pending.pid) {
Ok(pidfd) => {
self.members.lock().push(ExactTestMember {
pid: pending.pid,
pidfd,
phase: ExactCleanupPhase::NeedsSignal,
});
self.registration_error.lock().take();
}
Err(error) => {
*self.registration_error.lock() = Some(PendingExactRegistration {
pid: pending.pid,
error,
});
failures.push(format!(
"retry NewChild exact pidfd registration for {} after {}: {}",
pending.pid, pending.error, error
));
}
}
}
let registration_retry_failed = self.registration_error.lock().is_some();
if registration_retry_failed {
return Err(self.cleanup_error(&failures));
}
let members = mem::take(&mut *self.members.lock());
let mut retained = Vec::new();
for mut member in members {
if member.phase == ExactCleanupPhase::NeedsSignal {
let signal = Self::take_injected_error(&mut self.signal_errors, member.pid)
.map_or_else(
|| pidfd_send_signal(&member.pidfd, libc::SIGKILL),
|error| Err(io::Error::from_raw_os_error(error.into_raw())),
);
match signal {
Ok(()) => member.phase = ExactCleanupPhase::NeedsReap,
Err(error) if error.raw_os_error() == Some(libc::ESRCH) => {
member.phase = ExactCleanupPhase::NeedsReap;
}
Err(error) => {
failures.push(format!(
"signal exact member {} through pidfd: {error}",
member.pid
));
retained.push(member);
continue;
}
}
}
let reap = Self::take_injected_error(&mut self.reap_errors, member.pid)
.map_or_else(
|| {
let pid = Pid::from_raw(member.pid.as_raw());
reap_tracee_pidfd_bounded(pid, &member.pidfd)
},
|error| Err(io::Error::from_raw_os_error(error.into_raw())),
);
if let Err(error) = reap {
failures.push(format!("reap exact member {}: {error}", member.pid));
retained.push(member);
}
}
self.members.lock().extend(retained);
if failures.is_empty()
&& self.registration_error.lock().is_none()
&& self.members.lock().is_empty()
{
self.armed = false;
Ok(())
} else {
Err(self.cleanup_error(&failures))
}
}
fn emergency_cleanup_without_injection(&mut self) -> io::Result<()> {
if !self.armed {
return Ok(());
}
self.event.event().new_child_decode_pause.lock().take();
self.event.event().cleanup_return_pause.lock().take();
let mut failures = Vec::new();
if let Some(pending) = *self.registration_error.lock() {
match open_thread_pidfd_kernel(pending.pid) {
Ok(pidfd) => {
self.members.lock().push(ExactTestMember {
pid: pending.pid,
pidfd,
phase: ExactCleanupPhase::NeedsSignal,
});
self.registration_error.lock().take();
}
Err(error) => {
*self.registration_error.lock() = Some(PendingExactRegistration {
pid: pending.pid,
error,
});
failures.push(format!(
"emergency exact pidfd registration for {} after {}: {}",
pending.pid, pending.error, error
));
}
}
}
let registration_failed = self.registration_error.lock().is_some();
if registration_failed {
return Err(self.cleanup_error(&failures));
}
let members = mem::take(&mut *self.members.lock());
let mut retained = Vec::new();
for member in members {
let signal = pidfd_send_signal(&member.pidfd, libc::SIGKILL);
let signal_succeeded = match signal {
Ok(()) => true,
Err(error) if error.raw_os_error() == Some(libc::ESRCH) => true,
Err(error) => {
failures.push(format!(
"emergency signal exact member {} through pidfd: {error}",
member.pid
));
false
}
};
let pid = Pid::from_raw(member.pid.as_raw());
match reap_tracee_pidfd_bounded(pid, &member.pidfd) {
Ok(()) if signal_succeeded => {}
Ok(()) => {
failures.retain(|failure| {
!failure.starts_with(&format!(
"emergency signal exact member {} ",
member.pid
))
});
}
Err(error) => {
failures.push(format!(
"emergency reap exact member {}: {error}",
member.pid
));
retained.push(member);
}
}
}
self.members.lock().extend(retained);
if self.members.lock().is_empty()
&& self.registration_error.lock().is_none()
&& failures.is_empty()
{
self.armed = false;
Ok(())
} else {
Err(self.cleanup_error(&failures))
}
}
}
impl Drop for ExactNewChildCleanupGuard {
fn drop(&mut self) {
let mut retry_failures = Vec::new();
for attempt in 1..=Self::DROP_CLEANUP_ATTEMPTS {
match self.cleanup() {
Ok(()) => return,
Err(error) => retry_failures.push(format!("attempt {attempt}: {error}")),
}
if attempt < Self::DROP_CLEANUP_ATTEMPTS {
thread::sleep(SUBPROCESS_POLL_INTERVAL);
}
}
match self.emergency_cleanup_without_injection() {
Ok(()) => return,
Err(error) => retry_failures.push(format!("emergency: {error}")),
}
let terminal = self.cleanup_error(&retry_failures);
eprintln!(
"NewChild fixture exact cleanup remained armed after bounded retries and emergency cleanup: {terminal}"
);
std::process::abort();
}
}
fn spawn_exact_cleanup_member() -> (Pid, WorkerIdentity, ExactNewChildCleanupGuard) {
let mut command = Command::new("/bin/sleep");
command
.arg("60")
.stdin(Stdio::null())
.stdout(Stdio::null())
.stderr(Stdio::null());
unsafe {
command.pre_exec(|| {
if libc::prctl(libc::PR_SET_PDEATHSIG, libc::SIGKILL) == -1 {
Err(io::Error::last_os_error())
} else {
Ok(())
}
});
}
let mut child = crate::test_fork::spawn(&mut command).expect("spawn exact-cleanup member");
let pid = Pid::from_raw(child.id() as i32);
let mut guard =
ExactNewChildCleanupGuard::new(pid, EventHandle::new()).unwrap_or_else(|error| {
let _ = child.kill();
let _ = child.wait();
panic!("open exact-cleanup member guard: {error}");
});
let identity = WorkerIdentity::capture_process(pid.into()).unwrap_or_else(|error| {
guard.cleanup().unwrap_or_else(|cleanup_error| {
panic!("capture exact-cleanup member identity: {error}; cleanup: {cleanup_error}")
});
panic!("capture exact-cleanup member identity: {error}");
});
drop(child);
(pid, identity, guard)
}
#[test]
fn exact_new_child_cleanup_signal_failure_retains_authority_for_retry() {
let (pid, identity, mut guard) = spawn_exact_cleanup_member();
guard.inject_signal_error(pid.into(), Errno::EIO);
let error = guard
.cleanup()
.expect_err("injected exact signal failure must fail cleanup");
assert!(error.to_string().contains("signal exact member"), "{error}");
assert!(guard.armed);
{
let members = guard.members.lock();
assert_eq!(members.len(), 1);
assert_eq!(members[0].pid, pid.into());
assert_eq!(members[0].phase, ExactCleanupPhase::NeedsSignal);
assert!(
!pidfd_exited(&members[0].pidfd).expect("poll retained exact member"),
"signal failure unexpectedly lost the live member"
);
}
guard.cleanup().expect("retry exact signal cleanup");
assert!(!guard.armed);
assert!(guard.members.lock().is_empty());
wait_generation_retired_bounded(&identity, "retried exact signal cleanup")
.expect("no orphan after exact signal retry");
}
#[test]
fn exact_new_child_cleanup_reap_failure_resumes_without_duplicate_signal() {
let (pid, identity, mut guard) = spawn_exact_cleanup_member();
guard.inject_reap_error(pid.into(), Errno::EIO);
let error = guard
.cleanup()
.expect_err("injected exact reap failure must fail cleanup");
assert!(error.to_string().contains("reap exact member"), "{error}");
assert!(guard.armed);
{
let members = guard.members.lock();
assert_eq!(members.len(), 1);
assert_eq!(members[0].pid, pid.into());
assert_eq!(members[0].phase, ExactCleanupPhase::NeedsReap);
}
guard.inject_signal_error(pid.into(), Errno::EIO);
guard.cleanup().expect("retry exact reap cleanup");
assert!(!guard.armed);
assert!(guard.members.lock().is_empty());
wait_generation_retired_bounded(&identity, "retried exact reap cleanup")
.expect("no orphan after exact reap retry");
}
fn assert_unwind_retries_exact_member_cleanup(
inject: impl FnOnce(&mut ExactNewChildCleanupGuard, crate::Pid),
context: &str,
) {
let (pid, identity, mut guard) = spawn_exact_cleanup_member();
inject(&mut guard, pid.into());
let unwind = std::panic::catch_unwind(std::panic::AssertUnwindSafe(move || {
let _guard = guard;
panic!("exercise exact cleanup Drop during unwind");
}));
assert!(unwind.is_err(), "fixture panic unexpectedly returned");
wait_generation_retired_bounded(&identity, context)
.unwrap_or_else(|error| panic!("{context} left an exact member alive: {error}"));
}
#[test]
fn exact_cleanup_drop_retries_one_shot_signal_failure_during_unwind() {
assert_unwind_retries_exact_member_cleanup(
|guard, pid| guard.inject_signal_error(pid, Errno::EIO),
"unwind retry after exact signal failure",
);
}
#[test]
fn exact_cleanup_drop_retries_one_shot_reap_failure_during_unwind() {
assert_unwind_retries_exact_member_cleanup(
|guard, pid| guard.inject_reap_error(pid, Errno::EIO),
"unwind retry after exact reap failure",
);
}
#[test]
fn exact_cleanup_drop_emergency_bypasses_persistent_test_signal_failures() {
assert_unwind_retries_exact_member_cleanup(
|guard, pid| {
for _ in 0..ExactNewChildCleanupGuard::DROP_CLEANUP_ATTEMPTS {
guard.inject_signal_error(pid, Errno::EIO);
}
},
"uninjected emergency cleanup after persistent test signal failures",
);
}
#[test]
fn exact_cleanup_drop_retries_registration_failure_during_unwind() {
let child_op = crate::ChildOp::Clone;
let (pid, stopped, mut cleanup) =
spawn_new_child_tracee(child_op).expect("spawn unwind registration-failure tracee");
let parent_identity = WorkerIdentity::capture_process(pid.into())
.expect("capture unwind registration-failure parent");
let parent_event = stopped.1.event().clone();
let exact_cleanup = ExactNewChildCleanupGuard::new(pid, parent_event)
.expect("construct unwind exact guard while parent is stopped");
let (captured_tx, captured_rx) = mpsc::sync_channel(1);
let (_resume_tx, resume_rx) = mpsc::channel();
exact_cleanup.inject_pidfd_open_error(Errno::EMFILE);
exact_cleanup.install_new_child_decode_pause(captured_tx, resume_rx);
let running = stopped
.resume(None)
.expect("resume unwind registration-failure parent after guard install");
cleanup
.store_terminal(TerminalCleanup::new_unregistered(pid.into(), &running.1))
.expect("bind unwind registration-failure parent cleanup");
let error = running
.wait()
.expect_err("pidfd-open injection must abort NewChild materialization");
assert!(matches!(error, Error::Errno(Errno::EMFILE)), "{error:?}");
assert!(captured_rx.try_recv().is_err());
let pending = (*exact_cleanup.registration_error.lock())
.expect("unwind registration failure must remain retryable");
let child_pid = pending.pid;
let child_identity = WorkerIdentity::capture_process(child_pid)
.expect("capture unwind registration-failure child");
PIDFD_OPEN_ERRORS.lock().insert(child_pid, Errno::ENFILE);
let unwind = std::panic::catch_unwind(std::panic::AssertUnwindSafe(move || {
let _exact_cleanup = exact_cleanup;
panic!("exercise registration retry during exact cleanup Drop");
}));
assert!(unwind.is_err(), "fixture panic unexpectedly returned");
cleanup.disarm();
wait_generation_retired_bounded(
&parent_identity,
"unwind retry after parent registration failure",
)
.expect("registration unwind retry left parent alive");
wait_generation_retired_bounded(
&child_identity,
&format!("unwind retry after child {child_pid} registration failure"),
)
.expect("registration unwind retry left child alive");
}
#[test]
fn new_child_pidfd_open_failure_precedes_materialization_and_cleanup_retries() {
let child_op = crate::ChildOp::Clone;
let (pid, stopped, mut cleanup) =
spawn_new_child_tracee(child_op).expect("spawn pidfd-open failure tracee");
let parent_identity = WorkerIdentity::capture_process(pid.into())
.expect("capture pidfd-open failure parent identity");
let parent_event = stopped.1.event().clone();
let mut exact_cleanup = ExactNewChildCleanupGuard::new(pid, parent_event)
.expect("construct exact guard while parent is stopped");
let (captured_tx, captured_rx) = mpsc::sync_channel(1);
let (_resume_tx, resume_rx) = mpsc::channel();
exact_cleanup.inject_pidfd_open_error(Errno::EMFILE);
exact_cleanup.install_new_child_decode_pause(captured_tx, resume_rx);
let running = stopped
.resume(None)
.expect("resume pidfd-open failure parent after guard install");
cleanup
.store_terminal(TerminalCleanup::new_unregistered(pid.into(), &running.1))
.expect("bind pidfd-open failure parent cleanup");
let error = running
.wait()
.expect_err("pidfd-open failure must abort NewChild decoding");
assert!(matches!(error, Error::Errno(Errno::EMFILE)), "{error:?}");
assert!(
captured_rx.try_recv().is_err(),
"NewChild materialization pause ran after exact registration failed"
);
let pending = (*exact_cleanup.registration_error.lock())
.expect("failed child registration must remain retryable");
assert_eq!(pending.error, Errno::EMFILE);
assert_ne!(pending.pid.as_raw(), pid.as_raw());
let child_pid = pending.pid;
let child_identity = WorkerIdentity::capture_process(child_pid)
.expect("capture failed-registration child identity before cleanup retry");
PIDFD_OPEN_ERRORS.lock().insert(child_pid, Errno::ENFILE);
let retry_error = exact_cleanup
.cleanup()
.expect_err("second pidfd-open failure must retain the stopped parent authority");
assert!(
retry_error
.to_string()
.contains("retry NewChild exact pidfd registration"),
"{retry_error}"
);
assert!(exact_cleanup.armed);
assert!(parent_identity.is_same_process_generation());
assert!(child_identity.is_same_process_generation());
assert_eq!(
(*exact_cleanup.registration_error.lock())
.expect("failed retry remains registered")
.error,
Errno::ENFILE
);
{
let members = exact_cleanup.members.lock();
assert_eq!(members.len(), 1);
assert_eq!(members[0].pid, pid.into());
assert_eq!(members[0].phase, ExactCleanupPhase::NeedsSignal);
}
exact_cleanup
.cleanup()
.expect("exact cleanup retries child pidfd registration");
assert!(!exact_cleanup.armed);
assert!(exact_cleanup.members.lock().is_empty());
assert!(exact_cleanup.registration_error.lock().is_none());
cleanup.disarm();
wait_generation_retired_bounded(&parent_identity, "pidfd-open failure parent")
.expect("no parent orphan after pidfd-open registration retry");
wait_generation_retired_bounded(
&child_identity,
&format!("pidfd-open failure child {child_pid}"),
)
.expect("no child orphan after pidfd-open registration retry");
}
fn start_new_child_return_race(
child_op: crate::ChildOp,
expected_return_owner: u8,
) -> io::Result<NewChildReturnRace> {
let (pid, stopped, mut cleanup) = spawn_new_child_tracee(child_op)?;
let parent_identity = WorkerIdentity::capture_process(pid.into()).map_err(|error| {
io::Error::other(format!(
"capture {child_op:?} parent {pid} identity: {error}"
))
})?;
let parent_event = stopped.1.event().clone();
let exact_cleanup = ExactNewChildCleanupGuard::new(pid, parent_event.clone())?;
let (child_materialized_tx, child_materialized_rx) = mpsc::sync_channel(0);
let (resume_return_tx, resume_return_rx) = mpsc::channel();
exact_cleanup.install_new_child_decode_pause(child_materialized_tx, resume_return_rx);
let running = stopped
.resume(None)
.map_err(|error| io::Error::other(format!("resume {child_op:?} parent: {error}")))?;
cleanup.store_terminal(TerminalCleanup::new_unregistered(pid.into(), &running.1))?;
let coordinator = thread::spawn(move || {
if let Err(error) = child_materialized_rx.recv_timeout(TRACEE_WAIT_TIMEOUT) {
let cleanup_result = cleanup.cleanup();
return (
Err(io::Error::new(
io::ErrorKind::TimedOut,
format!(
"{child_op:?} decode did not materialize within bound: {error}; cleanup: {cleanup_result:?}"
),
)),
cleanup,
);
}
assert_eq!(
parent_event.event().wait_owner.load(Ordering::Acquire),
expected_return_owner,
"{child_op:?} decoded outside its return transaction"
);
let (cleanup_saw_return_tx, cleanup_saw_return_rx) = mpsc::sync_channel(0);
let (resume_cleanup_tx, resume_cleanup_rx) = mpsc::channel();
*parent_event.event().cleanup_return_pause.lock() = Some(BoundedTestPause {
captured: cleanup_saw_return_tx,
resume: resume_cleanup_rx,
});
let cleaning = thread::spawn(move || {
let result = cleanup.cleanup();
(result, cleanup)
});
if let Err(error) = cleanup_saw_return_rx.recv_timeout(TRACEE_WAIT_TIMEOUT) {
parent_event.event().cleanup_return_pause.lock().take();
drop(cleanup_saw_return_rx);
let _ = resume_return_tx.send(());
let (cleanup_result, cleanup) = cleaning.join().expect("join failed cleanup race");
return (
Err(io::Error::new(
io::ErrorKind::TimedOut,
format!(
"{child_op:?} cleanup did not observe RETURNING: {error}; cleanup: {cleanup_result:?}"
),
)),
cleanup,
);
}
assert!(
!parent_event
.event()
.cleanup_cancel_requested
.load(Ordering::Acquire),
"{child_op:?} cleanup cancelled after typed return had begun"
);
resume_cleanup_tx
.send(())
.expect("resume cleanup RETURNING observation");
resume_return_tx
.send(())
.expect("resume NewChild return transaction");
cleaning.join().expect("join new-child cleanup")
});
Ok(NewChildReturnRace {
pid,
running: Some(running),
parent_identity,
coordinator,
exact_cleanup,
})
}
fn wait_generation_retired_bounded(identity: &WorkerIdentity, context: &str) -> io::Result<()> {
let deadline = Instant::now() + TRACEE_WAIT_TIMEOUT;
loop {
if !identity.is_same_process_generation() {
return Ok(());
}
if Instant::now() >= deadline {
let status = fs::read_to_string(format!("/proc/{}/status", identity.pid))
.unwrap_or_else(|error| format!("unavailable: {error}"));
let status = status
.lines()
.filter(|line| {
line.starts_with("State:")
|| line.starts_with("Tgid:")
|| line.starts_with("TracerPid:")
})
.collect::<Vec<_>>()
.join(", ");
let pidfd_live = identity.pidfd_is_live();
return Err(io::Error::new(
io::ErrorKind::TimedOut,
format!(
"{context}: generation {} remained live; pidfd={pidfd_live:?}; proc=[{status}]",
identity.pid
),
));
}
thread::sleep(SUBPROCESS_POLL_INTERVAL);
}
}
fn finish_new_child_return_race(
child_op: crate::ChildOp,
mut race: NewChildReturnRace,
wait: Wait,
) -> io::Result<()> {
let (parent, event) = wait.assume_stopped();
let child = match event {
crate::Event::NewChild(observed, child) if observed == child_op => child,
event => {
return Err(io::Error::other(format!(
"expected {child_op:?} event, got {event:?}"
)));
}
};
if parent.pid() != race.pid.into() {
return Err(io::Error::other(format!(
"{child_op:?} returned parent {} instead of {}",
parent.pid(),
race.pid
)));
}
drop(parent);
let child_pid = Pid::from_raw(child.pid().as_raw());
let child_identity = WorkerIdentity::capture_process(child.pid()).map_err(|error| {
io::Error::other(format!(
"capture materialized {child_op:?} child {child_pid}: {error}"
))
})?;
reap_new_child_bounded(child)?;
let (cleanup_result, cleanup) = race
.coordinator
.join()
.map_err(|_| io::Error::other(format!("{child_op:?} cleanup coordinator panicked")))?;
cleanup_result?;
if !matches!(
cleanup.ownership,
TraceeCleanupOwnership::NotifierOwned {
cleanup_signal_sent: true,
..
}
) {
return Err(io::Error::other(format!(
"{child_op:?} parent cleanup did not send its exact cancellation signal"
)));
}
wait_generation_retired_bounded(
&race.parent_identity,
&format!("{child_op:?} parent cleanup"),
)?;
wait_generation_retired_bounded(
&child_identity,
&format!("{child_op:?} child {child_pid} reap"),
)?;
race.exact_cleanup.disarm_completed()?;
Ok(())
}
fn new_child_trace_option(child_op: crate::ChildOp) -> Options {
Options::PTRACE_O_EXITKILL
| match child_op {
crate::ChildOp::Fork => Options::PTRACE_O_TRACEFORK,
crate::ChildOp::Vfork => Options::PTRACE_O_TRACEVFORK,
crate::ChildOp::Clone => Options::PTRACE_O_TRACECLONE,
}
}
fn spawn_new_child_tracee(
child_op: crate::ChildOp,
) -> io::Result<(Pid, Stopped, TraceeCleanupGuard)> {
match unsafe { fork() }.map_err(io::Error::other)? {
ForkResult::Parent { child } => {
let mut cleanup = TraceeCleanupGuard::new(child).inspect_err(|_| {
let _ = unsafe { libc::kill(child.as_raw(), libc::SIGKILL) };
let _ = reap_tracee_bounded(child);
})?;
let mut running = match Running::seize(
child.into(),
new_child_trace_option(child_op),
) {
Ok(running) => running,
Err(error) => {
let snapshot = worker_proc_snapshot(child.into());
let cleanup_result = cleanup.cleanup();
return Err(io::Error::other(format!(
"seize {child_op:?} parent {child}: {error}; snapshot: {snapshot:?}; cleanup: {cleanup_result:?}"
)));
}
};
let stopped = loop {
match running.wait() {
Ok(Wait::Stopped(stopped, crate::Event::Stop))
| Ok(Wait::Stopped(stopped, crate::Event::Signal(Signal::SIGSTOP))) => {
break stopped;
}
Ok(Wait::Stopped(stopped, crate::Event::Signal(signal))) => {
running = stopped.resume(Some(signal)).map_err(io::Error::other)?;
}
Ok(Wait::Stopped(stopped, _)) => {
running = stopped.resume(None).map_err(io::Error::other)?;
}
Ok(Wait::Exited(_, status)) => {
cleanup.disarm();
return Err(io::Error::other(format!(
"{child_op:?} parent {child} exited before initial stop: {status:?}"
)));
}
Err(error) => {
let cleanup_result = cleanup.cleanup();
return Err(io::Error::other(format!(
"wait {child_op:?} parent {child} initial stop: {error}; cleanup: {cleanup_result:?}"
)));
}
}
};
Ok((child, stopped, cleanup))
}
ForkResult::Child => {
if unsafe { libc::raise(libc::SIGSTOP) } != 0 {
unsafe { libc::_exit(125) };
}
let flags = match child_op {
crate::ChildOp::Fork => libc::SIGCHLD,
crate::ChildOp::Vfork => libc::CLONE_VFORK | libc::SIGCHLD,
crate::ChildOp::Clone => 0,
};
let result = unsafe {
libc::syscall(libc::SYS_clone, flags, 0usize, 0usize, 0usize, 0usize)
};
if result == 0 {
unsafe { libc::_exit(0) };
}
unsafe { libc::_exit(i32::from(result == -1)) };
}
}
}
fn reap_new_child_bounded(child: Running) -> io::Result<()> {
let child_pid = Pid::from_raw(child.pid().as_raw());
let child_pidfd = pidfd_open(child_pid.as_raw())?;
let wait_bounded = |running: Running| -> Result<Wait, Error> {
let pid = running.0;
let token = running.1;
let flags = WaitPidFlag::from_bits_retain(
WaitPidFlag::WEXITED.bits()
| WaitPidFlag::WSTOPPED.bits()
| WaitPidFlag::WNOHANG.bits()
| libc::__WALL,
);
let deadline = Instant::now() + TRACEE_WAIT_TIMEOUT;
loop {
match waitid::waitpidfd(child_pidfd.as_raw_fd(), flags) {
Ok(Some(status)) => {
return Wait::from_raw_with_token(pid, status, token);
}
Ok(None) | Err(Errno::EINTR) => {}
Err(error) => return Err(error.into()),
}
if Instant::now() >= deadline {
return Err(Errno::ETIMEDOUT.into());
}
thread::sleep(SUBPROCESS_POLL_INTERVAL);
}
};
let first = match wait_bounded(child) {
Ok(wait) => wait,
Err(error) => {
let _ = pidfd_send_signal(&child_pidfd, libc::SIGKILL);
let cleanup = reap_tracee_pidfd_bounded(child_pid, &child_pidfd);
return Err(io::Error::other(format!(
"bounded initial child {child_pid} wait: {error}; cleanup: {cleanup:?}"
)));
}
};
let (stopped, event) = first.assume_stopped();
if !matches!(
event,
crate::Event::Stop | crate::Event::Signal(Signal::SIGSTOP)
) {
let _ = pidfd_send_signal(&child_pidfd, libc::SIGKILL);
let cleanup = reap_tracee_pidfd_bounded(child_pid, &child_pidfd);
return Err(io::Error::other(format!(
"unexpected initial child event: {event:?}; cleanup: {cleanup:?}"
)));
}
let running = match stopped.resume(None) {
Ok(running) => running,
Err(error) => {
let _ = pidfd_send_signal(&child_pidfd, libc::SIGKILL);
let cleanup = reap_tracee_pidfd_bounded(child_pid, &child_pidfd);
return Err(io::Error::other(format!(
"resume child {child_pid}: {error}; cleanup: {cleanup:?}"
)));
}
};
let (_, exit) = match wait_bounded(running) {
Ok(wait) => wait.assume_exited(),
Err(Error::Errno(Errno::ETIMEDOUT)) => {
pidfd_send_signal(&child_pidfd, libc::SIGKILL)?;
let cleanup = reap_tracee_pidfd_bounded(child_pid, &child_pidfd);
return Err(io::Error::new(
io::ErrorKind::TimedOut,
format!("child {child_pid} did not exit within bound; cleanup: {cleanup:?}"),
));
}
Err(error) => {
let _ = pidfd_send_signal(&child_pidfd, libc::SIGKILL);
let cleanup = reap_tracee_pidfd_bounded(child_pid, &child_pidfd);
return Err(io::Error::other(format!(
"bounded child {child_pid} exit wait: {error}; cleanup: {cleanup:?}"
)));
}
};
if exit != crate::ExitStatus::Exited(0) {
return Err(io::Error::other(format!("unexpected child exit: {exit:?}")));
}
Ok(())
}
fn spawn_traced_process(
requested_pid: Option<i32>,
) -> Option<(Pid, Stopped, TraceeCleanupGuard)> {
spawn_traced_process_as(requested_pid, Stopped::new_unchecked)
}
fn spawn_traced_process_as(
requested_pid: Option<i32>,
capability: impl FnOnce(crate::Pid) -> Stopped,
) -> Option<(Pid, Stopped, TraceeCleanupGuard)> {
let child = if let Some(requested_pid) = requested_pid {
#[repr(C)]
#[derive(Default)]
struct CloneArgs {
flags: u64,
pidfd: u64,
child_tid: u64,
parent_tid: u64,
exit_signal: u64,
stack: u64,
stack_size: u64,
tls: u64,
set_tid: u64,
set_tid_size: u64,
cgroup: u64,
}
let mut set_tid = requested_pid as u64;
let args = CloneArgs {
exit_signal: libc::SIGCHLD as u64,
set_tid: std::ptr::from_mut(&mut set_tid) as u64,
set_tid_size: 1,
..CloneArgs::default()
};
let result = crate::test_fork::exclusive(|| unsafe {
libc::syscall(
libc::SYS_clone3,
std::ptr::from_ref(&args),
mem::size_of::<CloneArgs>(),
)
});
if result == -1 {
return None;
}
if result == 0 {
crate::traceme_and_stop().expect("TRACEME requested-PID child");
unsafe { libc::_exit(42) };
}
Pid::from_raw(result as i32)
} else {
match unsafe { fork() }.expect("fork duplicate-exit tracee") {
ForkResult::Parent { child } => child,
ForkResult::Child => {
crate::traceme_and_stop().expect("TRACEME duplicate-exit child");
unsafe { libc::_exit(42) };
}
}
};
let mut cleanup = TraceeCleanupGuard::new(child).unwrap_or_else(|error| {
let _ = unsafe { libc::kill(child.as_raw(), libc::SIGKILL) };
let _ = reap_tracee_bounded(child);
panic!("open duplicate-exit tracee pidfd: {error}");
});
let stopped = stopped_tracee_bounded(child, capability).unwrap_or_else(|error| {
cleanup
.cleanup()
.unwrap_or_else(|cleanup_error| panic!("{error}; cleanup: {cleanup_error}"));
panic!("wait duplicate-exit initial stop: {error}");
});
Some((child, stopped, cleanup))
}
async fn duplicate_exit_waiter_rejects_replacement(requested_pid: Option<i32>) -> bool {
let Some((old_pid, old_stopped, mut old_cleanup)) = spawn_traced_process(requested_pid)
else {
return false;
};
old_stopped
.setoptions(Options::PTRACE_O_TRACEEXIT)
.expect("enable exit stop for duplicate waiter");
let mut cancelled = Box::pin(
old_cleanup
.exit_event(&old_stopped)
.expect("bind duplicate-waiter cleanup to notifier"),
);
let waker = futures::task::noop_waker();
let mut context = Context::from_waker(&waker);
assert_eq!(cancelled.as_mut().poll(&mut context), Poll::Pending);
drop(cancelled);
let winner = old_cleanup
.exit_event(&old_stopped)
.expect("create winner on bound notifier generation");
let duplicate = old_cleanup
.exit_event(&old_stopped)
.expect("create duplicate on bound notifier generation");
let late = old_cleanup
.exit_event(&old_stopped)
.expect("create late waiter on bound notifier generation");
old_stopped
.resume(None)
.expect("resume old duplicate-waiter tracee");
let exit_stopped = tokio::time::timeout(TRACEE_WAIT_TIMEOUT, winner)
.await
.expect("old exit-stop winner timed out")
.expect("claim old exit-stop capability");
old_cleanup.mark_claimed_exit();
assert_eq!(
tokio::time::timeout(TRACEE_WAIT_TIMEOUT, duplicate)
.await
.expect("old duplicate ExitFuture timed out"),
Err(Error::Errno(Errno::EALREADY))
);
let final_wait = exit_stopped
.resume(None)
.expect("resume claimed old exit stop");
let final_wait = tokio::time::timeout(TRACEE_WAIT_TIMEOUT, final_wait.next_state())
.await
.expect("old claimed exit final status timed out")
.expect("wait old final status");
old_cleanup.disarm();
assert_eq!(
final_wait.assume_exited(),
(old_pid.into(), crate::ExitStatus::Exited(42))
);
thread::sleep(Duration::from_millis(20));
let Some((replacement_pid, replacement, mut replacement_cleanup)) =
spawn_traced_process(requested_pid)
else {
return false;
};
if requested_pid.is_some() {
assert_eq!(
replacement_pid, old_pid,
"clone3 did not reuse requested PID"
);
}
assert_eq!(
tokio::time::timeout(TRACEE_WAIT_TIMEOUT, late)
.await
.expect("late old ExitFuture timed out"),
Err(Error::Errno(Errno::EALREADY))
);
replacement
.getregs()
.expect("late old waiter touched the stopped replacement");
replacement_cleanup
.bind_notifier(&replacement)
.expect("bind replacement cleanup to notifier");
let replacement = replacement.resume(None).expect("resume replacement tracee");
let replacement = tokio::time::timeout(TRACEE_WAIT_TIMEOUT, replacement.next_state())
.await
.expect("replacement final status timed out")
.expect("wait replacement tracee");
replacement_cleanup.disarm();
assert_eq!(
replacement.assume_exited(),
(replacement_pid.into(), crate::ExitStatus::Exited(42))
);
true
}
async fn unclaimed_exit_waiter_expires_before_cleanup_replacement(
requested_pid: Option<i32>,
) -> bool {
let Some((old_pid, old_stopped, mut old_cleanup)) = spawn_traced_process(requested_pid)
else {
return false;
};
old_stopped
.setoptions(Options::PTRACE_O_TRACEEXIT)
.expect("enable exit stop for cleanup expiration");
let mut late = Box::pin(
old_cleanup
.exit_event(&old_stopped)
.expect("bind unclaimed cleanup to notifier"),
);
let terminal = old_cleanup
.terminal()
.expect("bound unclaimed cleanup terminal");
let waker = futures::task::noop_waker();
let mut context = Context::from_waker(&waker);
assert_eq!(late.as_mut().poll(&mut context), Poll::Pending);
old_stopped
.resume(None)
.expect("resume unclaimed exit-stop tracee");
let deadline = Instant::now() + Duration::from_secs(1);
while !terminal.exit_stop_observed() {
assert!(
Instant::now() < deadline,
"unclaimed cleanup tracee did not reach exit stop"
);
thread::yield_now();
}
terminal
.revoke_unclaimed_exit_stop()
.expect("cleanup failed to expire unclaimed exit stop");
nix::sys::ptrace::cont(old_pid, None).expect("raw cleanup resume old exit stop");
assert!(
terminal.wait(Duration::from_secs(1)),
"cleanup notifier did not publish old final status"
);
old_cleanup.disarm();
thread::sleep(Duration::from_millis(20));
let Some((replacement_pid, replacement, mut replacement_cleanup)) =
spawn_traced_process(requested_pid)
else {
return false;
};
if requested_pid.is_some() {
assert_eq!(
replacement_pid, old_pid,
"clone3 did not reuse cleanup test PID"
);
}
assert_eq!(
tokio::time::timeout(TRACEE_WAIT_TIMEOUT, late)
.await
.expect("late unclaimed ExitFuture timed out"),
Err(Error::Errno(Errno::EALREADY))
);
replacement
.getregs()
.expect("late unclaimed waiter touched the stopped replacement");
replacement_cleanup
.bind_notifier(&replacement)
.expect("bind cleanup replacement to notifier");
let replacement = replacement
.resume(None)
.expect("resume cleanup replacement tracee");
let replacement = tokio::time::timeout(TRACEE_WAIT_TIMEOUT, replacement.next_state())
.await
.expect("cleanup replacement final status timed out")
.expect("wait cleanup replacement tracee");
replacement_cleanup.disarm();
assert_eq!(
replacement.assume_exited(),
(replacement_pid.into(), crate::ExitStatus::Exited(42))
);
true
}
#[tokio::test(flavor = "current_thread")]
async fn exit_waiters_never_target_reused_pid_after_claim_or_cleanup() {
const INNER: &str = "SAFEPTRACE_EXIT_REUSE_INNER";
if env::var_os(INNER).is_some() {
if duplicate_exit_waiter_rejects_replacement(Some(100)).await
&& unclaimed_exit_waiter_expires_before_cleanup_replacement(Some(100)).await
{
println!("ACTUAL_EXIT_PID_REUSE_EXERCISED");
} else {
println!("ACTUAL_EXIT_PID_REUSE_UNAVAILABLE");
}
return;
}
let inner = "notifier::test::exit_waiters_never_target_reused_pid_after_claim_or_cleanup";
let actual_reuse = run_exact_in_pid_namespace_bounded(inner, &[(INNER, "1")]);
match classify_exact_reuse_output(
actual_reuse.as_ref(),
"ACTUAL_EXIT_PID_REUSE_EXERCISED",
"ACTUAL_EXIT_PID_REUSE_UNAVAILABLE",
)
.unwrap_or_else(|error| panic!("actual exit PID-reuse regression failed: {error}"))
{
ExactReuseOutcome::Exercised => return,
ExactReuseOutcome::Unavailable => {}
}
assert!(duplicate_exit_waiter_rejects_replacement(None).await);
assert!(unclaimed_exit_waiter_expires_before_cleanup_replacement(None).await);
}
async fn terminal_generation_wait_leaves_replacement_untouched(
requested_pid: Option<i32>,
) -> bool {
let Some((old_pid, old_stopped, mut old_cleanup)) = spawn_traced_process(requested_pid)
else {
return false;
};
old_stopped
.setoptions(Options::PTRACE_O_TRACEEXIT)
.expect("enable exit stop for terminal generation");
let mut exit_stop = Box::pin(
old_cleanup
.exit_event(&old_stopped)
.expect("bind terminal-generation cleanup to notifier"),
);
let terminal = old_cleanup
.terminal()
.expect("bound terminal-generation cleanup terminal");
let waker = futures::task::noop_waker();
let mut context = Context::from_waker(&waker);
assert_eq!(exit_stop.as_mut().poll(&mut context), Poll::Pending);
let token = old_stopped.1.clone();
drop(old_stopped);
pidfd_send_signal(&old_cleanup.pidfd, libc::SIGKILL).expect("SIGKILL old generation");
let deadline = Instant::now() + Duration::from_secs(1);
while !terminal.exit_stop_observed() {
assert!(
Instant::now() < deadline,
"killed generation did not reach exit stop"
);
thread::yield_now();
}
terminal
.revoke_unclaimed_exit_stop()
.expect("expire the unclaimed exit stop");
nix::sys::ptrace::cont(old_pid, None).expect("raw resume old exit stop");
assert!(
terminal.wait(Duration::from_secs(1)),
"notifier did not publish old final status"
);
old_cleanup.disarm();
assert_eq!(
tokio::time::timeout(TRACEE_WAIT_TIMEOUT, exit_stop)
.await
.expect("expired ExitFuture timed out"),
Err(Error::Errno(Errno::EALREADY))
);
thread::sleep(Duration::from_millis(20));
let Some((replacement_pid, replacement, mut replacement_cleanup)) =
spawn_traced_process(requested_pid)
else {
return false;
};
let generation_pid = if requested_pid.is_some() {
assert_eq!(
replacement_pid, old_pid,
"clone3 did not reuse terminal-generation PID"
);
old_pid
} else {
replacement_pid
};
let generation = Stopped::from_token(generation_pid.into(), token);
replacement
.getregs()
.expect("the old generation's numeric PID names the stopped replacement");
assert!(
matches!(generation.getregs(), Err(Error::Died(_))),
"a request through the reaped generation reached the replacement"
);
let mut wait = generation.wait_owned();
let result = tokio::time::timeout(TRACEE_WAIT_TIMEOUT, &mut wait)
.await
.expect("terminal-generation wait timed out")
.expect("terminal-generation wait");
assert_eq!(
result.assume_exited(),
(
generation_pid.into(),
crate::ExitStatus::Signaled(Signal::SIGKILL, false)
)
);
assert!(matches!((&mut wait).await, Err(OwnedWaitError::Completed)));
let info = replacement
.getsiginfo()
.expect("terminal-generation wait touched the stopped replacement");
assert_eq!(info.si_signo, libc::SIGSTOP);
replacement_cleanup
.bind_notifier(&replacement)
.expect("bind replacement to notifier");
let replacement = replacement.resume(None).expect("resume replacement");
let replacement = tokio::time::timeout(TRACEE_WAIT_TIMEOUT, replacement.next_state())
.await
.expect("replacement final status timed out")
.expect("wait replacement");
replacement_cleanup.disarm();
assert_eq!(
replacement.assume_exited(),
(replacement_pid.into(), crate::ExitStatus::Exited(42))
);
true
}
#[tokio::test(flavor = "current_thread")]
async fn terminal_generation_wait_never_targets_a_replacement_pid() {
const INNER: &str = "SAFEPTRACE_TERMINAL_GENERATION_REUSE_INNER";
if env::var_os(INNER).is_some() {
if terminal_generation_wait_leaves_replacement_untouched(Some(100)).await {
println!("ACTUAL_TERMINAL_GENERATION_PID_REUSE_EXERCISED");
} else {
println!("ACTUAL_TERMINAL_GENERATION_PID_REUSE_UNAVAILABLE");
}
return;
}
let inner = "notifier::test::terminal_generation_wait_never_targets_a_replacement_pid";
let actual_reuse = run_exact_in_pid_namespace_bounded(inner, &[(INNER, "1")]);
match classify_exact_reuse_output(
actual_reuse.as_ref(),
"ACTUAL_TERMINAL_GENERATION_PID_REUSE_EXERCISED",
"ACTUAL_TERMINAL_GENERATION_PID_REUSE_UNAVAILABLE",
)
.unwrap_or_else(|error| panic!("terminal-generation PID-reuse regression failed: {error}"))
{
ExactReuseOutcome::Exercised => return,
ExactReuseOutcome::Unavailable => {}
}
assert!(terminal_generation_wait_leaves_replacement_untouched(None).await);
}
async fn claimed_exit_stop_refuses_after_reap(
requested_pid: Option<i32>,
primary: PrimaryRoute,
) -> bool {
let Some((old_pid, unchecked, mut old_cleanup)) = spawn_traced_process(requested_pid)
else {
return false;
};
let early = Stopped::new_unchecked(old_pid.into());
let old_stopped = match primary {
PrimaryRoute::Unchecked => unchecked,
PrimaryRoute::RawWait => {
drop(unchecked);
let stop = (libc::SIGSTOP << 8) | 0x7f;
let (raw, _) = Wait::from_raw(old_pid.into(), stop)
.expect("convert the actual stop's raw status")
.assume_stopped();
assert!(
Arc::ptr_eq(raw.1.event().event(), early.1.event().event()),
"Wait::from_raw made a capability outside the live generation's Event"
);
raw
}
PrimaryRoute::Unbound => {
drop(unchecked);
Stopped::from_token(old_pid.into(), crate::TraceeToken::new())
}
};
let absorbed_before = ABSORBED_UNREGISTERED
.lock()
.get(&old_pid.into())
.copied()
.unwrap_or(0);
old_stopped
.setoptions(Options::PTRACE_O_TRACEEXIT)
.expect("enable exit stop for claimed generation");
let exit_stop = old_cleanup
.exit_event(&old_stopped)
.expect("bind claimed-generation cleanup to notifier");
old_stopped
.resume(None)
.expect("resume claimed-generation tracee");
let claimed = tokio::time::timeout(TRACEE_WAIT_TIMEOUT, exit_stop)
.await
.expect("claimed-generation ExitFuture timed out")
.expect("claim the exit-stop capability");
assert!(
Arc::ptr_eq(claimed.1.event().event(), early.1.event().event()),
"the first registration left the earlier capability on another Event"
);
if matches!(primary, PrimaryRoute::Unbound) {
assert_eq!(
ABSORBED_UNREGISTERED.lock().get(&old_pid.into()).copied(),
Some(absorbed_before + 1),
"the earlier generation was not redirected at first registration"
);
}
old_cleanup.mark_claimed_exit();
let terminal = old_cleanup
.terminal()
.expect("bound claimed-generation cleanup terminal");
nix::sys::ptrace::cont(old_pid, None).expect("raw resume of the claimed exit stop");
assert!(
terminal.wait(Duration::from_secs(1)),
"notifier did not publish old final status"
);
old_cleanup.disarm();
thread::sleep(Duration::from_millis(20));
let Some((replacement_pid, replacement, mut replacement_cleanup)) =
spawn_traced_process(requested_pid)
else {
return false;
};
let (claimed, early) = if requested_pid.is_some() {
assert_eq!(
replacement_pid, old_pid,
"clone3 did not reuse claimed-generation PID"
);
(claimed, early)
} else {
(
Stopped::from_token(replacement_pid.into(), claimed.1),
Stopped::from_token(replacement_pid.into(), early.1),
)
};
replacement
.getregs()
.expect("the claimed capability's numeric PID names the stopped replacement");
assert!(
matches!(claimed.getregs(), Err(Error::Died(_))),
"a request through the claimed capability reached the replacement"
);
assert!(
matches!(early.getregs(), Err(Error::Died(_))),
"a request through the capability created before registration reached the replacement"
);
let (claimed, errno) = claimed
.resume_retaining(None)
.expect_err("the claimed capability resumed the replacement");
assert_eq!(errno, Errno::ESRCH);
#[cfg(feature = "memory")]
let replacement =
assert_user_access_write_refused_after_reap([claimed, early], replacement);
#[cfg(not(feature = "memory"))]
drop((claimed, early));
let info = replacement
.getsiginfo()
.expect("the claimed capability touched the stopped replacement");
assert_eq!(info.si_signo, libc::SIGSTOP);
replacement_cleanup
.bind_notifier(&replacement)
.expect("bind replacement to notifier");
let replacement = replacement.resume(None).expect("resume replacement");
let replacement = tokio::time::timeout(TRACEE_WAIT_TIMEOUT, replacement.next_state())
.await
.expect("replacement final status timed out")
.expect("wait replacement");
replacement_cleanup.disarm();
assert_eq!(
replacement.assume_exited(),
(replacement_pid.into(), crate::ExitStatus::Exited(42))
);
true
}
#[cfg(feature = "memory")]
static mut REUSE_PROBE: [u8; 16] = *b"replacement-data";
#[cfg(feature = "memory")]
fn assert_user_access_write_refused_after_reap(
stale: impl IntoIterator<Item = Stopped>,
mut replacement: Stopped,
) -> Stopped {
use reverie_memory::Addr;
use reverie_memory::AddrMut;
use reverie_memory::MemoryAccess;
let raw = std::ptr::addr_of!(REUSE_PROBE) as usize;
let probe = AddrMut::from_raw(raw).expect("probe address");
let original = *b"replacement-data";
let read_probe = |replacement: &Stopped| {
let mut observed = [0u8; 16];
replacement
.read_exact(
Addr::<u8>::from_raw(raw).expect("probe address"),
&mut observed,
)
.expect("read the replacement's probe");
observed
};
for mut stale in stale {
assert_eq!(
stale.write_with_user_access(probe, b"stale-generation"),
Err(Errno::ESRCH),
"a user-access write through a reaped generation's capability reached the replacement"
);
drop(stale);
assert_eq!(read_probe(&replacement), original);
}
assert_eq!(
replacement.write_with_user_access(probe, b"own-capability!!"),
Ok(16)
);
assert_eq!(&read_probe(&replacement), b"own-capability!!");
replacement
}
#[tokio::test(flavor = "current_thread")]
async fn claimed_exit_stop_never_resumes_a_replacement_pid() {
const INNER: &str = "SAFEPTRACE_CLAIMED_EXIT_REUSE_INNER";
if env::var_os(INNER).is_some() {
if claimed_exit_stop_refuses_after_reap(Some(100), PrimaryRoute::Unchecked).await {
println!("ACTUAL_CLAIMED_EXIT_PID_REUSE_EXERCISED");
} else {
println!("ACTUAL_CLAIMED_EXIT_PID_REUSE_UNAVAILABLE");
}
return;
}
let inner = "notifier::test::claimed_exit_stop_never_resumes_a_replacement_pid";
let actual_reuse = run_exact_in_pid_namespace_bounded(inner, &[(INNER, "1")]);
match classify_exact_reuse_output(
actual_reuse.as_ref(),
"ACTUAL_CLAIMED_EXIT_PID_REUSE_EXERCISED",
"ACTUAL_CLAIMED_EXIT_PID_REUSE_UNAVAILABLE",
)
.unwrap_or_else(|error| panic!("claimed exit PID-reuse regression failed: {error}"))
{
ExactReuseOutcome::Exercised => return,
ExactReuseOutcome::Unavailable => {}
}
assert!(claimed_exit_stop_refuses_after_reap(None, PrimaryRoute::Unchecked).await);
}
#[derive(Clone, Copy, Debug)]
enum PrimaryRoute {
Unchecked,
RawWait,
Unbound,
}
async fn claimed_exit_reuse_through(primary: PrimaryRoute, inner_env: &str, inner: &str) {
if env::var_os(inner_env).is_some() {
if claimed_exit_stop_refuses_after_reap(Some(100), primary).await {
println!("ACTUAL_CLAIMED_EXIT_PID_REUSE_EXERCISED");
} else {
println!("ACTUAL_CLAIMED_EXIT_PID_REUSE_UNAVAILABLE");
}
return;
}
let actual_reuse = run_exact_in_pid_namespace_bounded(inner, &[(inner_env, "1")]);
match classify_exact_reuse_output(
actual_reuse.as_ref(),
"ACTUAL_CLAIMED_EXIT_PID_REUSE_EXERCISED",
"ACTUAL_CLAIMED_EXIT_PID_REUSE_UNAVAILABLE",
)
.unwrap_or_else(|error| panic!("{primary:?} PID-reuse regression failed: {error}"))
{
ExactReuseOutcome::Exercised => return,
ExactReuseOutcome::Unavailable => {}
}
assert!(claimed_exit_stop_refuses_after_reap(None, primary).await);
}
#[tokio::test(flavor = "current_thread")]
async fn raw_wait_capability_shares_the_unchecked_generation_gate() {
claimed_exit_reuse_through(
PrimaryRoute::RawWait,
"SAFEPTRACE_RAW_WAIT_REUSE_INNER",
"notifier::test::raw_wait_capability_shares_the_unchecked_generation_gate",
)
.await;
}
#[tokio::test(flavor = "current_thread")]
async fn first_registration_redirects_an_earlier_unregistered_generation() {
claimed_exit_reuse_through(
PrimaryRoute::Unbound,
"SAFEPTRACE_UNBOUND_REUSE_INNER",
"notifier::test::first_registration_redirects_an_earlier_unregistered_generation",
)
.await;
}
#[test]
fn synchronous_first_registration_redirects_an_earlier_unregistered_generation() {
let (pid, _, mut cleanup) =
spawn_traced_process(None).expect("spawn synchronous registration tracee");
let early = Stopped::new_unchecked(pid.into());
let unbound = Stopped::from_token(pid.into(), crate::TraceeToken::new());
let token = unbound.1.clone();
let running = unbound.resume(None).expect("resume to exit");
assert_eq!(
running.wait().expect("synchronous wait").assume_exited(),
(pid.into(), crate::ExitStatus::Exited(42))
);
cleanup.disarm();
assert_eq!(
ABSORBED_UNREGISTERED.lock().get(&pid.into()).copied(),
Some(1),
"sync_handle did not redirect the earlier generation"
);
assert!(Arc::ptr_eq(early.1.event().event(), token.event().event()));
assert!(
early.1.event().hold_tid().is_none(),
"the earlier capability's TID gate stayed open after the reap"
);
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
enum FailedRegistration {
OwnWorkerStart,
AbsorbedWorkerStart,
AbsorbedWorkerStartEarlierDropped,
Absorption,
UncachedWorkerStart,
BoundLiveness,
SyncBoundLiveness,
BoundLivenessTwice,
SyncBoundLivenessTwice,
BoundLivenessProcfsError,
SyncBoundLivenessProcfsError,
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
enum RegistrationRetry {
Unchecked,
UncheckedSync,
Unbound,
}
async fn failed_registration_refuses_after_reap(
requested_pid: Option<i32>,
failure: FailedRegistration,
retry: RegistrationRetry,
) -> bool {
let Some((old_pid, early, mut old_cleanup)) = spawn_traced_process(requested_pid) else {
return false;
};
let absorbed = || {
ABSORBED_UNREGISTERED
.lock()
.get(&old_pid.into())
.copied()
.unwrap_or(0)
};
let absorbed_before = absorbed();
let mut early = if failure == FailedRegistration::UncachedWorkerStart {
drop(early);
None
} else {
Some(early)
};
let failed = if failure == FailedRegistration::OwnWorkerStart {
Stopped::new_unchecked(old_pid.into())
} else {
Stopped::from_token(old_pid.into(), crate::TraceeToken::new())
};
let expected = match failure {
FailedRegistration::Absorption => {
ABSORB_LIVENESS_ERRORS
.lock()
.insert(old_pid.into(), Errno::EIO);
Errno::EIO
}
FailedRegistration::BoundLiveness
| FailedRegistration::SyncBoundLiveness
| FailedRegistration::BoundLivenessTwice
| FailedRegistration::SyncBoundLivenessTwice
| FailedRegistration::BoundLivenessProcfsError
| FailedRegistration::SyncBoundLivenessProcfsError => {
let procfs_error = matches!(
failure,
FailedRegistration::BoundLivenessProcfsError
| FailedRegistration::SyncBoundLivenessProcfsError
);
let twice = procfs_error
|| matches!(
failure,
FailedRegistration::BoundLivenessTwice
| FailedRegistration::SyncBoundLivenessTwice
);
PIDFD_LIVENESS_ERRORS
.lock()
.entry(old_pid.into())
.or_default()
.extend(std::iter::repeat_n(Errno::EINVAL, 1 + usize::from(twice)));
if procfs_error {
PROCFS_OBSERVATION_ERRORS
.lock()
.insert(old_pid.into(), Errno::EMFILE);
}
Errno::EINVAL
}
_ => {
SPAWN_WORKER_ERRORS
.lock()
.insert(old_pid.into(), libc::EAGAIN);
Errno::EAGAIN
}
};
if failure != FailedRegistration::OwnWorkerStart {
assert!(
failed.1.event().identity().is_none(),
"{failure:?}: the failing capability's identity was bound before its registration"
);
}
let registration = if matches!(
failure,
FailedRegistration::SyncBoundLiveness
| FailedRegistration::SyncBoundLivenessTwice
| FailedRegistration::SyncBoundLivenessProcfsError
) {
NOTIFIER
.sync_handle(old_pid.into(), failed.1.event())
.map(drop)
} else {
TerminalCleanup::new_unregistered(old_pid.into(), &failed.1).ensure_registered()
};
assert_eq!(
registration,
Err(expected),
"{failure:?}: the first registration did not fail as injected"
);
assert!(
PIDFD_LIVENESS_ERRORS
.lock()
.get(&old_pid.into())
.is_none_or(VecDeque::is_empty),
"{failure:?}: the injected liveness error was not consumed"
);
assert!(
!PROCFS_OBSERVATION_ERRORS
.lock()
.contains_key(&old_pid.into()),
"{failure:?}: the injected procfs error was not consumed"
);
assert!(
failed.1.event().identity().is_some(),
"{failure:?}: the failed registration did not bind the task's identity"
);
assert!(
!NOTIFIER.pids.lock().contains_key(&old_pid.into()),
"{failure:?}: the failed registration stayed in the registry"
);
assert!(
failed.1.event().hold_tid().is_some(),
"{failure:?}: the failed registration closed a live task's TID gate"
);
failed.getregs().unwrap_or_else(|error| {
panic!("{failure:?}: the failed capability refused a live task's request: {error:?}")
});
let absorbed_after_failure = absorbed();
let redirected_before_failure = matches!(
failure,
FailedRegistration::AbsorbedWorkerStart
| FailedRegistration::AbsorbedWorkerStartEarlierDropped
);
assert_eq!(
absorbed_after_failure,
absorbed_before + usize::from(redirected_before_failure),
"{failure:?}: unexpected redirects before the failure"
);
if failure == FailedRegistration::AbsorbedWorkerStartEarlierDropped {
let earlier = early.take().expect("the earlier capability");
let generation = Arc::downgrade(&earlier.1.event().0);
drop(earlier);
assert_eq!(
generation.strong_count(),
0,
"the dropped earlier capability's generation is still alive"
);
}
let unjoined = matches!(
failure,
FailedRegistration::Absorption
| FailedRegistration::BoundLiveness
| FailedRegistration::SyncBoundLiveness
| FailedRegistration::BoundLivenessTwice
| FailedRegistration::SyncBoundLivenessTwice
| FailedRegistration::BoundLivenessProcfsError
| FailedRegistration::SyncBoundLivenessProcfsError
);
let redirects = match (unjoined, retry) {
(true, RegistrationRetry::Unbound) => 2,
(_, RegistrationRetry::Unbound) | (true, _) => 1,
_ => 0,
};
let constructed = Stopped::new_unchecked(old_pid.into());
let shared = early.as_ref().unwrap_or(&failed);
assert!(
Arc::ptr_eq(constructed.1.event().event(), shared.1.event().event()),
"{failure:?}: a capability made after the failed registration left the earlier generation's Event"
);
let later = if retry == RegistrationRetry::Unbound {
Stopped::from_token(old_pid.into(), crate::TraceeToken::new())
} else {
Stopped::new_unchecked(old_pid.into())
};
if retry == RegistrationRetry::UncheckedSync {
let running = later
.resume(None)
.expect("resume the later capability to exit");
assert_eq!(
running.wait().expect("synchronous wait").assume_exited(),
(old_pid.into(), crate::ExitStatus::Exited(42))
);
} else {
old_cleanup
.bind_notifier(&later)
.expect("register the later capability");
let running = later
.resume(None)
.expect("resume the later capability to exit");
let exited = tokio::time::timeout(TRACEE_WAIT_TIMEOUT, running.next_state())
.await
.expect("later capability's final status timed out")
.expect("wait the later capability");
assert_eq!(
exited.assume_exited(),
(old_pid.into(), crate::ExitStatus::Exited(42))
);
}
old_cleanup.disarm();
let mut stale = vec![("failed", failed), ("constructed", constructed)];
if let Some(early) = early {
stale.push(("earlier", early));
}
for (name, capability) in &stale {
assert!(
capability.1.event().hold_tid().is_none(),
"{failure:?}, {retry:?}: the {name} capability's TID gate stayed open after the reap"
);
}
assert_eq!(
absorbed(),
absorbed_after_failure + redirects,
"{failure:?}, {retry:?}: the retry did not redirect every earlier generation"
);
assert_stale_refused_beside_replacement(
requested_pid,
old_pid,
stale,
&format!("{failure:?}, {retry:?}"),
)
.await
}
async fn assert_stale_refused_beside_replacement(
requested_pid: Option<i32>,
old_pid: Pid,
mut stale: Vec<(&'static str, Stopped)>,
case: &str,
) -> bool {
thread::sleep(Duration::from_millis(20));
let Some((replacement_pid, replacement, mut replacement_cleanup)) =
spawn_traced_process(requested_pid)
else {
return false;
};
if requested_pid.is_some() {
assert_eq!(
replacement_pid, old_pid,
"clone3 did not reuse the reaped tracee's PID"
);
} else {
for (_, capability) in &mut stale {
let token = capability.1.clone();
*capability = Stopped::from_token(replacement_pid.into(), token);
}
}
replacement
.getregs()
.expect("the old capabilities' numeric PID names the stopped replacement");
for (name, capability) in &stale {
assert!(
matches!(capability.getregs(), Err(Error::Died(_))),
"{case}: a request through the {name} capability reached the replacement"
);
}
#[cfg(feature = "memory")]
let replacement = assert_user_access_write_refused_after_reap(
stale.into_iter().map(|(_, capability)| capability),
replacement,
);
#[cfg(not(feature = "memory"))]
drop(stale);
let info = replacement
.getsiginfo()
.expect("an old capability touched the stopped replacement");
assert_eq!(info.si_signo, libc::SIGSTOP);
replacement_cleanup
.bind_notifier(&replacement)
.expect("bind replacement to notifier");
let replacement = replacement.resume(None).expect("resume replacement");
let replacement = tokio::time::timeout(TRACEE_WAIT_TIMEOUT, replacement.next_state())
.await
.expect("replacement final status timed out")
.expect("wait replacement");
replacement_cleanup.disarm();
assert_eq!(
replacement.assume_exited(),
(replacement_pid.into(), crate::ExitStatus::Exited(42))
);
true
}
async fn failed_registration_reuse_through(
failure: FailedRegistration,
inner_env: &str,
inner: &str,
) {
const RETRIES: [RegistrationRetry; 3] = [
RegistrationRetry::Unchecked,
RegistrationRetry::UncheckedSync,
RegistrationRetry::Unbound,
];
if env::var_os(inner_env).is_some() {
for retry in RETRIES {
if !failed_registration_refuses_after_reap(Some(100), failure, retry).await {
println!("ACTUAL_FAILED_REGISTRATION_PID_REUSE_UNAVAILABLE");
return;
}
}
println!("ACTUAL_FAILED_REGISTRATION_PID_REUSE_EXERCISED");
return;
}
let actual_reuse = run_exact_in_pid_namespace_bounded(inner, &[(inner_env, "1")]);
match classify_exact_reuse_output(
actual_reuse.as_ref(),
"ACTUAL_FAILED_REGISTRATION_PID_REUSE_EXERCISED",
"ACTUAL_FAILED_REGISTRATION_PID_REUSE_UNAVAILABLE",
)
.unwrap_or_else(|error| panic!("{failure:?} PID-reuse regression failed: {error}"))
{
ExactReuseOutcome::Exercised => return,
ExactReuseOutcome::Unavailable => {}
}
for retry in RETRIES {
assert!(failed_registration_refuses_after_reap(None, failure, retry).await);
}
}
#[tokio::test(flavor = "current_thread")]
async fn worker_start_failure_keeps_the_generation_for_later_capabilities() {
failed_registration_reuse_through(
FailedRegistration::OwnWorkerStart,
"SAFEPTRACE_OWN_WORKER_START_REUSE_INNER",
"notifier::test::worker_start_failure_keeps_the_generation_for_later_capabilities",
)
.await;
}
#[tokio::test(flavor = "current_thread")]
async fn worker_start_failure_after_redirect_keeps_the_generation_for_later_capabilities() {
failed_registration_reuse_through(
FailedRegistration::AbsorbedWorkerStart,
"SAFEPTRACE_ABSORBED_WORKER_START_REUSE_INNER",
"notifier::test::worker_start_failure_after_redirect_keeps_the_generation_for_later_capabilities",
)
.await;
}
#[tokio::test(flavor = "current_thread")]
async fn failed_redirect_keeps_both_generations_for_the_next_registration() {
failed_registration_reuse_through(
FailedRegistration::Absorption,
"SAFEPTRACE_FAILED_ABSORPTION_REUSE_INNER",
"notifier::test::failed_redirect_keeps_both_generations_for_the_next_registration",
)
.await;
}
#[tokio::test(flavor = "current_thread")]
async fn worker_start_failure_keeps_the_generation_after_its_predecessor_is_dropped() {
failed_registration_reuse_through(
FailedRegistration::AbsorbedWorkerStartEarlierDropped,
"SAFEPTRACE_DROPPED_PREDECESSOR_REUSE_INNER",
"notifier::test::worker_start_failure_keeps_the_generation_after_its_predecessor_is_dropped",
)
.await;
}
#[tokio::test(flavor = "current_thread")]
async fn liveness_failure_after_binding_keeps_the_generation_for_later_capabilities() {
failed_registration_reuse_through(
FailedRegistration::BoundLiveness,
"SAFEPTRACE_BOUND_LIVENESS_REUSE_INNER",
"notifier::test::liveness_failure_after_binding_keeps_the_generation_for_later_capabilities",
)
.await;
}
#[tokio::test(flavor = "current_thread")]
async fn synchronous_liveness_failure_after_binding_keeps_the_generation_for_later_capabilities()
{
failed_registration_reuse_through(
FailedRegistration::SyncBoundLiveness,
"SAFEPTRACE_SYNC_BOUND_LIVENESS_REUSE_INNER",
"notifier::test::synchronous_liveness_failure_after_binding_keeps_the_generation_for_later_capabilities",
)
.await;
}
#[tokio::test(flavor = "current_thread")]
async fn repeated_liveness_failure_keeps_a_live_generation_open() {
failed_registration_reuse_through(
FailedRegistration::BoundLivenessTwice,
"SAFEPTRACE_REPEATED_LIVENESS_REUSE_INNER",
"notifier::test::repeated_liveness_failure_keeps_a_live_generation_open",
)
.await;
}
#[tokio::test(flavor = "current_thread")]
async fn synchronous_repeated_liveness_failure_keeps_a_live_generation_open() {
failed_registration_reuse_through(
FailedRegistration::SyncBoundLivenessTwice,
"SAFEPTRACE_SYNC_REPEATED_LIVENESS_REUSE_INNER",
"notifier::test::synchronous_repeated_liveness_failure_keeps_a_live_generation_open",
)
.await;
}
#[tokio::test(flavor = "current_thread")]
async fn procfs_error_after_liveness_failure_keeps_a_live_generation_open() {
failed_registration_reuse_through(
FailedRegistration::BoundLivenessProcfsError,
"SAFEPTRACE_PROCFS_ERROR_LIVENESS_REUSE_INNER",
"notifier::test::procfs_error_after_liveness_failure_keeps_a_live_generation_open",
)
.await;
}
#[test]
fn only_a_missing_or_changed_thread_shows_exit_in_procfs() {
for errno in [libc::ENOENT, libc::ESRCH] {
assert!(procfs_error_shows_exit(&io::Error::from_raw_os_error(
errno
)));
}
assert!(procfs_error_shows_exit(&io::Error::new(
io::ErrorKind::NotFound,
"tracee identity changed while reading procfs",
)));
for errno in [libc::EMFILE, libc::ENFILE, libc::EACCES, libc::EIO] {
assert!(!procfs_error_shows_exit(&io::Error::from_raw_os_error(
errno
)));
}
assert!(!procfs_error_shows_exit(&io::Error::new(
io::ErrorKind::InvalidData,
"malformed stat",
)));
}
#[tokio::test(flavor = "current_thread")]
async fn synchronous_procfs_error_after_liveness_failure_keeps_a_live_generation_open() {
failed_registration_reuse_through(
FailedRegistration::SyncBoundLivenessProcfsError,
"SAFEPTRACE_SYNC_PROCFS_ERROR_LIVENESS_REUSE_INNER",
"notifier::test::synchronous_procfs_error_after_liveness_failure_keeps_a_live_generation_open",
)
.await;
}
#[tokio::test(flavor = "current_thread")]
async fn liveness_failure_beside_a_registered_generation_follows_it() {
let (pid, registered, mut cleanup) =
spawn_traced_process(None).expect("spawn registered-generation tracee");
cleanup
.bind_notifier(®istered)
.expect("register the first capability");
let failed = Stopped::from_token(pid.into(), crate::TraceeToken::new());
PIDFD_LIVENESS_ERRORS
.lock()
.entry(pid.into())
.or_default()
.push_back(Errno::EINVAL);
assert_eq!(
TerminalCleanup::new_unregistered(pid.into(), &failed.1).ensure_registered(),
Err(Errno::EINVAL),
"the registration did not fail as injected"
);
assert!(
failed.1.event().identity().is_some(),
"the failed registration did not bind the task's identity"
);
assert!(
Arc::ptr_eq(failed.1.event().event(), registered.1.event().event()),
"the failed generation does not follow the registered one"
);
let running = registered.resume(None).expect("resume to exit");
let exited = tokio::time::timeout(TRACEE_WAIT_TIMEOUT, running.next_state())
.await
.expect("final status timed out")
.expect("wait the registered capability");
cleanup.disarm();
assert_eq!(
exited.assume_exited(),
(pid.into(), crate::ExitStatus::Exited(42))
);
assert!(
failed.1.event().hold_tid().is_none(),
"the failed capability's TID gate stayed open after the reap"
);
}
async fn retirement_before_retention_refuses_after_reap(
requested_pid: Option<i32>,
sync: bool,
) -> bool {
let Some((pid, registered, mut cleanup)) = spawn_traced_process(requested_pid) else {
return false;
};
cleanup
.bind_notifier(®istered)
.expect("register the first capability");
let failed = Stopped::from_token(pid.into(), crate::TraceeToken::new());
PIDFD_LIVENESS_ERRORS
.lock()
.entry(pid.into())
.or_default()
.push_back(Errno::EINVAL);
let (reached_tx, reached_rx) = mpsc::sync_channel(1);
let (resume_tx, resume_rx) = mpsc::channel();
RETENTION_PAUSES.lock().insert(
pid.into(),
RetentionPause {
reached: reached_tx,
resume: resume_rx,
},
);
let token = failed.1.clone();
let registering = thread::spawn(move || {
if sync {
NOTIFIER.sync_handle(pid.into(), token.event()).map(drop)
} else {
TerminalCleanup::new_unregistered(pid.into(), &token).ensure_registered()
}
});
reached_rx
.recv_timeout(TRACEE_WAIT_TIMEOUT)
.expect("the failed registration did not reach retention");
assert!(
failed.1.event().identity().is_some(),
"sync={sync}: the failed registration did not bind the task's identity"
);
let running = registered
.resume(None)
.expect("resume the registered capability to exit");
let exited = tokio::time::timeout(TRACEE_WAIT_TIMEOUT, running.next_state())
.await
.expect("registered capability's final status timed out")
.expect("wait the registered capability");
cleanup.disarm();
assert_eq!(
exited.assume_exited(),
(pid.into(), crate::ExitStatus::Exited(42))
);
let deadline = Instant::now() + TRACEE_WAIT_TIMEOUT;
while NOTIFIER.pids.lock().contains_key(&pid.into()) {
assert!(
Instant::now() < deadline,
"the registered generation's worker did not leave the registry"
);
thread::sleep(Duration::from_millis(1));
}
drop(resume_tx);
assert_eq!(
registering.join().expect("join the failed registration"),
Err(Errno::EINVAL),
"sync={sync}: the registration did not fail as injected"
);
assert!(
PIDFD_LIVENESS_ERRORS
.lock()
.get(&pid.into())
.is_none_or(VecDeque::is_empty),
"sync={sync}: the injected liveness error was not consumed"
);
assert!(
failed.1.event().hold_tid().is_none(),
"sync={sync}: the failed capability's TID gate stayed open after the reap"
);
assert_stale_refused_beside_replacement(
requested_pid,
pid,
vec![("failed", failed)],
&format!("retirement before retention, sync={sync}"),
)
.await
}
async fn retirement_before_retention_reuse_through(sync: bool, inner_env: &str, inner: &str) {
if env::var_os(inner_env).is_some() {
if retirement_before_retention_refuses_after_reap(Some(100), sync).await {
println!("ACTUAL_RETIRED_AUTHORITY_PID_REUSE_EXERCISED");
} else {
println!("ACTUAL_RETIRED_AUTHORITY_PID_REUSE_UNAVAILABLE");
}
return;
}
let actual_reuse = run_exact_in_pid_namespace_bounded(inner, &[(inner_env, "1")]);
match classify_exact_reuse_output(
actual_reuse.as_ref(),
"ACTUAL_RETIRED_AUTHORITY_PID_REUSE_EXERCISED",
"ACTUAL_RETIRED_AUTHORITY_PID_REUSE_UNAVAILABLE",
)
.unwrap_or_else(|error| panic!("sync={sync} PID-reuse regression failed: {error}"))
{
ExactReuseOutcome::Exercised => return,
ExactReuseOutcome::Unavailable => {}
}
assert!(retirement_before_retention_refuses_after_reap(None, sync).await);
}
#[tokio::test(flavor = "current_thread")]
async fn failed_registration_after_its_authority_retired_refuses_after_reap() {
retirement_before_retention_reuse_through(
false,
"SAFEPTRACE_RETIRED_AUTHORITY_REUSE_INNER",
"notifier::test::failed_registration_after_its_authority_retired_refuses_after_reap",
)
.await;
}
#[tokio::test(flavor = "current_thread")]
async fn synchronous_failed_registration_after_its_authority_retired_refuses_after_reap() {
retirement_before_retention_reuse_through(
true,
"SAFEPTRACE_SYNC_RETIRED_AUTHORITY_REUSE_INNER",
"notifier::test::synchronous_failed_registration_after_its_authority_retired_refuses_after_reap",
)
.await;
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
enum StaleEntryRetry {
Unchecked,
Unbound,
}
async fn stale_entry_refuses_after_reap(
requested_pid: Option<i32>,
retry: StaleEntryRetry,
) -> bool {
let Some((old_pid, old, mut old_cleanup)) = spawn_traced_process(requested_pid) else {
return false;
};
old_cleanup
.bind_notifier(&old)
.expect("register the old tracee");
let old_handle = old.1.event().clone();
let old_event = Arc::clone(old_handle.event());
let (removing_tx, removing_rx) = mpsc::sync_channel(1);
let (resume_tx, resume_rx) = mpsc::channel();
if requested_pid.is_some() {
*old_event.registry_retirement_pause.lock() = Some(BoundedTestPause {
captured: removing_tx,
resume: resume_rx,
});
}
let running = old.resume(None).expect("resume the old tracee to exit");
let exited = tokio::time::timeout(TRACEE_WAIT_TIMEOUT, running.next_state())
.await
.expect("old tracee's final status timed out")
.expect("wait the old tracee");
old_cleanup.disarm();
assert_eq!(
exited.assume_exited(),
(old_pid.into(), crate::ExitStatus::Exited(42))
);
if requested_pid.is_some() {
removing_rx
.recv_timeout(TRACEE_WAIT_TIMEOUT)
.expect("the old tracee's worker did not reach its registry removal");
}
thread::sleep(Duration::from_millis(20));
let Some((pid, failed, mut cleanup)) = spawn_traced_process_as(requested_pid, |pid| {
Stopped::from_token(pid, crate::TraceeToken::new())
}) else {
return false;
};
if requested_pid.is_some() {
assert_eq!(pid, old_pid, "clone3 did not reuse the old tracee's PID");
} else {
let identity = Arc::clone(
old_handle
.identity()
.expect("the old tracee's registered identity"),
);
NOTIFIER.pids.lock().insert(
pid.into(),
NotifierEntry {
handle: old_handle.clone(),
identity,
},
);
}
let stale_entry = || {
NOTIFIER
.pids
.lock()
.get(&pid.into())
.is_some_and(|entry| Arc::ptr_eq(entry.handle.event(), &old_event))
};
PIDFD_LIVENESS_ERRORS
.lock()
.entry(pid.into())
.or_default()
.push_back(Errno::EINVAL);
assert_eq!(
TerminalCleanup::new_unregistered(pid.into(), &failed.1).ensure_registered(),
Err(Errno::EINVAL),
"{retry:?}: the registration did not fail as injected"
);
assert!(
failed.1.event().identity().is_some(),
"{retry:?}: the failed registration did not bind the task's identity"
);
assert!(
failed.1.event().hold_tid().is_some(),
"{retry:?}: the failed registration closed a live task's TID gate"
);
assert!(
stale_entry(),
"{retry:?}: the old tracee's entry left the registry before the retry"
);
let absorbed = || {
ABSORBED_UNREGISTERED
.lock()
.get(&pid.into())
.copied()
.unwrap_or(0)
};
let absorbed_before = absorbed();
let later = match retry {
StaleEntryRetry::Unchecked => {
let later = Stopped::new_unchecked(pid.into());
assert!(
Arc::ptr_eq(later.1.event().event(), failed.1.event().event()),
"a capability made beside the stale entry left the failed generation's Event"
);
later
}
StaleEntryRetry::Unbound => Stopped::from_token(pid.into(), crate::TraceeToken::new()),
};
cleanup
.bind_notifier(&later)
.expect("register the later capability");
drop(resume_tx);
if retry == StaleEntryRetry::Unbound {
assert_eq!(
absorbed(),
absorbed_before + 1,
"registering over the stale entry did not redirect the failed generation"
);
assert!(Arc::ptr_eq(
later.1.event().event(),
failed.1.event().event()
));
}
let running = later
.resume(None)
.expect("resume the later capability to exit");
let exited = tokio::time::timeout(TRACEE_WAIT_TIMEOUT, running.next_state())
.await
.expect("later capability's final status timed out")
.expect("wait the later capability");
cleanup.disarm();
assert_eq!(
exited.assume_exited(),
(pid.into(), crate::ExitStatus::Exited(42))
);
assert!(
failed.1.event().hold_tid().is_none(),
"{retry:?}: the failed capability's TID gate stayed open after the reap"
);
assert_stale_refused_beside_replacement(
requested_pid,
pid,
vec![("failed", failed)],
&format!("stale entry, {retry:?}"),
)
.await
}
async fn stale_entry_reuse_through(retry: StaleEntryRetry, inner_env: &str, inner: &str) {
if env::var_os(inner_env).is_some() {
if stale_entry_refuses_after_reap(Some(100), retry).await {
println!("ACTUAL_STALE_ENTRY_PID_REUSE_EXERCISED");
} else {
println!("ACTUAL_STALE_ENTRY_PID_REUSE_UNAVAILABLE");
}
return;
}
let actual_reuse = run_exact_in_pid_namespace_bounded(inner, &[(inner_env, "1")]);
match classify_exact_reuse_output(
actual_reuse.as_ref(),
"ACTUAL_STALE_ENTRY_PID_REUSE_EXERCISED",
"ACTUAL_STALE_ENTRY_PID_REUSE_UNAVAILABLE",
)
.unwrap_or_else(|error| panic!("{retry:?} PID-reuse regression failed: {error}"))
{
ExactReuseOutcome::Exercised => return,
ExactReuseOutcome::Unavailable => {}
}
assert!(stale_entry_refuses_after_reap(None, retry).await);
}
#[tokio::test(flavor = "current_thread")]
async fn constructor_beside_a_stale_entry_shares_the_failed_generation() {
stale_entry_reuse_through(
StaleEntryRetry::Unchecked,
"SAFEPTRACE_STALE_ENTRY_UNCHECKED_REUSE_INNER",
"notifier::test::constructor_beside_a_stale_entry_shares_the_failed_generation",
)
.await;
}
#[tokio::test(flavor = "current_thread")]
async fn registration_over_a_stale_entry_redirects_the_failed_generation() {
stale_entry_reuse_through(
StaleEntryRetry::Unbound,
"SAFEPTRACE_STALE_ENTRY_UNBOUND_REUSE_INNER",
"notifier::test::registration_over_a_stale_entry_redirects_the_failed_generation",
)
.await;
}
#[tokio::test(flavor = "current_thread")]
async fn worker_start_failure_keeps_an_uncached_generation_for_later_capabilities() {
failed_registration_reuse_through(
FailedRegistration::UncachedWorkerStart,
"SAFEPTRACE_UNCACHED_WORKER_START_REUSE_INNER",
"notifier::test::worker_start_failure_keeps_an_uncached_generation_for_later_capabilities",
)
.await;
}
fn distinct_running(pid: crate::Pid) -> Running {
Running::from_token(pid, crate::TraceeToken::new())
}
fn spawn_stopped_process(requested_pid: Option<i32>) -> Option<(Pid, TraceeCleanupGuard)> {
let child = if let Some(requested_pid) = requested_pid {
#[repr(C)]
#[derive(Default)]
struct CloneArgs {
flags: u64,
pidfd: u64,
child_tid: u64,
parent_tid: u64,
exit_signal: u64,
stack: u64,
stack_size: u64,
tls: u64,
set_tid: u64,
set_tid_size: u64,
cgroup: u64,
}
let mut set_tid = requested_pid as u64;
let args = CloneArgs {
exit_signal: libc::SIGCHLD as u64,
set_tid: std::ptr::from_mut(&mut set_tid) as u64,
set_tid_size: 1,
..CloneArgs::default()
};
let result = crate::test_fork::exclusive(|| unsafe {
libc::syscall(
libc::SYS_clone3,
std::ptr::from_ref(&args),
mem::size_of::<CloneArgs>(),
)
});
if result == -1 {
return None;
}
if result == 0 {
let _ = nix::sys::signal::raise(Signal::SIGSTOP);
unsafe { libc::_exit(0) };
}
Pid::from_raw(result as i32)
} else {
match unsafe { fork() }.expect("fork replacement-generation tracee") {
ForkResult::Parent { child } => child,
ForkResult::Child => {
let _ = nix::sys::signal::raise(Signal::SIGSTOP);
unsafe { libc::_exit(0) };
}
}
};
let mut cleanup = TraceeCleanupGuard::new(child).unwrap_or_else(|error| {
let _ = unsafe { libc::kill(child.as_raw(), libc::SIGKILL) };
let _ = reap_tracee_bounded(child);
panic!("open replacement-generation pidfd: {error}");
});
let status = waitpid_status_bounded(child, libc::WUNTRACED, TRACEE_WAIT_TIMEOUT)
.unwrap_or_else(|error| {
let cleanup_result = cleanup.cleanup();
panic!("wait replacement-generation tracee: {error}; cleanup: {cleanup_result:?}")
});
assert!(libc::WIFSTOPPED(status));
Some((child, cleanup))
}
fn reap_stopped_process(mut cleanup: TraceeCleanupGuard) {
cleanup.cleanup().expect("bounded stopped-process cleanup");
}
fn assert_live_replacement_rejected(first_pid: Option<i32>) -> bool {
let Some((old_pid, old_cleanup)) = spawn_stopped_process(first_pid) else {
return false;
};
let old_identity = WorkerIdentity::capture_process(old_pid.into())
.expect("capture old notifier worker generation");
assert!(old_identity.is_same_process_generation());
reap_stopped_process(old_cleanup);
thread::sleep(Duration::from_millis(20));
let Some((new_pid, new_cleanup)) = spawn_stopped_process(first_pid) else {
return false;
};
let new_identity = WorkerIdentity::capture_process(new_pid.into())
.expect("capture replacement notifier worker generation");
assert!(new_identity.is_same_process_generation());
assert!(
!old_identity.is_same_process_generation(),
"an ECHILD worker accepted a live replacement proc generation"
);
assert!(!old_identity.is_active_tracee());
reap_stopped_process(new_cleanup);
true
}
#[test]
fn worker_echild_rejects_live_replacement_generation() {
const INNER: &str = "SAFEPTRACE_ACTUAL_REUSE_INNER";
if env::var_os(INNER).is_some() {
if assert_live_replacement_rejected(Some(100)) {
println!("ACTUAL_PID_REUSE_EXERCISED");
} else {
println!("ACTUAL_PID_REUSE_UNAVAILABLE");
}
return;
}
let inner = "notifier::test::worker_echild_rejects_live_replacement_generation";
let actual_reuse = run_exact_in_pid_namespace_bounded(inner, &[(INNER, "1")]);
match classify_exact_reuse_output(
actual_reuse.as_ref(),
"ACTUAL_PID_REUSE_EXERCISED",
"ACTUAL_PID_REUSE_UNAVAILABLE",
)
.unwrap_or_else(|error| panic!("actual worker PID-reuse regression failed: {error}"))
{
ExactReuseOutcome::Exercised => return,
ExactReuseOutcome::Unavailable => {}
}
assert!(assert_live_replacement_rejected(None));
}
#[test]
fn registry_rejects_terminal_event_after_actual_pid_reuse() {
const INNER: &str = "SAFEPTRACE_REGISTRY_REUSE_INNER";
if env::var_os(INNER).is_some() {
let Some((old_pid, old_cleanup)) = spawn_stopped_process(Some(100)) else {
println!("ACTUAL_REGISTRY_PID_REUSE_UNAVAILABLE");
return;
};
let old_identity = Arc::new(
WorkerIdentity::capture_process(old_pid.into())
.expect("capture old registry generation"),
);
let old_handle = EventHandle::with_identity(Arc::clone(&old_identity));
NOTIFIER.pids.lock().insert(
old_pid.into(),
NotifierEntry {
handle: old_handle.clone(),
identity: old_identity,
},
);
assert!(old_handle.event().try_begin_unstarted_completion());
old_handle.event().mark_echild();
old_handle.event().mark_worker_done();
reap_stopped_process(old_cleanup);
thread::sleep(Duration::from_millis(20));
let Some((new_pid, new_cleanup)) = spawn_stopped_process(Some(100)) else {
NOTIFIER.pids.lock().remove(&old_pid.into());
println!("ACTUAL_REGISTRY_PID_REUSE_UNAVAILABLE");
return;
};
let selected = EventHandle::current_or_new(new_pid.into())
.expect("select replacement registry generation");
assert_ne!(
selected, old_handle,
"registry rebound a reused numeric PID to the old terminal Event"
);
NOTIFIER.pids.lock().remove(&new_pid.into());
reap_stopped_process(new_cleanup);
println!("ACTUAL_REGISTRY_PID_REUSE_EXERCISED");
return;
}
let inner = "notifier::test::registry_rejects_terminal_event_after_actual_pid_reuse";
let actual_reuse = run_exact_in_pid_namespace_bounded(inner, &[(INNER, "1")]);
match classify_exact_reuse_output(
actual_reuse.as_ref(),
"ACTUAL_REGISTRY_PID_REUSE_EXERCISED",
"ACTUAL_REGISTRY_PID_REUSE_UNAVAILABLE",
)
.unwrap_or_else(|error| panic!("actual registry PID-reuse regression failed: {error}"))
{
ExactReuseOutcome::Exercised => return,
ExactReuseOutcome::Unavailable => {}
}
assert_projected_registry_reuse_rejected();
}
fn assert_projected_registry_reuse_rejected() {
let (old_pid, old_cleanup) =
spawn_stopped_process(None).expect("spawn old projected generation");
let mut old_identity = WorkerIdentity::capture_process(old_pid.into())
.expect("capture old projected generation");
reap_stopped_process(old_cleanup);
thread::sleep(Duration::from_millis(20));
let (new_pid, new_cleanup) =
spawn_stopped_process(None).expect("spawn new projected generation");
old_identity.pid = new_pid.into();
let old_identity = Arc::new(old_identity);
let old_handle = EventHandle::with_identity(Arc::clone(&old_identity));
assert!(old_handle.event().try_begin_unstarted_completion());
old_handle.event().mark_echild();
old_handle.event().mark_worker_done();
NOTIFIER.pids.lock().insert(
new_pid.into(),
NotifierEntry {
handle: old_handle.clone(),
identity: old_identity,
},
);
let selected = EventHandle::current_or_new(new_pid.into())
.expect("select projected replacement registry generation");
assert_ne!(
selected, old_handle,
"registry selected a terminal Event using only the projected numeric PID"
);
NOTIFIER.pids.lock().remove(&new_pid.into());
reap_stopped_process(new_cleanup);
}
#[test]
fn registry_rejects_projected_pid_reuse_without_user_namespaces() {
assert_projected_registry_reuse_rejected();
}
}