use std::ffi::c_int;
use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering};
use std::sync::{Arc, Mutex, MutexGuard, Weak};
use snare_interpose::RaceCell;
use crate::readiness::{WakeKey, WakeKeys, readiness};
use crate::scope::SimShared;
use crate::{HashMap, HashSet};
#[derive(Default)]
pub(crate) struct Pipes {
state: Mutex<PipeState>,
}
#[derive(Default)]
struct PipeState {
ends: HashMap<(u64, u64), u64>,
lent: HashSet<u64>,
next: u64,
}
impl Pipes {
fn state(&self) -> MutexGuard<'_, PipeState> {
self.state.lock().unwrap_or_else(|e| e.into_inner())
}
pub(crate) fn add(&self, fds: [c_int; 2]) {
let ends = fds.map(identity);
let mut state = self.state();
state.next += 1;
let pipe = state.next;
for end in ends.into_iter().flatten() {
state.ends.insert(end, pipe);
}
}
pub(crate) fn lend(&self, fd: Option<c_int>) {
let end = fd.map(identity);
let mut state = self.state();
let state = &mut *state;
match end {
Some(end) => {
if let Some(&pipe) = end.and_then(|end| state.ends.get(&end)) {
state.lent.insert(pipe);
}
}
None => state.lent.extend(state.ends.values().copied()),
}
}
pub(crate) fn lent(&self, fd: c_int) -> Option<bool> {
if self.state().ends.is_empty() {
return None;
}
let end = identity(fd)?;
let state = self.state();
let pipe = state.ends.get(&end)?;
Some(state.lent.contains(pipe))
}
}
#[allow(clippy::unnecessary_cast)]
fn identity(fd: c_int) -> Option<(u64, u64)> {
snare_interpose::real(|| {
let mut stat = std::mem::MaybeUninit::<libc::stat>::uninit();
if unsafe { libc::fstat(fd, stat.as_mut_ptr()) } != 0 {
return None;
}
let stat = unsafe { stat.assume_init() };
Some((stat.st_dev as u64, stat.st_ino as u64))
})
}
struct Entry {
id: u64,
domain: usize,
probe: c_int,
waiting: bool,
armed: bool,
outside: bool,
child: Option<(libc::pid_t, Weak<SimShared>)>,
}
static WATCHES: Mutex<Vec<Entry>> = Mutex::new(Vec::new());
static NEXT_ID: AtomicU64 = AtomicU64::new(1);
static WATCH_COUNT: AtomicUsize = AtomicUsize::new(0);
fn watches() -> MutexGuard<'static, Vec<Entry>> {
WATCHES.lock().unwrap_or_else(|e| e.into_inner())
}
fn with_entry<R>(id: u64, f: impl FnOnce(&mut Entry) -> R) -> Option<R> {
snare_interpose::real(|| watches().iter_mut().find(|e| e.id == id).map(f))
}
pub(crate) const OUTSIDE_POLL: std::time::Duration = std::time::Duration::from_millis(200);
pub(crate) struct Watch {
id: u64,
probe: c_int,
fds: Mutex<Vec<(c_int, i16)>>,
edges: Mutex<HashMap<c_int, (u64, u64)>>,
}
impl Watch {
pub(crate) fn new(fds: &[(c_int, i16)]) -> Option<Arc<Watch>> {
let probe = snare_interpose::real(probe_set);
if probe < 0 {
return None;
}
let watch = Arc::new(Watch {
id: NEXT_ID.fetch_add(1, Ordering::Relaxed),
probe,
fds: Mutex::new(Vec::new()),
edges: Mutex::new(HashMap::default()),
});
watch.set(fds);
snare_interpose::real(|| {
watches().push(Entry {
id: watch.id,
domain: 0,
probe,
waiting: false,
armed: false,
outside: false,
child: None,
});
});
WATCH_COUNT.fetch_add(1, Ordering::SeqCst);
Some(watch)
}
pub(crate) fn key(&self) -> WakeKey {
WakeKey::Real(self.id)
}
pub(crate) fn set(&self, fds: &[(c_int, i16)]) {
let mut wanted: Vec<(c_int, i16)> = Vec::with_capacity(fds.len());
for &(fd, events) in fds {
match wanted.iter_mut().find(|(f, _)| *f == fd) {
Some(entry) => entry.1 |= events,
None => wanted.push((fd, events)),
}
}
let mut held = self.fds.lock().unwrap_or_else(|e| e.into_inner());
if *held == wanted {
return;
}
snare_interpose::real(|| {
for &(fd, _) in held
.iter()
.filter(|(fd, _)| !wanted.iter().any(|w| w.0 == *fd))
{
unwatch(self.probe, fd);
}
for &(fd, events) in &wanted {
if !held.contains(&(fd, events)) {
watch(self.probe, fd, events);
}
}
});
let mut edges = self.edges.lock().unwrap_or_else(|e| e.into_inner());
edges.retain(|fd, _| wanted.iter().any(|w| w.0 == *fd));
*held = wanted;
}
pub(crate) fn drain(&self) {
let mut edges = self.edges.lock().unwrap_or_else(|e| e.into_inner());
snare_interpose::real(|| {
drain(self.probe, &mut |fd, read, write| {
let counts = edges.entry(fd).or_default();
counts.0 += u64::from(read);
counts.1 += u64::from(write);
});
});
}
pub(crate) fn edges(&self, fd: c_int) -> (u64, u64) {
let edges = self.edges.lock().unwrap_or_else(|e| e.into_inner());
edges.get(&fd).copied().unwrap_or_default()
}
pub(crate) fn wait(self: &Arc<Self>, shared: &Arc<SimShared>) -> Waiting {
let domain = shared.domain_key();
with_entry(self.id, |entry| {
entry.domain = domain;
entry.waiting = true;
});
let waiting = Waiting {
watch: self.clone(),
shared: shared.clone(),
outside: AtomicBool::new(false),
};
waiting.arm();
waiting
}
}
impl Drop for Watch {
fn drop(&mut self) {
let armed = snare_interpose::real(|| {
let mut watches = watches();
let at = watches.iter().position(|e| e.id == self.id)?;
let entry = watches.swap_remove(at);
unsafe { libc::close(entry.probe) };
Some(entry.armed)
});
if let Some(armed) = armed {
WATCH_COUNT.fetch_sub(1, Ordering::SeqCst);
if armed {
watcher().poke();
}
}
}
}
pub(crate) struct Waiting {
watch: Arc<Watch>,
shared: Arc<SimShared>,
outside: AtomicBool,
}
impl Waiting {
pub(crate) fn watch(&self) -> &Arc<Watch> {
&self.watch
}
pub(crate) fn look(&self) {
self.watch.drain();
self.arm();
}
fn arm(&self) {
if !snare_interpose::det_active() {
let rearmed = with_entry(self.watch.id, |entry| {
!std::mem::replace(&mut entry.armed, true)
});
if rearmed == Some(true) {
watcher().poke();
}
}
let outside = {
let fds = self.watch.fds.lock().unwrap_or_else(|e| e.into_inner());
fds.iter()
.any(|&(fd, _)| self.shared.pipes.lent(fd) == Some(true) && !hung_up(fd))
};
if self.outside.swap(outside, Ordering::AcqRel) != outside {
if let Some(clock) = &self.shared.clock {
clock.hold_outside(outside);
}
with_entry(self.watch.id, |entry| entry.outside = outside);
}
}
}
impl Drop for Waiting {
fn drop(&mut self) {
if self.outside.load(Ordering::Acquire)
&& let Some(clock) = &self.shared.clock
{
clock.hold_outside(false);
}
with_entry(self.watch.id, |entry| {
entry.waiting = false;
entry.armed = false;
entry.outside = false;
});
}
}
fn hung_up(fd: c_int) -> bool {
let mut pfd = libc::pollfd {
fd,
events: libc::POLLIN | libc::POLLOUT,
revents: 0,
};
let polled = snare_interpose::real(|| unsafe { libc::poll(&mut pfd, 1, 0) });
polled < 0 || pfd.revents & (libc::POLLHUP | libc::POLLERR | libc::POLLNVAL) != 0
}
fn probe_ready(probe: c_int) -> bool {
let mut pfd = libc::pollfd {
fd: probe,
events: libc::POLLIN,
revents: 0,
};
let polled = snare_interpose::real(|| unsafe { libc::poll(&mut pfd, 1, 0) });
polled > 0 && pfd.revents & libc::POLLIN != 0
}
#[cfg(target_os = "linux")]
fn probe_set() -> c_int {
unsafe { libc::epoll_create1(libc::EPOLL_CLOEXEC) }
}
#[cfg(target_os = "macos")]
fn probe_set() -> c_int {
let kq = unsafe { libc::kqueue() };
if kq >= 0 {
unsafe { libc::fcntl(kq, libc::F_SETFD, libc::FD_CLOEXEC) };
}
kq
}
#[cfg(target_os = "linux")]
fn watch(probe: c_int, fd: c_int, events: i16) {
let mask = (events as u16 as u32)
& (libc::EPOLLIN | libc::EPOLLOUT | libc::EPOLLPRI | libc::EPOLLRDHUP) as u32;
let mut event = libc::epoll_event {
events: mask | libc::EPOLLET as u32,
u64: fd as u64,
};
unsafe {
if libc::epoll_ctl(probe, libc::EPOLL_CTL_ADD, fd, &mut event) != 0 {
libc::epoll_ctl(probe, libc::EPOLL_CTL_MOD, fd, &mut event);
}
}
}
#[cfg(target_os = "linux")]
fn unwatch(probe: c_int, fd: c_int) {
unsafe { libc::epoll_ctl(probe, libc::EPOLL_CTL_DEL, fd, std::ptr::null_mut()) };
}
#[cfg(target_os = "linux")]
fn drain(probe: c_int, edge: &mut dyn FnMut(c_int, bool, bool)) {
let mut events = [libc::epoll_event { events: 0, u64: 0 }; 16];
loop {
let n = unsafe { libc::epoll_wait(probe, events.as_mut_ptr(), 16, 0) };
for event in &events[..n.max(0) as usize] {
let (bits, fd) = (event.events, event.u64 as c_int);
let broken = bits & (libc::EPOLLHUP | libc::EPOLLERR) as u32 != 0;
let read = bits & (libc::EPOLLIN | libc::EPOLLPRI | libc::EPOLLRDHUP) as u32 != 0;
edge(
fd,
read || broken,
bits & libc::EPOLLOUT as u32 != 0 || broken,
);
}
if n < 16 {
break;
}
}
}
#[cfg(target_os = "macos")]
fn kevent_filters(events: i16) -> impl Iterator<Item = i16> {
[
(
libc::POLLIN | libc::POLLPRI | libc::POLLRDNORM,
libc::EVFILT_READ,
),
(libc::POLLOUT | libc::POLLWRNORM, libc::EVFILT_WRITE),
]
.into_iter()
.filter(move |(bits, _)| events & bits != 0)
.map(|(_, filter)| filter)
}
#[cfg(target_os = "macos")]
fn change(probe: c_int, fd: c_int, events: i16, flags: u16) {
for filter in kevent_filters(events) {
let change = libc::kevent {
ident: fd as usize,
filter,
flags: flags | libc::EV_RECEIPT,
fflags: 0,
data: 0,
udata: std::ptr::null_mut(),
};
let mut receipt: libc::kevent = unsafe { std::mem::zeroed() };
unsafe { libc::kevent(probe, &change, 1, &mut receipt, 1, std::ptr::null()) };
}
}
#[cfg(target_os = "macos")]
fn watch(probe: c_int, fd: c_int, events: i16) {
change(probe, fd, events, libc::EV_ADD | libc::EV_CLEAR);
}
#[cfg(target_os = "macos")]
fn unwatch(probe: c_int, fd: c_int) {
change(probe, fd, libc::POLLIN | libc::POLLOUT, libc::EV_DELETE);
}
#[cfg(target_os = "macos")]
fn drain(probe: c_int, edge: &mut dyn FnMut(c_int, bool, bool)) {
let mut events: [libc::kevent; 16] = unsafe { std::mem::zeroed() };
let zero = libc::timespec {
tv_sec: 0,
tv_nsec: 0,
};
loop {
let n = unsafe { libc::kevent(probe, std::ptr::null(), 0, events.as_mut_ptr(), 16, &zero) };
for event in &events[..n.max(0) as usize] {
let fd = event.ident as c_int;
edge(
fd,
event.filter == libc::EVFILT_READ,
event.filter == libc::EVFILT_WRITE,
);
}
if n < 16 {
break;
}
}
}
pub(crate) fn watch_child(shared: &Arc<SimShared>, pid: libc::pid_t) {
snare_interpose::real(|| snare_interpose::adopt_child(pid));
let probe = snare_interpose::real(|| child_probe(pid));
if probe < 0 {
child_exited(shared, pid, None);
return;
}
snare_interpose::real(|| {
watches().push(Entry {
id: NEXT_ID.fetch_add(1, Ordering::Relaxed),
domain: shared.domain_key(),
probe,
waiting: false,
armed: false,
outside: false,
child: Some((pid, Arc::downgrade(shared))),
});
});
WATCH_COUNT.fetch_add(1, Ordering::SeqCst);
watcher().poke();
}
#[cfg(target_os = "linux")]
pub(crate) fn child_probe(pid: libc::pid_t) -> c_int {
let fd = unsafe { libc::syscall(libc::SYS_pidfd_open, pid, 0) } as c_int;
if fd >= 0 {
unsafe { libc::fcntl(fd, libc::F_SETFD, libc::FD_CLOEXEC) };
}
fd
}
#[cfg(target_os = "macos")]
pub(crate) fn child_probe(pid: libc::pid_t) -> c_int {
let kq = probe_set();
if kq < 0 {
return kq;
}
let change = libc::kevent {
ident: pid as usize,
filter: libc::EVFILT_PROC,
flags: libc::EV_ADD | libc::EV_ONESHOT,
fflags: libc::NOTE_EXIT | libc::NOTE_EXITSTATUS,
data: 0,
udata: std::ptr::null_mut(),
};
if unsafe { libc::kevent(kq, &change, 1, std::ptr::null_mut(), 0, std::ptr::null()) } != 0 {
unsafe { libc::close(kq) };
return -1;
}
kq
}
#[cfg(target_os = "linux")]
fn child_status(_probe: c_int) -> Option<c_int> {
None
}
#[cfg(target_os = "macos")]
fn child_status(probe: c_int) -> Option<c_int> {
let mut event: libc::kevent = unsafe { std::mem::zeroed() };
let zero = libc::timespec {
tv_sec: 0,
tv_nsec: 0,
};
let n = unsafe { libc::kevent(probe, std::ptr::null(), 0, &mut event, 1, &zero) };
(n == 1 && event.filter == libc::EVFILT_PROC && event.fflags & libc::NOTE_EXIT != 0)
.then_some(event.data as c_int)
}
const REAP_NOTE_GRACE: std::time::Duration = std::time::Duration::from_millis(100);
fn child_exited(shared: &Arc<SimShared>, pid: libc::pid_t, status: Option<c_int>) {
let action = snare_interpose::real(|| shared.signals.child_action());
if action.sa_sigaction == libc::SIG_IGN || action.sa_flags & libc::SA_NOCLDWAIT != 0 {
snare_interpose::real(|| unsafe {
libc::waitpid(pid, std::ptr::null_mut(), libc::WNOHANG)
});
}
let handled = action.sa_sigaction != libc::SIG_DFL && action.sa_sigaction != libc::SIG_IGN;
if let Some(domain) = shared.domain.get().and_then(|d| d.upgrade())
&& (handled || (held_by_default(&action) && domain.holds_signal(libc::SIGCHLD)))
{
let _ = domain.spawn_injected("snare-sigchld", move || {
let uid = unsafe { libc::getuid() };
let exit = snare_interpose::real(|| child_exit(pid, uid, status));
snare_interpose::deliver_child_exit(exit);
});
}
if let Some(clock) = &shared.clock {
clock.release_child(pid);
}
}
fn held_by_default(action: &libc::sigaction) -> bool {
cfg!(target_os = "linux") && action.sa_sigaction == libc::SIG_DFL
}
fn child_exit(
pid: libc::pid_t,
uid: libc::uid_t,
status: Option<c_int>,
) -> snare_interpose::ChildExit {
if let Some(exit) =
status.and_then(|status| snare_interpose::ChildExit::from_wait_status(pid, uid, status))
{
return exit;
}
let mut info: libc::siginfo_t = unsafe { std::mem::zeroed() };
let waited = unsafe {
libc::waitid(
libc::P_PID,
pid as libc::id_t,
&mut info,
libc::WEXITED | libc::WNOHANG | libc::WNOWAIT,
)
};
if waited == 0 && unsafe { info.si_pid() } == pid {
return snare_interpose::ChildExit {
pid,
uid,
code: info.si_code,
status: unsafe { info.si_status() },
};
}
let since = std::time::Instant::now();
loop {
if let Some(exit) = snare_interpose::child_exit_of(pid, uid) {
return exit;
}
if since.elapsed() >= REAP_NOTE_GRACE {
return snare_interpose::ChildExit {
pid,
uid,
code: libc::CLD_EXITED,
status: 0,
};
}
std::thread::sleep(std::time::Duration::from_millis(1));
}
}
pub(crate) fn settling(domain: usize) -> bool {
if WATCH_COUNT.load(Ordering::SeqCst) == 0 {
return false;
}
let mut woken = WakeKeys::default();
snare_interpose::real(|| {
for entry in watches().iter() {
if entry.domain == domain && entry.waiting && probe_ready(entry.probe) {
woken.push(WakeKey::Real(entry.id));
}
}
});
if woken.as_slice().is_empty() {
return false;
}
readiness().bump_keys_unscheduled(domain, woken.as_slice());
true
}
pub(crate) fn recheck(domain: usize) {
if WATCH_COUNT.load(Ordering::SeqCst) == 0 {
return;
}
let mut keys = WakeKeys::default();
snare_interpose::real(|| {
for entry in watches().iter() {
if entry.domain == domain && entry.waiting {
keys.push(WakeKey::Real(entry.id));
}
}
});
if !keys.as_slice().is_empty() {
readiness().bump_keys(domain, keys.as_slice());
}
}
pub(crate) fn outside(domain: usize) -> bool {
WATCH_COUNT.load(Ordering::SeqCst) > 0
&& snare_interpose::real(|| {
watches()
.iter()
.any(|e| e.domain == domain && e.waiting && e.outside)
})
}
pub(crate) fn real_ready(domain: usize, ready: &mut dyn FnMut(WakeKey)) {
if WATCH_COUNT.load(Ordering::SeqCst) == 0 {
return;
}
snare_interpose::real(|| {
for entry in watches().iter() {
if entry.domain == domain && entry.waiting && probe_ready(entry.probe) {
ready(WakeKey::Real(entry.id));
}
}
});
}
struct Watcher {
wake: [c_int; 2],
}
static WATCHER: RaceCell<Watcher> = RaceCell::new();
fn watcher() -> &'static Watcher {
let (instance, started) = WATCHER.get_or_init(|| {
let mut wake = [-1; 2];
snare_interpose::real(|| {
if unsafe { libc::pipe(wake.as_mut_ptr()) } == 0 {
for fd in wake {
unsafe {
libc::fcntl(fd, libc::F_SETFD, libc::FD_CLOEXEC);
libc::fcntl(fd, libc::F_SETFL, libc::O_NONBLOCK);
}
}
}
});
Watcher { wake }
});
if started {
snare_interpose::real(|| {
std::thread::Builder::new()
.name("snare-real-fds".into())
.spawn(|| {
let _service = snare_interpose::service_thread();
watcher().run();
})
.expect("start the real-descriptor watcher");
});
}
instance
}
impl Watcher {
fn poke(&self) {
if self.wake[1] >= 0 {
snare_interpose::real(|| unsafe { libc::write(self.wake[1], b"w".as_ptr().cast(), 1) });
}
}
fn run(&self) {
unsafe {
let mut all: libc::sigset_t = std::mem::zeroed();
libc::sigfillset(&mut all);
libc::pthread_sigmask(libc::SIG_BLOCK, &all, std::ptr::null_mut());
}
let mut set: Vec<libc::pollfd> = Vec::new();
let mut ids: Vec<u64> = Vec::new();
let mut woken: Vec<(usize, WakeKey)> = Vec::new();
let mut exited: Vec<(libc::pid_t, Weak<SimShared>, Option<c_int>)> = Vec::new();
loop {
set.clear();
ids.clear();
set.push(libc::pollfd {
fd: self.wake[0],
events: libc::POLLIN,
revents: 0,
});
{
let watches = watches();
for entry in watches.iter() {
if entry.child.is_some() || entry.armed && entry.waiting {
set.push(libc::pollfd {
fd: entry.probe,
events: libc::POLLIN,
revents: 0,
});
ids.push(entry.id);
}
}
}
let polled = unsafe { libc::poll(set.as_mut_ptr(), set.len() as _, -1) };
if polled < 0
&& std::io::Error::last_os_error().kind() != std::io::ErrorKind::Interrupted
{
std::thread::sleep(std::time::Duration::from_millis(1));
}
let mut byte = [0u8; 64];
while unsafe { libc::read(self.wake[0], byte.as_mut_ptr().cast(), 64) } > 0 {}
{
let mut watches = watches();
for (pfd, id) in set[1..].iter().zip(&ids) {
if pfd.revents == 0 {
continue;
}
let Some(at) = watches.iter().position(|e| e.id == *id) else {
continue;
};
if watches[at].child.is_some() {
let entry = watches.swap_remove(at);
let status = child_status(entry.probe);
unsafe { libc::close(entry.probe) };
WATCH_COUNT.fetch_sub(1, Ordering::SeqCst);
exited.extend(entry.child.map(|(pid, shared)| (pid, shared, status)));
} else {
watches[at].armed = false;
woken.push((watches[at].domain, WakeKey::Real(*id)));
}
}
}
for (domain, key) in woken.drain(..) {
readiness().bump_keys_unscheduled(domain, &[key]);
}
for (pid, shared, status) in exited.drain(..) {
if let Some(shared) = shared.upgrade() {
child_exited(&shared, pid, status);
}
}
}
}
}