use std::collections::{BTreeMap, HashMap};
use std::os::raw::{c_char, c_int, c_void};
use std::sync::atomic::{AtomicBool, AtomicI32, AtomicU32, AtomicU64, Ordering};
use std::sync::{Mutex, OnceLock};
pub use crate::sentry_portable::{netstack, proctree, state_snap};
#[cfg(all(target_os = "linux", target_arch = "x86_64"))]
pub mod memimage;
const PR_SET_SYSCALL_USER_DISPATCH: c_int = 59;
const PR_SYS_DISPATCH_ON: libc::c_ulong = 1;
const PR_SET_PDEATHSIG: u64 = 1;
const PR_GET_DUMPABLE: u64 = 3;
const PR_SET_DUMPABLE: u64 = 4;
const PR_SET_NAME: u64 = 15;
const PR_GET_NAME: u64 = 16;
const PR_SET_SECCOMP: u64 = 22;
const PR_SET_CHILD_SUBREAPER_SUPERVISOR: libc::c_int = 36;
const PR_CAPBSET_READ: u64 = 23;
const PR_SET_NO_NEW_PRIVS: u64 = 38;
const PR_SET_PTRACER: u64 = 0x5961_6d61;
const PR_SET_VMA: u64 = 0x5356_4d41;
const PR_SET_VMA_ANON_NAME: u64 = 0;
const FILTER_BLOCK: u8 = 1;
static mut SELECTOR: u8 = FILTER_BLOCK;
const WINDOW_FLOOR: u64 = 0x20_0000_0000;
const USER_TOP: u64 = 0x0000_7fff_ffff_f000;
const EXE_DYN_BASE: u64 = 0x0040_0000; const INTERP_BASE: u64 = 0x4000_0000; const ARENA_TOP: u64 = WINDOW_FLOOR; const ARENA_LIMIT: u64 = 0x4100_0000;
const REG_R8: usize = 0;
const REG_R9: usize = 1;
const REG_R10: usize = 2;
const REG_R11: usize = 3;
const REG_R12: usize = 4;
const REG_R13: usize = 5;
const REG_R14: usize = 6;
const REG_R15: usize = 7;
const REG_RDI: usize = 8;
const REG_RSI: usize = 9;
const REG_RBP: usize = 10;
const REG_RBX: usize = 11;
const REG_RDX: usize = 12;
const REG_RAX: usize = 13;
const REG_RCX: usize = 14;
const REG_RSP: usize = 15;
const REG_RIP: usize = 16;
const ARCH_SET_FS: u64 = 0x1002;
const ARCH_GET_FS: u64 = 0x1003;
const SYS_READ: i64 = 0;
const SYS_WRITE: i64 = 1;
const SYS_OPEN: i64 = 2;
const SYS_CLOSE: i64 = 3;
const SYS_STAT: i64 = 4;
const SYS_FSTAT: i64 = 5;
const SYS_LSTAT: i64 = 6;
const SYS_LSEEK: i64 = 8;
const SYS_MMAP: i64 = 9;
const SYS_MPROTECT: i64 = 10;
const SYS_MUNMAP: i64 = 11;
const SYS_MSYNC: i64 = 26;
const SYS_BRK: i64 = 12;
const SYS_RT_SIGACTION: i64 = 13;
const SYS_RT_SIGPROCMASK: i64 = 14;
const SYS_RT_SIGRETURN: i64 = 15;
const SYS_PAUSE: i64 = 34;
const SYS_RT_SIGPENDING: i64 = 127;
const SYS_RT_SIGTIMEDWAIT: i64 = 128;
const SYS_RT_SIGSUSPEND: i64 = 130;
const SYS_IOCTL: i64 = 16;
const SYS_SHMGET: i64 = 29;
const SYS_SHMAT: i64 = 30;
const SYS_SHMCTL: i64 = 31;
const SYS_GETITIMER: i64 = 36;
const SYS_ALARM: i64 = 37;
const SYS_SETITIMER: i64 = 38;
const SYS_KILL: i64 = 62;
const SYS_CREAT: i64 = 85;
const SYS_PREAD64: i64 = 17;
const SYS_WRITEV: i64 = 20;
const SYS_ACCESS: i64 = 21;
const SYS_MREMAP: i64 = 25;
const SYS_MINCORE: i64 = 27;
const SYS_MADVISE: i64 = 28;
const SYS_CLONE: i64 = 56;
const SYS_CLONE3: i64 = 435;
const SYS_FORK: i64 = 57;
const SYS_VFORK: i64 = 58;
const SYS_EXECVE: i64 = 59;
const SYS_WAIT4: i64 = 61;
const SYS_SHMDT: i64 = 67;
const SIGCHLD_FLAG: u64 = 17;
const CLONE_VM: u64 = 0x0000_0100;
const CLONE_PIDFD: u64 = 0x0000_1000;
const CLONE_VFORK: u64 = 0x0000_4000;
const CLONE_THREAD: u64 = 0x0001_0000;
const CLONE_CLEAR_SIGHAND: u64 = 0x1_0000_0000;
const SYS_GETPID: i64 = 39;
const SYS_SENDFILE: i64 = 40;
const SYS_EXIT: i64 = 60;
const SYS_UNAME: i64 = 63;
const SYS_FCNTL: i64 = 72;
const SYS_FLOCK: i64 = 73;
const F_GETLK: u64 = 5;
const F_SETLK: u64 = 6;
const F_SETLKW: u64 = 7;
const F_OFD_GETLK: u64 = 36;
const F_OFD_SETLK: u64 = 37;
const F_OFD_SETLKW: u64 = 38;
const F_GETFD: u64 = 1;
const F_SETFD: u64 = 2;
const F_DUPFD_CLOEXEC: u64 = 1030;
const FD_CLOEXEC: i64 = 1;
const FLOCK_SZ: usize = 32;
const SYS_CHDIR: i64 = 80;
const SYS_FCHDIR: i64 = 81;
const SYS_GETCWD: i64 = 79;
const SYS_PRCTL: i64 = 157;
const SYS_READLINK: i64 = 89;
const SYS_UMASK: i64 = 95;
const SYS_GETRUSAGE: i64 = 98;
const SYS_SYSINFO: i64 = 99;
const SYS_TIMES: i64 = 100;
const SYS_WAITID: i64 = 247;
const SIGINFO_SZ: usize = 128;
const SIGINFO_SI_PID_OFF: usize = 16;
const _: () = assert!(SIGINFO_SI_PID_OFF + 4 <= SIGINFO_SZ);
const RUSAGE_SZ: usize = 144;
const WNOWAIT: u64 = 0x0100_0000;
const SYS_GETUID: i64 = 102;
const SYS_GETGID: i64 = 104;
const SYS_SETUID: i64 = 105;
const SYS_SETGID: i64 = 106;
const SYS_GETGROUPS: i64 = 115;
const SYS_SETGROUPS: i64 = 116;
const SYS_SETREUID: i64 = 113;
const SYS_SETREGID: i64 = 114;
const SYS_SETRESUID: i64 = 117;
const SYS_GETRESUID: i64 = 118;
const SYS_SETRESGID: i64 = 119;
const SYS_GETRESGID: i64 = 120;
const SYS_SETFSUID: i64 = 122;
const SYS_SETFSGID: i64 = 123;
const SYS_GETPPID: i64 = 110;
const SYS_GETEUID: i64 = 107;
const SYS_GETEGID: i64 = 108;
const SYS_SETPGID: i64 = 109;
const SYS_GETPGRP: i64 = 111;
const SYS_SETSID: i64 = 112;
const SYS_GETPGID: i64 = 121;
const SYS_GETSID: i64 = 124;
const SYS_GETTIMEOFDAY: i64 = 96;
const SYS_STATFS: i64 = 137;
const SYS_GETPRIORITY: i64 = 140;
const SYS_SETPRIORITY: i64 = 141;
const SYS_ARCH_PRCTL: i64 = 158;
const SYS_GETTID: i64 = 186;
const SYS_GETXATTR: i64 = 191;
const SYS_LGETXATTR: i64 = 192;
const SYS_FGETXATTR: i64 = 193;
const SYS_CAPGET: i64 = 125;
const SYS_CAPSET: i64 = 126;
const SYS_FUTEX: i64 = 202;
const SYS_TGKILL: i64 = 234;
const SYS_TKILL: i64 = 200;
const SYS_GETDENTS64: i64 = 217;
const SYS_POLL: i64 = 7;
const SYS_PPOLL: i64 = 271;
const SYS_PSELECT6: i64 = 270;
const SYS_SELECT: i64 = 23;
const SYS_SCHED_YIELD: i64 = 24;
const SYS_SCHED_SETPARAM: i64 = 142;
const SYS_SCHED_GETPARAM: i64 = 143;
const SYS_SCHED_SETSCHEDULER: i64 = 144;
const SYS_SCHED_GETSCHEDULER: i64 = 145;
const SYS_SCHED_GET_PRIORITY_MAX: i64 = 146;
const SYS_SCHED_GET_PRIORITY_MIN: i64 = 147;
const SYS_SCHED_RR_GET_INTERVAL: i64 = 148;
const SYS_DUP: i64 = 32;
const SYS_DUP2: i64 = 33;
const SYS_NANOSLEEP: i64 = 35;
const SYS_FSYNC: i64 = 74;
const SYS_FDATASYNC: i64 = 75;
const SYS_FTRUNCATE: i64 = 77;
const SYS_SIGALTSTACK: i64 = 131;
const SYS_PIPE: i64 = 22;
const SYS_PIPE2: i64 = 293;
const SYS_TIME: i64 = 201;
const SYS_SCHED_SETAFFINITY: i64 = 203;
const SYS_SCHED_GETAFFINITY: i64 = 204;
const SYS_FADVISE64: i64 = 221;
const SYS_EPOLL_WAIT: i64 = 232;
const SYS_EPOLL_CTL: i64 = 233;
const SYS_CLOCK_GETRES: i64 = 229;
const SYS_CLOCK_NANOSLEEP: i64 = 230;
const SYS_EPOLL_PWAIT: i64 = 281;
const SYS_TIMERFD_CREATE: i64 = 283;
const SYS_TIMERFD_SETTIME: i64 = 286;
const SYS_TIMERFD_GETTIME: i64 = 287;
const SYS_EVENTFD2: i64 = 290;
const SYS_EPOLL_CREATE1: i64 = 291;
const SYS_GETCPU: i64 = 309;
const SYS_MEMBARRIER: i64 = 324;
const SYS_PKEY_MPROTECT: i64 = 329;
const SYS_PKEY_ALLOC: i64 = 330;
const SYS_PKEY_FREE: i64 = 331;
const SYS_PWRITE64: i64 = 18;
const SYS_FALLOCATE: i64 = 285;
const SYS_PREADV: i64 = 295;
const SYS_PWRITEV: i64 = 296;
const SYS_PREADV2: i64 = 327;
const SYS_PWRITEV2: i64 = 328;
const SYS_COPY_FILE_RANGE: i64 = 326;
const SYS_SPLICE: i64 = 275;
const SYS_TEE: i64 = 276;
const SYS_VMSPLICE: i64 = 278;
const SYS_READV: i64 = 19;
const SYS_SOCKET: i64 = 41;
const SYS_CONNECT: i64 = 42;
const SYS_ACCEPT: i64 = 43;
const SYS_SENDTO: i64 = 44;
const SYS_RECVFROM: i64 = 45;
const SYS_SENDMSG: i64 = 46;
const SYS_RECVMSG: i64 = 47;
const SYS_RECVMMSG: i64 = 299;
const SYS_SENDMMSG: i64 = 307;
const SYS_SHUTDOWN: i64 = 48;
const SYS_BIND: i64 = 49;
const SYS_LISTEN: i64 = 50;
const SYS_GETSOCKNAME: i64 = 51;
const SYS_GETPEERNAME: i64 = 52;
const SYS_SOCKETPAIR: i64 = 53;
const SYS_SETSOCKOPT: i64 = 54;
const SYS_GETSOCKOPT: i64 = 55;
const SYS_ACCEPT4: i64 = 288;
const SYS_UNSHARE: i64 = 272;
const SYS_SETNS: i64 = 308;
const SYS_SET_TID_ADDRESS: i64 = 218;
const SYS_CLOCK_GETTIME: i64 = 228;
const SYS_EXIT_GROUP: i64 = 231;
const SYS_OPENAT: i64 = 257;
const SYS_NEWFSTATAT: i64 = 262;
const SYS_READLINKAT: i64 = 267;
const SYS_TRUNCATE: i64 = 76;
const SYS_RENAME: i64 = 82;
const SYS_MKDIR: i64 = 83;
const SYS_RMDIR: i64 = 84;
const SYS_LINK: i64 = 86;
const SYS_UNLINK: i64 = 87;
const SYS_SYMLINK: i64 = 88;
const SYS_CHMOD: i64 = 90;
const SYS_CHOWN: i64 = 92;
const SYS_LCHOWN: i64 = 94;
const SYS_UTIMENSAT: i64 = 280;
const SYS_FCHOWNAT: i64 = 260;
const SYS_RENAMEAT: i64 = 264;
const SYS_LINKAT: i64 = 265;
const SYS_SYMLINKAT: i64 = 266;
const SYS_FCHMODAT: i64 = 268;
const SYS_FCHMOD: i64 = 91;
const SYS_FCHOWN: i64 = 93;
const SYS_MKDIRAT: i64 = 258;
const SYS_UNLINKAT: i64 = 263;
const SYS_RENAMEAT2: i64 = 316;
const AT_FDCWD: i32 = -100;
const AT_REMOVEDIR: u64 = 0x200;
const RESOLVE_BENEATH: u64 = 0x08;
const SYS_FACCESSAT: i64 = 269;
const SYS_FACCESSAT2: i64 = 439;
const SYS_SET_ROBUST_LIST: i64 = 273;
const SYS_PRLIMIT64: i64 = 302;
const SYS_GETRANDOM: i64 = 318;
const SYS_STATX: i64 = 332;
const SYS_RSEQ: i64 = 334;
const SYS_PROCESS_VM_READV: i64 = 310;
const SYS_PROCESS_VM_WRITEV: i64 = 311;
const SYS_DUP3: i64 = 292;
const SYS_INOTIFY_INIT: i64 = 253;
const SYS_INOTIFY_ADD_WATCH: i64 = 254;
const SYS_INOTIFY_RM_WATCH: i64 = 255;
const SYS_INOTIFY_INIT1: i64 = 294;
const SYS_EVENTFD: i64 = 284;
const SYS_EPOLL_CREATE: i64 = 213;
const SYS_SIGNALFD: i64 = 282;
const SYS_SIGNALFD4: i64 = 289;
const SYS_MKNOD: i64 = 133;
const SYS_MKNODAT: i64 = 259;
const SYS_CLOSE_RANGE: i64 = 436;
const CLOSE_RANGE_UNSHARE: u64 = 1 << 1;
const CLOSE_RANGE_CLOEXEC: u64 = 1 << 2;
const S_IFMT: u32 = 0o170000;
const S_IFREG: u32 = 0o100000;
const S_IFDIR: u32 = 0o040000;
const S_IFCHR: u32 = 0o020000;
const S_IFBLK: u32 = 0o060000;
const S_IFIFO: u32 = 0o010000;
const S_IFSOCK: u32 = 0o140000;
const CTL_REAP: i64 = 0x53_0001;
const CTL_FORK_TABLE: i64 = 0x53_0002;
const CTL_FORK_CANCEL: i64 = 0x53_0003;
const CTL_ENSURE_SERVICER: i64 = 0x53_0004;
const CTL_BIND_SLOT: i64 = 0x53_000a;
const CTL_SET_EXE: i64 = 0x53_0005;
const CTL_SET_CMDLINE: i64 = 0x53_0006;
const CTL_CLOSE_CLOEXEC: i64 = 0x53_0007;
const CTL_DETHREAD_FOR_EXEC: i64 = 0x53_0008;
const CTL_SHM_FLUSH: i64 = 0x53_0009;
const CTL_SHM_ALIAS: i64 = 0x53_000e;
const CTL_SHM_PROTECT: i64 = 0x53_0010;
const CTL_SHM_MOVE: i64 = 0x53_0011;
const CTL_ENVDBG: i64 = 0x53_000b;
const CTL_COREDUMP: i64 = 0x53_000c;
const CTL_WATCHADDR: i64 = 0x53_000d;
const CTL_PROCEXE_EXEC_FAIL: i64 = 0x53_000f;
const CTL_LOGLINE: i64 = 0x53_0014;
const CTL_SET_CREDS: i64 = 0x53_0012;
const CTL_SET_UMASK: i64 = 0x53_0013;
const CANCEL_SENTINEL: i64 = -255;
const EINTR: i64 = -4;
const EAGAIN: i64 = -11;
const CANCEL_REAP_PROCESS: u8 = 0;
const CANCEL_SLOT_ONLY: u8 = 1;
const POLLIN_BIT: i16 = 0x001;
const POLLPRI_BIT: i16 = 0x002;
const EFD_NONBLOCK_CLOEXEC: u64 = 0o4000 | 0o2000000;
const FUTEX_WAIT: u64 = 0;
const FUTEX_WAKE: u64 = 1;
const FUTEX_PRIVATE_FLAG: u64 = 128;
const FUTEX_CLOCK_REALTIME: u64 = 256;
const FUTEX_WAIT_BITSET: u64 = 9;
const FUTEX_BITSET_MATCH_ANY: u64 = 0xffff_ffff;
const RING_SPIN: u32 = 4096;
#[inline(always)]
unsafe fn host(nr: i64, a: u64, b: u64, c: u64, d: u64, e: u64, f: u64) -> i64 {
unsafe {
let ret: i64;
std::arch::asm!(
"syscall",
inlateout("rax") nr => ret,
in("rdi") a, in("rsi") b, in("rdx") c,
in("r10") d, in("r8") e, in("r9") f,
lateout("rcx") _, lateout("r11") _,
options(nostack),
);
ret
}
}
fn raw_write(fd: c_int, b: &[u8]) {
unsafe {
host(
SYS_WRITE,
fd as u64,
b.as_ptr() as u64,
b.len() as u64,
0,
0,
0,
)
};
}
fn log(b: &[u8]) {
raw_write(2, b);
}
fn fmt_i64(n: i64, buf: &mut [u8; 24]) -> usize {
let neg = n < 0;
let mut m = (n as i128).unsigned_abs();
let mut i = 24;
if m == 0 {
i -= 1;
buf[i] = b'0';
}
while m > 0 {
i -= 1;
buf[i] = b'0' + (m % 10) as u8;
m /= 10;
}
if neg {
i -= 1;
buf[i] = b'-';
}
i
}
fn logn(prefix: &[u8], n: i64, suffix: &[u8]) {
let mut b = [0u8; 24];
let i = fmt_i64(n, &mut b);
log(prefix);
log(&b[i..]);
log(suffix);
}
fn ipc_logf(parts: &[(&[u8], i64)], hex: &[u8]) {
let mut line: Vec<u8> = Vec::with_capacity(128);
for (k, v) in parts {
line.extend_from_slice(k);
let mut nb = [0u8; 24];
let i = fmt_i64(*v, &mut nb);
line.extend_from_slice(&nb[i..]);
line.push(b' ');
}
if !hex.is_empty() {
line.extend_from_slice(b"hex=");
const H: &[u8; 16] = b"0123456789abcdef";
for &byte in hex.iter().take(80) {
line.push(H[(byte >> 4) as usize]);
line.push(H[(byte & 0xf) as usize]);
}
line.push(b' ');
}
line.push(b'\n');
let flags = (libc::O_WRONLY | libc::O_CREAT | libc::O_APPEND) as u64;
let write_to = |dirfd: i64, path: &[u8]| {
let fd = unsafe {
host(
SYS_OPENAT,
dirfd as u64,
path.as_ptr() as u64,
flags,
0o644,
0,
0,
)
};
if fd < 0 {
return;
}
unsafe {
host(
SYS_WRITE,
fd as u64,
line.as_ptr() as u64,
line.len() as u64,
0,
0,
0,
);
host(SYS_CLOSE, fd as u64, 0, 0, 0, 0, 0);
}
};
write_to(-100, b"/tmp/sentry_ipc.log\0"); let rfd = ROOT_FD.load(Ordering::Relaxed);
if rfd >= 0 {
write_to(rfd as i64, b"tmp/sentry_ipc.log\0"); }
}
fn ipc_logn(parts: &[(&[u8], i64)]) {
let mut buf = [0u8; 320];
let mut pos = 0usize;
for (k, v) in parts {
if pos + k.len() + 24 + 2 > buf.len() {
break;
}
buf[pos..pos + k.len()].copy_from_slice(k);
pos += k.len();
let mut nb = [0u8; 24];
let i = fmt_i64(*v, &mut nb);
let n = 24 - i;
buf[pos..pos + n].copy_from_slice(&nb[i..]);
pos += n;
buf[pos] = b' ';
pos += 1;
}
let _ = delegate(CTL_LOGLINE, buf.as_ptr() as u64, pos as u64, 0, 0, 0, 0);
}
fn ipc_logf_raw(bytes: &[u8]) {
let mut line = bytes.to_vec();
line.push(b'\n');
let flags = (libc::O_WRONLY | libc::O_CREAT | libc::O_APPEND) as u64;
let write_to = |dirfd: i64, path: &[u8]| {
let fd = unsafe {
host(
SYS_OPENAT,
dirfd as u64,
path.as_ptr() as u64,
flags,
0o644,
0,
0,
)
};
if fd < 0 {
return;
}
unsafe {
host(
SYS_WRITE,
fd as u64,
line.as_ptr() as u64,
line.len() as u64,
0,
0,
0,
);
host(SYS_CLOSE, fd as u64, 0, 0, 0, 0, 0);
}
};
write_to(-100, b"/tmp/sentry_ipc.log\0"); let rfd = ROOT_FD.load(Ordering::Relaxed);
if rfd >= 0 {
write_to(rfd as i64, b"tmp/sentry_ipc.log\0"); }
}
fn emfile_reserve_open() {
if EMFILE_LOG_RESERVE.load(Ordering::Relaxed) >= 0 {
return;
}
let fd = unsafe {
host(
SYS_OPENAT,
(-100i64) as u64,
b"/dev/null\0".as_ptr() as u64,
(libc::O_RDONLY | libc::O_CLOEXEC) as u64,
0,
0,
0,
)
};
if fd >= 0 {
if EMFILE_LOG_RESERVE
.compare_exchange(-1, fd as i32, Ordering::AcqRel, Ordering::Relaxed)
.is_err()
{
unsafe {
host(SYS_CLOSE, fd as u64, 0, 0, 0, 0, 0);
}
}
}
}
fn emfile_reserve_close_one() {
let fd = EMFILE_LOG_RESERVE.swap(-1, Ordering::AcqRel);
if fd >= 0 {
unsafe {
host(SYS_CLOSE, fd as u64, 0, 0, 0, 0, 0);
}
}
}
fn fd_table_stats(pid: i32) -> (usize, usize, i32) {
let t = fdt().lock().unwrap();
let proc_fds = t.get(&pid).map(|m| m.len()).unwrap_or(0);
let total_fds = t.values().map(|m| m.len()).sum();
let max_guest = t
.get(&pid)
.and_then(|m| m.keys().next_back().copied())
.unwrap_or(-1);
(proc_fds, total_fds, max_guest)
}
fn pending_fd_count() -> usize {
pending().lock().unwrap().values().map(|m| m.len()).sum()
}
fn proc_self_fd_count() -> i64 {
std::fs::read_dir("/proc/self/fd")
.map(|it| it.count() as i64)
.unwrap_or(-1)
}
fn log_open_emfile(pid: i32, nr: i64, path: &[u8]) {
emfile_reserve_close_one();
let (proc_fds, total_fds, max_guest) = fd_table_stats(pid);
let pending_fds = pending_fd_count();
let host_fds = proc_self_fd_count();
let mut cur = -1i64;
let mut max = -1i64;
unsafe {
let mut lim: libc::rlimit = std::mem::zeroed();
if libc::getrlimit(libc::RLIMIT_NOFILE, &mut lim) == 0 {
cur = lim.rlim_cur as i64;
max = lim.rlim_max as i64;
}
}
let mut line: Vec<u8> = Vec::with_capacity(320);
macro_rules! field {
($name:expr, $value:expr) => {{
let mut nb = [0u8; 24];
let i = fmt_i64($value as i64, &mut nb);
line.extend_from_slice($name);
line.extend_from_slice(&nb[i..]);
line.push(b' ');
}};
}
line.extend_from_slice(b"EMFILE_OPEN ");
field!(b"pid=", pid);
field!(b"nr=", nr);
field!(b"proc_fds=", proc_fds);
field!(b"total_fds=", total_fds);
field!(b"pending_fds=", pending_fds);
field!(b"max_guest=", max_guest);
field!(b"host_fds=", host_fds);
field!(b"rlim_cur=", cur);
field!(b"rlim_max=", max);
line.extend_from_slice(b"cmd=");
let cmd = cmdline_for(pid);
for &b in cmd.iter().take(80) {
line.push(if b == 0 { b' ' } else { b });
}
line.extend_from_slice(b" path=");
let bare = match path.split_last() {
Some((0, head)) => head,
_ => path,
};
line.extend_from_slice(&bare[..bare.len().min(160)]);
ipc_logf_raw(&line);
emfile_reserve_open();
}
fn die(msg: &[u8]) -> ! {
log(msg);
unsafe { libc::_exit(1) }
}
fn die_code(msg: &[u8], code: c_int) -> ! {
log(msg);
unsafe { libc::_exit(code) }
}
static mut TRACE: bool = false;
static mut IPCTRACE: bool = false;
static REGDIFF: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false);
static ENVDBG: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false);
static ENVSCAN: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false);
static SCM_FDS_SENT: AtomicU64 = AtomicU64::new(0);
static SCM_FDS_RECVD: AtomicU64 = AtomicU64::new(0);
static EMFILE_LOG_RESERVE: AtomicI32 = AtomicI32::new(-1);
static SUPERVISOR_SELF_PID: AtomicI32 = AtomicI32::new(0);
static GUEST_NOFILE_DEFAULT_CUR: AtomicU64 = AtomicU64::new(1024);
static GUEST_NOFILE_DEFAULT_MAX: AtomicU64 = AtomicU64::new(1024);
static SCM_RECV_TRUNC: AtomicU64 = AtomicU64::new(0);
static mut WAITTRACE: bool = false;
static mut SYSCALLTRACE: bool = false;
static mut FDTRACE: bool = false;
static mut SLOTDUMP: bool = false;
const MAX_SLOTS: usize = 512;
const SLOT_WORDS: usize = MAX_SLOTS / 64;
const DELEG_SLOTS: usize = 8;
const PROCESS_SLOTS: usize = MAX_SLOTS - DELEG_SLOTS;
static RSEQ_ADDR: [AtomicU64; MAX_SLOTS] = [const { AtomicU64::new(0) }; MAX_SLOTS];
static RSEQ_LEN: [AtomicU64; MAX_SLOTS] = [const { AtomicU64::new(0) }; MAX_SLOTS];
static RSEQ_SIG: [AtomicU64; MAX_SLOTS] = [const { AtomicU64::new(0) }; MAX_SLOTS];
static mut GUEST_SIGALTSTACKS: [[u8; 24]; MAX_SLOTS] = [[0u8; 24]; MAX_SLOTS];
static mut VFORK_EFDS: [i32; MAX_SLOTS] = [-1; MAX_SLOTS];
const MAX_DEPTH: usize = 8;
const RING_PROCESS: u8 = 0;
const RING_DELEG: u8 = 1;
#[repr(C)]
struct Ring {
request: u32, response: u32, pid: i32, tid: i32, fork_parent: i32, nr: i64,
args: [u64; 6],
ret: i64,
kind: u8, cancel_mode: u8,
_kind_pad: [u8; 2],
cancel_efd: i32,
intr_efd: i32,
cancel_pending: u8,
intr_pending: u8,
_pad: [u8; 2], dbg_rip: u64,
dbg_caller: u64,
snap_valid: u32,
_snap_pad: u32,
snap_fs: u64,
snap_gregs: [i64; 18],
}
#[inline(always)]
fn set_slot(slot: u64) {
unsafe {
std::arch::asm!("wrgsbase {}", in(reg) slot, options(nostack, nomem, preserves_flags))
};
}
static mut RINGS: *mut Ring = std::ptr::null_mut();
static mut LAST_TRAP_RIP: [u64; MAX_SLOTS] = [0u64; MAX_SLOTS];
static mut LAST_TRAP_CALLER: [u64; MAX_SLOTS] = [0u64; MAX_SLOTS];
static mut SLOT_BM: *mut u64 = std::ptr::null_mut();
#[inline]
fn slot_word(w: usize) -> &'static std::sync::atomic::AtomicU64 {
unsafe { std::sync::atomic::AtomicU64::from_ptr(SLOT_BM.add(w)) }
}
#[inline]
fn slot_is_set(i: u64) -> bool {
slot_word((i / 64) as usize).load(Ordering::Relaxed) & (1u64 << (i % 64)) != 0
}
fn alloc_slot() -> u32 {
alloc_range(0, PROCESS_SLOTS as u32)
}
fn alloc_deleg_slot() -> u32 {
alloc_range(PROCESS_SLOTS as u32, MAX_SLOTS as u32)
}
#[inline]
fn alloc_range(lo: u32, hi: u32) -> u32 {
let first_word = (lo / 64) as usize;
let last_word = ((hi - 1) / 64) as usize;
for w in first_word..=last_word {
let word = slot_word(w);
let wbase = (w as u32) * 64;
let lobit = if w == first_word { lo - wbase } else { 0 };
let hibit = if w == last_word { hi - wbase } else { 64 };
let rangemask: u64 = if hibit - lobit == 64 {
u64::MAX
} else {
((1u64 << (hibit - lobit)) - 1) << lobit
};
loop {
let cur = word.load(Ordering::Relaxed);
let free = !cur & rangemask;
if free == 0 {
break; }
let bit = free.trailing_zeros();
if word
.compare_exchange(
cur,
cur | (1u64 << bit),
Ordering::Relaxed,
Ordering::Relaxed,
)
.is_ok()
{
let i = wbase + bit;
unsafe {
std::ptr::write_volatile(
std::ptr::addr_of_mut!((*slot_stack(i as u64)).depth),
0,
);
std::ptr::write_volatile(
std::ptr::addr_of_mut!((*ring_at(i as u64)).kind),
RING_PROCESS,
);
std::ptr::write_volatile(
std::ptr::addr_of_mut!((*ring_at(i as u64)).cancel_mode),
CANCEL_REAP_PROCESS,
);
}
reset_guest_sigaltstack(i as u64);
set_vfork_efd(i as u64, -1);
return i;
}
}
}
MAX_SLOTS as u32
}
fn free_slot(i: u32) {
unsafe {
clear_ring_owner(i as u64);
std::ptr::write_volatile(
std::ptr::addr_of_mut!((*ring_at(i as u64)).cancel_mode),
CANCEL_REAP_PROCESS,
);
}
set_vfork_efd(i as u64, -1);
slot_word((i / 64) as usize).fetch_and(!(1u64 << (i % 64)), Ordering::Relaxed);
}
fn free_deleg_slot(i: u32) {
free_slot(i)
}
#[repr(C)]
struct SlotStack {
depth: u32,
frames: [u32; MAX_DEPTH], }
const _: () = assert!(std::mem::size_of::<SlotStack>() == 4 * (MAX_DEPTH + 1));
const _: () = assert!(std::mem::align_of::<SlotStack>() == 4);
static mut SLOT_STACKS: *mut SlotStack = std::ptr::null_mut();
#[inline]
fn slot_stack(base: u64) -> *mut SlotStack {
unsafe { SLOT_STACKS.add(base as usize) }
}
#[inline]
fn ring_word(p: *mut u32) -> &'static AtomicU32 {
unsafe { AtomicU32::from_ptr(p) }
}
#[inline]
fn ring_at(slot: u64) -> *mut Ring {
unsafe { RINGS.add(slot as usize) }
}
#[inline]
unsafe fn set_ring_owner(slot: u64, pid: i32, tid: i32) {
unsafe {
let r = ring_at(slot);
std::ptr::write_volatile(std::ptr::addr_of_mut!((*r).pid), pid);
std::ptr::write_volatile(std::ptr::addr_of_mut!((*r).tid), tid);
}
}
#[inline]
unsafe fn clear_ring_owner(slot: u64) {
unsafe {
set_ring_owner(slot, 0, 0);
}
}
#[inline(always)]
fn current_slot() -> u64 {
let s: u64;
unsafe { std::arch::asm!("rdgsbase {}", out(reg) s, options(nostack, nomem, preserves_flags)) };
s
}
unsafe fn store_slot_snapshot_regs(slot: u64, g: &[libc::greg_t; 23], rax: i64) {
unsafe {
if slot as usize >= MAX_SLOTS {
return;
}
let r = ring_at(slot);
for i in 0..18 {
std::ptr::write_volatile(std::ptr::addr_of_mut!((*r).snap_gregs[i]), g[i] as i64);
}
std::ptr::write_volatile(std::ptr::addr_of_mut!((*r).snap_gregs[REG_RAX]), rax);
let mut fs = 0u64;
host(
SYS_ARCH_PRCTL,
ARCH_GET_FS,
std::ptr::addr_of_mut!(fs) as u64,
0,
0,
0,
0,
);
std::ptr::write_volatile(std::ptr::addr_of_mut!((*r).snap_fs), fs);
std::ptr::write_volatile(std::ptr::addr_of_mut!((*r).snap_valid), 1);
}
}
unsafe fn update_slot_snapshot_rax(slot: u64, rax: i64) {
unsafe {
if slot as usize >= MAX_SLOTS {
return;
}
let r = ring_at(slot);
if std::ptr::read_volatile(std::ptr::addr_of!((*r).snap_valid)) == 0 {
return;
}
std::ptr::write_volatile(std::ptr::addr_of_mut!((*r).snap_gregs[REG_RAX]), rax);
}
}
#[cfg(all(target_os = "linux", target_arch = "x86_64"))]
fn ring_snapshot_regs_for_pid(pid: i32, vtid: state_snap::Vpid) -> Option<state_snap::ThreadRegs> {
if pid <= 0 {
return None;
}
for i in 0..MAX_SLOTS as u64 {
let r = ring_at(i);
unsafe {
if std::ptr::read_volatile(std::ptr::addr_of!((*r).pid)) != pid {
continue;
}
if std::ptr::read_volatile(std::ptr::addr_of!((*r).snap_valid)) == 0 {
continue;
}
let mut gregs = [0i64; 18];
for j in 0..18 {
gregs[j] = std::ptr::read_volatile(std::ptr::addr_of!((*r).snap_gregs[j]));
}
let fs = std::ptr::read_volatile(std::ptr::addr_of!((*r).snap_fs));
let req = ring_word(std::ptr::addr_of_mut!((*r).request)).load(Ordering::Acquire);
let resp = ring_word(std::ptr::addr_of_mut!((*r).response)).load(Ordering::Acquire);
let xsave = if req != resp {
let nr = std::ptr::read_volatile(std::ptr::addr_of!((*r).nr));
let mut args = [0u64; 6];
for (idx, arg) in args.iter_mut().enumerate() {
*arg = std::ptr::read_volatile(std::ptr::addr_of!((*r).args[idx]));
}
encode_pending_syscall(nr, args)
} else {
Vec::new()
};
return Some(state_snap::ThreadRegs {
vtid,
gregs,
fs,
xsave,
});
}
}
None
}
const PENDING_SYSCALL_MAGIC: &[u8; 8] = b"SMPEND1\0";
fn encode_pending_syscall(nr: i64, args: [u64; 6]) -> Vec<u8> {
let mut out = Vec::with_capacity(8 + 8 + 6 * 8);
out.extend_from_slice(PENDING_SYSCALL_MAGIC);
out.extend_from_slice(&nr.to_le_bytes());
for arg in args {
out.extend_from_slice(&arg.to_le_bytes());
}
out
}
fn decode_pending_syscall(buf: &[u8]) -> Option<(i64, [u64; 6])> {
if buf.len() != 8 + 8 + 6 * 8 || &buf[..8] != PENDING_SYSCALL_MAGIC {
return None;
}
let nr = i64::from_le_bytes(buf[8..16].try_into().ok()?);
let mut args = [0u64; 6];
let mut off = 16;
for arg in &mut args {
*arg = u64::from_le_bytes(buf[off..off + 8].try_into().ok()?);
off += 8;
}
Some((nr, args))
}
fn record_rseq_result(addr: u64, len: u64, flags: u64, sig: u64, ret: i64) {
if ret != 0 {
return;
}
let slot = current_slot() as usize;
if slot >= MAX_SLOTS {
return;
}
const RSEQ_FLAG_UNREGISTER: u64 = 1;
if (flags & RSEQ_FLAG_UNREGISTER) != 0 {
RSEQ_ADDR[slot].store(0, Ordering::Release);
RSEQ_LEN[slot].store(0, Ordering::Release);
RSEQ_SIG[slot].store(0, Ordering::Release);
} else {
RSEQ_LEN[slot].store(len, Ordering::Release);
RSEQ_SIG[slot].store(sig, Ordering::Release);
RSEQ_ADDR[slot].store(addr, Ordering::Release);
}
}
fn register_rseq_for_child(parent_slot: u64, child_slot: u64) {
let (p, c) = (parent_slot as usize, child_slot as usize);
if p >= MAX_SLOTS || c >= MAX_SLOTS {
return;
}
RSEQ_ADDR[c].store(RSEQ_ADDR[p].load(Ordering::Acquire), Ordering::Release);
RSEQ_LEN[c].store(RSEQ_LEN[p].load(Ordering::Acquire), Ordering::Release);
RSEQ_SIG[c].store(RSEQ_SIG[p].load(Ordering::Acquire), Ordering::Release);
}
fn unregister_current_rseq_for_exec() {
let slot = current_slot() as usize;
if slot >= MAX_SLOTS {
return;
}
let addr = RSEQ_ADDR[slot].swap(0, Ordering::AcqRel);
if addr == 0 {
return;
}
let len = RSEQ_LEN[slot].swap(0, Ordering::AcqRel);
let sig = RSEQ_SIG[slot].swap(0, Ordering::AcqRel);
const RSEQ_FLAG_UNREGISTER: u64 = 1;
unsafe {
host(SYS_RSEQ, addr, len, RSEQ_FLAG_UNREGISTER, sig, 0, 0);
}
}
fn unregister_host_libc_rseq() {
unsafe {
let offset_ptr = libc::dlsym(
std::ptr::null_mut(),
b"__rseq_offset\0".as_ptr() as *const c_char,
) as *const isize;
let size_ptr = libc::dlsym(
std::ptr::null_mut(),
b"__rseq_size\0".as_ptr() as *const c_char,
) as *const u32;
if offset_ptr.is_null() || size_ptr.is_null() {
return;
}
let offset = std::ptr::read_volatile(offset_ptr);
let len = std::ptr::read_volatile(size_ptr);
if len == 0 {
return;
}
let mut fs = 0u64;
if host(
SYS_ARCH_PRCTL,
ARCH_GET_FS,
std::ptr::addr_of_mut!(fs) as u64,
0,
0,
0,
0,
) != 0
{
return;
}
let addr = if offset >= 0 {
fs.wrapping_add(offset as u64)
} else {
fs.wrapping_sub(offset.wrapping_neg() as u64)
};
const RSEQ_FLAG_UNREGISTER: u64 = 1;
host(
SYS_RSEQ,
addr,
len as u64,
RSEQ_FLAG_UNREGISTER,
0x5305_3053,
0,
0,
);
}
}
fn disabled_sigaltstack() -> [u8; 24] {
let mut old = [0u8; 24];
old[8..12].copy_from_slice(&(libc::SS_DISABLE as u32).to_le_bytes());
old
}
fn reset_guest_sigaltstack(slot: u64) {
if (slot as usize) < MAX_SLOTS {
unsafe {
GUEST_SIGALTSTACKS[slot as usize] = disabled_sigaltstack();
}
}
}
fn guest_sigaltstack(slot: u64) -> [u8; 24] {
if (slot as usize) >= MAX_SLOTS {
return disabled_sigaltstack();
}
unsafe {
let cur = GUEST_SIGALTSTACKS[slot as usize];
if cur == [0u8; 24] {
disabled_sigaltstack()
} else {
cur
}
}
}
fn set_vfork_efd(slot: u64, efd: i32) {
if (slot as usize) < MAX_SLOTS {
unsafe {
VFORK_EFDS[slot as usize] = efd;
}
}
}
fn take_vfork_efd(slot: u64) -> i32 {
if (slot as usize) >= MAX_SLOTS {
return -1;
}
unsafe {
let efd = VFORK_EFDS[slot as usize];
VFORK_EFDS[slot as usize] = -1;
efd
}
}
fn notify_vfork_release() {
let efd = take_vfork_efd(current_slot());
if efd < 0 {
return;
}
let one: u64 = 1;
unsafe {
host(
SYS_WRITE,
efd as u64,
std::ptr::addr_of!(one) as u64,
8,
0,
0,
0,
);
host(SYS_CLOSE, efd as u64, 0, 0, 0, 0, 0);
}
}
fn wait_vfork_release(efd: i32) {
if efd < 0 {
return;
}
let mut word = 0u64;
loop {
let n = unsafe {
host(
SYS_READ,
efd as u64,
std::ptr::addr_of_mut!(word) as u64,
8,
0,
0,
0,
)
};
if n == 8 || n != -4 {
break;
}
}
unsafe { host(SYS_CLOSE, efd as u64, 0, 0, 0, 0, 0) };
}
const ALT_SZ: usize = 128 * 1024;
#[no_mangle]
static mut ALT_STACKS: [[u8; ALT_SZ]; MAX_SLOTS] = [[0u8; ALT_SZ]; MAX_SLOTS];
unsafe fn register_altstack() {
unsafe {
let slot = current_slot() as usize;
if slot < MAX_SLOTS {
let ss = libc::stack_t {
ss_sp: std::ptr::addr_of_mut!(ALT_STACKS[slot]) as *mut c_void,
ss_flags: 0,
ss_size: ALT_SZ,
};
libc::sigaltstack(&ss, std::ptr::null_mut());
}
}
}
#[no_mangle]
extern "C" fn child_thread_init() {
unsafe {
let slot = current_slot();
set_ring_owner(
slot,
host(SYS_GETPID, 0, 0, 0, 0, 0, 0) as i32,
host(SYS_GETTID, 0, 0, 0, 0, 0, 0) as i32,
);
}
unsafe { register_altstack() };
}
fn delegate_on(
slot: u64,
nr: i64,
a: u64,
b: u64,
c: u64,
d: u64,
e: u64,
f: u64,
saved_mask: Option<u64>,
) -> i64 {
unsafe {
if saved_mask.is_some() {
block_guest_signals_until_sigreturn();
}
let r = ring_at(slot);
let req = ring_word(std::ptr::addr_of_mut!((*r).request));
let resp = ring_word(std::ptr::addr_of_mut!((*r).response));
let k = req.load(Ordering::Relaxed) + 1;
std::ptr::write_volatile(std::ptr::addr_of_mut!((*r).nr), nr);
let ap = std::ptr::addr_of_mut!((*r).args) as *mut u64;
ap.write_volatile(a);
ap.add(1).write_volatile(b);
ap.add(2).write_volatile(c);
ap.add(3).write_volatile(d);
ap.add(4).write_volatile(e);
ap.add(5).write_volatile(f);
if SYSCALLTRACE || SLOTDUMP {
let s = current_slot() as usize;
let (rip, caller) = if s < MAX_SLOTS {
(LAST_TRAP_RIP[s], LAST_TRAP_CALLER[s])
} else {
(0, 0)
};
std::ptr::write_volatile(std::ptr::addr_of_mut!((*r).dbg_rip), rip);
std::ptr::write_volatile(std::ptr::addr_of_mut!((*r).dbg_caller), caller);
}
req.store(k, Ordering::Release);
host(SYS_FUTEX, req.as_ptr() as u64, FUTEX_WAKE, 1, 0, 0, 0);
let mut spins = 0u32;
loop {
let cur = resp.load(Ordering::Acquire);
if cur == k {
break;
}
if spins < RING_SPIN {
spins += 1;
std::hint::spin_loop();
continue;
}
host(
SYS_FUTEX,
resp.as_ptr() as u64,
FUTEX_WAIT,
cur as u64,
0,
0,
0,
);
}
std::ptr::read_volatile(std::ptr::addr_of!((*r).ret))
}
}
fn delegate(nr: i64, a: u64, b: u64, c: u64, d: u64, e: u64, f: u64) -> i64 {
delegate_masked(nr, a, b, c, d, e, f, None)
}
fn delegate_masked(
nr: i64,
a: u64,
b: u64,
c: u64,
d: u64,
e: u64,
f: u64,
saved_mask: Option<u64>,
) -> i64 {
let base = current_slot();
let ss = slot_stack(base);
let d_now = unsafe { std::ptr::read_volatile(std::ptr::addr_of!((*ss).depth)) };
if d_now == 0 {
unsafe {
std::ptr::write_volatile(std::ptr::addr_of_mut!((*ss).frames[0]), base as u32);
std::ptr::write_volatile(std::ptr::addr_of_mut!((*ss).depth), 1);
}
let ret = delegate_on(base, nr, a, b, c, d, e, f, saved_mask);
unsafe { std::ptr::write_volatile(std::ptr::addr_of_mut!((*ss).depth), 0) };
ret
} else if (d_now as usize) >= MAX_DEPTH {
-11 } else {
let fresh = alloc_deleg_slot();
if fresh as usize >= MAX_SLOTS {
return -11; }
unsafe {
let fr = ring_at(fresh as u64);
set_ring_owner(
fresh as u64,
host(SYS_GETPID, 0, 0, 0, 0, 0, 0) as i32,
host(SYS_GETTID, 0, 0, 0, 0, 0, 0) as i32,
);
std::ptr::write_volatile(std::ptr::addr_of_mut!((*fr).kind), RING_DELEG);
std::ptr::write_volatile(std::ptr::addr_of_mut!((*fr).fork_parent), 0);
}
unsafe {
std::ptr::write_volatile(std::ptr::addr_of_mut!((*ss).frames[d_now as usize]), fresh);
std::ptr::write_volatile(std::ptr::addr_of_mut!((*ss).depth), d_now + 1);
}
let ret = delegate_on(fresh as u64, nr, a, b, c, d, e, f, saved_mask);
unsafe { std::ptr::write_volatile(std::ptr::addr_of_mut!((*ss).depth), d_now) };
free_deleg_slot(fresh);
ret
}
}
static mut CLONE_REGS: [[u64; 15]; MAX_SLOTS] = [[0; 15]; MAX_SLOTS];
std::arch::global_asm!(
".globl do_clone",
"do_clone:",
" push %r12",
" push %r13",
" push %r14",
" push %r15",
" mov 40(%rsp), %r14", " mov 48(%rsp), %r13", " mov 56(%rsp), %r12", " mov 64(%rsp), %r15", " mov %rcx, %r10", " mov $56, %rax", " syscall",
" test %rax, %rax",
" jz 1f",
" pop %r15", " pop %r14",
" pop %r13",
" pop %r12",
" ret",
"1: wrgsbase %r14", " mov $157, %rax", " mov $59, %rdi",
" mov $1, %rsi",
" movabs $0x2000000000, %rdx", " movabs $0x00007fdffffff000, %r10", " mov %r13, %r8", " syscall",
" sub $32, %rsp",
" mov %r15, (%rsp)",
" movl $0, 8(%rsp)",
" movq $131072, 16(%rsp)",
" mov $131, %rax", " mov %rsp, %rdi",
" xor %rsi, %rsi",
" syscall",
" add $32, %rsp",
" mov 0(%r12), %r8",
" mov 8(%r12), %r9",
" mov 16(%r12), %r10",
" mov 24(%r12), %r11",
" mov 40(%r12), %r13",
" mov 48(%r12), %r14",
" mov 56(%r12), %r15",
" mov 64(%r12), %rbx",
" mov 72(%r12), %rbp",
" mov 80(%r12), %rcx",
" mov 88(%r12), %rdx",
" mov 96(%r12), %rsi",
" mov 104(%r12), %rdi",
" push 112(%r12)", " mov 32(%r12), %r12", " xor %rax, %rax", " ret", options(att_syntax)
);
std::arch::global_asm!(
".globl sig_restorer",
"sig_restorer:",
" mov $15, %rax", " syscall",
options(att_syntax)
);
std::arch::global_asm!(
".globl unmap_and_exit",
"unmap_and_exit:",
" mov $11, %eax", " syscall",
" xor %edi, %edi", " mov $60, %eax", " syscall",
" ud2", options(att_syntax)
);
std::arch::global_asm!(
".globl rewrite_dispatch",
"rewrite_dispatch:",
" cmp $0x5359, %rax", " je 2f",
" cmp $56, %rax", " je 2f",
" cmp $435, %rax", " je 2f",
" cmp $59, %rax", " je 2f",
" push %r9", " push %r8",
" push %r10",
" push %rdx",
" push %rsi",
" push %rdi",
" push %rax",
" mov %rsp, %rdi", " mov %rsp, %rax", " and $-16, %rsp", " push %rax", " push %rax", " call sentry_dispatch_simple",
" pop %rcx", " pop %rcx", " mov %rcx, %rsp",
" mov 8(%rsp), %rdi", " mov 16(%rsp), %rsi",
" mov 24(%rsp), %rdx",
" mov 32(%rsp), %r10",
" mov 40(%rsp), %r8",
" mov 48(%rsp), %r9",
" add $56, %rsp", " ret", "2:", " mov $600, %r11d",
" jmp *%r11",
options(att_syntax)
);
extern "C" {
fn sig_restorer();
fn rewrite_dispatch();
fn do_clone(
flags: u64,
stack: u64,
ptid: u64,
ctid: u64,
tls: u64,
rip: u64,
slot: u64,
sel: u64,
regs: u64,
alt_stack: u64,
) -> i64;
fn unmap_and_exit(base: u64, len: u64) -> !;
}
unsafe fn clone_resume_rip(g: &[libc::greg_t; 23]) -> u64 {
unsafe {
let trapped_rip = g[REG_RIP] as u64;
if trapped_rip <= 0x1000 {
*(g[REG_RSP] as *const u64)
} else {
trapped_rip
}
}
}
unsafe fn capture_clone_regs(g: &[libc::greg_t; 23], slot: usize, rip: u64) -> *mut u64 {
unsafe {
let cr = std::ptr::addr_of_mut!(CLONE_REGS[slot]) as *mut u64;
*cr.add(0) = g[REG_R8] as u64;
*cr.add(1) = g[REG_R9] as u64;
*cr.add(2) = g[REG_R10] as u64;
*cr.add(3) = g[REG_R11] as u64;
*cr.add(4) = g[REG_R12] as u64;
*cr.add(5) = g[REG_R13] as u64;
*cr.add(6) = g[REG_R14] as u64;
*cr.add(7) = g[REG_R15] as u64;
*cr.add(8) = g[REG_RBX] as u64;
*cr.add(9) = g[REG_RBP] as u64;
*cr.add(10) = g[REG_RCX] as u64;
*cr.add(11) = g[REG_RDX] as u64;
*cr.add(12) = g[REG_RSI] as u64;
*cr.add(13) = g[REG_RDI] as u64;
*cr.add(14) = rip;
cr
}
}
unsafe fn handle_clone(
g: &mut [libc::greg_t; 23],
flags: u64,
stack: u64,
parent_tid: u64,
child_tid: u64,
tls: u64,
) -> i64 {
unsafe {
let slot = alloc_slot();
if slot as usize >= MAX_SLOTS {
return -11; }
if delegate(CTL_ENSURE_SERVICER, slot as u64, 0, 0, 0, 0, 0) < 0 {
free_slot(slot);
return -11; }
if (flags & CLONE_VM) != 0 {
MM_USERS.fetch_add(1, Ordering::AcqRel);
}
if (flags & CLONE_THREAD) != 0 {
if stack == 0 {
free_slot(slot);
return -22; }
set_ring_owner(slot as u64, host(SYS_GETPID, 0, 0, 0, 0, 0, 0) as i32, 0);
let rip = clone_resume_rip(g);
let cr = capture_clone_regs(g, slot as usize, rip);
let alt_stack = std::ptr::addr_of_mut!(ALT_STACKS[slot as usize]) as u64;
let r = do_clone(
flags & !CLONE_CLEAR_SIGHAND,
stack,
parent_tid,
child_tid,
tls,
rip,
slot as u64,
std::ptr::addr_of!(SELECTOR) as u64,
cr as u64,
alt_stack,
);
if r < 0 {
free_slot(slot);
} else {
set_ring_owner(
slot as u64,
host(SYS_GETPID, 0, 0, 0, 0, 0, 0) as i32,
r as i32,
);
}
return r;
}
register_rseq_for_child(current_slot(), slot as u64);
let vfork_like = (flags & CLONE_VFORK) != 0;
let vfork_efd = if vfork_like {
host(SYS_EVENTFD2, 0, 0, 0, 0, 0, 0) as i32
} else {
-1
};
if vfork_efd >= 0 {
set_vfork_efd(slot as u64, vfork_efd);
}
let pflags = flags & !(CLONE_VM | CLONE_VFORK | CLONE_CLEAR_SIGHAND);
delegate(
CTL_FORK_TABLE,
slot as u64,
SYS_CLONE as u64,
flags,
0,
0,
0,
);
(*ring_at(slot as u64)).fork_parent = host(SYS_GETPID, 0, 0, 0, 0, 0, 0) as i32;
let child = if stack != 0 {
let rip = clone_resume_rip(g);
let cr = capture_clone_regs(g, slot as usize, rip);
let alt_stack = std::ptr::addr_of_mut!(ALT_STACKS[slot as usize]) as u64;
do_clone(
pflags,
stack,
parent_tid,
child_tid,
tls,
rip,
slot as u64,
std::ptr::addr_of!(SELECTOR) as u64,
cr as u64,
alt_stack,
)
} else if pflags == SIGCHLD_FLAG {
host(SYS_FORK, 0, 0, 0, 0, 0, 0)
} else {
host(SYS_CLONE, pflags, 0, parent_tid, child_tid, tls, 0)
};
if child == 0 && stack == 0 {
set_slot(slot as u64);
MM_USERS.store(1, Ordering::Release);
set_ring_owner(
slot as u64,
host(SYS_GETPID, 0, 0, 0, 0, 0, 0) as i32,
host(SYS_GETTID, 0, 0, 0, 0, 0, 0) as i32,
);
host(
157,
PR_SET_SYSCALL_USER_DISPATCH as u64,
PR_SYS_DISPATCH_ON,
WINDOW_FLOOR,
USER_TOP - WINDOW_FLOOR,
std::ptr::addr_of!(SELECTOR) as u64,
0,
);
0
} else if child < 0 {
if vfork_efd >= 0 {
set_vfork_efd(slot as u64, -1);
host(SYS_CLOSE, vfork_efd as u64, 0, 0, 0, 0, 0);
}
(*ring_at(slot as u64)).fork_parent = 0;
delegate(CTL_FORK_CANCEL, slot as u64, 0, 0, 0, 0, 0);
child
} else {
set_ring_owner(slot as u64, child as i32, child as i32);
delegate(CTL_BIND_SLOT, slot as u64, child as u64, 0, 0, 0, 0);
wait_vfork_release(vfork_efd);
child
}
}
}
fn setup_trampoline() {
let p = unsafe {
libc::mmap(
std::ptr::null_mut(),
4096,
libc::PROT_READ | libc::PROT_WRITE | libc::PROT_EXEC,
libc::MAP_PRIVATE | libc::MAP_ANONYMOUS | libc::MAP_FIXED,
-1,
0,
)
};
if p != std::ptr::null_mut() {
REWRITE_ON.store(false, Ordering::Relaxed);
return;
}
unsafe {
let base = std::hint::black_box(p) as *mut u8;
std::ptr::write_bytes(base, 0x90, 512); let jmp = [0xff_u8, 0x25, 0, 0, 0, 0];
std::ptr::copy_nonoverlapping(jmp.as_ptr(), base.add(512), 6);
let disp = (rewrite_dispatch as usize as u64).to_le_bytes();
std::ptr::copy_nonoverlapping(disp.as_ptr(), base.add(518), 8);
let cf = [0x0f_u8, 0x05, 0xc3];
std::ptr::copy_nonoverlapping(cf.as_ptr(), base.add(600), 3);
libc::mprotect(base as *mut c_void, 4096, libc::PROT_READ | libc::PROT_EXEC);
}
}
fn maybe_patch_site(rip: u64, nr: i64) {
if !REWRITE_ON.load(Ordering::Relaxed) || WARM_MODE.load(Ordering::Relaxed) {
return;
}
if nr == SENTINEL || nr == SYS_CLONE || nr == SYS_CLONE3 || nr == SYS_EXECVE {
return;
}
if rip <= 0x1000 {
return; }
let site = rip - 2; unsafe {
let p = site as *const u8;
if *p != 0x0f || *p.add(1) != 0x05 {
return; }
let page = site & !0xfff;
let span: u64 = if site & 0xfff == 0xfff {
0x2000
} else {
0x1000
};
host(
SYS_MPROTECT,
page,
span,
(libc::PROT_READ | libc::PROT_WRITE | libc::PROT_EXEC) as u64,
0,
0,
0,
);
let pp = site as *mut u8;
*pp = 0xff; *pp.add(1) = 0xd0;
host(
SYS_MPROTECT,
page,
span,
(libc::PROT_READ | libc::PROT_EXEC) as u64,
0,
0,
0,
);
}
}
static mut HEAP_BASE: u64 = 0;
static HEAP_CUR: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
static mut HEAP_END: u64 = 0;
const HEAP_SIZE: u64 = 64 * 1024 * 1024;
static ARENA_CUR: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(ARENA_TOP);
static MM_USERS: std::sync::atomic::AtomicU32 = std::sync::atomic::AtomicU32::new(1);
const MAP_FIXED_BIT: u64 = libc::MAP_FIXED as u64;
const MAP_FIXED_NOREPLACE_BIT: u64 = 0x100000;
const PROT_EXEC_BIT: u64 = libc::PROT_EXEC as u64;
fn reset_brk_heap_for_exec() -> i64 {
let (old_base, old_end) = unsafe { (HEAP_BASE, HEAP_END) };
let h = unsafe {
host(
SYS_MMAP,
0,
HEAP_SIZE,
PROT_RW,
(libc::MAP_PRIVATE | libc::MAP_ANONYMOUS) as u64,
u64::MAX,
0,
)
};
if h < 0 {
return h;
}
let new_base = h as u64;
unsafe {
HEAP_BASE = new_base;
HEAP_END = new_base + HEAP_SIZE;
}
HEAP_CUR.store(new_base, Ordering::Release);
if old_base != 0 && old_end > old_base {
unsafe {
host(SYS_MUNMAP, old_base, old_end - old_base, 0, 0, 0, 0);
}
}
0
}
fn overlaps_sud_host_region(addr: u64, len: u64) -> bool {
let end = page_up(addr.saturating_add(len));
addr < USER_TOP && end > WINDOW_FLOOR
}
#[repr(C)]
struct GuestVmaSlot {
lo: std::sync::atomic::AtomicU64,
hi: std::sync::atomic::AtomicU64,
}
const GUEST_VMA_MAX: usize = 4096;
static GUEST_VMA_NEXT: std::sync::atomic::AtomicU32 = std::sync::atomic::AtomicU32::new(0);
static GUEST_VMAS: [GuestVmaSlot; GUEST_VMA_MAX] = [const {
GuestVmaSlot {
lo: std::sync::atomic::AtomicU64::new(0),
hi: std::sync::atomic::AtomicU64::new(0),
}
}; GUEST_VMA_MAX];
fn guest_vma_note_mapping(addr: u64, len: u64) {
if len == 0 {
return;
}
let start = page_down(addr);
let end = page_up(addr.saturating_add(len));
if end <= WINDOW_FLOOR || start >= USER_TOP || end <= start {
return;
}
let idx = GUEST_VMA_NEXT.fetch_add(1, Ordering::AcqRel) as usize;
if idx >= GUEST_VMA_MAX {
return;
}
GUEST_VMAS[idx]
.hi
.store(end.min(USER_TOP), Ordering::Release);
GUEST_VMAS[idx]
.lo
.store(start.max(WINDOW_FLOOR), Ordering::Release);
}
fn guest_vma_forget_range(addr: u64, len: u64) {
if len == 0 {
return;
}
let start = page_down(addr);
let end = page_up(addr.saturating_add(len));
if end <= WINDOW_FLOOR || start >= USER_TOP || end <= start {
return;
}
let n = (GUEST_VMA_NEXT.load(Ordering::Acquire) as usize).min(GUEST_VMA_MAX);
for slot in &GUEST_VMAS[..n] {
let lo = slot.lo.load(Ordering::Acquire);
let hi = slot.hi.load(Ordering::Acquire);
if lo != 0 && start < hi && end > lo {
slot.lo.store(0, Ordering::Release);
slot.hi.store(0, Ordering::Release);
}
}
}
const LOOP_WAIT_TICK_MS: i64 = 25;
const LOOP_INSURANCE_TICK_MS: i64 = 200;
const SENTINEL: i64 = 0x5359; static WARM_MODE: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false);
static REWRITE_ON: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false);
static ZYG_DRIVEN: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false);
static mut CAPTURED: [i64; 18] = [0; 18]; static mut GUEST_FS: u64 = 0; static RESTORE_PENDING_SYSCALL: AtomicBool = AtomicBool::new(false);
static mut RESTORE_PENDING_NR: i64 = 0;
static mut RESTORE_PENDING_ARGS: [u64; 6] = [0; 6];
static mut JMPBUF: [u64; 32] = [0; 32]; extern "C" {
fn __sigsetjmp(env: *mut c_void, savesigs: c_int) -> c_int;
fn siglongjmp(env: *mut c_void, val: c_int) -> !;
}
#[repr(C)]
struct ZygShared {
ready: u32,
req: u32,
done: u32,
exit: i32,
mean_ns: u64, run_mode: u32,
inst_pid: i32, }
static mut ZYG: *mut ZygShared = std::ptr::null_mut();
#[no_mangle]
static mut CELL_FS: u64 = 0;
static mut CELL_ROOT: Option<std::path::PathBuf> = None;
fn page_up(x: u64) -> u64 {
(x + 0xFFF) & !0xFFF
}
fn page_down(x: u64) -> u64 {
x & !0xFFF
}
fn arena_alignment(need: u64) -> u64 {
const GIB: u64 = 1 << 30;
if need >= 32 * GIB {
32 * GIB
} else if need >= 4 * GIB {
4 * GIB
} else if need >= GIB {
GIB
} else {
0x1000
}
}
fn arena_alloc(len: u64) -> u64 {
use std::sync::atomic::Ordering;
let need = page_up(len);
let align = arena_alignment(need);
let reused = arena_free_take(need, align);
if reused != 0 {
return reused;
}
let mut cur = ARENA_CUR.load(Ordering::Relaxed);
loop {
let end = cur & !(align - 1);
let nc = end.saturating_sub(need);
if nc < ARENA_LIMIT {
return 0; }
match ARENA_CUR.compare_exchange_weak(cur, nc, Ordering::AcqRel, Ordering::Relaxed) {
Ok(_) => return nc,
Err(actual) => cur = actual,
}
}
}
static ARENA_EXHAUST_LOGS: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
static ARENA_OVERSIZE_LOGS: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
fn log_arena_exhausted(len: u64, prot: u64, flags: u64) {
use std::sync::atomic::Ordering;
let oversize = page_up(len) >= (ARENA_TOP - ARENA_LIMIT) / 2;
let n = if oversize {
ARENA_OVERSIZE_LOGS.fetch_add(1, Ordering::Relaxed)
} else {
ARENA_EXHAUST_LOGS.fetch_add(1, Ordering::Relaxed)
};
let should = if oversize {
n < 2
} else {
n < 16 || n % 4096 == 0
};
if should {
ipc_logn(&[
(
if oversize {
b"ARENA-OVERSIZE n=".as_slice()
} else {
b"ARENA-EXHAUST n=".as_slice()
},
n as i64,
),
(b"len=", len as i64),
(b"prot=", prot as i64),
(b"flags=", flags as i64),
(b"cur=", ARENA_CUR.load(Ordering::Relaxed) as i64),
]);
}
}
const ARENA_FREE_MAX: usize = 512;
static ARENA_FREE: [std::sync::atomic::AtomicU64; ARENA_FREE_MAX] =
[const { std::sync::atomic::AtomicU64::new(0) }; ARENA_FREE_MAX];
fn af_pack(lo: u64, hi: u64) -> u64 {
((lo >> 12) << 32) | (hi >> 12)
}
fn af_lo(w: u64) -> u64 {
(w >> 32) << 12
}
fn af_hi(w: u64) -> u64 {
(w & 0xffff_ffff) << 12
}
fn arena_free_insert(start: u64, end: u64) {
use std::sync::atomic::Ordering;
let start = start.max(ARENA_LIMIT);
let end = end.min(ARENA_TOP);
if end <= start {
return;
}
let w = af_pack(start, end);
for slot in &ARENA_FREE {
if slot
.compare_exchange(0, w, Ordering::AcqRel, Ordering::Relaxed)
.is_ok()
{
return;
}
}
}
fn arena_free_take(need: u64, align: u64) -> u64 {
use std::sync::atomic::Ordering;
for slot in &ARENA_FREE {
loop {
let w = slot.load(Ordering::Acquire);
if w == 0 {
break;
}
let lo = af_lo(w);
let hi = af_hi(w);
let end = hi & !(align - 1);
if end < need {
break;
}
let start = end - need;
if start < lo {
break;
}
let rem = if start > lo { af_pack(lo, start) } else { 0 };
if slot
.compare_exchange(w, rem, Ordering::AcqRel, Ordering::Relaxed)
.is_ok()
{
if hi > end {
arena_free_insert(end, hi);
}
return start;
}
}
}
0
}
fn arena_free_purge(start: u64, end: u64) {
use std::sync::atomic::Ordering;
if end <= start {
return;
}
for slot in &ARENA_FREE {
loop {
let w = slot.load(Ordering::Acquire);
if w == 0 {
break;
}
let lo = af_lo(w);
let hi = af_hi(w);
if start >= hi || end <= lo {
break;
}
let lower = if start > lo { af_pack(lo, start) } else { 0 };
if slot
.compare_exchange(w, lower, Ordering::AcqRel, Ordering::Relaxed)
.is_ok()
{
if end < hi {
arena_free_insert(end, hi);
}
break;
}
}
}
}
fn arena_free_clear() {
use std::sync::atomic::Ordering;
for slot in &ARENA_FREE {
slot.store(0, Ordering::Release);
}
}
fn arena_reclaim_mapping(addr: u64, len: u64) {
if len == 0 {
return;
}
let start = page_down(addr);
let end = page_up(addr.saturating_add(len)).min(ARENA_TOP);
if start < ARENA_LIMIT || end <= start || end > ARENA_TOP {
return;
}
let mut cur = ARENA_CUR.load(std::sync::atomic::Ordering::Relaxed);
while cur == start {
match ARENA_CUR.compare_exchange_weak(
cur,
end,
std::sync::atomic::Ordering::AcqRel,
std::sync::atomic::Ordering::Relaxed,
) {
Ok(_) => return,
Err(actual) => cur = actual,
}
}
arena_free_insert(start, end);
}
fn arena_note_mapping(addr: u64, len: u64) {
if len == 0 {
return;
}
let start = page_down(addr);
let end = page_up(addr.saturating_add(len));
if end <= ARENA_LIMIT || start >= ARENA_TOP {
return;
}
let next = start.max(ARENA_LIMIT);
let mut cur = ARENA_CUR.load(std::sync::atomic::Ordering::Relaxed);
while next < cur {
match ARENA_CUR.compare_exchange_weak(
cur,
next,
std::sync::atomic::Ordering::AcqRel,
std::sync::atomic::Ordering::Relaxed,
) {
Ok(_) => break,
Err(actual) => cur = actual,
}
}
}
const MAP_ANON_BIT: u64 = libc::MAP_ANONYMOUS as u64;
const PROT_RW: u64 = (libc::PROT_READ | libc::PROT_WRITE) as u64;
static EXE_SPAN_LO: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
static EXE_SPAN_HI: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
static INTERP_SPAN_LO: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
static INTERP_SPAN_HI: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
fn record_loader_span(base: u64, lo: u64, hi: u64) {
let (slo, shi) = if base == INTERP_BASE {
(&INTERP_SPAN_LO, &INTERP_SPAN_HI)
} else {
if hi > INTERP_BASE {
die(b"sentry: main exe top exceeds INTERP_BASE (ld.so base) - raise INTERP_BASE/ARENA_LIMIT\n");
}
(&EXE_SPAN_LO, &EXE_SPAN_HI)
};
shi.store(hi, Ordering::Relaxed);
slo.store(lo, Ordering::Relaxed); }
fn overlaps_loader(start: u64, end: u64) -> bool {
for (slo, shi) in [
(&EXE_SPAN_LO, &EXE_SPAN_HI),
(&INTERP_SPAN_LO, &INTERP_SPAN_HI),
] {
let lo = slo.load(Ordering::Relaxed);
if lo != 0 && start < shi.load(Ordering::Relaxed) && end > lo {
return true;
}
}
false
}
const MAP_SHARED_BIT: u64 = libc::MAP_SHARED as u64;
const SHM_POOL_SIZE: u64 = 64 << 30;
static SHM_POOL_BASE: AtomicU64 = AtomicU64::new(0);
static SHM_POOL_NEXT: AtomicU64 = AtomicU64::new(0);
fn in_shm_pool(va: u64) -> bool {
let base = SHM_POOL_BASE.load(Ordering::Relaxed);
base != 0 && va >= base && va < base + SHM_POOL_SIZE
}
#[derive(Clone, Copy)]
struct ShmSlice {
pool_off: u64,
capacity: u64,
backing_fd: i32,
}
static SHM_ENOMEM_DECLINES: AtomicU64 = AtomicU64::new(0);
#[derive(Clone, Copy)]
struct ShmMap {
addr: u64,
len: u64,
file_off: u64,
key: (u64, u64),
writable: bool,
}
fn shm_slices() -> &'static Mutex<HashMap<(u64, u64), ShmSlice>> {
static S: OnceLock<Mutex<HashMap<(u64, u64), ShmSlice>>> = OnceLock::new();
S.get_or_init(|| Mutex::new(HashMap::new()))
}
fn shm_free_slices() -> &'static Mutex<Vec<(u64, u64)>> {
static F: OnceLock<Mutex<Vec<(u64, u64)>>> = OnceLock::new();
F.get_or_init(|| Mutex::new(Vec::new()))
}
fn shm_pool_alloc(aligned: u64) -> Option<u64> {
if aligned == 0 || aligned > SHM_POOL_SIZE {
return None;
}
{
let mut free = shm_free_slices().lock().unwrap();
if let Some(i) = free.iter().position(|(_, cap)| *cap >= aligned) {
let (off, cap) = free.remove(i);
if cap > aligned {
free.push((off + aligned, cap - aligned));
}
return Some(off);
}
}
let mut cur = SHM_POOL_NEXT.load(Ordering::Relaxed);
loop {
let next = cur.checked_add(aligned)?;
if next > SHM_POOL_SIZE {
return None;
}
match SHM_POOL_NEXT.compare_exchange(cur, next, Ordering::Relaxed, Ordering::Relaxed) {
Ok(_) => return Some(cur),
Err(actual) => cur = actual,
}
}
}
fn shm_pool_free(off: u64, capacity: u64) {
if capacity == 0 || off >= SHM_POOL_SIZE {
return;
}
let end = off.saturating_add(capacity).min(SHM_POOL_SIZE);
if end <= off {
return;
}
let mut free = shm_free_slices().lock().unwrap();
free.push((off, end - off));
free.sort_unstable_by_key(|(o, _)| *o);
let mut merged: Vec<(u64, u64)> = Vec::with_capacity(free.len());
for (o, cap) in free.drain(..) {
if let Some((last_o, last_cap)) = merged.last_mut() {
let last_end = last_o.saturating_add(*last_cap);
if o <= last_end {
let new_end = o.saturating_add(cap).max(last_end);
*last_cap = new_end - *last_o;
continue;
}
}
merged.push((o, cap));
}
*free = merged;
}
fn shm_maps() -> &'static Mutex<HashMap<i32, Vec<ShmMap>>> {
static M: OnceLock<Mutex<HashMap<i32, Vec<ShmMap>>>> = OnceLock::new();
M.get_or_init(|| Mutex::new(HashMap::new()))
}
fn pending_shm_maps() -> &'static Mutex<HashMap<u64, Vec<ShmMap>>> {
static M: OnceLock<Mutex<HashMap<u64, Vec<ShmMap>>>> = OnceLock::new();
M.get_or_init(|| Mutex::new(HashMap::new()))
}
fn retired_exec_shm_keys() -> &'static Mutex<HashMap<i32, Vec<(u64, u64)>>> {
static M: OnceLock<Mutex<HashMap<i32, Vec<(u64, u64)>>>> = OnceLock::new();
M.get_or_init(|| Mutex::new(HashMap::new()))
}
fn retire_exec_shm_keys(pid: i32, keys: &[(u64, u64)]) {
if keys.is_empty() {
return;
}
let mut retired = retired_exec_shm_keys().lock().unwrap();
let entry = retired.entry(pid).or_default();
for key in keys {
if !entry.contains(key) {
entry.push(*key);
}
}
}
fn take_retired_exec_shm_keys(pid: i32) -> Vec<(u64, u64)> {
retired_exec_shm_keys()
.lock()
.unwrap()
.remove(&pid)
.unwrap_or_default()
}
fn shm_pool_map(pid: i32, h: i32, want_len: u64, prot: u64, off: u64) -> i64 {
let base = SHM_POOL_BASE.load(Ordering::Relaxed);
if base == 0 {
SHM_ENOMEM_DECLINES.fetch_add(1, Ordering::Relaxed);
ipc_logf(&[(b"SHMPOOL-DECLINE reason=nopool pid=", pid as i64)], &[]);
return -1;
}
let mut st = [0u8; 144];
let fr = unsafe { host(SYS_FSTAT, h as u64, st.as_mut_ptr() as u64, 0, 0, 0, 0) };
if fr != 0 {
SHM_ENOMEM_DECLINES.fetch_add(1, Ordering::Relaxed);
ipc_logf(
&[
(b"SHMPOOL-DECLINE reason=fstat pid=", pid as i64),
(b"hfd=", h as i64),
(b"err=", fr),
],
&[],
);
return -1;
}
let dev = u64::from_le_bytes(st[0..8].try_into().unwrap());
let ino = u64::from_le_bytes(st[8..16].try_into().unwrap());
let size = u64::from_le_bytes(st[48..56].try_into().unwrap());
let need = want_len.max(size).saturating_add(off);
let raw = (need + 0xfff) & !0xfff;
let aligned = {
let floor = raw.max(4 << 20);
let bucket = (floor + ((1 << 20) - 1)) & !((1u64 << 20) - 1);
if raw >= (4 << 20) {
bucket.saturating_mul(2)
} else {
bucket
}
};
let key = (dev, ino);
let mut slices = shm_slices().lock().unwrap();
let existed = slices.get(&key).copied();
let slice = if let Some(s) = existed {
if raw > s.capacity {
SHM_ENOMEM_DECLINES.fetch_add(1, Ordering::Relaxed);
ipc_logf(
&[
(b"SHMPOOL-DECLINE reason=grow raw=", raw as i64),
(b"cap=", s.capacity as i64),
(b"ino=", ino as i64),
(
b"enomem=",
SHM_ENOMEM_DECLINES.load(Ordering::Relaxed) as i64,
),
],
&[],
);
return -1; }
s
} else {
let Some(o) = shm_pool_alloc(aligned) else {
SHM_ENOMEM_DECLINES.fetch_add(1, Ordering::Relaxed);
let free_total: u64 = shm_free_slices()
.lock()
.unwrap()
.iter()
.map(|(_, cap)| *cap)
.sum();
ipc_logf(
&[
(b"SHMPOOL-DECLINE reason=full aligned=", aligned as i64),
(b"next=", SHM_POOL_NEXT.load(Ordering::Relaxed) as i64),
(b"free=", free_total as i64),
(
b"enomem=",
SHM_ENOMEM_DECLINES.load(Ordering::Relaxed) as i64,
),
],
&[],
);
return -1;
};
let dup = unsafe {
host(
SYS_FCNTL,
h as u64,
libc::F_DUPFD_CLOEXEC as u64,
3,
0,
0,
0,
)
};
if dup < 0 {
ipc_logf(
&[(b"SHMPOOL-NOFLUSH dup err=", dup), (b"ino=", ino as i64)],
&[],
);
}
let mut done = 0u64;
while done < want_len {
let chunk = (want_len - done).min(4 * 1024 * 1024);
let n = unsafe {
host(
SYS_PREAD64,
h as u64,
base + o + off + done,
chunk,
off + done,
0,
0,
)
};
if n <= 0 {
break;
}
done += n as u64;
}
let s = ShmSlice {
pool_off: o,
capacity: aligned,
backing_fd: if dup >= 0 { dup as i32 } else { -1 },
};
slices.insert(key, s);
s
};
drop(slices);
let addr = base + slice.pool_off + off;
shm_maps()
.lock()
.unwrap()
.entry(pid)
.or_default()
.push(ShmMap {
addr,
len: want_len,
file_off: off,
key,
writable: (prot & libc::PROT_WRITE as u64) != 0,
});
if unsafe { IPCTRACE } {
ipc_logf(
&[
(b"SHMMAP sup=", unsafe {
host(SYS_GETPID, 0, 0, 0, 0, 0, 0)
}),
(b"hfd=", h as i64),
(b"dev=", dev as i64),
(b"ino=", ino as i64),
(b"sz=", size as i64),
(b"len=", want_len as i64),
(b"prot=", prot as i64),
(b"off=", off as i64),
(b"writable=", ((prot & libc::PROT_WRITE as u64) != 0) as i64),
(
b"existed=",
existed.map(|s| s.pool_off as i64).unwrap_or(-1),
),
(b"slice=", slice.pool_off as i64),
(b"base=", base as i64),
(b"addr=", addr as i64),
(
b"enomem=",
SHM_ENOMEM_DECLINES.load(Ordering::Relaxed) as i64,
),
],
&[],
);
}
addr as i64
}
fn shm_alias_pid_range(pid: i32, from: u64, to: u64, len: u64) -> i64 {
let mut maps = shm_maps().lock().unwrap();
let Some(entries) = maps.get_mut(&pid) else {
return -2;
};
for m in entries.iter_mut().rev() {
if m.addr == from && m.len == len {
m.addr = to;
return 0;
}
}
-2
}
fn shm_protect_pid_range(pid: i32, addr: u64, len: u64, prot: u64) -> i64 {
let mut maps = shm_maps().lock().unwrap();
let Some(entries) = maps.get_mut(&pid) else {
return 0;
};
let writable = (prot & libc::PROT_WRITE as u64) != 0;
for m in entries.iter_mut() {
if !ranges_overlap(m.addr, m.len, addr, len) {
continue;
}
if writable {
m.writable = true;
} else if addr <= m.addr && addr.saturating_add(len) >= m.addr.saturating_add(m.len) {
m.writable = false;
}
}
0
}
fn shm_move_pid_range(pid: i32, from: u64, old_len: u64, to: u64, new_len: u64) -> i64 {
let mut maps = shm_maps().lock().unwrap();
let Some(entries) = maps.get_mut(&pid) else {
return 0;
};
for m in entries.iter_mut().rev() {
if m.addr == from && m.len == old_len {
m.addr = to;
m.len = new_len;
return 0;
}
}
0
}
fn shm_snapshot(parent: i32, slot: u64) {
let src = shm_maps()
.lock()
.unwrap()
.get(&parent)
.cloned()
.unwrap_or_default();
if src.is_empty() {
pending_shm_maps().lock().unwrap().remove(&slot);
} else {
pending_shm_maps().lock().unwrap().insert(slot, src);
}
}
fn shm_adopt(slot: u64, child: i32) {
let Some(entries) = pending_shm_maps().lock().unwrap().remove(&slot) else {
return;
};
let old = shm_maps().lock().unwrap().insert(child, entries);
if let Some(old) = old {
let keys: Vec<(u64, u64)> = old.iter().map(|m| m.key).collect();
shm_reclaim_unreferenced_keys(&keys);
}
}
fn shm_reclaim_unreferenced_keys(keys: &[(u64, u64)]) {
if keys.is_empty() {
return;
}
let mut uniq = Vec::new();
for key in keys {
if !uniq.contains(key) {
uniq.push(*key);
}
}
for key in uniq {
let still_mapped = {
let maps = shm_maps().lock().unwrap();
maps.values()
.any(|entries| entries.iter().any(|m| m.key == key))
} || {
let pending = pending_shm_maps().lock().unwrap();
pending
.values()
.any(|entries| entries.iter().any(|m| m.key == key))
} || {
let retired = retired_exec_shm_keys().lock().unwrap();
retired
.values()
.any(|entries| entries.iter().any(|retired_key| *retired_key == key))
};
if still_mapped {
continue;
}
let slice = {
let mut slices = shm_slices().lock().unwrap();
slices.remove(&key)
};
let Some(slice) = slice else {
continue;
};
let base = SHM_POOL_BASE.load(Ordering::Relaxed);
if base != 0 {
unsafe {
host(
SYS_MADVISE,
base + slice.pool_off,
slice.capacity,
libc::MADV_DONTNEED as u64,
0,
0,
0,
);
}
}
if slice.backing_fd >= 0 {
unsafe { host(SYS_CLOSE, slice.backing_fd as u64, 0, 0, 0, 0, 0) };
}
shm_pool_free(slice.pool_off, slice.capacity);
}
}
fn ranges_overlap(a: u64, alen: u64, b: u64, blen: u64) -> bool {
let ae = a.saturating_add(alen);
let be = b.saturating_add(blen);
a < be && b < ae
}
fn shm_pwrite_all(fd: i32, mut src: u64, mut len: u64, mut off: u64) -> i64 {
while len > 0 {
let chunk = len.min(4 * 1024 * 1024);
let n = unsafe { host(SYS_PWRITE64, fd as u64, src, chunk, off, 0, 0) };
if n < 0 {
return n;
}
if n == 0 {
return -5; }
let n = n as u64;
src = src.saturating_add(n);
off = off.saturating_add(n);
len -= n;
}
0
}
fn shm_flush_pid_range(pid: i32, addr: u64, len: u64, remove: bool) -> i64 {
shm_flush_pid_range_inner(pid, addr, len, remove, false)
}
fn shm_flush_pid_range_for_exec(pid: i32, addr: u64, len: u64) -> i64 {
shm_flush_pid_range_inner(pid, addr, len, true, true)
}
fn shm_reap_pid(pid: i32) {
shm_flush_pid_range(pid, 0, u64::MAX, true);
let retired = take_retired_exec_shm_keys(pid);
shm_reclaim_unreferenced_keys(&retired);
}
fn shm_flush_pid_range_inner(
pid: i32,
addr: u64,
len: u64,
remove: bool,
retire_until_reap: bool,
) -> i64 {
let all = addr == 0 && len == u64::MAX;
let mut removed_keys = Vec::new();
let maps_to_flush: Vec<ShmMap> = {
let mut maps = shm_maps().lock().unwrap();
let Some(entries) = maps.get_mut(&pid) else {
return 0;
};
let mut selected = Vec::new();
entries.retain(|m| {
let hit = all || ranges_overlap(m.addr, m.len, addr, len);
if hit {
selected.push(*m);
if remove {
removed_keys.push(m.key);
}
}
!(remove && hit)
});
if entries.is_empty() {
maps.remove(&pid);
}
selected
};
if maps_to_flush.is_empty() {
return 0;
}
let slices = shm_slices().lock().unwrap();
let mut first_err = 0;
for m in maps_to_flush {
if !m.writable {
if unsafe { IPCTRACE } && m.len <= 4096 {
ipc_logf(
&[
(b"SHMFLUSH-SKIP pid=", pid as i64),
(b"addr=", m.addr as i64),
(b"len=", m.len as i64),
(b"file_off=", m.file_off as i64),
(b"writable=0", 0),
],
&[],
);
}
continue;
}
let Some(slice) = slices.get(&m.key).copied() else {
continue;
};
if slice.backing_fd < 0 {
continue;
}
let flush_start = if all { m.addr } else { m.addr.max(addr) };
let flush_end = if all {
m.addr.saturating_add(m.len)
} else {
m.addr.saturating_add(m.len).min(addr.saturating_add(len))
};
if flush_end <= flush_start {
continue;
}
let rel = flush_start.saturating_sub(m.addr);
let flush_len = flush_end - flush_start;
let file_off = m.file_off.saturating_add(rel);
let pool_src = SHM_POOL_BASE
.load(Ordering::Relaxed)
.saturating_add(slice.pool_off)
.saturating_add(file_off);
if file_off.saturating_add(flush_len) > slice.capacity {
if first_err == 0 {
first_err = -22; }
continue;
}
let n = shm_pwrite_all(slice.backing_fd, pool_src, flush_len, file_off);
if unsafe { IPCTRACE } && flush_len <= 4096 {
let bytes = unsafe {
std::slice::from_raw_parts(pool_src as *const u8, flush_len.min(16) as usize)
};
ipc_logf(
&[
(b"SHMFLUSH pid=", pid as i64),
(b"addr=", flush_start as i64),
(b"src=", pool_src as i64),
(b"len=", flush_len as i64),
(b"file_off=", file_off as i64),
(b"ret=", n),
],
bytes,
);
}
if n < 0 && first_err == 0 {
first_err = n;
}
}
drop(slices);
if remove {
if retire_until_reap {
retire_exec_shm_keys(pid, &removed_keys);
} else {
shm_reclaim_unreferenced_keys(&removed_keys);
}
}
first_err
}
#[derive(Clone, Copy)]
struct SysvShmSegment {
id: i32,
key: u64,
size: u64,
addr: u64,
nattch: u64,
removed: bool,
}
fn sysv_shm_segments() -> &'static Mutex<HashMap<i32, SysvShmSegment>> {
static S: OnceLock<Mutex<HashMap<i32, SysvShmSegment>>> = OnceLock::new();
S.get_or_init(|| Mutex::new(HashMap::new()))
}
fn sysv_shm_next_id() -> &'static AtomicU32 {
static N: AtomicU32 = AtomicU32::new(1);
&N
}
fn sysv_shmget(key: u64, size: u64, flags: u64) -> i64 {
const IPC_PRIVATE: u64 = 0;
const IPC_CREAT: u64 = 0o1000;
const IPC_EXCL: u64 = 0o2000;
let base = SHM_POOL_BASE.load(Ordering::Relaxed);
if base == 0 {
return -38; }
let mut segments = sysv_shm_segments().lock().unwrap();
if key != IPC_PRIVATE {
if let Some(existing) = segments.values().find(|s| s.key == key && !s.removed) {
if (flags & IPC_CREAT) != 0 && (flags & IPC_EXCL) != 0 {
return -17; }
if size > existing.size {
return -22; }
return existing.id as i64;
}
if (flags & IPC_CREAT) == 0 {
return -2; }
}
if size == 0 {
return -22; }
let aligned = page_up(size);
let Some(off) = shm_pool_alloc(aligned) else {
return -28; };
let id = sysv_shm_next_id().fetch_add(1, Ordering::Relaxed) as i32;
let addr = base + off;
unsafe { std::ptr::write_bytes(addr as *mut u8, 0, aligned as usize) };
segments.insert(
id,
SysvShmSegment {
id,
key,
size,
addr,
nattch: 0,
removed: false,
},
);
id as i64
}
fn sysv_shmat(shmid: i32, shmaddr: u64, flags: u64) -> i64 {
const SHM_RDONLY: u64 = 0o10000;
if shmaddr != 0 || (flags & !SHM_RDONLY) != 0 {
return -22; }
let mut segments = sysv_shm_segments().lock().unwrap();
let Some(segment) = segments.get_mut(&shmid) else {
return -22; };
if segment.removed {
return -22;
}
segment.nattch = segment.nattch.saturating_add(1);
segment.addr as i64
}
fn sysv_shmdt(addr: u64) -> i64 {
let mut segments = sysv_shm_segments().lock().unwrap();
let Some((&id, segment)) = segments.iter_mut().find(|(_, s)| s.addr == addr) else {
return -22; };
segment.nattch = segment.nattch.saturating_sub(1);
if segment.removed && segment.nattch == 0 {
segments.remove(&id);
}
0
}
fn sysv_shmctl(pid: i32, shmid: i32, cmd: u64, buf: u64) -> i64 {
const IPC_RMID: u64 = 0;
const IPC_STAT: u64 = 2;
let mut segments = sysv_shm_segments().lock().unwrap();
match cmd {
IPC_RMID => {
let Some(segment) = segments.get_mut(&shmid) else {
return -22; };
segment.removed = true;
if segment.nattch == 0 {
segments.remove(&shmid);
}
0
}
IPC_STAT => {
let Some(segment) = segments.get(&shmid) else {
return -22; };
if buf == 0 {
return -14; }
let mut out = [0u8; 112];
out[32..40].copy_from_slice(&segment.size.to_le_bytes());
vm_write(pid, buf, &out);
0
}
_ => -22, }
}
fn guest_mmap(addr: u64, len: u64, prot: u64, flags: u64, fd: u64, off: u64) -> i64 {
if len == 0 {
return -22; }
if (flags & MAP_SHARED_BIT) != 0 && (flags & MAP_ANON_BIT) == 0 && fd != u64::MAX {
let r = delegate(SYS_MMAP, addr, len, prot, flags, fd, off);
if r > 0 {
let alias = unsafe {
host(
SYS_MREMAP,
r as u64,
0,
len,
libc::MREMAP_MAYMOVE as u64,
0,
0,
)
};
if alias <= 0 {
ipc_logn(&[
(b"SHMALIAS-FAIL pool=", r),
(b"len=", len as i64),
(b"ret=", alias),
]);
let _ = delegate(CTL_SHM_FLUSH, r as u64, len, 1, 0, 0, 0);
return -12; }
let vis = alias as u64;
let _ = unsafe { host(SYS_MPROTECT, vis, len, prot, 0, 0, 0) };
let aliased = delegate(CTL_SHM_ALIAS, r as u64, vis, len, 0, 0, 0);
if aliased != 0 {
let _ = unsafe { host(SYS_MUNMAP, vis, len, 0, 0, 0, 0) };
let _ = delegate(CTL_SHM_FLUSH, r as u64, len, 1, 0, 0, 0);
return -12;
}
guest_vma_note_mapping(vis, len);
return vis as i64;
}
ipc_logn(&[
(b"SHMMAP-DECLINE fd=", fd as i64),
(b"len=", len as i64),
(b"prot=", prot as i64),
(b"ret=", r),
]);
return -12; }
let guest_fixed = (flags & (MAP_FIXED_BIT | MAP_FIXED_NOREPLACE_BIT)) != 0;
let guest_noreplace = (flags & MAP_FIXED_NOREPLACE_BIT) != 0;
let target = if guest_fixed {
if addr == 0 {
return -1; }
arena_free_purge(page_down(addr), page_up(addr.saturating_add(len)));
addr
} else {
const ARENA_BYPASS_MIN: u64 = 2 << 30;
if len >= ARENA_BYPASS_MIN && (flags & MAP_ANON_BIT) != 0 && (prot & PROT_EXEC_BIT) == 0 {
let host_flags = flags & !(MAP_FIXED_BIT | MAP_FIXED_NOREPLACE_BIT);
let r = unsafe { host(SYS_MMAP, 0, len, prot, host_flags, u64::MAX, 0) };
if r >= 0 {
guest_vma_note_mapping(r as u64, len);
arena_note_mapping(r as u64, len);
arena_free_purge(page_down(r as u64), page_up((r as u64).saturating_add(len)));
return r;
}
}
let t = arena_alloc(len);
if t == 0 {
log_arena_exhausted(len, prot, flags);
if (flags & MAP_ANON_BIT) != 0 && (prot & PROT_EXEC_BIT) == 0 {
let host_flags = flags & !(MAP_FIXED_BIT | MAP_FIXED_NOREPLACE_BIT);
let r = unsafe { host(SYS_MMAP, addr, len, prot, host_flags, u64::MAX, 0) };
if r >= 0 {
guest_vma_note_mapping(r as u64, len);
arena_note_mapping(r as u64, len);
arena_free_purge(page_down(r as u64), page_up((r as u64).saturating_add(len)));
}
return r;
}
if (prot & PROT_EXEC_BIT) == 0 {
let m = unsafe {
host(
SYS_MMAP,
0,
len,
PROT_RW,
(libc::MAP_PRIVATE | libc::MAP_ANONYMOUS) as u64,
u64::MAX,
0,
)
};
if m < 0 {
return m;
}
let base = m as u64;
let mut done = 0u64;
while done < len {
let chunk = (len - done).min(4 * 1024 * 1024);
let n = delegate(SYS_PREAD64, fd, base + done, chunk, off + done, 0, 0);
if n < 0 {
unsafe { host(SYS_MUNMAP, base, len, 0, 0, 0, 0) };
return n;
}
if n == 0 {
break;
}
done += n as u64;
}
if prot != PROT_RW {
unsafe { host(SYS_MPROTECT, base, len, prot, 0, 0, 0) };
}
guest_vma_note_mapping(base, len);
arena_note_mapping(base, len);
arena_free_purge(page_down(base), page_up(base.saturating_add(len)));
return base as i64;
}
return -12;
}
t
};
if (prot & PROT_EXEC_BIT) != 0 && overlaps_sud_host_region(target, len) {
ipc_logn(&[
(b"MMAP-EXEC-DENY target=", target as i64),
(b"len=", len as i64),
]);
return -1; }
if guest_fixed && overlaps_loader(target, page_up(target + len)) {
if guest_noreplace {
return -17; }
return -12; }
if (flags & MAP_ANON_BIT) != 0 {
let host_flags = if guest_noreplace {
(flags & !MAP_FIXED_BIT) | MAP_FIXED_NOREPLACE_BIT
} else {
flags | MAP_FIXED_BIT
};
let r = unsafe { host(SYS_MMAP, target, len, prot, host_flags, u64::MAX, 0) };
if r >= 0 {
arena_note_mapping(target, len);
guest_vma_note_mapping(r as u64, len);
} else if r == -12 {
ipc_logn(&[
(b"MMAP-HOST-ENOMEM len=", len as i64),
(b"prot=", prot as i64),
(b"target=", target as i64),
]);
}
return r;
}
let reserve_flags = if guest_noreplace {
(libc::MAP_PRIVATE as u64) | (libc::MAP_ANONYMOUS as u64) | MAP_FIXED_NOREPLACE_BIT
} else {
(libc::MAP_PRIVATE | libc::MAP_ANONYMOUS | libc::MAP_FIXED) as u64
};
let m = unsafe { host(SYS_MMAP, target, len, PROT_RW, reserve_flags, u64::MAX, 0) };
if m < 0 {
if m == -12 {
ipc_logn(&[
(b"MMAP-HOST-ENOMEM file len=", len as i64),
(b"target=", target as i64),
]);
}
return m;
}
let mut done = 0u64;
while done < len {
let chunk = (len - done).min(4 * 1024 * 1024);
let n = delegate(SYS_PREAD64, fd, target + done, chunk, off + done, 0, 0);
if n < 0 {
unsafe { host(SYS_MUNMAP, target, len, 0, 0, 0, 0) };
return n;
}
if n == 0 {
break; }
done += n as u64;
}
if prot != PROT_RW {
unsafe { host(SYS_MPROTECT, target, len, prot, 0, 0, 0) };
}
arena_note_mapping(target, len);
guest_vma_note_mapping(target, len);
target as i64
}
fn brk(req: u64) -> i64 {
use std::sync::atomic::Ordering;
let (base, end) = unsafe { (HEAP_BASE, HEAP_END) };
if req == 0 || req < base || req > end {
return HEAP_CUR.load(Ordering::Acquire) as i64;
}
loop {
let cur = HEAP_CUR.load(Ordering::Acquire);
if req <= cur {
match HEAP_CUR.compare_exchange_weak(cur, req, Ordering::AcqRel, Ordering::Acquire) {
Ok(_) => return req as i64,
Err(_) => continue,
}
}
match HEAP_CUR.compare_exchange_weak(cur, req, Ordering::AcqRel, Ordering::Acquire) {
Ok(_) => {
unsafe { std::ptr::write_bytes(cur as *mut u8, 0, (req - cur) as usize) };
return req as i64;
}
Err(_) => continue,
}
}
}
struct CellCreds {
ruid: std::sync::atomic::AtomicU32,
euid: std::sync::atomic::AtomicU32,
suid: std::sync::atomic::AtomicU32,
fsuid: std::sync::atomic::AtomicU32,
rgid: std::sync::atomic::AtomicU32,
egid: std::sync::atomic::AtomicU32,
sgid: std::sync::atomic::AtomicU32,
fsgid: std::sync::atomic::AtomicU32,
}
#[derive(Clone, Copy)]
struct CellCredSnapshot {
ruid: u32,
euid: u32,
suid: u32,
fsuid: u32,
rgid: u32,
egid: u32,
sgid: u32,
fsgid: u32,
}
impl Default for CellCredSnapshot {
fn default() -> Self {
Self {
ruid: 0,
euid: 0,
suid: 0,
fsuid: 0,
rgid: 0,
egid: 0,
sgid: 0,
fsgid: 0,
}
}
}
static CELL_CREDS: CellCreds = CellCreds {
ruid: std::sync::atomic::AtomicU32::new(0),
euid: std::sync::atomic::AtomicU32::new(0),
suid: std::sync::atomic::AtomicU32::new(0),
fsuid: std::sync::atomic::AtomicU32::new(0),
rgid: std::sync::atomic::AtomicU32::new(0),
egid: std::sync::atomic::AtomicU32::new(0),
sgid: std::sync::atomic::AtomicU32::new(0),
fsgid: std::sync::atomic::AtomicU32::new(0),
};
#[inline]
fn cred_get(a: &std::sync::atomic::AtomicU32) -> u32 {
a.load(std::sync::atomic::Ordering::Relaxed)
}
#[inline]
fn cred_set(a: &std::sync::atomic::AtomicU32, v: u32) {
a.store(v, std::sync::atomic::Ordering::Relaxed)
}
fn cred_reset(uid: u32, gid: u32) {
cred_set(&CELL_CREDS.ruid, uid);
cred_set(&CELL_CREDS.euid, uid);
cred_set(&CELL_CREDS.suid, uid);
cred_set(&CELL_CREDS.fsuid, uid);
cred_set(&CELL_CREDS.rgid, gid);
cred_set(&CELL_CREDS.egid, gid);
cred_set(&CELL_CREDS.sgid, gid);
cred_set(&CELL_CREDS.fsgid, gid);
}
fn cred_snapshot_current() -> CellCredSnapshot {
CellCredSnapshot {
ruid: cred_get(&CELL_CREDS.ruid),
euid: cred_get(&CELL_CREDS.euid),
suid: cred_get(&CELL_CREDS.suid),
fsuid: cred_get(&CELL_CREDS.fsuid),
rgid: cred_get(&CELL_CREDS.rgid),
egid: cred_get(&CELL_CREDS.egid),
sgid: cred_get(&CELL_CREDS.sgid),
fsgid: cred_get(&CELL_CREDS.fsgid),
}
}
fn pack_u32_pair(lo: u32, hi: u32) -> u64 {
lo as u64 | ((hi as u64) << 32)
}
fn unpack_u32_pair(v: u64) -> (u32, u32) {
(v as u32, (v >> 32) as u32)
}
fn supervisor_creds() -> &'static Mutex<HashMap<i32, CellCredSnapshot>> {
static T: OnceLock<Mutex<HashMap<i32, CellCredSnapshot>>> = OnceLock::new();
T.get_or_init(|| Mutex::new(HashMap::new()))
}
fn supervisor_cred_for(pid: i32) -> CellCredSnapshot {
supervisor_creds()
.lock()
.unwrap()
.get(&pid)
.copied()
.unwrap_or_default()
}
fn sync_creds_to_supervisor() {
let c = cred_snapshot_current();
let _ = delegate(
CTL_SET_CREDS,
pack_u32_pair(c.ruid, c.euid),
pack_u32_pair(c.suid, c.fsuid),
pack_u32_pair(c.rgid, c.egid),
pack_u32_pair(c.sgid, c.fsgid),
0,
0,
);
}
const DEFAULT_UMASK: u32 = 0o022;
static CELL_UMASK: std::sync::atomic::AtomicU32 = std::sync::atomic::AtomicU32::new(DEFAULT_UMASK);
fn cell_umask_get() -> u32 {
CELL_UMASK.load(std::sync::atomic::Ordering::Relaxed) & 0o777
}
fn cell_umask_set(mask: u32) -> u32 {
CELL_UMASK.swap(mask & 0o777, std::sync::atomic::Ordering::Relaxed) & 0o777
}
fn cell_umask_reset() {
CELL_UMASK.store(DEFAULT_UMASK, std::sync::atomic::Ordering::Relaxed);
}
fn supervisor_umasks() -> &'static Mutex<HashMap<i32, u32>> {
static T: OnceLock<Mutex<HashMap<i32, u32>>> = OnceLock::new();
T.get_or_init(|| Mutex::new(HashMap::new()))
}
fn supervisor_umask_for(pid: i32) -> u32 {
supervisor_umasks()
.lock()
.unwrap()
.get(&pid)
.copied()
.unwrap_or(DEFAULT_UMASK)
& 0o777
}
fn sync_umask_to_supervisor() {
let _ = delegate(CTL_SET_UMASK, cell_umask_get() as u64, 0, 0, 0, 0, 0);
}
fn apply_umask(pid: i32, mode: u64) -> u64 {
mode & !(supervisor_umask_for(pid) as u64)
}
#[inline]
fn cred_privileged() -> bool {
cred_get(&CELL_CREDS.euid) == 0
}
const ID_UNCHANGED: u32 = u32::MAX; fn cred_setuid(u: u32) -> i64 {
if cred_privileged() {
cred_set(&CELL_CREDS.ruid, u);
cred_set(&CELL_CREDS.euid, u);
cred_set(&CELL_CREDS.suid, u);
cred_set(&CELL_CREDS.fsuid, u);
0
} else if u == cred_get(&CELL_CREDS.ruid)
|| u == cred_get(&CELL_CREDS.suid)
|| u == cred_get(&CELL_CREDS.euid)
{
cred_set(&CELL_CREDS.euid, u);
cred_set(&CELL_CREDS.fsuid, u);
0
} else {
-1
}
}
fn cred_setgid(g: u32) -> i64 {
if cred_privileged() {
cred_set(&CELL_CREDS.rgid, g);
cred_set(&CELL_CREDS.egid, g);
cred_set(&CELL_CREDS.sgid, g);
cred_set(&CELL_CREDS.fsgid, g);
0
} else if g == cred_get(&CELL_CREDS.rgid)
|| g == cred_get(&CELL_CREDS.sgid)
|| g == cred_get(&CELL_CREDS.egid)
{
cred_set(&CELL_CREDS.egid, g);
cred_set(&CELL_CREDS.fsgid, g);
0
} else {
-1
}
}
fn cred_setresuid(r: u32, eu: u32, s: u32) -> i64 {
let held = |v: u32| {
v == cred_get(&CELL_CREDS.ruid)
|| v == cred_get(&CELL_CREDS.euid)
|| v == cred_get(&CELL_CREDS.suid)
};
if !cred_privileged() {
for &v in &[r, eu, s] {
if v != ID_UNCHANGED && !held(v) {
return -1;
}
}
}
if r != ID_UNCHANGED {
cred_set(&CELL_CREDS.ruid, r);
}
if eu != ID_UNCHANGED {
cred_set(&CELL_CREDS.euid, eu);
cred_set(&CELL_CREDS.fsuid, eu);
}
if s != ID_UNCHANGED {
cred_set(&CELL_CREDS.suid, s);
}
0
}
fn cred_setresgid(r: u32, eg: u32, s: u32) -> i64 {
let held = |v: u32| {
v == cred_get(&CELL_CREDS.rgid)
|| v == cred_get(&CELL_CREDS.egid)
|| v == cred_get(&CELL_CREDS.sgid)
};
if !cred_privileged() {
for &v in &[r, eg, s] {
if v != ID_UNCHANGED && !held(v) {
return -1;
}
}
}
if r != ID_UNCHANGED {
cred_set(&CELL_CREDS.rgid, r);
}
if eg != ID_UNCHANGED {
cred_set(&CELL_CREDS.egid, eg);
cred_set(&CELL_CREDS.fsgid, eg);
}
if s != ID_UNCHANGED {
cred_set(&CELL_CREDS.sgid, s);
}
0
}
fn cred_getresuid(r: u64, e: u64, s: u64) -> i64 {
if r == 0 || e == 0 || s == 0 {
return -14; }
unsafe {
std::ptr::write_unaligned(r as *mut u32, cred_get(&CELL_CREDS.ruid));
std::ptr::write_unaligned(e as *mut u32, cred_get(&CELL_CREDS.euid));
std::ptr::write_unaligned(s as *mut u32, cred_get(&CELL_CREDS.suid));
}
0
}
fn cred_getresgid(r: u64, e: u64, s: u64) -> i64 {
if r == 0 || e == 0 || s == 0 {
return -14; }
unsafe {
std::ptr::write_unaligned(r as *mut u32, cred_get(&CELL_CREDS.rgid));
std::ptr::write_unaligned(e as *mut u32, cred_get(&CELL_CREDS.egid));
std::ptr::write_unaligned(s as *mut u32, cred_get(&CELL_CREDS.sgid));
}
0
}
fn sentry_reserved_signal_mask() -> u64 {
(1u64 << (31 - 1)) | (1u64 << ((SIG_DETHREAD as u64) - 1))
}
fn sanitize_guest_sigmask(mask: u64) -> u64 {
mask & !sentry_reserved_signal_mask()
}
unsafe fn block_guest_signals_until_sigreturn() {
unsafe {
const SIG_BLOCK: u64 = 0;
let mut set = sanitize_guest_sigmask(!0u64); set &= !(1u64 << (9 - 1)); set &= !(1u64 << (19 - 1)); host(
SYS_RT_SIGPROCMASK,
SIG_BLOCK,
std::ptr::addr_of!(set) as u64,
0,
8,
0,
0,
);
}
}
unsafe fn pending_unmasked_guest_signal(saved_mask: u64) -> bool {
unsafe {
let mut pending = 0u64;
if host(
SYS_RT_SIGPENDING,
std::ptr::addr_of_mut!(pending) as u64,
8,
0,
0,
0,
0,
) != 0
{
return false;
}
let ignored = (1u64 << (9 - 1)) | (1u64 << (19 - 1)) | sentry_reserved_signal_mask();
(pending & !saved_mask & !ignored) != 0
}
}
unsafe fn reset_exec_signal_dispositions() {
unsafe {
let dfl: [u64; 4] = [0, 0, 0, 0]; for sig in 1u64..=64 {
if sig == 31 || sig == 32 || sig == 33 || sig == SIG_DETHREAD as u64 {
continue;
}
host(SYS_RT_SIGACTION, sig, dfl.as_ptr() as u64, 0, 8, 0, 0);
}
}
}
unsafe fn short_futex_timeout_for(op: u64) -> [i64; 2] {
unsafe {
const ONE_MS_NS: i64 = 1_000_000;
let cmd = op & !(FUTEX_PRIVATE_FLAG | FUTEX_CLOCK_REALTIME);
if cmd == FUTEX_WAIT_BITSET {
let clock = if (op & FUTEX_CLOCK_REALTIME) != 0 {
0
} else {
1
};
let mut ts = [0i64; 2];
host(SYS_CLOCK_GETTIME, clock, ts.as_mut_ptr() as u64, 0, 0, 0, 0);
ts[1] += ONE_MS_NS;
if ts[1] >= 1_000_000_000 {
ts[0] += 1;
ts[1] -= 1_000_000_000;
}
ts
} else {
[0, ONE_MS_NS]
}
}
}
unsafe fn host_monotonic_futex_deadline(op: u64, d: u64, out: &mut [i64; 2]) -> u64 {
unsafe {
if d == 0 {
return 0;
}
let cmd = op & !(FUTEX_PRIVATE_FLAG | FUTEX_CLOCK_REALTIME);
if cmd != FUTEX_WAIT_BITSET || (op & FUTEX_CLOCK_REALTIME) != 0 {
return d; }
let Some(&base) = GUEST_MONOTONIC_BASE_NS.get() else {
return d; };
let g = d as *const i64;
let total = (g.read() as i128) * 1_000_000_000 + g.add(1).read() as i128 + base as i128;
out[0] = (total / 1_000_000_000) as i64;
out[1] = (total % 1_000_000_000) as i64;
out.as_ptr() as u64
}
}
unsafe fn futex_from_sigsys_trap(
a: u64,
b: u64,
c: u64,
d: u64,
e: u64,
f: u64,
saved_mask: Option<u64>,
) -> i64 {
unsafe {
let mut adj = [0i64; 2];
let d = host_monotonic_futex_deadline(b, d, &mut adj);
let Some(saved_mask) = saved_mask else {
return host(SYS_FUTEX, a, b, c, d, e, f);
};
let cmd = b & !(FUTEX_PRIVATE_FLAG | FUTEX_CLOCK_REALTIME);
let interruptible_wait = (cmd == FUTEX_WAIT || cmd == FUTEX_WAIT_BITSET) && d == 0;
if !interruptible_wait {
return host(SYS_FUTEX, a, b, c, d, e, f);
}
block_guest_signals_until_sigreturn();
loop {
if pending_unmasked_guest_signal(saved_mask) {
return EINTR;
}
let mut ts = short_futex_timeout_for(b);
let timeout = ts.as_mut_ptr() as u64;
let bitset = if cmd == FUTEX_WAIT_BITSET {
if f == 0 {
FUTEX_BITSET_MATCH_ANY
} else {
f
}
} else {
f
};
let r = host(SYS_FUTEX, a, b, c, timeout, e, bitset);
if r == -110 {
continue;
}
if r == EINTR && pending_unmasked_guest_signal(saved_mask) {
return EINTR;
}
return r;
}
}
}
fn dispatch_simple(
nr: i64,
a: u64,
b: u64,
c: u64,
d: u64,
e: u64,
f: u64,
trap_saved_mask: Option<u64>,
) -> i64 {
match nr {
SYS_BRK => brk(a),
SYS_MMAP => guest_mmap(a, b, c, d, e, f),
SYS_MUNMAP => {
let _ = delegate(CTL_SHM_FLUSH, a, b, 1, 0, 0, 0);
if overlaps_loader(a, page_up(a + b)) || in_shm_pool(a) {
0
} else {
let r = unsafe { host(SYS_MUNMAP, a, b, 0, 0, 0, 0) };
if r == 0 {
guest_vma_forget_range(a, b);
arena_reclaim_mapping(a, b);
}
r
}
}
SYS_MSYNC => delegate(CTL_SHM_FLUSH, a, b, 0, 0, 0, 0),
SYS_MREMAP => {
if overlaps_loader(a, page_up(a + b)) || in_shm_pool(a) {
-22 } else {
let r = unsafe { host(SYS_MREMAP, a, b, c, d, e, f) };
if r == -12 {
ipc_logn(&[
(b"MREMAP-ENOMEM old=", a as i64),
(b"oldlen=", b as i64),
(b"newlen=", c as i64),
]);
}
if r >= 0 {
guest_vma_forget_range(a, b);
guest_vma_note_mapping(r as u64, c);
let _ = delegate(CTL_SHM_MOVE, a, b, r as u64, c, 0, 0);
let ra = r as u64;
if ra != a {
arena_reclaim_mapping(a, b);
arena_free_purge(page_down(ra), page_up(ra.saturating_add(c)));
} else if c < b {
arena_reclaim_mapping(a.saturating_add(c), b - c);
} else if c > b {
arena_free_purge(
page_down(a.saturating_add(b)),
page_up(a.saturating_add(c)),
);
}
}
r
}
}
SYS_MPROTECT => {
if (c & PROT_EXEC_BIT) != 0 && overlaps_sud_host_region(a, b) {
ipc_logn(&[(b"MPROTECT-EXEC-DENY addr=", a as i64), (b"len=", b as i64)]);
-1 } else {
let r = unsafe { host(SYS_MPROTECT, a, b, c, d, e, f) };
if r == 0 {
let _ = delegate(CTL_SHM_PROTECT, a, b, c, 0, 0, 0);
}
r
}
}
SYS_MADVISE | SYS_MINCORE | SYS_ARCH_PRCTL => unsafe { host(nr, a, b, c, d, e, f) },
SYS_PKEY_MPROTECT => {
if (c & PROT_EXEC_BIT) != 0 && overlaps_sud_host_region(a, b) {
ipc_logn(&[
(b"PKEYMPROTECT-EXEC-DENY addr=", a as i64),
(b"len=", b as i64),
]);
-1 } else {
unsafe { host(SYS_PKEY_MPROTECT, a, b, c, d, e, f) }
}
}
SYS_PKEY_ALLOC | SYS_PKEY_FREE => unsafe { host(nr, a, b, c, d, e, f) },
SYS_FUTEX => unsafe { futex_from_sigsys_trap(a, b, c, d, e, f, trap_saved_mask) },
SYS_GETPID | SYS_GETTID | SYS_GETPPID => unsafe { host(nr, 0, 0, 0, 0, 0, 0) },
SYS_GETUID => cred_get(&CELL_CREDS.ruid) as i64,
SYS_GETEUID => cred_get(&CELL_CREDS.euid) as i64,
SYS_GETGID => cred_get(&CELL_CREDS.rgid) as i64,
SYS_GETEGID => cred_get(&CELL_CREDS.egid) as i64,
SYS_GETRESUID => cred_getresuid(a, b, c),
SYS_GETRESGID => cred_getresgid(a, b, c),
SYS_UMASK => {
let old = cell_umask_set(a as u32) as i64;
sync_umask_to_supervisor();
old
}
SYS_GETGROUPS => 0,
SYS_SETUID => {
let r = cred_setuid(a as u32);
if r == 0 {
sync_creds_to_supervisor();
}
r
}
SYS_SETGID => {
let r = cred_setgid(a as u32);
if r == 0 {
sync_creds_to_supervisor();
}
r
}
SYS_SETRESUID => {
let r = cred_setresuid(a as u32, b as u32, c as u32);
if r == 0 {
sync_creds_to_supervisor();
}
r
}
SYS_SETRESGID => {
let r = cred_setresgid(a as u32, b as u32, c as u32);
if r == 0 {
sync_creds_to_supervisor();
}
r
}
SYS_SETREUID => {
let r = cred_setresuid(a as u32, b as u32, ID_UNCHANGED);
if r == 0 {
sync_creds_to_supervisor();
}
r
}
SYS_SETREGID => {
let r = cred_setresgid(a as u32, b as u32, ID_UNCHANGED);
if r == 0 {
sync_creds_to_supervisor();
}
r
}
SYS_SETFSUID => {
let old = cred_get(&CELL_CREDS.fsuid) as i64;
let u = a as u32;
if cred_privileged()
|| u == cred_get(&CELL_CREDS.ruid)
|| u == cred_get(&CELL_CREDS.euid)
|| u == cred_get(&CELL_CREDS.suid)
|| u == cred_get(&CELL_CREDS.fsuid)
{
cred_set(&CELL_CREDS.fsuid, u);
sync_creds_to_supervisor();
}
old
}
SYS_SETFSGID => {
let old = cred_get(&CELL_CREDS.fsgid) as i64;
let g = a as u32;
if cred_privileged()
|| g == cred_get(&CELL_CREDS.rgid)
|| g == cred_get(&CELL_CREDS.egid)
|| g == cred_get(&CELL_CREDS.sgid)
|| g == cred_get(&CELL_CREDS.fsgid)
{
cred_set(&CELL_CREDS.fsgid, g);
sync_creds_to_supervisor();
}
old
}
SYS_SETGROUPS => 0,
SYS_SET_TID_ADDRESS => unsafe { host(SYS_SET_TID_ADDRESS, a, 0, 0, 0, 0, 0) },
SYS_SET_ROBUST_LIST => unsafe { host(SYS_SET_ROBUST_LIST, a, b, 0, 0, 0, 0) },
SYS_RSEQ => -38, SYS_PAUSE => unsafe { host(SYS_PAUSE, 0, 0, 0, 0, 0, 0) },
SYS_RT_SIGTIMEDWAIT => unsafe { host(SYS_RT_SIGTIMEDWAIT, a, b, c, d, 0, 0) },
SYS_RT_SIGSUSPEND => unsafe {
if a == 0 || b != 8 {
host(SYS_RT_SIGSUSPEND, a, b, 0, 0, 0, 0)
} else {
let set = sanitize_guest_sigmask(std::ptr::read_unaligned(a as *const u64));
host(
SYS_RT_SIGSUSPEND,
std::ptr::addr_of!(set) as u64,
8,
0,
0,
0,
0,
)
}
},
SYS_PRCTL => match a {
PR_SET_NAME | PR_GET_NAME | PR_SET_PDEATHSIG => unsafe {
host(SYS_PRCTL, a, b, c, d, e, f)
},
PR_CAPBSET_READ => 0,
PR_GET_DUMPABLE | PR_SET_DUMPABLE => unsafe { host(SYS_PRCTL, a, b, c, d, e, f) },
PR_SET_VMA if b == PR_SET_VMA_ANON_NAME => {
let r = unsafe { host(SYS_PRCTL, a, b, c, d, e, f) };
if r == -22 || r == -38 {
0
} else {
r
}
}
PR_SET_SECCOMP | PR_SET_NO_NEW_PRIVS => -1,
x if x == PR_SET_SYSCALL_USER_DISPATCH as u64 => -1,
_ => -38,
},
SYS_SETSID | SYS_GETPGRP => unsafe { host(nr, 0, 0, 0, 0, 0, 0) },
SYS_SETPGID => {
if a as i32 == 0 || pid_in_sandbox(a as i32) {
unsafe { host(SYS_SETPGID, a, b, 0, 0, 0, 0) }
} else {
-1 }
}
SYS_GETPGID | SYS_GETSID => {
if a as i32 == 0 || pid_in_sandbox(a as i32) {
unsafe { host(nr, a, 0, 0, 0, 0, 0) }
} else {
unsafe { host(nr, 0, 0, 0, 0, 0, 0) }
}
}
SYS_CLONE3 => -38,
SYS_FORK | SYS_VFORK => {
let parent_slot = current_slot();
let slot = alloc_slot();
if slot as usize >= MAX_SLOTS {
-11
} else if delegate(CTL_ENSURE_SERVICER, slot as u64, 0, 0, 0, 0, 0) < 0 {
free_slot(slot);
-11
} else {
delegate(CTL_FORK_TABLE, slot as u64, nr as u64, 0, 0, 0, 0);
unsafe {
(*ring_at(slot as u64)).fork_parent = host(SYS_GETPID, 0, 0, 0, 0, 0, 0) as i32;
}
let child = unsafe { host(SYS_FORK, 0, 0, 0, 0, 0, 0) };
if child < 0 {
unsafe { (*ring_at(slot as u64)).fork_parent = 0 };
delegate(CTL_FORK_CANCEL, slot as u64, 0, 0, 0, 0, 0);
child
} else if child == 0 {
set_slot(slot as u64);
register_rseq_for_child(parent_slot, slot as u64);
MM_USERS.store(1, Ordering::Release);
unsafe {
set_ring_owner(
slot as u64,
host(SYS_GETPID, 0, 0, 0, 0, 0, 0) as i32,
host(SYS_GETTID, 0, 0, 0, 0, 0, 0) as i32,
);
host(
157,
PR_SET_SYSCALL_USER_DISPATCH as u64,
PR_SYS_DISPATCH_ON,
WINDOW_FLOOR,
USER_TOP - WINDOW_FLOOR,
std::ptr::addr_of!(SELECTOR) as u64,
0,
);
register_altstack();
}
0
} else {
delegate(CTL_BIND_SLOT, slot as u64, child as u64, 0, 0, 0, 0);
child
}
}
}
SYS_WAIT4 => {
let r = unsafe { host(SYS_WAIT4, a, b, c, d, e, f) };
if r > 0 {
delegate(CTL_REAP, r as u64, 0, 0, 0, 0, 0);
}
r
}
SYS_WAITID => {
let r = unsafe { host(SYS_WAITID, a, b, c, d, e, f) };
if r == 0 && c != 0 && (d & WNOWAIT) == 0 {
let si_code = unsafe { ((c as usize + 8) as *const i32).read() };
let reaped = unsafe { ((c as usize + SIGINFO_SI_PID_OFF) as *const i32).read() };
let terminal = (1..=3).contains(&si_code); if reaped > 0 && terminal {
delegate(CTL_REAP, reaped as u64, 0, 0, 0, 0, 0);
}
}
r
}
SYS_EXIT => {
let slot = current_slot();
notify_vfork_release();
if slot != 0 {
free_slot(slot as u32);
}
unsafe { host(SYS_EXIT, a, 0, 0, 0, 0, 0) }
}
SYS_EXIT_GROUP => {
let me = unsafe { host(SYS_GETPID, 0, 0, 0, 0, 0, 0) };
notify_vfork_release();
delegate(CTL_REAP, me as u64, 0, 0, 0, 0, 0);
unsafe { host(SYS_EXIT_GROUP, a, 0, 0, 0, 0, 0) }
}
SYS_RT_SIGACTION => {
if a as i32 == 31 || a as i32 == SIG_DETHREAD {
0
} else if b != 0 {
unsafe {
let src = b as *const u64;
let flags = src.add(1).read() & 0xffff_ffff;
let k = [
src.read(),
(flags | 0x0400_0000) & !0x0800_0000,
sig_restorer as u64,
sanitize_guest_sigmask(src.add(3).read()),
];
host(SYS_RT_SIGACTION, a, k.as_ptr() as u64, c, d, 0, 0)
}
} else {
unsafe { host(SYS_RT_SIGACTION, a, 0, c, d, 0, 0) }
}
}
SYS_RT_SIGPROCMASK => unsafe {
let r = host(SYS_RT_SIGPROCMASK, a, b, c, d, 0, 0);
let mut s: u64 = sentry_reserved_signal_mask();
host(
SYS_RT_SIGPROCMASK,
1,
std::ptr::addr_of_mut!(s) as u64,
0,
8,
0,
0,
);
r
},
SYS_SETITIMER | SYS_GETITIMER | SYS_ALARM => unsafe { host(nr, a, b, c, d, e, f) },
SYS_CLOCK_GETTIME
if a as i32 == libc::CLOCK_MONOTONIC || a as i32 == libc::CLOCK_BOOTTIME =>
{
if b == 0 {
-14
} else {
let ts = guest_monotonic_timespec();
unsafe { std::ptr::copy_nonoverlapping(ts.as_ptr(), b as *mut u8, ts.len()) };
0
}
}
SYS_CLOCK_GETTIME => unsafe { host(SYS_CLOCK_GETTIME, a, b, 0, 0, 0, 0) },
SYS_GETRANDOM => unsafe { host(SYS_GETRANDOM, a, b, c, 0, 0, 0) },
SYS_MEMBARRIER => unsafe { host(SYS_MEMBARRIER, a, b, c, d, e, f) },
SYS_PPOLL => unsafe { delegate_with_sigmask(nr, [a, b, c, d, e, f], d, e) },
SYS_EPOLL_PWAIT => unsafe { delegate_with_sigmask(nr, [a, b, c, d, e, f], e, f) },
SYS_PSELECT6 => unsafe {
let (sigmask_ptr, sigset_size) = if f == 0 {
(0, 0)
} else {
(
std::ptr::read_unaligned(f as *const u64),
std::ptr::read_unaligned((f + 8) as *const u64),
)
};
delegate_with_sigmask(nr, [a, b, c, d, e, f], sigmask_ptr, sigset_size)
},
_ => delegate_masked(nr, a, b, c, d, e, f, trap_saved_mask),
}
}
unsafe fn delegate_with_sigmask(
nr: i64,
args: [u64; 6],
sigmask_ptr: u64,
sigset_size: u64,
) -> i64 {
unsafe {
const SIG_SETMASK: u64 = 2;
if sigmask_ptr == 0 || sigset_size != 8 {
return delegate(nr, args[0], args[1], args[2], args[3], args[4], args[5]);
}
let set = sanitize_guest_sigmask(std::ptr::read_unaligned(sigmask_ptr as *const u64));
let mut old = 0u64;
let r = host(
SYS_RT_SIGPROCMASK,
SIG_SETMASK,
std::ptr::addr_of!(set) as u64,
std::ptr::addr_of_mut!(old) as u64,
8,
0,
0,
);
if r < 0 {
return r;
}
let ret = delegate(nr, args[0], args[1], args[2], args[3], args[4], args[5]);
host(
SYS_RT_SIGPROCMASK,
SIG_SETMASK,
std::ptr::addr_of!(old) as u64,
0,
8,
0,
0,
);
let mut sigsys = sentry_reserved_signal_mask();
host(
SYS_RT_SIGPROCMASK,
1,
std::ptr::addr_of_mut!(sigsys) as u64,
0,
8,
0,
0,
);
ret
}
}
unsafe fn rt_sigprocmask_in_ucontext(
uc: *mut libc::ucontext_t,
how: u64,
set_ptr: u64,
old_ptr: u64,
sigset_size: u64,
) -> i64 {
unsafe {
const SIG_BLOCK: u64 = 0;
const SIG_UNBLOCK: u64 = 1;
const SIG_SETMASK: u64 = 2;
if sigset_size != 8 {
return -22; }
let saved = std::ptr::addr_of_mut!((*uc).uc_sigmask) as *mut u64;
let current = std::ptr::read_unaligned(saved);
if old_ptr != 0 {
std::ptr::write_unaligned(old_ptr as *mut u64, current);
}
if set_ptr == 0 {
return 0;
}
let set = std::ptr::read_unaligned(set_ptr as *const u64);
let mut next = match how {
SIG_BLOCK => current | set,
SIG_UNBLOCK => current & !set,
SIG_SETMASK => set,
_ => return -22, };
next = sanitize_guest_sigmask(next); std::ptr::write_unaligned(saved, next);
0
}
}
#[no_mangle]
extern "C" fn sentry_dispatch_simple(regs: *const u64) -> i64 {
let r = |i: usize| unsafe { *regs.add(i) };
dispatch_simple(r(0) as i64, r(1), r(2), r(3), r(4), r(5), r(6), None)
}
extern "C" fn cell_layer1(_sig: c_int, _info: *mut libc::siginfo_t, ctx: *mut c_void) {
let uc = ctx as *mut libc::ucontext_t;
let g = unsafe { &mut (*uc).uc_mcontext.gregs };
let nr = g[REG_RAX];
let regdiff_snap: Option<[libc::greg_t; 17]> =
if std::hint::black_box(REGDIFF.load(std::sync::atomic::Ordering::Relaxed)) {
let mut s = [0 as libc::greg_t; 17];
let mut i = 0;
while i < 17 {
s[i] = unsafe { std::ptr::read_volatile(&g[i]) };
i += 1;
}
Some(s)
} else {
None
};
if unsafe { SYSCALLTRACE || SLOTDUMP } {
let s = current_slot() as usize;
if s < MAX_SLOTS {
unsafe { LAST_TRAP_RIP[s] = g[REG_RIP] as u64 };
let rsp = g[REG_RSP] as u64;
let mut caller = 0u64;
if rsp != 0 && rsp < USER_TOP {
for i in 0..32usize {
let w =
unsafe { std::ptr::read_volatile((rsp + (i as u64) * 8) as *const u64) };
if w >= EXE_DYN_BASE && w < 0x3f00_0000 {
caller = w;
break;
}
}
}
unsafe { LAST_TRAP_CALLER[s] = caller };
}
}
let (a, b, c, d, e, f) = (
g[REG_RDI] as u64,
g[REG_RSI] as u64,
g[REG_RDX] as u64,
g[REG_R10] as u64,
g[REG_R8] as u64,
g[REG_R9] as u64,
);
if nr == SYS_RT_SIGPROCMASK {
let ret = unsafe { rt_sigprocmask_in_ucontext(uc, a, b, c, d) };
g[REG_RAX] = ret;
return;
}
if nr == SENTINEL && WARM_MODE.load(Ordering::Relaxed) {
unsafe {
host(
SYS_ARCH_PRCTL,
ARCH_GET_FS,
std::ptr::addr_of_mut!(GUEST_FS) as u64,
0,
0,
0,
0,
);
host(SYS_ARCH_PRCTL, ARCH_SET_FS, CELL_FS, 0, 0, 0, 0);
for i in 0..18 {
CAPTURED[i] = g[i];
}
siglongjmp(std::ptr::addr_of_mut!(JMPBUF) as *mut c_void, 1);
}
}
let ret: i64 = match nr {
SYS_CLONE3 => unsafe {
if a == 0 || b < 64 {
-22 } else {
let p = a as *const u64;
let flags = std::ptr::read_unaligned(p.add(0));
let pidfd = std::ptr::read_unaligned(p.add(1));
let child_tid = std::ptr::read_unaligned(p.add(2));
let parent_tid = std::ptr::read_unaligned(p.add(3));
let exit_signal = std::ptr::read_unaligned(p.add(4));
let stack_base = std::ptr::read_unaligned(p.add(5));
let stack_size = std::ptr::read_unaligned(p.add(6));
let tls = std::ptr::read_unaligned(p.add(7));
let old_clone_stack = if stack_base != 0 && stack_size != 0 {
stack_base.wrapping_add(stack_size)
} else {
stack_base
};
let old_clone_parent_tid = if (flags & CLONE_PIDFD) != 0 {
pidfd
} else {
parent_tid
};
handle_clone(
g,
flags | (exit_signal & 0xff),
old_clone_stack,
old_clone_parent_tid,
child_tid,
tls,
)
}
},
SYS_CLONE => unsafe { handle_clone(g, a, b, c, d, e) },
SYS_EXECVE => {
emulate_execve(g, a, b, c);
return; }
SYS_MUNMAP
if b != 0 && {
let gsp = g[REG_RSP] as u64;
gsp >= a && gsp < a.wrapping_add(b)
} =>
{
let slot = current_slot();
notify_vfork_release();
if slot != 0 {
free_slot(slot as u32);
}
unsafe { unmap_and_exit(a, b) }
}
_ => {
let saved_mask = unsafe {
std::ptr::read_unaligned(std::ptr::addr_of!((*uc).uc_sigmask) as *const u64)
};
unsafe { store_slot_snapshot_regs(current_slot(), g, EINTR) };
dispatch_simple(nr, a, b, c, d, e, f, Some(saved_mask))
}
};
unsafe { update_slot_snapshot_rax(current_slot(), ret) };
maybe_patch_site(g[REG_RIP] as u64, nr);
g[REG_RAX] = ret;
if let Some(snap) = regdiff_snap {
let mut changed = 0u32;
for i in 0..17usize {
let cur = unsafe { std::ptr::read_volatile(&g[i]) };
if i != REG_RAX && cur != snap[i] {
changed |= 1u32 << i;
}
}
if std::hint::black_box(changed) != 0 {
use std::sync::atomic::{AtomicU32, Ordering};
static ANOM: AtomicU32 = AtomicU32::new(0);
if ANOM.fetch_add(1, Ordering::Relaxed) < 48 {
let mut buf = [0u8; 512];
let mut p = 0usize;
macro_rules! puts {
($s:expr) => {{
let s: &[u8] = $s;
for &b in s {
if p < buf.len() {
buf[p] = b;
p += 1;
}
}
}};
}
macro_rules! puthex {
($v:expr) => {{
let v = $v as u64;
puts!(b"0x");
for sh in (0..16u32).rev() {
let nib = ((v >> (sh * 4)) & 0xf) as usize;
if p < buf.len() {
buf[p] = b"0123456789abcdef"[nib];
p += 1;
}
}
}};
}
puts!(b"REGDIFF nr=");
puthex!(nr);
puts!(b" rip=");
puthex!(g[REG_RIP]);
puts!(b" changed:");
for i in 0..17usize {
if (changed & (1u32 << i)) != 0 {
puts!(b" [");
let mut nb = [0u8; 4];
let mut k = 4;
let mut x = i;
loop {
k -= 1;
nb[k] = b'0' + (x % 10) as u8;
x /= 10;
if x == 0 {
break;
}
}
puts!(&nb[k..]);
puts!(b"]");
puthex!(snap[i]);
puts!(b"->");
puthex!(g[i]);
}
}
puts!(b"\n");
delegate(CTL_ENVDBG, buf.as_ptr() as u64, p as u64, 0, 0, 0, 0);
}
}
}
}
unsafe fn read_cstr(p: u64) -> Vec<u8> {
unsafe {
let mut v = Vec::new();
if p == 0 {
return v;
}
let mut q = p as *const u8;
loop {
let b = *q;
if b == 0 || v.len() > 65536 {
break;
}
v.push(b);
q = q.add(1);
}
v
}
}
unsafe fn read_cstr_array(p: u64) -> Vec<Vec<u8>> {
unsafe {
let mut out = Vec::new();
if p == 0 {
return out;
}
let mut q = p as *const u64;
loop {
let ptr = *q;
if ptr == 0 || out.len() > 4096 {
break;
}
out.push(read_cstr(ptr));
q = q.add(1);
}
out
}
}
fn delegated_read(guest_path: &[u8]) -> Option<Vec<u8>> {
let mut cpath = guest_path.to_vec();
cpath.push(0);
let fd = delegate(
SYS_OPENAT,
(-100i64) as u64,
cpath.as_ptr() as u64,
0,
0,
0,
0,
);
if fd < 0 {
return None;
}
let mut st = [0u8; 144];
let r = delegate(SYS_FSTAT, fd as u64, st.as_mut_ptr() as u64, 0, 0, 0, 0);
let size = if r == 0 {
u64::from_le_bytes(st[48..56].try_into().unwrap()) as usize } else {
0
};
let mut buf = vec![0u8; size];
let mut off = 0usize;
while off < buf.len() {
let want = (buf.len() - off).min(4 * 1024 * 1024);
let n = delegate(
SYS_PREAD64,
fd as u64,
buf[off..].as_mut_ptr() as u64,
want as u64,
off as u64,
0,
0,
);
if n <= 0 {
break;
}
off += n as usize;
}
buf.truncate(off);
delegate(SYS_CLOSE, fd as u64, 0, 0, 0, 0, 0);
if off == 0 {
None
} else {
Some(buf)
}
}
fn passwd_entry(spec: &str) -> Option<(u32, u32, String)> {
let wanted_uid = spec.parse::<u32>().ok();
let bytes = delegated_read(b"/etc/passwd")?;
for line in String::from_utf8_lossy(&bytes).lines() {
let fields: Vec<&str> = line.split(':').collect();
if fields.len() < 7 {
continue;
}
let uid = fields[2].parse::<u32>().ok()?;
let gid = fields[3].parse::<u32>().ok()?;
if fields[0] == spec || wanted_uid == Some(uid) {
return Some((uid, gid, fields[5].to_owned()));
}
}
None
}
fn group_id(spec: &str) -> Option<u32> {
if let Ok(gid) = spec.parse::<u32>() {
return Some(gid);
}
let bytes = delegated_read(b"/etc/group")?;
String::from_utf8_lossy(&bytes).lines().find_map(|line| {
let mut fields = line.split(':');
let name = fields.next()?;
fields.next()?;
let gid = fields.next()?.parse().ok()?;
(name == spec).then_some(gid)
})
}
fn execvp_path(command: &str, envp: &[Vec<u8>]) -> String {
if command.contains('/') {
return command.to_owned();
}
let path = envp
.iter()
.find_map(|v| v.strip_prefix(b"PATH="))
.map(String::from_utf8_lossy)
.unwrap_or_else(|| "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin".into());
for dir in path.split(':') {
let candidate = format!("{dir}/{command}");
if delegated_read(candidate.as_bytes()).is_some() {
return candidate;
}
}
command.to_owned()
}
fn emulate_execve(g: &mut [libc::greg_t], path_ptr: u64, argv_ptr: u64, envp_ptr: u64) {
let mut old_guest_fs = 0u64;
unsafe {
host(
SYS_ARCH_PRCTL,
ARCH_GET_FS,
std::ptr::addr_of_mut!(old_guest_fs) as u64,
0,
0,
0,
0,
);
host(SYS_ARCH_PRCTL, ARCH_SET_FS, CELL_FS, 0, 0, 0, 0);
}
macro_rules! execve_fail {
($errno:expr) => {{
unsafe { host(SYS_ARCH_PRCTL, ARCH_SET_FS, old_guest_fs, 0, 0, 0, 0) };
g[REG_RAX] = $errno;
return;
}};
}
let (path, argv, mut envp) = unsafe {
(
read_cstr(path_ptr),
read_cstr_array(argv_ptr),
read_cstr_array(envp_ptr),
)
};
if ENVDBG.load(Ordering::Relaxed) {
let mut dbidx: i64 = -1;
let mut dblen: i64 = 0;
for (i, e) in envp.iter().enumerate() {
if e.starts_with(b"DATABASE_URL=") {
dbidx = i as i64;
dblen = e.len().saturating_sub(b"DATABASE_URL=".len()) as i64;
break;
}
}
if dbidx >= 0 {
let a0 = argv.first().map(|a| a.as_slice()).unwrap_or(b"?");
let mut m: Vec<u8> = Vec::with_capacity(220);
m.extend_from_slice(b"DBPOS a0=");
m.extend_from_slice(&a0[..a0.len().min(40)]);
m.extend_from_slice(b" idx=");
let mut nb = [0u8; 24];
let i = fmt_i64(dbidx, &mut nb);
m.extend_from_slice(&nb[i..]);
m.extend_from_slice(b" len=");
let mut nb = [0u8; 24];
let i = fmt_i64(dblen, &mut nb);
m.extend_from_slice(&nb[i..]);
m.extend_from_slice(b" total=");
let mut nb = [0u8; 24];
let i = fmt_i64(envp.len() as i64, &mut nb);
m.extend_from_slice(&nb[i..]);
m.push(b'\n');
delegate(CTL_ENVDBG, m.as_ptr() as u64, m.len() as u64, 0, 0, 0, 0);
if dblen < 70 {
{
use std::sync::atomic::{AtomicBool, Ordering};
static DUMPED: AtomicBool = AtomicBool::new(false);
if false && !DUMPED.swap(true, Ordering::Relaxed) {
delegate(CTL_COREDUMP, 0, 0, 0, 0, 0, 0);
}
}
unsafe {
let entry_ptr = *((envp_ptr as *const u64).add(dbidx as usize));
if entry_ptr != 0 {
let raw = std::slice::from_raw_parts(entry_ptr as *const u8, 90);
let mut hx: Vec<u8> = Vec::with_capacity(300);
const H: &[u8; 16] = b"0123456789abcdef";
hx.extend_from_slice(b"DBRAW ptr=");
for s in (0..16u32).rev() {
hx.push(H[((entry_ptr >> (s * 4)) & 0xf) as usize]);
}
hx.extend_from_slice(b" hbase=");
for s in (0..16u32).rev() {
hx.push(H[((HEAP_BASE >> (s * 4)) & 0xf) as usize]);
}
hx.extend_from_slice(b" hend=");
for s in (0..16u32).rev() {
hx.push(H[((HEAP_END >> (s * 4)) & 0xf) as usize]);
}
hx.extend_from_slice(b" bytes=");
for &b in raw {
hx.push(H[(b >> 4) as usize]);
hx.push(H[(b & 0xf) as usize]);
}
hx.push(b'\n');
delegate(CTL_ENVDBG, hx.as_ptr() as u64, hx.len() as u64, 0, 0, 0, 0);
}
}
}
}
}
if ENVDBG.load(Ordering::Relaxed) {
let dbs: Vec<&Vec<u8>> = envp
.iter()
.filter(|v| v.starts_with(b"DATABASE_URL="))
.collect();
if !dbs.is_empty() {
let a0 = argv.first().map(|a| a.as_slice()).unwrap_or(b"?");
let mut msg: Vec<u8> = Vec::with_capacity(400);
msg.extend_from_slice(b"ENVDBG2 a0=");
msg.extend_from_slice(&a0[..a0.len().min(40)]);
msg.extend_from_slice(b" count=");
let mut nb = [0u8; 24];
let i = fmt_i64(dbs.len() as i64, &mut nb);
msg.extend_from_slice(&nb[i..]);
for du in &dbs {
let val = &du[b"DATABASE_URL=".len()..];
msg.extend_from_slice(b" len=");
let mut nb = [0u8; 24];
let i = fmt_i64(val.len() as i64, &mut nb);
msg.extend_from_slice(&nb[i..]);
}
msg.push(b'\n');
delegate(
CTL_ENVDBG,
msg.as_ptr() as u64,
msg.len() as u64,
0,
0,
0,
0,
);
let truncated = dbs
.iter()
.any(|d| d.len().saturating_sub(b"DATABASE_URL=".len()) < 70);
if truncated {
let mut m: Vec<u8> = Vec::with_capacity(7800);
m.extend_from_slice(b"ENVFULL total=");
let mut nb = [0u8; 24];
let i = fmt_i64(envp.len() as i64, &mut nb);
m.extend_from_slice(&nb[i..]);
for (idx, e) in envp.iter().enumerate() {
let eq = e.iter().position(|&c| c == b'=').unwrap_or(e.len());
m.push(b' ');
let mut nb = [0u8; 24];
let i = fmt_i64(idx as i64, &mut nb);
m.extend_from_slice(&nb[i..]);
m.push(b':');
m.extend_from_slice(&e[..eq.min(22)]);
m.push(b'=');
let vlen = e.len().saturating_sub(eq + 1);
let mut nb = [0u8; 24];
let i = fmt_i64(vlen as i64, &mut nb);
m.extend_from_slice(&nb[i..]);
if m.len() > 7600 {
break;
}
}
m.push(b'\n');
delegate(CTL_ENVDBG, m.as_ptr() as u64, m.len() as u64, 0, 0, 0, 0);
{
use std::sync::atomic::{AtomicBool, Ordering};
static RAW_DONE: AtomicBool = AtomicBool::new(false);
if !RAW_DONE.swap(true, Ordering::Relaxed) {
let mut r: Vec<u8> = Vec::with_capacity(8192);
r.extend_from_slice(b"ENVRAW ");
for e in envp.iter() {
if r.len() + e.len() + 1 > 8000 {
break;
}
r.extend_from_slice(e);
r.push(0x01);
}
r.push(b'\n');
delegate(CTL_ENVDBG, r.as_ptr() as u64, r.len() as u64, 0, 0, 0, 0);
}
}
}
}
}
let mut path_s = String::from_utf8_lossy(&path).into_owned();
let mut argv_s: Vec<String> = argv
.iter()
.map(|a| String::from_utf8_lossy(a).into_owned())
.collect();
if matches!(path_s.rsplit('/').next(), Some("su-exec" | "gosu")) && argv_s.len() >= 3 {
let mut parts = argv_s[1].splitn(2, ':');
let user = parts.next().unwrap_or_default();
let explicit_group = parts.next();
let entry = passwd_entry(user);
let uid = user
.parse::<u32>()
.ok()
.or_else(|| entry.as_ref().map(|e| e.0));
let gid = explicit_group
.and_then(group_id)
.or_else(|| entry.as_ref().map(|e| e.1));
let (Some(uid), Some(gid)) = (uid, gid) else {
execve_fail!(-2); };
if cred_setgid(gid) < 0 || cred_setuid(uid) < 0 {
execve_fail!(-1); }
if let Some((_, _, home)) = entry {
envp.retain(|v| !v.starts_with(b"HOME="));
envp.push(format!("HOME={home}").into_bytes());
}
argv_s = argv_s.split_off(2);
path_s = execvp_path(&argv_s[0], &envp);
}
let effective = if path_s == "/proc/self/exe" {
String::from_utf8_lossy(&guest_exe().lock().unwrap()).into_owned()
} else {
path_s.clone()
};
let log_proc_self_exe_exec_fail = |stage: &[u8], errno: i64| {
if path_s != "/proc/self/exe" {
return;
}
let mut line = Vec::new();
line.extend_from_slice(b"errno=");
line.extend_from_slice(errno.to_string().as_bytes());
line.extend_from_slice(b" stage=");
line.extend_from_slice(stage);
line.extend_from_slice(b" effective=");
line.extend_from_slice(effective.as_bytes());
line.extend_from_slice(b" argv0=");
if let Some(a0) = argv_s.first() {
line.extend_from_slice(a0.as_bytes());
}
delegate(
CTL_PROCEXE_EXEC_FAIL,
line.as_ptr() as u64,
line.len() as u64,
0,
0,
0,
0,
);
};
let (path, bytes, argv) =
match resolve_shebang(&effective, &argv_s, |p| delegated_read(p.as_bytes())) {
Some((p, b, a)) => (
p.into_bytes(),
b,
a.into_iter()
.map(String::into_bytes)
.collect::<Vec<Vec<u8>>>(),
),
None => {
log_proc_self_exe_exec_fail(b"resolve", -2);
execve_fail!(-2); }
};
if bytes.len() < 64 || &bytes[0..4] != b"\x7fELF" || bytes[4] != 2 {
log_proc_self_exe_exec_fail(b"elf", -8);
execve_fail!(-8); }
*guest_exe().lock().unwrap() = path.clone();
delegate(
CTL_SET_EXE,
path.as_ptr() as u64,
path.len() as u64,
0,
0,
0,
0,
);
{
let mut joined: Vec<u8> = Vec::with_capacity(256);
for a in &argv {
joined.extend_from_slice(a);
joined.push(0);
if joined.len() >= 4096 {
break;
}
}
delegate(
CTL_SET_CMDLINE,
joined.as_ptr() as u64,
joined.len() as u64,
0,
0,
0,
0,
);
}
delegate(CTL_CLOSE_CLOEXEC, 0, 0, 0, 0, 0, 0);
notify_vfork_release();
delegate(CTL_SHM_FLUSH, 0, u64::MAX, 2, 0, 0, 0);
let de = delegate(
CTL_DETHREAD_FOR_EXEC,
current_slot(),
unsafe { host(SYS_GETTID, 0, 0, 0, 0, 0, 0) as u64 },
unsafe { host(SYS_GETPID, 0, 0, 0, 0, 0, 0) as u64 },
0,
0,
0,
);
if de < 0 {
execve_fail!(de);
}
MM_USERS.store(1, Ordering::Release);
unsafe {
reset_exec_signal_dispositions();
}
if MM_USERS.load(std::sync::atomic::Ordering::Acquire) <= 1 {
ARENA_CUR.store(ARENA_TOP, std::sync::atomic::Ordering::Release);
arena_free_clear();
}
let heap_reset = reset_brk_heap_for_exec();
if heap_reset < 0 {
execve_fail!(heap_reset);
}
let exe_base = if rd_u16(&bytes, 16) == 3 {
EXE_DYN_BASE
} else {
0
};
let exe = load_elf(&bytes, exe_base);
let (jump_to, interp_base) = if let Some(ref ip) = exe.interp {
let ib = match delegated_read(ip) {
Some(b) => b,
None => {
execve_fail!(-8);
}
};
let interp = load_elf(&ib, INTERP_BASE);
(interp.entry, Some(INTERP_BASE))
} else {
(exe.entry, None)
};
unregister_current_rseq_for_exec();
let argvv: Vec<Vec<u8>> = if argv.is_empty() {
vec![path.clone()]
} else {
argv
};
let envpv: Vec<Vec<u8>> = if envp.is_empty() {
vec![b"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin".to_vec()]
} else {
envp
};
reset_guest_sigaltstack(current_slot());
let sp = build_stack(&exe, interp_base, &argvv, &envpv);
g[REG_RIP] = jump_to as libc::greg_t;
g[REG_RSP] = sp as libc::greg_t;
g[REG_RDX] = 0;
g[REG_RBP] = 0;
}
fn iov(base: u64, len: usize) -> [u64; 2] {
[base, len as u64]
}
static SHARED_ZONE_LO: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
static SHARED_ZONE_HI: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
fn shared_zone() -> (u64, u64) {
(
SHARED_ZONE_LO.load(Ordering::Relaxed),
SHARED_ZONE_HI.load(Ordering::Relaxed),
)
}
fn vm_arg_ok(va: u64, len: u64) -> bool {
if va == 0 || va >= USER_TOP {
return false;
}
let end = match va.checked_add(len) {
Some(e) if e <= USER_TOP => e,
_ => return false, };
let (lo, hi) = shared_zone();
if hi > lo && va < hi && end > lo {
return false;
}
true
}
fn vm_self_pid_for_target_check() -> i32 {
let sr = SERVICER_RING.with(|cell| cell.get());
if sr.is_null() {
return unsafe { host(SYS_GETPID, 0, 0, 0, 0, 0, 0) } as i32;
}
let cached = SUPERVISOR_SELF_PID.load(Ordering::Relaxed);
if cached > 1 {
return cached;
}
let pid = unsafe { host(SYS_GETPID, 0, 0, 0, 0, 0, 0) } as i32;
if pid > 1 {
SUPERVISOR_SELF_PID.store(pid, Ordering::Relaxed);
}
pid
}
fn vm_target_ok(pid: i32) -> bool {
pid > 1 && pid != vm_self_pid_for_target_check()
}
fn vm_read(pid: i32, remote: u64, buf: &mut [u8]) -> i64 {
if !vm_target_ok(pid) {
return -14; }
if !vm_arg_ok(remote, buf.len() as u64) {
return -14; }
let total = buf.len();
let base = buf.as_mut_ptr() as u64;
let mut done = 0usize;
while done < total {
let l = iov(base + done as u64, total - done);
let r = iov(remote + done as u64, total - done);
let n = unsafe {
host(
SYS_PROCESS_VM_READV,
pid as u64,
l.as_ptr() as u64,
1,
r.as_ptr() as u64,
1,
0,
)
};
if n <= 0 {
return if done == 0 { n } else { done as i64 };
}
done += n as usize;
}
done as i64
}
fn vm_write(pid: i32, remote: u64, buf: &[u8]) -> i64 {
if !vm_target_ok(pid) {
return -14; }
{
let sr = SERVICER_RING.with(|c| c.get());
if !sr.is_null() {
let cur = unsafe { std::ptr::read_volatile(std::ptr::addr_of!((*sr).pid)) };
if cur != pid {
return -14; }
}
}
if !vm_arg_ok(remote, buf.len() as u64) {
return -14; }
if ENVDBG.load(Ordering::Relaxed)
&& buf.len() <= 64
&& remote < 0x7fff_b800_0000
&& remote + buf.len() as u64 > 0x7fff_b000_0000
{
let nr = {
let sr = SERVICER_RING.with(|c| c.get());
if sr.is_null() {
-1i64
} else {
unsafe { std::ptr::read_volatile(std::ptr::addr_of!((*sr).nr)) }
}
};
let zeros = buf.iter().filter(|&&b| b == 0).count();
ipc_logf_raw(
&[
b"VMWRITE_ENVRGN pid=".as_slice(),
pid.to_string().as_bytes(),
b" remote=0x".as_slice(),
format!("{remote:x}").as_bytes(),
b" len=".as_slice(),
buf.len().to_string().as_bytes(),
b" zeros=".as_slice(),
zeros.to_string().as_bytes(),
b" nr=".as_slice(),
nr.to_string().as_bytes(),
]
.concat(),
);
}
if ENVDBG.load(Ordering::Relaxed)
&& buf.len() <= 32
&& remote >= 0x41_00_0000
&& remote < 0x20_0000_0000
{
let nr = {
let sr = SERVICER_RING.with(|c| c.get());
if sr.is_null() {
-1i64
} else {
unsafe { std::ptr::read_volatile(std::ptr::addr_of!((*sr).nr)) }
}
};
let has3b = buf.contains(&0x3b);
let mut m: Vec<u8> = Vec::with_capacity(96);
m.extend_from_slice(b"VMWRITE_ARENA pid=");
m.extend_from_slice(pid.to_string().as_bytes());
m.extend_from_slice(b" remote=0x");
m.extend_from_slice(format!("{remote:x}").as_bytes());
m.extend_from_slice(b" len=");
m.extend_from_slice(buf.len().to_string().as_bytes());
m.extend_from_slice(b" nr=");
m.extend_from_slice(nr.to_string().as_bytes());
m.extend_from_slice(if has3b {
b" has3b=1 b="
} else {
b" has3b=0 b="
});
const HX: &[u8; 16] = b"0123456789abcdef";
for &b in buf.iter().take(16) {
m.push(HX[(b >> 4) as usize]);
m.push(HX[(b & 0xf) as usize]);
}
ipc_logf_raw(&m);
}
let total = buf.len();
let base = buf.as_ptr() as u64;
let mut done = 0usize;
while done < total {
let l = iov(base + done as u64, total - done);
let r = iov(remote + done as u64, total - done);
let n = unsafe {
host(
SYS_PROCESS_VM_WRITEV,
pid as u64,
l.as_ptr() as u64,
1,
r.as_ptr() as u64,
1,
0,
)
};
if n <= 0 {
return if done == 0 { n } else { done as i64 };
}
done += n as usize;
}
done as i64
}
fn pull_path(pid: i32, remote: u64) -> Vec<u8> {
let mut out = Vec::new();
let mut off = 0u64;
loop {
let mut chunk = [0u8; 256];
let n = vm_read(pid, remote + off, &mut chunk);
if n <= 0 {
break;
}
for &byte in &chunk[..n as usize] {
if byte == 0 {
out.push(0);
return out;
}
out.push(byte);
}
off += n as u64;
if out.len() > 4096 {
break;
}
}
out.push(0);
out
}
const CAP: u64 = 64 * 1024 * 1024;
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
#[allow(dead_code)] enum FdVal {
Host(i32),
Loop(netstack::SockId),
}
fn fdt() -> &'static Mutex<HashMap<i32, BTreeMap<i32, FdVal>>> {
static T: OnceLock<Mutex<HashMap<i32, BTreeMap<i32, FdVal>>>> = OnceLock::new();
T.get_or_init(|| Mutex::new(HashMap::new()))
}
fn fd_desc_flags() -> &'static Mutex<HashMap<(i32, i32), i32>> {
static T: OnceLock<Mutex<HashMap<(i32, i32), i32>>> = OnceLock::new();
T.get_or_init(|| Mutex::new(HashMap::new()))
}
fn guest_nofile_limits() -> &'static Mutex<HashMap<i32, (u64, u64)>> {
static T: OnceLock<Mutex<HashMap<i32, (u64, u64)>>> = OnceLock::new();
T.get_or_init(|| Mutex::new(HashMap::new()))
}
fn pending_nofile_limits() -> &'static Mutex<HashMap<u64, (u64, u64)>> {
static T: OnceLock<Mutex<HashMap<u64, (u64, u64)>>> = OnceLock::new();
T.get_or_init(|| Mutex::new(HashMap::new()))
}
fn guest_nofile_limit(pid: i32) -> (u64, u64) {
guest_nofile_limits()
.lock()
.unwrap()
.get(&pid)
.copied()
.unwrap_or_else(|| {
(
GUEST_NOFILE_DEFAULT_CUR.load(Ordering::Relaxed),
GUEST_NOFILE_DEFAULT_MAX.load(Ordering::Relaxed),
)
})
}
fn set_guest_nofile_limit(pid: i32, cur: u64, max: u64) {
guest_nofile_limits()
.lock()
.unwrap()
.insert(pid, (cur, max));
}
fn nice_values() -> &'static Mutex<HashMap<i32, i32>> {
static T: OnceLock<Mutex<HashMap<i32, i32>>> = OnceLock::new();
T.get_or_init(|| Mutex::new(HashMap::new()))
}
fn host_fd_flags(hfd: i32) -> i32 {
let flags = unsafe { host(SYS_FCNTL, hfd as u64, F_GETFD, 0, 0, 0, 0) };
if flags >= 0 {
flags as i32
} else {
0
}
}
fn fd_default_desc_flags(v: FdVal) -> i32 {
match v {
FdVal::Host(h) => host_fd_flags(h),
FdVal::Loop(_) => 0,
}
}
fn fd_get_desc_flags(pid: i32, g: i32, v: FdVal) -> i32 {
fd_desc_flags()
.lock()
.unwrap()
.get(&(pid, g))
.copied()
.unwrap_or_else(|| fd_default_desc_flags(v))
}
fn fd_set_desc_flags(pid: i32, g: i32, flags: i32) {
let v = fdt()
.lock()
.unwrap()
.get(&pid)
.and_then(|m| m.get(&g))
.copied();
if let Some(FdVal::Host(h)) = v {
unsafe {
host(SYS_FCNTL, h as u64, F_SETFD, flags as u64, 0, 0, 0);
}
}
fd_desc_flags().lock().unwrap().insert((pid, g), flags);
}
fn fd_guest_for_host(pid: i32, hfd: i32) -> Option<i32> {
fdt().lock().unwrap().get(&pid)?.iter().find_map(|(&g, v)| {
if *v == FdVal::Host(hfd) {
Some(g)
} else {
None
}
})
}
fn fd_host(pid: i32, gfd: i32) -> Option<i32> {
match fdt().lock().unwrap().get(&pid)?.get(&gfd)? {
FdVal::Host(h) => Some(*h),
FdVal::Loop(_) => None,
}
}
#[allow(dead_code)] fn fd_loop(pid: i32, gfd: i32) -> Option<netstack::SockId> {
match fdt().lock().unwrap().get(&pid)?.get(&gfd)? {
FdVal::Loop(s) => Some(*s),
FdVal::Host(_) => None,
}
}
fn fd_install(pid: i32, hfd: i32, min: i32) -> i64 {
fd_install_val(pid, FdVal::Host(hfd), min)
}
#[allow(dead_code)] fn fd_install_loop(pid: i32, sid: netstack::SockId, min: i32) -> i64 {
fd_install_val(pid, FdVal::Loop(sid), min)
}
fn fd_install_val(pid: i32, v: FdVal, min: i32) -> i64 {
let flags = fd_default_desc_flags(v);
fd_install_val_with_flags(pid, v, min, flags)
}
fn fd_install_val_with_flags(pid: i32, v: FdVal, min: i32, flags: i32) -> i64 {
let nofile_cur = guest_nofile_limit(pid).0;
let mut t = fdt().lock().unwrap();
let m = t.entry(pid).or_default();
let mut g = min.max(0);
while m.contains_key(&g) {
g += 1;
}
if (g as u64) >= nofile_cur {
drop(t);
fd_close_val(v);
return -24; }
m.insert(g, v);
drop(t);
fd_set_desc_flags(pid, g, flags);
g as i64
}
fn fd_close_val(v: FdVal) {
match v {
FdVal::Host(p) => unsafe {
fd_forget_host_side_tables(p);
host(SYS_CLOSE, p as u64, 0, 0, 0, 0, 0);
},
FdVal::Loop(s) => loop_close(s),
}
}
fn fd_forget_host_side_tables(h: i32) {
synth_fd_dirs().lock().unwrap().remove(&h);
synth_proc_dirs().lock().unwrap().remove(&h);
signalfd_close(h);
unix_alias_placeholders_remove(h);
if !host_v6_route() {
dns_dest_fds().lock().unwrap().remove(&h);
}
epoll_loops().lock().unwrap().remove(&h);
}
fn fd_install_val_at(pid: i32, g: i32, v: FdVal) {
let flags = fd_default_desc_flags(v);
fd_install_val_at_with_flags(pid, g, v, flags);
}
fn fd_install_val_at_with_flags(pid: i32, g: i32, v: FdVal, flags: i32) {
let prev = fdt().lock().unwrap().entry(pid).or_default().insert(g, v);
fd_set_desc_flags(pid, g, flags);
if let Some(prev) = prev {
fd_close_val(prev);
}
}
fn fd_install_at(pid: i32, g: i32, hfd: i32) {
fd_install_val_at(pid, g, FdVal::Host(hfd));
}
fn fd_detach_stdio_to_dev_null(pid: i32) {
let rd = unsafe {
host(
SYS_OPENAT,
(-100i64) as u64,
b"/dev/null\0".as_ptr() as u64,
libc::O_RDONLY as u64,
0,
0,
0,
)
};
if rd >= 0 {
fd_install_at(pid, 0, rd as i32);
}
for g in [1, 2] {
let wr = unsafe {
host(
SYS_OPENAT,
(-100i64) as u64,
b"/dev/null\0".as_ptr() as u64,
libc::O_WRONLY as u64,
0,
0,
0,
)
};
if wr >= 0 {
fd_install_at(pid, g, wr as i32);
}
}
}
fn fd_install_loop_dup(pid: i32, sid: netstack::SockId, min: i32) -> i64 {
loop_state().lock().unwrap().net.dup(sid);
fd_install_val(pid, FdVal::Loop(sid), min)
}
fn fd_install_loop_dup_at(pid: i32, sid: netstack::SockId, g: i32) {
loop_state().lock().unwrap().net.dup(sid);
fd_install_val_at(pid, g, FdVal::Loop(sid));
}
fn fd_remove(pid: i32, g: i32) -> Option<FdVal> {
fd_desc_flags().lock().unwrap().remove(&(pid, g));
fdt().lock().unwrap().get_mut(&pid)?.remove(&g)
}
fn signalfd_writers() -> &'static Mutex<HashMap<i32, i32>> {
static S: OnceLock<Mutex<HashMap<i32, i32>>> = OnceLock::new();
S.get_or_init(|| Mutex::new(HashMap::new()))
}
fn signalfd_close(read_h: i32) {
if let Some(write_h) = signalfd_writers().lock().unwrap().remove(&read_h) {
unsafe { host(SYS_CLOSE, write_h as u64, 0, 0, 0, 0, 0) };
}
}
fn signalfd_notify_sigchld(dead_pid: i32) {
let writers: Vec<i32> = signalfd_writers()
.lock()
.unwrap()
.values()
.copied()
.collect();
if writers.is_empty() {
return;
}
let mut rec = [0u8; 128];
rec[0..4].copy_from_slice(&(libc::SIGCHLD as u32).to_le_bytes());
rec[8..12].copy_from_slice(&1u32.to_le_bytes()); rec[12..16].copy_from_slice(&(dead_pid as u32).to_le_bytes());
for write_h in writers {
unsafe {
let _ = host(
SYS_WRITE,
write_h as u64,
rec.as_ptr() as u64,
rec.len() as u64,
0,
0,
0,
);
}
}
}
fn pending() -> &'static Mutex<HashMap<u64, BTreeMap<i32, FdVal>>> {
static T: OnceLock<Mutex<HashMap<u64, BTreeMap<i32, FdVal>>>> = OnceLock::new();
T.get_or_init(|| Mutex::new(HashMap::new()))
}
fn pending_fd_flags() -> &'static Mutex<HashMap<u64, BTreeMap<i32, i32>>> {
static T: OnceLock<Mutex<HashMap<u64, BTreeMap<i32, i32>>>> = OnceLock::new();
T.get_or_init(|| Mutex::new(HashMap::new()))
}
fn pending_creds() -> &'static Mutex<HashMap<u64, CellCredSnapshot>> {
static T: OnceLock<Mutex<HashMap<u64, CellCredSnapshot>>> = OnceLock::new();
T.get_or_init(|| Mutex::new(HashMap::new()))
}
fn pending_umasks() -> &'static Mutex<HashMap<u64, u32>> {
static T: OnceLock<Mutex<HashMap<u64, u32>>> = OnceLock::new();
T.get_or_init(|| Mutex::new(HashMap::new()))
}
fn fd_snapshot(parent: i32, slot: u64) {
shm_snapshot(parent, slot);
let src = fdt()
.lock()
.unwrap()
.get(&parent)
.cloned()
.unwrap_or_default();
let mut dst = BTreeMap::new();
let mut dst_flags = BTreeMap::new();
for (g, v) in src {
let desc_flags = fd_get_desc_flags(parent, g, v);
match v {
FdVal::Host(h) => {
let dup_cmd = if (desc_flags & FD_CLOEXEC as i32) != 0 {
F_DUPFD_CLOEXEC
} else {
libc::F_DUPFD as u64
};
let d = unsafe { host(SYS_FCNTL, h as u64, dup_cmd, 0, 0, 0, 0) };
if d >= 0 {
dst.insert(g, FdVal::Host(d as i32));
dst_flags.insert(g, desc_flags);
let mut el = epoll_loops().lock().unwrap();
if let Some(w) = el.get(&h).cloned() {
el.insert(d as i32, w);
}
}
}
FdVal::Loop(s) => {
loop_state().lock().unwrap().net.dup(s);
dst.insert(g, FdVal::Loop(s));
dst_flags.insert(g, desc_flags);
}
}
}
if unsafe { TRACE } {
logn(b" [fd_snapshot ", parent as i64, b"");
logn(b" slot=", slot as i64, b"");
logn(b" fds=", dst.len() as i64, b"]\n");
}
if let Some(old) = pending().lock().unwrap().insert(slot, dst) {
for (_, v) in old {
fd_close_val(v);
}
}
pending_fd_flags().lock().unwrap().insert(slot, dst_flags);
pending_creds()
.lock()
.unwrap()
.insert(slot, supervisor_cred_for(parent));
pending_umasks()
.lock()
.unwrap()
.insert(slot, supervisor_umask_for(parent));
pending_cwd().lock().unwrap().insert(slot, cwd_of(parent));
pending_exe().lock().unwrap().insert(slot, exe_for(parent));
pending_cmdline()
.lock()
.unwrap()
.insert(slot, cmdline_for(parent));
pending_nofile_limits()
.lock()
.unwrap()
.insert(slot, guest_nofile_limit(parent));
proctree::alloc_for_slot(slot, parent);
}
fn fd_adopt(slot: u64, child: i32) {
shm_adopt(slot, child);
if let Some(m) = pending().lock().unwrap().remove(&slot) {
let flags = pending_fd_flags()
.lock()
.unwrap()
.remove(&slot)
.unwrap_or_default();
if let Some(old) = fdt().lock().unwrap().insert(child, m) {
for (_, v) in old {
fd_close_val(v);
}
}
fd_desc_flags()
.lock()
.unwrap()
.retain(|(p, _), _| *p != child);
for (g, fl) in flags {
fd_set_desc_flags(child, g, fl);
}
}
if let Some(c) = pending_cwd().lock().unwrap().remove(&slot) {
cwds().lock().unwrap().insert(child, c);
}
if let Some(c) = pending_creds().lock().unwrap().remove(&slot) {
supervisor_creds().lock().unwrap().insert(child, c);
}
if let Some(mask) = pending_umasks().lock().unwrap().remove(&slot) {
supervisor_umasks()
.lock()
.unwrap()
.insert(child, mask & 0o777);
}
if let Some(e) = pending_exe().lock().unwrap().remove(&slot) {
if !e.is_empty() {
proc_exe().lock().unwrap().insert(child, e);
}
}
if let Some(c) = pending_cmdline().lock().unwrap().remove(&slot) {
if !c.is_empty() {
proc_cmdline().lock().unwrap().insert(child, c);
}
}
fdtrace_dump_table(child, b"ADOPT");
if let Some(lim) = pending_nofile_limits().lock().unwrap().remove(&slot) {
guest_nofile_limits().lock().unwrap().insert(child, lim);
}
proctree::bind_slot(slot, child);
}
fn fd_drop(pid: i32) {
if let Some(m) = fdt().lock().unwrap().remove(&pid) {
for (_, v) in m {
match v {
FdVal::Host(h) => {
let had_epoll_loop = epoll_loops().lock().unwrap().contains_key(&h);
fd_forget_host_side_tables(h);
if had_epoll_loop && epdbg_on() {
ipc_logf(&[(b"FDDROP pid=", pid as i64), (b" h=", h as i64)], b"");
}
unsafe { host(SYS_CLOSE, h as u64, 0, 0, 0, 0, 0) };
}
FdVal::Loop(s) => loop_close(s),
}
}
}
supervisor_umasks().lock().unwrap().remove(&pid);
fd_desc_flags()
.lock()
.unwrap()
.retain(|(p, _), _| *p != pid);
cwds().lock().unwrap().remove(&pid);
proc_exe().lock().unwrap().remove(&pid);
proc_cmdline().lock().unwrap().remove(&pid);
guest_nofile_limits().lock().unwrap().remove(&pid);
supervisor_creds().lock().unwrap().remove(&pid);
proctree::mark_dead(pid);
}
struct LoopState {
net: netstack::LoopNet,
efds: HashMap<netstack::SockId, i32>,
nonblock: std::collections::HashSet<netstack::SockId>,
}
fn loop_state() -> &'static Mutex<LoopState> {
static T: OnceLock<Mutex<LoopState>> = OnceLock::new();
T.get_or_init(|| {
Mutex::new(LoopState {
net: netstack::LoopNet::new(),
efds: HashMap::new(),
nonblock: std::collections::HashSet::new(),
})
})
}
fn servicer_cancel_efd() -> i32 {
let r = SERVICER_RING.with(|c| c.get());
if r.is_null() {
-1
} else {
unsafe { std::ptr::read_volatile(std::ptr::addr_of!((*r).cancel_efd)) }
}
}
fn servicer_intr_efd() -> i32 {
let r = SERVICER_RING.with(|c| c.get());
if r.is_null() {
-1
} else {
unsafe { std::ptr::read_volatile(std::ptr::addr_of!((*r).intr_efd)) }
}
}
fn loop_new_efd() -> i32 {
unsafe { host(SYS_EVENTFD2, 0, EFD_NONBLOCK_CLOEXEC, 0, 0, 0, 0) as i32 }
}
const LOOPRING_CAP: usize = 1 << 16; static LOOPRING_CUR: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
static LOOPRING: [[std::sync::atomic::AtomicU64; 2]; LOOPRING_CAP] = [const {
[
std::sync::atomic::AtomicU64::new(0),
std::sync::atomic::AtomicU64::new(0),
]
}; LOOPRING_CAP];
fn loopring_on() -> bool {
static O: std::sync::atomic::AtomicU8 = std::sync::atomic::AtomicU8::new(2);
let v = O.load(std::sync::atomic::Ordering::Relaxed);
if v != 2 {
return v == 1;
}
let on = std::env::var("SENTRY_LOOPRING").is_ok_and(|s| s == "1");
O.store(on as u8, std::sync::atomic::Ordering::Relaxed);
on
}
#[inline]
fn loopring_ev(op: u8, sid: netstack::SockId, aux: i64) {
use std::sync::atomic::Ordering;
if !loopring_on() {
return;
}
let i = LOOPRING_CUR.fetch_add(1, Ordering::Relaxed) & (LOOPRING_CAP - 1);
let word = ((sid as u64) << 32) | ((op as u64) << 24) | ((aux as u64) & 0xff_ffff);
LOOPRING[i][0].store(word, Ordering::Relaxed);
LOOPRING[i][1].store(now_ns() / 1_000, Ordering::Relaxed); }
fn loopring_dump() {
let total = LOOPRING_CUR.load(std::sync::atomic::Ordering::Relaxed);
if total == 0 {
return;
}
let path = format!("/tmp/sentry_loopring_{}.bin", std::process::id());
let mut buf: Vec<u8> = Vec::with_capacity(LOOPRING_CAP.min(total) * 16);
let n = total.min(LOOPRING_CAP);
for i in 0..n {
buf.extend_from_slice(
&LOOPRING[i][0]
.load(std::sync::atomic::Ordering::Relaxed)
.to_le_bytes(),
);
buf.extend_from_slice(
&LOOPRING[i][1]
.load(std::sync::atomic::Ordering::Relaxed)
.to_le_bytes(),
);
}
let _ = std::fs::write(path, &buf);
}
fn loop_wake(ls: &LoopState, sid: netstack::SockId) {
if let Some(&efd) = ls.efds.get(&sid) {
let one: u64 = 1;
unsafe {
host(
SYS_WRITE,
efd as u64,
std::ptr::addr_of!(one) as u64,
8,
0,
0,
0,
)
};
}
}
fn loop_close(sid: netstack::SockId) {
loopring_ev(6, sid, 0);
let torn_down = {
let mut ls = loop_state().lock().unwrap();
let peer = ls.net.peer_of(sid);
let torn = ls.net.close(sid).unwrap_or(true); if torn {
if let Some(efd) = ls.efds.remove(&sid) {
unsafe { host(SYS_CLOSE, efd as u64, 0, 0, 0, 0, 0) };
}
if let Some(p) = peer {
loop_wake(&ls, p);
}
}
torn
}; if torn_down {
epoll_drop_sid(sid);
}
}
fn ep_from_sockaddr(addr: &[u8]) -> Option<netstack::Endpoint> {
if addr.len() < 2 {
return None;
}
let fam = u16::from_le_bytes([addr[0], addr[1]]);
match fam {
2 => {
if addr.len() < 8 {
return None;
}
let port = u16::from_be_bytes([addr[2], addr[3]]);
let mut ip = [0u8; 16];
ip[..4].copy_from_slice(&addr[4..8]);
Some(netstack::Endpoint {
v6: false,
ip,
port,
})
}
10 => {
if addr.len() < 24 {
return None;
}
let port = u16::from_be_bytes([addr[2], addr[3]]);
let mut ip = [0u8; 16];
ip.copy_from_slice(&addr[8..24]);
Some(netstack::Endpoint { v6: true, ip, port })
}
_ => None,
}
}
fn ep_is_loopback(ep: &netstack::Endpoint) -> bool {
if ep.v6 {
ep.ip == [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1]
} else {
ep.ip[0] == 127
}
}
fn host_sock_is_stream(h: i32) -> bool {
let mut ty: i32 = 0;
let mut len: u32 = 4;
let r = unsafe {
host(
SYS_GETSOCKOPT,
h as u64,
libc::SOL_SOCKET as u64,
libc::SO_TYPE as u64,
std::ptr::addr_of_mut!(ty) as u64,
std::ptr::addr_of_mut!(len) as u64,
0,
)
};
r == 0 && ty == libc::SOCK_STREAM
}
fn write_sockaddr(pid: i32, addr_ptr: u64, len_ptr: u64, ep: &netstack::Endpoint) {
if addr_ptr == 0 {
return;
}
let mut cap = 128usize;
if len_ptr != 0 {
let mut lb = [0u8; 4];
vm_read(pid, len_ptr, &mut lb);
cap = (u32::from_le_bytes(lb) as usize).min(128);
}
let sa = if ep.v6 {
let mut s = vec![0u8; 28];
s[0..2].copy_from_slice(&10u16.to_le_bytes());
s[2..4].copy_from_slice(&ep.port.to_be_bytes());
s[8..24].copy_from_slice(&ep.ip);
s
} else {
let mut s = vec![0u8; 16];
s[0..2].copy_from_slice(&2u16.to_le_bytes());
s[2..4].copy_from_slice(&ep.port.to_be_bytes());
s[4..8].copy_from_slice(&ep.ip[..4]);
s
};
let n = sa.len().min(cap);
if n > 0 {
vm_write(pid, addr_ptr, &sa[..n]);
}
if len_ptr != 0 {
vm_write(pid, len_ptr, &(sa.len() as u32).to_le_bytes());
}
}
fn loop_socket_for(pid: i32, gfd: i32) -> netstack::SockId {
if let Some(sid) = fd_loop(pid, gfd) {
return sid;
}
let mut nonblock = false;
if let Some(h) = fd_host(pid, gfd) {
let fl = unsafe { host(SYS_FCNTL, h as u64, libc::F_GETFL as u64, 0, 0, 0, 0) };
nonblock = fl >= 0 && (fl as u64 & libc::O_NONBLOCK as u64) != 0;
unsafe { host(SYS_CLOSE, h as u64, 0, 0, 0, 0, 0) };
}
let sid = loop_state().lock().unwrap().net.socket();
let efd = loop_new_efd();
{
let mut ls = loop_state().lock().unwrap();
ls.efds.insert(sid, efd);
if nonblock {
ls.nonblock.insert(sid);
}
}
fdt()
.lock()
.unwrap()
.entry(pid)
.or_default()
.insert(gfd, FdVal::Loop(sid));
sid
}
fn loop_recv_bytes(sid: netstack::SockId, max: usize, nonblock: bool) -> (i64, Vec<u8>) {
let max = max.min(CAP as usize);
loop {
let res = loop_state().lock().unwrap().net.recv(sid, max);
match res {
Ok(data) => {
if !data.is_empty() {
let ls = loop_state().lock().unwrap();
if let Some(p) = ls.net.peer_of(sid) {
loop_wake(&ls, p);
}
}
loopring_ev(2, sid, data.len() as i64);
return (data.len() as i64, data);
}
Err(netstack::EAGAIN) => {
if nonblock {
return (-(netstack::EAGAIN as i64), Vec::new());
}
let efd = match loop_state().lock().unwrap().efds.get(&sid).copied() {
Some(e) => e,
None => return (-9, Vec::new()),
};
let w = unsafe {
wait_or_cancel(
efd,
servicer_cancel_efd(),
servicer_intr_efd(),
LOOP_WAIT_TICK_MS,
)
};
if w == -1 {
return (CANCEL_SENTINEL, Vec::new());
}
if w == -3 {
return (EINTR, Vec::new());
}
if w != 0 {
let mut sink = [0u8; 8];
unsafe { host(SYS_READ, efd as u64, sink.as_mut_ptr() as u64, 8, 0, 0, 0) };
}
}
Err(e) => return (-(e as i64), Vec::new()),
}
}
}
fn loop_recv(pid: i32, sid: netstack::SockId, buf_ptr: u64, len: u64, nonblock: bool) -> i64 {
let (st, data) = loop_recv_bytes(sid, len as usize, nonblock);
if st >= 0 && !data.is_empty() {
let w = vm_write(pid, buf_ptr, &data);
let w = if w < 0 {
0
} else {
(w as usize).min(data.len())
};
if w < data.len() {
loop_state().lock().unwrap().net.unrecv(sid, &data[w..]);
loopring_ev(7, sid, (data.len() - w) as i64);
return if w > 0 { w as i64 } else { -14 }; }
}
st
}
fn loop_send_bytes(sid: netstack::SockId, buf: &[u8]) -> i64 {
let n = buf.len().min(CAP as usize);
let mut ls = loop_state().lock().unwrap();
let r = match ls.net.send(sid, &buf[..n]) {
Ok(sent) => {
if let Some(p) = ls.net.peer_of(sid) {
loop_wake(&ls, p);
}
sent as i64
}
Err(e) => -(e as i64),
};
drop(ls);
loopring_ev(1, sid, r);
r
}
fn loop_send_all(sid: netstack::SockId, buf: &[u8], nonblock: bool) -> i64 {
if nonblock {
return loop_send_bytes(sid, buf);
}
let mut off = 0usize;
loop {
let r = loop_send_bytes(sid, &buf[off..]);
if r > 0 {
off += r as usize;
if off >= buf.len() {
return off as i64;
}
continue;
}
if r == -(netstack::EAGAIN as i64) {
let efd = match loop_state().lock().unwrap().efds.get(&sid).copied() {
Some(e) => e,
None => return if off > 0 { off as i64 } else { -9 },
};
let w = unsafe {
wait_or_cancel(
efd,
servicer_cancel_efd(),
servicer_intr_efd(),
LOOP_WAIT_TICK_MS,
)
};
if w == -1 {
return CANCEL_SENTINEL;
}
if w == -3 {
return if off > 0 { off as i64 } else { EINTR };
}
if w != 0 {
let mut sink = [0u8; 8];
unsafe { host(SYS_READ, efd as u64, sink.as_mut_ptr() as u64, 8, 0, 0, 0) };
}
continue;
}
return if off > 0 { off as i64 } else { r };
}
}
fn loop_send(pid: i32, sid: netstack::SockId, buf_ptr: u64, len: u64, nonblock: bool) -> i64 {
let n = (len as usize).min(CAP as usize);
let mut buf = vec![0u8; n];
let got = vm_read(pid, buf_ptr, &mut buf);
if got < 0 {
return -14; }
let got = (got as usize).min(n);
loop_send_all(sid, &buf[..got], nonblock)
}
fn loop_sendfile(
pid: i32,
out_sid: netstack::SockId,
in_gfd: i32,
off_ptr: u64,
count: u64,
) -> i64 {
let in_h = match fd_host(pid, in_gfd) {
Some(h) => h,
None => return -9, };
let n = (count as usize).min(CAP as usize);
if n == 0 {
return 0;
}
let mut buf = vec![0u8; n];
let got = if off_ptr != 0 {
let mut ob = [0u8; 8];
vm_read(pid, off_ptr, &mut ob);
let off = u64::from_le_bytes(ob);
unsafe {
host(
SYS_PREAD64,
in_h as u64,
buf.as_mut_ptr() as u64,
n as u64,
off,
0,
0,
)
}
} else {
unsafe {
host(
SYS_READ,
in_h as u64,
buf.as_mut_ptr() as u64,
n as u64,
0,
0,
0,
)
}
};
if got <= 0 {
return got; }
let sent = loop_send_bytes(out_sid, &buf[..got as usize]);
if sent > 0 && off_ptr != 0 {
let mut ob = [0u8; 8];
vm_read(pid, off_ptr, &mut ob);
let new_off = u64::from_le_bytes(ob).wrapping_add(sent as u64);
vm_write(pid, off_ptr, &new_off.to_le_bytes());
}
sent
}
fn iov_total(pid: i32, iov_ptr: u64, iovcnt: u64) -> usize {
let cnt = (iovcnt as usize).min(1024);
let mut total = 0usize;
for i in 0..cnt {
let mut iv = [0u8; 16];
vm_read(pid, iov_ptr + (i * 16) as u64, &mut iv);
total = total.saturating_add(u64::from_le_bytes(iv[8..16].try_into().unwrap()) as usize);
}
total
}
fn gather_iov(pid: i32, iov_ptr: u64, iovcnt: u64) -> Vec<u8> {
let cnt = (iovcnt as usize).min(1024);
let mut out = Vec::new();
for i in 0..cnt {
if out.len() >= CAP as usize {
break;
}
let mut iv = [0u8; 16];
vm_read(pid, iov_ptr + (i * 16) as u64, &mut iv);
let base = u64::from_le_bytes(iv[0..8].try_into().unwrap());
let ilen = u64::from_le_bytes(iv[8..16].try_into().unwrap()) as usize;
let take = ilen.min(CAP as usize - out.len());
if take == 0 {
continue;
}
let mut chunk = vec![0u8; take];
let got = vm_read(pid, base, &mut chunk);
if got <= 0 {
break;
}
let got = (got as usize).min(take);
out.extend_from_slice(&chunk[..got]);
if got < take {
break;
}
}
out
}
fn scatter_iov(pid: i32, iov_ptr: u64, iovcnt: u64, data: &[u8]) -> usize {
let cnt = (iovcnt as usize).min(1024);
let mut off = 0usize;
for i in 0..cnt {
if off >= data.len() {
break;
}
let mut iv = [0u8; 16];
vm_read(pid, iov_ptr + (i * 16) as u64, &mut iv);
let base = u64::from_le_bytes(iv[0..8].try_into().unwrap());
let ilen = u64::from_le_bytes(iv[8..16].try_into().unwrap()) as usize;
let take = ilen.min(data.len() - off);
if take == 0 {
continue;
}
let w = vm_write(pid, base, &data[off..off + take]);
let w = if w < 0 { 0 } else { (w as usize).min(take) };
off += w;
if w < take {
break;
}
}
off
}
fn loop_writev(pid: i32, sid: netstack::SockId, iov_ptr: u64, iovcnt: u64, nonblock: bool) -> i64 {
loop_send_all(sid, &gather_iov(pid, iov_ptr, iovcnt), nonblock)
}
fn loop_readv(pid: i32, sid: netstack::SockId, iov_ptr: u64, iovcnt: u64, nonblock: bool) -> i64 {
let (st, data) = loop_recv_bytes(sid, iov_total(pid, iov_ptr, iovcnt), nonblock);
if st < 0 {
return st;
}
let w = scatter_iov(pid, iov_ptr, iovcnt, &data);
if w < data.len() {
loop_state().lock().unwrap().net.unrecv(sid, &data[w..]);
if w == 0 && !data.is_empty() {
return -14; }
}
w as i64
}
fn loop_sendmsg(pid: i32, sid: netstack::SockId, msg_ptr: u64, nonblock: bool) -> i64 {
let mut mh = [0u8; 32];
vm_read(pid, msg_ptr, &mut mh);
let iov = u64::from_le_bytes(mh[16..24].try_into().unwrap());
let iovlen = u64::from_le_bytes(mh[24..32].try_into().unwrap());
loop_send_all(sid, &gather_iov(pid, iov, iovlen), nonblock)
}
fn loop_recvmsg(pid: i32, sid: netstack::SockId, msg_ptr: u64, nonblock: bool) -> i64 {
let mut mh = [0u8; 32];
vm_read(pid, msg_ptr, &mut mh);
let iov = u64::from_le_bytes(mh[16..24].try_into().unwrap());
let iovlen = u64::from_le_bytes(mh[24..32].try_into().unwrap());
let (st, data) = loop_recv_bytes(sid, iov_total(pid, iov, iovlen), nonblock);
if st < 0 {
return st;
}
scatter_iov(pid, iov, iovlen, &data) as i64
}
fn loop_accept(
pid: i32,
sid: netstack::SockId,
addr_ptr: u64,
len_ptr: u64,
nonblock: bool,
newsock_nonblock: bool,
) -> i64 {
loop {
let res = loop_state().lock().unwrap().net.accept(sid);
match res {
Ok(server) => {
let efd = loop_new_efd();
{
let mut ls = loop_state().lock().unwrap();
ls.efds.insert(server, efd);
if newsock_nonblock {
ls.nonblock.insert(server);
}
}
if addr_ptr != 0 {
if let Some(ep) = loop_state().lock().unwrap().net.peer_addr(server) {
write_sockaddr(pid, addr_ptr, len_ptr, &ep);
}
}
loopring_ev(4, sid, server as i64);
return fd_install_loop(pid, server, 0);
}
Err(netstack::EAGAIN) => {
if nonblock {
return -(netstack::EAGAIN as i64);
}
let efd = match loop_state().lock().unwrap().efds.get(&sid).copied() {
Some(e) => e,
None => return -9,
};
let w = unsafe {
wait_or_cancel(
efd,
servicer_cancel_efd(),
servicer_intr_efd(),
LOOP_WAIT_TICK_MS,
)
};
if w == -1 {
return CANCEL_SENTINEL;
}
if w == -3 {
return EINTR;
}
if w != 0 {
let mut sink = [0u8; 8];
unsafe { host(SYS_READ, efd as u64, sink.as_mut_ptr() as u64, 8, 0, 0, 0) };
}
}
Err(e) => return -(e as i64),
}
}
}
fn loop_service(
pid: i32,
nr: i64,
gfd: i32,
sid: netstack::SockId,
b: u64,
c: u64,
d: u64,
e: u64,
f: u64,
) -> i64 {
loop_service_inner(pid, nr, gfd, sid, b, c, d, e, f)
}
#[allow(clippy::too_many_arguments)]
fn loop_service_inner(
pid: i32,
nr: i64,
gfd: i32,
sid: netstack::SockId,
b: u64,
c: u64,
d: u64,
e: u64,
f: u64,
) -> i64 {
let nb = loop_state().lock().unwrap().nonblock.contains(&sid);
match nr {
SYS_READ => loop_recv(pid, sid, b, c, nb),
SYS_RECVFROM => {
let nonblock = nb || (d & 0x40) != 0; let r = loop_recv(pid, sid, b, c, nonblock);
if r >= 0 && e != 0 {
if let Some(ep) = loop_state().lock().unwrap().net.peer_addr(sid) {
write_sockaddr(pid, e, f, &ep);
}
}
r
}
SYS_WRITE => loop_send(pid, sid, b, c, nb),
SYS_SENDTO => loop_send(pid, sid, b, c, nb || (d & 0x40) != 0), SYS_WRITEV => loop_writev(pid, sid, b, c, nb),
SYS_READV => loop_readv(pid, sid, b, c, nb),
SYS_SENDFILE => loop_sendfile(pid, sid, b as i32, c, d),
SYS_SENDMSG => loop_sendmsg(pid, sid, b, nb || (c & 0x40) != 0), SYS_RECVMSG => loop_recvmsg(pid, sid, b, nb || (c & 0x40) != 0), SYS_LISTEN => loop_state()
.lock()
.unwrap()
.net
.listen(sid)
.map(|_| 0)
.unwrap_or_else(|e| -(e as i64)),
SYS_ACCEPT => loop_accept(pid, sid, b, c, nb, false),
SYS_ACCEPT4 => loop_accept(pid, sid, b, c, nb, (d & 0x800) != 0), SYS_SHUTDOWN => {
let mut ls = loop_state().lock().unwrap();
let peer = ls.net.peer_of(sid);
let r = ls
.net
.shutdown(sid, b as i32)
.map(|_| 0)
.unwrap_or_else(|e| -(e as i64));
if let Some(p) = peer {
loop_wake(&ls, p);
}
drop(ls);
loopring_ev(5, sid, b as i64);
r
}
SYS_GETSOCKNAME => match loop_state().lock().unwrap().net.local_addr(sid) {
Some(ep) => {
write_sockaddr(pid, b, c, &ep);
0
}
None => -(netstack::ENOTCONN as i64),
},
SYS_GETPEERNAME => match loop_state().lock().unwrap().net.peer_addr(sid) {
Some(ep) => {
write_sockaddr(pid, b, c, &ep);
0
}
None => -(netstack::ENOTCONN as i64),
},
SYS_SETSOCKOPT => 0, SYS_GETSOCKOPT => {
if d != 0 && e != 0 {
let mut lb = [0u8; 4];
vm_read(pid, e, &mut lb);
if u32::from_le_bytes(lb) as usize >= 4 {
vm_write(pid, d, &0i32.to_le_bytes());
vm_write(pid, e, &4u32.to_le_bytes());
}
}
0
}
SYS_DUP => fd_install_loop_dup(pid, sid, 0),
SYS_DUP2 | SYS_DUP3 => {
let gnew = b as i32;
if nr == SYS_DUP3 && (c & !(libc::O_CLOEXEC as u64)) != 0 {
return -22; }
if gfd == gnew {
if nr == SYS_DUP2 {
gnew as i64
} else {
-22 }
} else {
loop_state().lock().unwrap().net.dup(sid);
let flags = if nr == SYS_DUP3 && (c & libc::O_CLOEXEC as u64) != 0 {
FD_CLOEXEC as i32
} else {
0
};
fd_install_val_at_with_flags(pid, gnew, FdVal::Loop(sid), flags);
gnew as i64
}
}
SYS_FCNTL => match b {
0 => fd_install_loop_dup(pid, sid, c as i32),
1030 => {
loop_state().lock().unwrap().net.dup(sid);
fd_install_val_with_flags(pid, FdVal::Loop(sid), c as i32, FD_CLOEXEC as i32)
}
F_GETFD => fd_get_desc_flags(pid, gfd, FdVal::Loop(sid)) as i64,
F_SETFD => {
fd_set_desc_flags(pid, gfd, c as i32);
0
}
3 => {
if nb {
2 | 0x800
} else {
2
}
}
4 => {
let mut ls = loop_state().lock().unwrap();
if (c & 0x800) != 0 {
ls.nonblock.insert(sid);
} else {
ls.nonblock.remove(&sid);
}
0
}
_ => 0,
},
_ => -95, }
}
fn cwds() -> &'static Mutex<HashMap<i32, Vec<u8>>> {
static T: OnceLock<Mutex<HashMap<i32, Vec<u8>>>> = OnceLock::new();
T.get_or_init(|| Mutex::new(HashMap::new()))
}
fn cwd_of(pid: i32) -> Vec<u8> {
cwds()
.lock()
.unwrap()
.get(&pid)
.cloned()
.unwrap_or_else(|| b"/".to_vec())
}
fn pending_cwd() -> &'static Mutex<HashMap<u64, Vec<u8>>> {
static T: OnceLock<Mutex<HashMap<u64, Vec<u8>>>> = OnceLock::new();
T.get_or_init(|| Mutex::new(HashMap::new()))
}
fn pending_exe() -> &'static Mutex<HashMap<u64, Vec<u8>>> {
static T: OnceLock<Mutex<HashMap<u64, Vec<u8>>>> = OnceLock::new();
T.get_or_init(|| Mutex::new(HashMap::new()))
}
fn pending_cmdline() -> &'static Mutex<HashMap<u64, Vec<u8>>> {
static T: OnceLock<Mutex<HashMap<u64, Vec<u8>>>> = OnceLock::new();
T.get_or_init(|| Mutex::new(HashMap::new()))
}
fn readlink_fd(fd: i32) -> Option<Vec<u8>> {
let mut link = b"/proc/self/fd/".to_vec();
let mut nb = [0u8; 24];
let i = fmt_i64(fd as i64, &mut nb);
link.extend_from_slice(&nb[i..]);
link.push(0);
let mut buf = [0u8; 4096];
let n = unsafe {
host(
SYS_READLINK,
link.as_ptr() as u64,
buf.as_mut_ptr() as u64,
buf.len() as u64,
0,
0,
0,
)
};
if n <= 0 {
None
} else {
Some(buf[..n as usize].to_vec())
}
}
fn canonical_guest_cwd(fd: i32) -> Option<Vec<u8>> {
let hp = readlink_fd(fd)?;
host_to_guest_path(hp.as_slice())
}
fn host_to_guest_path(host_path: &[u8]) -> Option<Vec<u8>> {
{
let mt = mounts().lock().unwrap();
for m in mt.iter() {
if let Some(g) = map_host_into(&m.host_canonical, host_path, &m.guest_prefix) {
return Some(g);
}
}
}
let root = root_path().lock().unwrap();
if root.is_empty() {
return Some(host_path.to_vec());
}
map_host_into(root.as_slice(), host_path, b"")
}
fn map_host_into(host_root: &[u8], host_path: &[u8], guest_prefix: &[u8]) -> Option<Vec<u8>> {
if !prefix_covers(host_root, host_path) {
return None;
}
let rest = &host_path[host_root.len()..]; if rest.is_empty() {
Some(if guest_prefix.is_empty() {
b"/".to_vec()
} else {
guest_prefix.to_vec()
})
} else {
let mut g = guest_prefix.to_vec();
g.extend_from_slice(rest);
Some(g)
}
}
fn pull_cwd_path(pid: i32, remote: u64) -> Vec<u8> {
let p = pull_path(pid, remote); if p.first() == Some(&b'/') {
return p; }
let mut full = cwd_of(pid);
full.push(b'/');
full.extend_from_slice(&p[..p.len().saturating_sub(1)]); full.push(0);
full
}
fn openat_abs_guest_path(dirfd_h: i32, rel: &[u8]) -> Option<Vec<u8>> {
let r = match rel.split_last() {
Some((0, head)) => head,
_ => rel,
};
if r.first() == Some(&b'/') {
let mut a = r.to_vec();
a.push(0);
return Some(a);
}
if synth_proc_dirs().lock().unwrap().contains_key(&dirfd_h) {
let mut abs = b"/proc".to_vec();
if !r.is_empty() {
abs.push(b'/');
abs.extend_from_slice(r);
}
abs.push(0);
return Some(abs);
}
let host_dir = readlink_fd(dirfd_h)?;
let mut abs = if let Some(abs) = host_to_guest_path(&host_dir) {
abs
} else {
let rest = host_dir.strip_prefix(b"/proc/")?;
let first_slash = rest.iter().position(|&b| b == b'/');
let pid_part = first_slash.map_or(rest, |i| &rest[..i]);
let pid = parse_u32(pid_part)? as i32;
if pid <= 0
|| (pid != MAIN_CELL_PID.load(Ordering::Relaxed) as i32
&& proctree::vpid_for(pid).is_none())
{
return None;
}
host_dir
};
if abs.last() != Some(&b'/') {
abs.push(b'/');
}
abs.extend_from_slice(r);
abs.push(0);
Some(abs)
}
fn free_slots_of(pid: i32) {
for i in 1..MAX_SLOTS as u64 {
let r = ring_at(i);
unsafe {
if std::ptr::read_volatile(std::ptr::addr_of!((*r).pid)) != pid {
continue;
}
let req = ring_word(std::ptr::addr_of_mut!((*r).request)).load(Ordering::Acquire);
let resp = ring_word(std::ptr::addr_of_mut!((*r).response)).load(Ordering::Acquire);
if req == resp {
pending_drain_slot(i);
free_slot(i as u32);
continue;
}
let kind = std::ptr::read_volatile(std::ptr::addr_of!((*r).kind));
let efd = std::ptr::read_volatile(std::ptr::addr_of!((*r).cancel_efd));
if (kind != RING_PROCESS && kind != RING_DELEG) || efd < 0 {
continue;
}
let one: u64 = 1;
std::ptr::write_volatile(
std::ptr::addr_of_mut!((*r).cancel_mode),
CANCEL_REAP_PROCESS,
);
host(
SYS_WRITE,
efd as u64,
std::ptr::addr_of!(one) as u64,
8,
0,
0,
0,
);
std::ptr::write_volatile(std::ptr::addr_of_mut!((*r).cancel_pending), 1);
for _ in 0..64 {
if !slot_is_set(i) {
break; }
let rq = ring_word(std::ptr::addr_of_mut!((*r).request)).load(Ordering::Acquire);
let rp = ring_word(std::ptr::addr_of_mut!((*r).response)).load(Ordering::Acquire);
if rq == rp {
pending_drain_slot(i); free_slot(i as u32);
break;
}
let ts = [0i64, 1_000_000]; host(SYS_NANOSLEEP, ts.as_ptr() as u64, 0, 0, 0, 0, 0);
}
}
}
}
fn pid_is_dead(pid: i32) -> bool {
let mut nb = [0u8; 24];
let i = fmt_i64(pid as i64, &mut nb);
let mut p = Vec::with_capacity(24);
p.extend_from_slice(b"/proc/");
p.extend_from_slice(&nb[i..]);
p.extend_from_slice(b"/stat\0");
let fd = unsafe { host(SYS_OPENAT, (-100i64) as u64, p.as_ptr() as u64, 0, 0, 0, 0) };
if fd < 0 {
return true;
}
let mut buf = [0u8; 256];
let got = unsafe {
host(
SYS_READ,
fd as u64,
buf.as_mut_ptr() as u64,
buf.len() as u64,
0,
0,
0,
)
};
unsafe { host(SYS_CLOSE, fd as u64, 0, 0, 0, 0, 0) };
if got <= 0 {
return true;
}
let s = &buf[..got as usize];
if let Some(rp) = s.iter().rposition(|&b| b == b')') {
if let Some(&st) = s.get(rp + 2) {
return st == b'Z' || st == b'X' || st == b'x';
}
}
false
}
fn pending_drain_slot(slot: u64) {
if let Some(m) = pending().lock().unwrap().remove(&slot) {
for (_, v) in m {
fd_close_val(v);
}
}
pending_fd_flags().lock().unwrap().remove(&slot);
pending_creds().lock().unwrap().remove(&slot);
pending_cwd().lock().unwrap().remove(&slot);
pending_exe().lock().unwrap().remove(&slot);
pending_cmdline().lock().unwrap().remove(&slot);
pending_shm_maps().lock().unwrap().remove(&slot);
}
fn reap_dead_pid(pid: i32) {
if pid <= 1 {
return;
}
shm_reap_pid(pid);
fd_drop(pid);
for i in 0..MAX_SLOTS as u64 {
unsafe {
let r = ring_at(i);
if std::ptr::read_volatile(std::ptr::addr_of!((*r).pid)) == pid
&& std::ptr::read_volatile(std::ptr::addr_of!((*r).fork_parent)) != 0
{
pending_drain_slot(i);
}
}
}
free_slots_of(pid);
}
fn sweep_dead_cells() {
let main = MAIN_CELL_PID.load(Ordering::Relaxed) as i32;
let mut seen: [i32; MAX_SLOTS] = [0; MAX_SLOTS];
let mut n = 0usize;
for i in 1..MAX_SLOTS as u64 {
let p = unsafe { std::ptr::read_volatile(std::ptr::addr_of!((*ring_at(i)).pid)) };
if p <= 1 || p == main || seen[..n].contains(&p) {
continue;
}
seen[n] = p;
n += 1;
if pid_is_dead(p) {
reap_dead_pid(p);
}
}
}
fn fd_leak_sweeper() -> ! {
loop {
sweep_dead_cells();
let ts = [0i64, 20_000_000]; unsafe { host(SYS_NANOSLEEP, ts.as_ptr() as u64, 0, 0, 0, 0, 0) };
}
}
fn install_child_subreaper(context: &'static [u8]) {
let rc = unsafe { libc::prctl(PR_SET_CHILD_SUBREAPER_SUPERVISOR, 1, 0, 0, 0) };
ipc_logf(&[(b"SUBREAPER rc=", rc as i64)], context);
}
fn log_adopted_orphan_reap(pid: i32, status: c_int) {
ipc_logf(
&[
(b"SUBREAPER_REAP pid=", pid as i64),
(b" status=", status as i64),
],
b"",
);
}
fn adopted_orphan_reaper(launcher_pid: i32) -> ! {
loop {
loop {
let mut status: c_int = 0;
let r = unsafe { libc::waitpid(-1, &mut status, libc::WNOHANG) };
if r <= 0 {
break;
}
if r == launcher_pid {
ipc_logf(
&[
(b"SUBREAPER_LAUNCHER_EXIT pid=", r as i64),
(b" status=", status as i64),
],
b"",
);
unsafe { libc::_exit(127) };
}
log_adopted_orphan_reap(r, status);
reap_dead_pid(r);
}
let ts = [0i64, 20_000_000]; unsafe { host(SYS_NANOSLEEP, ts.as_ptr() as u64, 0, 0, 0, 0, 0) };
}
}
fn launcher_kill_owned_processes_on_eof() {
unsafe {
let me = libc::getpid();
let mut victims: [i32; MAX_SLOTS] = [0; MAX_SLOTS];
let mut n = 0usize;
for i in 0..MAX_SLOTS as u64 {
let p = std::ptr::read_volatile(std::ptr::addr_of!((*ring_at(i)).pid));
if p <= 1 || p == me || victims[..n].contains(&p) {
continue;
}
victims[n] = p;
n += 1;
}
for &p in &victims[..n] {
if libc::getpgid(p) == p {
libc::kill(-p, libc::SIGKILL);
}
libc::kill(p, libc::SIGKILL);
}
loop {
let mut st: c_int = 0;
let r = libc::waitpid(-1, &mut st, libc::WNOHANG);
if r <= 0 {
break;
}
reap_dead_pid(r);
}
}
}
fn pid_in_sandbox(target: i32) -> bool {
if target <= 0 {
return false;
}
if proctree::vpid_for(target).is_some() {
return true;
}
(0..MAX_SLOTS as u64).any(|i| unsafe {
std::ptr::read_volatile(std::ptr::addr_of!((*ring_at(i)).pid)) == target
})
}
fn interrupt_slots_for_pid(target: i32) {
if target <= 0 {
return;
}
unsafe {
let one: u64 = 1;
for i in 0..MAX_SLOTS as u64 {
let r = ring_at(i);
if std::ptr::read_volatile(std::ptr::addr_of!((*r).pid)) != target {
continue;
}
let efd = std::ptr::read_volatile(std::ptr::addr_of!((*r).intr_efd));
if efd >= 0 {
host(
SYS_WRITE,
efd as u64,
std::ptr::addr_of!(one) as u64,
8,
0,
0,
0,
);
std::ptr::write_volatile(std::ptr::addr_of_mut!((*r).intr_pending), 1);
}
}
}
}
#[cfg(all(target_os = "linux", target_arch = "x86_64"))]
fn slots_quiescent_for_pid(target: i32) -> bool {
if target <= 0 {
return true;
}
for i in 0..MAX_SLOTS as u64 {
let r = ring_at(i);
unsafe {
if std::ptr::read_volatile(std::ptr::addr_of!((*r).pid)) != target {
continue;
}
let req = ring_word(std::ptr::addr_of_mut!((*r).request)).load(Ordering::Acquire);
let resp = ring_word(std::ptr::addr_of_mut!((*r).response)).load(Ordering::Acquire);
if req != resp {
return false;
}
}
}
true
}
#[cfg(all(target_os = "linux", target_arch = "x86_64"))]
fn launcher_quiesce_live_targets(targets: &[LauncherCaptureTarget]) -> std::io::Result<()> {
for _ in 0..3 {
for target in targets {
if !slots_quiescent_for_pid(target.host_pid) {
interrupt_slots_for_pid(target.host_pid);
}
}
let ts = [0i64, 1_000_000]; unsafe {
host(SYS_NANOSLEEP, ts.as_ptr() as u64, 0, 0, 0, 0, 0);
}
}
Ok(())
}
fn read_task_tids(tgid: i32, out: &mut Vec<i32>) -> bool {
out.clear();
if tgid <= 0 {
return false;
}
let mut nb = [0u8; 24];
let i = fmt_i64(tgid as i64, &mut nb);
let mut path = Vec::with_capacity(32);
path.extend_from_slice(b"/proc/");
path.extend_from_slice(&nb[i..]);
path.extend_from_slice(b"/task\0");
const O_RDONLY: u64 = 0;
const O_DIRECTORY: u64 = 0o200000;
const AT_FDCWD: u64 = (-100i64) as u64;
let fd = unsafe {
host(
SYS_OPENAT,
AT_FDCWD,
path.as_ptr() as u64,
O_RDONLY | O_DIRECTORY,
0,
0,
0,
)
};
if fd < 0 {
return false;
}
let mut buf = [0u8; 8192];
loop {
let n = unsafe {
host(
SYS_GETDENTS64,
fd as u64,
buf.as_mut_ptr() as u64,
buf.len() as u64,
0,
0,
0,
)
};
if n <= 0 {
break;
}
let mut off = 0usize;
while off + 19 < n as usize {
let reclen = u16::from_ne_bytes([buf[off + 16], buf[off + 17]]) as usize;
if reclen == 0 {
break;
}
let mut tid: i64 = 0;
let mut valid = false;
let mut p = off + 19;
while p < n as usize {
let c = buf[p];
if c == 0 {
break;
}
if !c.is_ascii_digit() {
valid = false;
break;
}
tid = tid * 10 + (c - b'0') as i64;
valid = true;
p += 1;
}
if valid && tid > 0 && tid <= i32::MAX as i64 {
out.push(tid as i32);
}
off += reclen;
}
}
unsafe { host(SYS_CLOSE, fd as u64, 0, 0, 0, 0, 0) };
true
}
unsafe fn clear_or_free_thread_slot(slot: u64) {
unsafe {
if slot == 0 {
clear_ring_owner(0);
std::ptr::write_volatile(
std::ptr::addr_of_mut!((*ring_at(0)).cancel_mode),
CANCEL_REAP_PROCESS,
);
} else {
free_slot(slot as u32);
}
}
}
fn reclaim_dethread_slots(tgid: i32, keep_slot: u64, keep_tid: i32) {
if tgid <= 0 {
return;
}
for i in 0..MAX_SLOTS as u64 {
let r = ring_at(i);
unsafe {
if std::ptr::read_volatile(std::ptr::addr_of!((*r).pid)) != tgid {
continue;
}
let tid = std::ptr::read_volatile(std::ptr::addr_of!((*r).tid));
if i == keep_slot || (keep_tid > 0 && tid == keep_tid) {
continue;
}
let req = ring_word(std::ptr::addr_of_mut!((*r).request)).load(Ordering::Acquire);
let resp = ring_word(std::ptr::addr_of_mut!((*r).response)).load(Ordering::Acquire);
if req == resp {
clear_or_free_thread_slot(i);
continue;
}
let kind = std::ptr::read_volatile(std::ptr::addr_of!((*r).kind));
let efd = std::ptr::read_volatile(std::ptr::addr_of!((*r).cancel_efd));
if (kind != RING_PROCESS && kind != RING_DELEG) || efd < 0 {
continue;
}
std::ptr::write_volatile(std::ptr::addr_of_mut!((*r).cancel_mode), CANCEL_SLOT_ONLY);
let one: u64 = 1;
host(
SYS_WRITE,
efd as u64,
std::ptr::addr_of!(one) as u64,
8,
0,
0,
0,
);
std::ptr::write_volatile(std::ptr::addr_of_mut!((*r).cancel_pending), 1);
for _ in 0..64 {
if i != 0 && !slot_is_set(i) {
break;
}
let rq = ring_word(std::ptr::addr_of_mut!((*r).request)).load(Ordering::Acquire);
let rp = ring_word(std::ptr::addr_of_mut!((*r).response)).load(Ordering::Acquire);
if rq == rp {
clear_or_free_thread_slot(i);
break;
}
let ts = [0i64, 1_000_000]; host(SYS_NANOSLEEP, ts.as_ptr() as u64, 0, 0, 0, 0, 0);
}
}
}
}
fn supervisor_dethread_for_exec(keep_slot: u64, keep_tid: i32, tgid: i32) -> i64 {
if keep_slot as usize >= MAX_SLOTS || keep_tid <= 0 || tgid <= 0 {
return -22; }
let mut tids = Vec::new();
for _round in 0..200 {
if !read_task_tids(tgid, &mut tids) {
return -3; }
let mut remaining = false;
for &tid in &tids {
if tid == keep_tid {
continue;
}
remaining = true;
unsafe {
host(
SYS_TGKILL,
tgid as u64,
tid as u64,
SIG_DETHREAD as u64,
0,
0,
0,
);
}
}
if !remaining {
reclaim_dethread_slots(tgid, keep_slot, keep_tid);
return 0;
}
let ts = [0i64, 100_000]; unsafe { host(SYS_NANOSLEEP, ts.as_ptr() as u64, 0, 0, 0, 0, 0) };
}
-11 }
const SYS_OPENAT2: i64 = 437;
const SYS_FSTATFS: i64 = 138;
const O_PATH: u64 = 0x20_0000;
const O_NOFOLLOW: u64 = 0x2_0000;
const O_DIRECTORY: u64 = 0x1_0000;
const AT_EMPTY_PATH: u64 = 0x1000;
const RESOLVE_IN_ROOT: u64 = 0x10;
static ROOT_FD: std::sync::atomic::AtomicI32 = std::sync::atomic::AtomicI32::new(-1);
fn root_path() -> &'static Mutex<Vec<u8>> {
static P: OnceLock<Mutex<Vec<u8>>> = OnceLock::new();
P.get_or_init(|| Mutex::new(Vec::new()))
}
struct Mount {
guest_prefix: Vec<u8>, dirfd: i32,
host_canonical: Vec<u8>, readonly: bool,
}
fn mounts() -> &'static Mutex<Vec<Mount>> {
static M: OnceLock<Mutex<Vec<Mount>>> = OnceLock::new();
M.get_or_init(|| Mutex::new(Vec::new()))
}
fn prefix_covers(prefix: &[u8], p: &[u8]) -> bool {
p.len() >= prefix.len()
&& &p[..prefix.len()] == prefix
&& (p.len() == prefix.len() || p[prefix.len()] == b'/')
}
fn resolve_mount(p: &[u8]) -> (i32, Vec<u8>, bool) {
let root = ROOT_FD.load(Ordering::Relaxed);
let mt = mounts().lock().unwrap();
let mut best: Option<&Mount> = None;
for m in mt.iter() {
if prefix_covers(&m.guest_prefix, p)
&& best.map_or(true, |b| m.guest_prefix.len() > b.guest_prefix.len())
{
best = Some(m);
}
}
match best {
Some(m) => {
let rest = &p[m.guest_prefix.len()..]; let mut r = if rest.is_empty() {
b"/".to_vec()
} else {
rest.to_vec()
};
r.push(0);
(m.dirfd, r, m.readonly)
}
None => {
let mut r = p.to_vec();
r.push(0);
(root, r, false)
}
}
}
fn guest_to_host_path(gpath: &[u8]) -> Vec<u8> {
{
let mt = mounts().lock().unwrap();
for m in mt.iter() {
if prefix_covers(&m.guest_prefix, gpath) {
let rest = &gpath[m.guest_prefix.len()..];
let mut h = m.host_canonical.clone();
h.extend_from_slice(rest);
return h;
}
}
}
let root = root_path().lock().unwrap();
if root.is_empty() {
return gpath.to_vec();
}
let mut h = root.clone();
h.extend_from_slice(gpath);
h
}
fn guest_to_host_resolved_parent_path(gpath: &[u8]) -> Option<Vec<u8>> {
if gpath.first() != Some(&b'/') {
return None;
}
if ROOT_FD.load(Ordering::Relaxed) < 0 {
return Some(guest_to_host_path(gpath));
}
let mut nul = gpath.to_vec();
nul.push(0);
let (parent, base) = split_parent_base(&nul);
let parent_bare = match parent.split_last() {
Some((0, head)) => head,
_ => parent.as_slice(),
};
let base_bare = match base.split_last() {
Some((0, head)) => head,
_ => base.as_slice(),
};
if base_bare.is_empty() || parent_bare.is_empty() {
return None;
}
let (dirfd, resolved, _) = resolve_mount(parent_bare);
if dirfd < 0 {
return None;
}
let how = OpenHow {
flags: O_PATH | O_DIRECTORY,
mode: 0,
resolve: RESOLVE_IN_ROOT,
};
let pfd = unsafe {
host(
SYS_OPENAT2,
dirfd as u64,
resolved.as_ptr() as u64,
&how as *const _ as u64,
std::mem::size_of::<OpenHow>() as u64,
0,
0,
)
};
if pfd < 0 {
return None;
}
let mut host_parent = readlink_fd(pfd as i32)?;
unsafe {
host(SYS_CLOSE, pfd as u64, 0, 0, 0, 0, 0);
}
if host_parent.last() != Some(&b'/') {
host_parent.push(b'/');
}
host_parent.extend_from_slice(base_bare);
Some(host_parent)
}
fn unix_socket_aliases() -> &'static Mutex<HashMap<Vec<u8>, Vec<u8>>> {
static A: OnceLock<Mutex<HashMap<Vec<u8>, Vec<u8>>>> = OnceLock::new();
A.get_or_init(|| Mutex::new(HashMap::new()))
}
fn unix_socket_alias(host_path: &[u8]) -> Option<Vec<u8>> {
use std::hash::{Hash, Hasher};
use std::os::unix::ffi::OsStrExt;
let mut aliases = unix_socket_aliases().lock().unwrap();
if let Some(alias) = aliases.get(host_path) {
return Some(alias.clone());
}
let mut hasher = std::collections::hash_map::DefaultHasher::new();
host_path.hash(&mut hasher);
let dir = std::env::temp_dir().join(format!("supermachine-unix-{}", unsafe {
host(SYS_GETPID, 0, 0, 0, 0, 0, 0)
}));
std::fs::create_dir_all(&dir).ok()?;
let alias = dir.join(format!("{:016x}.sock", hasher.finish()));
let bytes = alias.as_os_str().as_bytes().to_vec();
if bytes.len() + 1 > 108 {
return None;
}
aliases.insert(host_path.to_vec(), bytes.clone());
Some(bytes)
}
fn unix_socket_alias_for_guest_path(path: &[u8]) -> Option<Vec<u8>> {
let bare = match path.split_last() {
Some((0, head)) => head,
_ => path,
};
let host = guest_to_host_resolved_parent_path(bare).unwrap_or_else(|| guest_to_host_path(bare));
unix_socket_aliases().lock().unwrap().get(&host).cloned()
}
fn rewrite_unix_sun_path(addr: &[u8]) -> Option<Vec<u8>> {
if addr.len() < 3 {
return None;
}
if u16::from_le_bytes([addr[0], addr[1]]) != libc::AF_UNIX as u16 {
return None;
}
let path = &addr[2..];
if path.first() == Some(&0) {
return None; }
let end = path.iter().position(|&c| c == 0).unwrap_or(path.len());
let gpath = &path[..end];
if gpath.first() != Some(&b'/') {
return None; }
let mut host = guest_to_host_resolved_parent_path(gpath)?;
if host.len() + 1 > 108 {
host = unix_socket_alias(&host)?;
}
let mut out = vec![0u8; 2 + host.len() + 1];
out[0..2].copy_from_slice(&(libc::AF_UNIX as u16).to_le_bytes());
out[2..2 + host.len()].copy_from_slice(&host);
Some(out)
}
fn unix_sockaddr_host_path(addr: &[u8]) -> Option<Vec<u8>> {
if addr.len() < 3 {
return None;
}
if u16::from_le_bytes([addr[0], addr[1]]) != libc::AF_UNIX as u16 {
return None;
}
let path = &addr[2..];
if path.first() == Some(&0) {
return None;
}
let end = path.iter().position(|&c| c == 0).unwrap_or(path.len());
let gpath = &path[..end];
if gpath.first() != Some(&b'/') {
return None;
}
guest_to_host_resolved_parent_path(gpath)
}
fn unix_sockaddr_path(addr: &[u8]) -> Option<&[u8]> {
if addr.len() < 3 {
return None;
}
if u16::from_le_bytes([addr[0], addr[1]]) != libc::AF_UNIX as u16 {
return None;
}
let path = &addr[2..];
if path.first() == Some(&0) {
return None;
}
let end = path.iter().position(|&c| c == 0).unwrap_or(path.len());
Some(&path[..end])
}
fn publish_unix_socket_alias_placeholder(host_path: &[u8]) {
let Ok(c) = std::ffi::CString::new(host_path) else {
return;
};
unsafe {
libc::unlink(c.as_ptr());
if libc::mknod(c.as_ptr(), libc::S_IFSOCK | 0o777, 0) != 0 {
let fd = libc::open(
c.as_ptr(),
libc::O_CREAT | libc::O_EXCL | libc::O_RDONLY,
0o666,
);
if fd >= 0 {
libc::close(fd);
}
}
}
}
fn unix_alias_placeholders() -> &'static Mutex<HashMap<i32, Vec<Vec<u8>>>> {
static A: OnceLock<Mutex<HashMap<i32, Vec<Vec<u8>>>>> = OnceLock::new();
A.get_or_init(|| Mutex::new(HashMap::new()))
}
fn unix_alias_placeholder_record(hfd: i32, host_path: &[u8]) {
unix_alias_placeholders()
.lock()
.unwrap()
.entry(hfd)
.or_default()
.push(host_path.to_vec());
}
fn unix_alias_placeholders_remove(hfd: i32) {
let paths = unix_alias_placeholders().lock().unwrap().remove(&hfd);
let Some(paths) = paths else {
return;
};
for path in paths {
let Ok(c) = std::ffi::CString::new(path) else {
continue;
};
unsafe {
libc::unlink(c.as_ptr());
}
}
}
const O_ACCMODE: u64 = 0o3;
const O_RDONLY: u64 = 0;
const O_WRONLY: u64 = 0o1;
const O_RDWR: u64 = 0o2;
const O_CREAT: u64 = 0o100;
const O_EXCL: u64 = 0o200;
const O_TRUNC: u64 = 0o1000;
const R_OK: u64 = 4;
const W_OK: u64 = 2;
const X_OK: u64 = 1;
const AT_SYMLINK_NOFOLLOW: u64 = 0x100;
const AT_EACCESS: u64 = 0x200;
fn is_write_open(flags: u64) -> bool {
let acc = flags & O_ACCMODE;
acc == O_WRONLY || acc == O_RDWR || (flags & (O_CREAT | O_TRUNC)) != 0
}
fn host_fstat_meta(fd: i32) -> Option<(u32, u32, u32)> {
let mut st: libc::stat = unsafe { std::mem::zeroed() };
let r = unsafe { host(SYS_FSTAT, fd as u64, &mut st as *mut _ as u64, 0, 0, 0, 0) };
if r == 0 {
Some((st.st_mode as u32, st.st_uid as u32, st.st_gid as u32))
} else {
None
}
}
fn virtual_ids_for_access(pid: i32, use_effective: bool) -> (u32, u32) {
let c = supervisor_cred_for(pid);
if use_effective {
(c.euid, c.egid)
} else {
(c.ruid, c.rgid)
}
}
fn virtual_fs_ids(pid: i32) -> (u32, u32) {
let c = supervisor_cred_for(pid);
(c.fsuid, c.fsgid)
}
fn access_bits_allowed(mode: u32, st_uid: u32, st_gid: u32, uid: u32, gid: u32, req: u64) -> bool {
if req == 0 {
return true;
}
if uid == 0 {
return (req & X_OK) == 0 || (mode & 0o111) != 0;
}
let shift = if uid == st_uid {
6
} else if gid == st_gid {
3
} else {
0
};
let allowed = ((mode >> shift) & 0o7) as u64;
(allowed & req) == req
}
fn faccess_fd(pid: i32, fd: i32, req: u64, use_effective: bool) -> i64 {
if (req & !(R_OK | W_OK | X_OK)) != 0 {
return -22; }
let Some((mode, st_uid, st_gid)) = host_fstat_meta(fd) else {
return -9; };
let (uid, gid) = virtual_ids_for_access(pid, use_effective);
if access_bits_allowed(mode, st_uid, st_gid, uid, gid, req) {
0
} else {
-13 }
}
fn chown_created_fd_to_virtual(pid: i32, fd: i32) {
let (uid, gid) = virtual_fs_ids(pid);
if uid == 0 && gid == 0 {
return;
}
unsafe {
host(SYS_FCHOWN, fd as u64, uid as u64, gid as u64, 0, 0, 0);
}
}
fn chown_created_at_to_virtual(pid: i32, pfd: i32, base: &[u8], flags: u64) {
let (uid, gid) = virtual_fs_ids(pid);
if uid == 0 && gid == 0 {
return;
}
unsafe {
host(
SYS_FCHOWNAT,
pfd as u64,
base.as_ptr() as u64,
uid as u64,
gid as u64,
flags,
0,
);
}
}
const CGROUP_BASE: &str = "/sys/fs/cgroup";
fn cgroup_path() -> &'static Mutex<Option<std::path::PathBuf>> {
static P: OnceLock<Mutex<Option<std::path::PathBuf>>> = OnceLock::new();
P.get_or_init(|| Mutex::new(None))
}
fn cell_limits() -> &'static Mutex<Limits> {
static L: OnceLock<Mutex<Limits>> = OnceLock::new();
L.get_or_init(|| Mutex::new(Limits::default()))
}
fn run_uid() -> &'static Mutex<Option<(u32, u32)>> {
static U: OnceLock<Mutex<Option<(u32, u32)>>> = OnceLock::new();
U.get_or_init(|| Mutex::new(None))
}
fn guest_env() -> &'static Mutex<Vec<Vec<u8>>> {
static E: OnceLock<Mutex<Vec<Vec<u8>>>> = OnceLock::new();
E.get_or_init(|| Mutex::new(Vec::new()))
}
fn guest_cwd_init() -> &'static Mutex<Option<Vec<u8>>> {
static C: OnceLock<Mutex<Option<Vec<u8>>>> = OnceLock::new();
C.get_or_init(|| Mutex::new(None))
}
fn published_ports() -> &'static Mutex<Vec<u16>> {
static P: OnceLock<Mutex<Vec<u16>>> = OnceLock::new();
P.get_or_init(|| Mutex::new(Vec::new()))
}
fn host_v6_route() -> bool {
crate::utils::net::host_ipv6_route()
}
fn resolv_nameserver_from(path: &str, allow_loopback: bool) -> Option<String> {
let text = std::fs::read_to_string(path).ok()?;
text.lines()
.map(str::trim)
.filter_map(|line| line.strip_prefix("nameserver "))
.map(str::trim)
.find(|ns| {
if ns.is_empty() {
return false;
}
if allow_loopback {
return true;
}
ns.parse::<std::net::IpAddr>()
.map(|ip| !(ip.is_loopback() || ip.is_unspecified()))
.unwrap_or(true)
})
.map(str::to_string)
}
fn host_resolv_nameserver() -> String {
resolv_nameserver_from("/run/systemd/resolve/resolv.conf", false)
.or_else(|| resolv_nameserver_from("/etc/resolv.conf", false))
.or_else(|| resolv_nameserver_from("/etc/resolv.conf", true))
.unwrap_or_else(|| "127.0.0.53".to_string())
}
fn dns_dest_fds() -> &'static Mutex<std::collections::HashSet<i32>> {
static D: OnceLock<Mutex<std::collections::HashSet<i32>>> = OnceLock::new();
D.get_or_init(|| Mutex::new(std::collections::HashSet::new()))
}
fn note_dns_dest(hfd: i32, sa: &std::net::SocketAddr) {
if host_v6_route() {
return;
}
let mut set = dns_dest_fds().lock().unwrap();
if sa.port() == 53 {
set.insert(hfd);
} else {
set.remove(&hfd);
}
}
fn is_dns_dest_fd(hfd: i32) -> bool {
if host_v6_route() {
return false;
}
dns_dest_fds().lock().unwrap().contains(&hfd)
}
fn maybe_strip_aaaa(hfd: i32, buf: &mut [u8], n: i64) -> i64 {
if n <= 0 || !is_dns_dest_fd(hfd) {
return n;
}
let got = n as usize;
if got > buf.len() {
return n;
}
match crate::devices::virtio::vsock::dns_filter::rewrite_aaaa_response_to_nodata(&buf[..got]) {
Some(rw) => {
let m = rw.len().min(buf.len());
buf[..m].copy_from_slice(&rw[..m]);
m as i64
}
None => n,
}
}
fn unwrap_v4mapped_sockaddr(addr: &mut [u8]) -> Option<usize> {
if addr.len() < 24 {
return None;
}
if u16::from_ne_bytes([addr[0], addr[1]]) != 10 {
return None; }
let mut o = [0u8; 16];
o.copy_from_slice(&addr[8..24]);
let v4 = std::net::Ipv6Addr::from(o).to_ipv4_mapped()?;
let octets = v4.octets();
addr[0..2].copy_from_slice(&2u16.to_ne_bytes());
addr[4..8].copy_from_slice(&octets);
for b in &mut addr[8..16] {
*b = 0;
}
Some(16)
}
fn log_connect_debug(pid: i32, hfd: i32, addr: &[u8], ret: i64) {
let sa = match parse_connect_addr(addr) {
Some(sa) if ret < 0 || sa.port() == 443 => sa,
_ => return,
};
let mut line = Vec::with_capacity(160);
line.extend_from_slice(b"CONNDBG pid=");
let mut nb = [0u8; 24];
let i = fmt_i64(pid as i64, &mut nb);
line.extend_from_slice(&nb[i..]);
line.extend_from_slice(b" hfd=");
let i = fmt_i64(hfd as i64, &mut nb);
line.extend_from_slice(&nb[i..]);
line.extend_from_slice(b" ret=");
let i = fmt_i64(ret, &mut nb);
line.extend_from_slice(&nb[i..]);
line.extend_from_slice(b" addr=");
line.extend_from_slice(sa.to_string().as_bytes());
ipc_logf_raw(&line);
}
fn setup_cgroup(name: &str, limits: &Limits, clear_stale: bool) -> Option<std::path::PathBuf> {
let base = std::path::Path::new(CGROUP_BASE);
if !base.join("cgroup.controllers").is_file() {
return None; }
let dir = base.join(name);
if clear_stale {
let _ = std::fs::remove_dir(&dir);
}
match std::fs::create_dir(&dir) {
Ok(()) => {}
Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {}
Err(_) => return None, }
if let Some(m) = limits.memory_max {
let _ = std::fs::write(dir.join("memory.max"), m.to_string());
let _ = std::fs::write(dir.join("memory.swap.max"), "0");
}
if let Some(p) = limits.pids_max {
let _ = std::fs::write(dir.join("pids.max"), p.to_string());
}
if let Some(c) = &limits.cpu_max {
let _ = std::fs::write(dir.join("cpu.max"), c);
}
Some(dir)
}
fn remove_cgroup(pid: i32) {
remove_cgroup_named(&format!("sentry-{pid}"));
}
fn remove_cgroup_named(name: &str) {
let dir = std::path::Path::new(CGROUP_BASE).join(name);
for _ in 0..5 {
if std::fs::remove_dir(&dir).is_ok() || !dir.exists() {
return;
}
std::thread::sleep(std::time::Duration::from_millis(2));
}
}
#[repr(C)]
struct OpenHow {
flags: u64,
mode: u64,
resolve: u64,
}
fn pull_open_how(pid: i32, remote: u64, size: u64) -> Result<OpenHow, i64> {
if remote == 0 || size < std::mem::size_of::<OpenHow>() as u64 {
return Err(-22); }
let mut buf = [0u8; std::mem::size_of::<OpenHow>()];
if vm_read(pid, remote, &mut buf) != buf.len() as i64 {
return Err(-14); }
Ok(OpenHow {
flags: u64::from_le_bytes(buf[0..8].try_into().unwrap()),
mode: u64::from_le_bytes(buf[8..16].try_into().unwrap()),
resolve: u64::from_le_bytes(buf[16..24].try_into().unwrap()),
})
}
fn mode_for_open_flags(pid: i32, flags: u64, mode: u64) -> u64 {
if (flags & O_CREAT) != 0 {
apply_umask(pid, mode & 0o7777)
} else {
0
}
}
fn host_device_for(p: &[u8]) -> Option<&'static [u8]> {
Some(match p {
b"/dev/null" => b"/dev/null\0",
b"/dev/zero" => b"/dev/zero\0",
b"/dev/full" => b"/dev/full\0",
b"/dev/random" => b"/dev/random\0",
b"/dev/urandom" => b"/dev/urandom\0",
_ => return None,
})
}
fn dev_fd_rel(bare: &[u8]) -> Option<Vec<u8>> {
if bare == b"/dev/fd" || bare == b"/dev/fd/" {
return Some(b"fd".to_vec());
}
match bare {
b"/dev/stdin" => return Some(b"fd/0".to_vec()),
b"/dev/stdout" => return Some(b"fd/1".to_vec()),
b"/dev/stderr" => return Some(b"fd/2".to_vec()),
_ => {}
}
let n = bare.strip_prefix(b"/dev/fd/")?;
if n.is_empty() || !n.iter().all(|c| c.is_ascii_digit()) {
return None;
}
let mut rel = b"fd/".to_vec();
rel.extend_from_slice(n);
Some(rel)
}
fn cgroup_cpu_count() -> Option<u64> {
let l = cell_limits().lock().unwrap();
let c = l.cpu_max.as_ref()?;
let mut it = c.split_ascii_whitespace();
let quota = it.next()?;
if quota == "max" {
return None; }
let quota: u64 = quota.parse().ok()?;
let period: u64 = it.next().and_then(|p| p.parse().ok()).unwrap_or(100_000);
if period == 0 {
return None;
}
Some(((quota + period - 1) / period).max(1))
}
fn cgroup_mem_bytes() -> Option<u64> {
let l = cell_limits().lock().unwrap();
match l.memory_max {
Some(m) if m > 0 && m < (1u64 << 62) => Some(m),
_ => None,
}
}
fn synth_cpuinfo(n: u64) -> Vec<u8> {
let host = std::fs::read("/proc/cpuinfo").unwrap_or_default();
let text = String::from_utf8_lossy(&host);
let template = text.split("\n\n").next().unwrap_or("").trim_end();
if template.is_empty() {
return host;
}
let mut out = String::new();
for i in 0..n {
for line in template.lines() {
if let Some(rest) = line.split_once(':') {
let key = rest.0.trim_end();
match key {
"processor" => {
out.push_str(&format!("processor\t: {i}\n"));
continue;
}
"core id" => {
out.push_str(&format!("core id\t\t: {i}\n"));
continue;
}
"apicid" | "initial apicid" => {
out.push_str(&format!("{key}\t\t: {i}\n"));
continue;
}
_ => {}
}
}
out.push_str(line);
out.push('\n');
}
out.push('\n');
}
out.into_bytes()
}
fn synth_stat(n: u64) -> Vec<u8> {
let host = std::fs::read("/proc/stat").unwrap_or_default();
let text = String::from_utf8_lossy(&host);
let mut out = String::new();
let mut saw_cpu = false;
let mut emitted_cpus = 0u64;
for line in text.lines() {
if line.starts_with("cpu ") {
saw_cpu = true;
out.push_str(line);
out.push('\n');
continue;
}
if let Some(rest) = line.strip_prefix("cpu") {
let is_cpu_line = rest
.as_bytes()
.first()
.map(|b| b.is_ascii_digit())
.unwrap_or(false);
if is_cpu_line {
if emitted_cpus < n {
out.push_str(&format!("cpu{emitted_cpus}"));
if let Some(fields) = rest
.trim_start_matches(|c: char| c.is_ascii_digit())
.strip_prefix(' ')
{
out.push(' ');
out.push_str(fields);
}
out.push('\n');
emitted_cpus += 1;
}
continue;
}
}
out.push_str(line);
out.push('\n');
}
if saw_cpu && emitted_cpus == n {
out.into_bytes()
} else {
host
}
}
fn synth_meminfo(cap: u64) -> Vec<u8> {
let host = std::fs::read("/proc/meminfo").unwrap_or_default();
let text = String::from_utf8_lossy(&host);
let total_kb = cap / 1024;
let free_kb = (total_kb / 10) * 9;
let mut out = String::new();
for line in text.lines() {
let key = line.split(':').next().unwrap_or("");
match key {
"MemTotal" => out.push_str(&format!("MemTotal: {total_kb:>8} kB\n")),
"MemFree" => out.push_str(&format!("MemFree: {free_kb:>8} kB\n")),
"MemAvailable" => out.push_str(&format!("MemAvailable: {free_kb:>8} kB\n")),
"SwapTotal" => out.push_str("SwapTotal: 0 kB\n"),
"SwapFree" => out.push_str("SwapFree: 0 kB\n"),
_ => {
out.push_str(line);
out.push('\n');
}
}
}
if out.is_empty() {
out = format!(
"MemTotal: {total_kb:>8} kB\nMemFree: {free_kb:>8} kB\nMemAvailable: {free_kb:>8} kB\nSwapTotal: 0 kB\nSwapFree: 0 kB\n"
);
}
out.into_bytes()
}
fn proc_sys_synth(p: &[u8]) -> Option<Vec<u8>> {
Some(match p {
b"/proc/sys/kernel/random/uuid" => {
let mut b = random_uuid_v4();
b.push(b'\n');
b
}
b"/proc/sys/kernel/random/boot_id" => {
let mut b = boot_id().clone();
b.push(b'\n');
b
}
b"/proc/sys/net/core/somaxconn" => b"4096\n".to_vec(),
b"/proc/sys/net/ipv4/ip_unprivileged_port_start" => b"0\n".to_vec(),
b"/proc/sys/vm/overcommit_memory" => b"1\n".to_vec(),
b"/proc/sys/vm/max_map_count" => b"262144\n".to_vec(),
b"/proc/sys/kernel/pid_max" => b"4194304\n".to_vec(),
b"/proc/sys/kernel/threads-max" => b"63988\n".to_vec(),
b"/proc/sys/fs/inotify/max_user_watches" => b"1048576\n".to_vec(),
b"/proc/sys/fs/inotify/max_user_instances" => b"1024\n".to_vec(),
b"/proc/sys/fs/inotify/max_queued_events" => b"16384\n".to_vec(),
_ => return None,
})
}
fn random_uuid_v4() -> Vec<u8> {
let mut r = [0u8; 16];
unsafe {
host(
SYS_GETRANDOM,
r.as_mut_ptr() as u64,
r.len() as u64,
0,
0,
0,
0,
);
}
r[6] = (r[6] & 0x0f) | 0x40; r[8] = (r[8] & 0x3f) | 0x80; let hex = b"0123456789abcdef";
let mut out = Vec::with_capacity(36);
for (i, byte) in r.iter().enumerate() {
if i == 4 || i == 6 || i == 8 || i == 10 {
out.push(b'-');
}
out.push(hex[(byte >> 4) as usize]);
out.push(hex[(byte & 0x0f) as usize]);
}
out
}
fn boot_id() -> &'static Vec<u8> {
static B: OnceLock<Vec<u8>> = OnceLock::new();
B.get_or_init(random_uuid_v4)
}
fn synth_self_cgroup() -> Vec<u8> {
let rel = cgroup_path()
.lock()
.unwrap()
.as_ref()
.and_then(|d| d.strip_prefix(CGROUP_BASE).ok().map(|r| r.to_path_buf()))
.map(|r| {
let mut s = b"/".to_vec();
s.extend_from_slice(r.to_string_lossy().as_bytes());
s
})
.unwrap_or_else(|| b"/".to_vec());
let mut out = b"0::".to_vec();
out.extend_from_slice(&rel);
out.push(b'\n');
out
}
fn synth_self_mountinfo() -> Vec<u8> {
let mut out = String::new();
out.push_str("1 0 0:1 / / rw,relatime - overlay rootfs rw\n");
out.push_str("2 1 0:2 / /proc rw,nosuid,nodev,noexec,relatime - proc proc rw\n");
out.push_str("3 1 0:3 / /dev rw,nosuid - tmpfs tmpfs rw,mode=755\n");
out.push_str("4 3 0:4 / /dev/shm rw,nosuid,nodev - tmpfs shm rw\n");
out.push_str("5 3 0:5 / /dev/fd rw,relatime - proc proc rw\n");
out.into_bytes()
}
fn proc_synth(p: &[u8]) -> Option<Vec<u8>> {
match p {
b"/proc/cpuinfo" => cgroup_cpu_count().map(synth_cpuinfo),
b"/proc/stat" => cgroup_cpu_count().map(synth_stat),
b"/proc/meminfo" => cgroup_mem_bytes().map(synth_meminfo),
b"/proc/mounts" => Some(synth_mounts()),
b"/proc/uptime" => Some(synth_uptime()),
b"/proc/net/dev" => Some(synth_net_dev()),
_ => proc_sys_synth(p),
}
}
fn synth_mounts() -> Vec<u8> {
let mut out = String::new();
out.push_str("overlay / overlay rw,relatime 0 0\n");
out.push_str("proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0\n");
out.push_str("tmpfs /dev tmpfs rw,nosuid,mode=755 0 0\n");
out.push_str("shm /dev/shm tmpfs rw,nosuid,nodev 0 0\n");
out.into_bytes()
}
fn synth_net_dev() -> Vec<u8> {
let mut out = String::new();
out.push_str("Inter-| Receive | Transmit\n");
out.push_str(
" face |bytes packets errs drop fifo frame compressed multicast|bytes packets errs drop fifo colls carrier compressed\n",
);
out.push_str(
" lo: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
);
out.into_bytes()
}
static GUEST_MONOTONIC_BASE_NS: OnceLock<u128> = OnceLock::new();
fn host_monotonic_ns() -> u128 {
let mut ts = [0u64; 2];
unsafe {
host(
SYS_CLOCK_GETTIME,
libc::CLOCK_MONOTONIC as u64,
ts.as_mut_ptr() as u64,
0,
0,
0,
0,
);
}
(ts[0] as u128) * 1_000_000_000u128 + ts[1] as u128
}
fn seed_guest_monotonic_clock() {
let _ = GUEST_MONOTONIC_BASE_NS.get_or_init(host_monotonic_ns);
}
fn guest_monotonic_timespec() -> [u8; 16] {
let now = host_monotonic_ns();
let base = *GUEST_MONOTONIC_BASE_NS.get_or_init(|| now);
let elapsed = now.saturating_sub(base);
let sec = (elapsed / 1_000_000_000u128) as u64;
let nsec = (elapsed % 1_000_000_000u128) as u64;
let mut out = [0u8; 16];
out[..8].copy_from_slice(&sec.to_le_bytes());
out[8..].copy_from_slice(&nsec.to_le_bytes());
out
}
fn synth_uptime() -> Vec<u8> {
let ts = guest_monotonic_timespec();
let sec = u64::from_le_bytes(ts[..8].try_into().unwrap_or([0; 8]));
let nsec = u64::from_le_bytes(ts[8..].try_into().unwrap_or([0; 8]));
format!("{}.{:02} 0.00\n", sec, nsec / 10_000_000).into_bytes()
}
fn proc_redirect(p: &[u8]) -> Option<&'static [u8]> {
Some(match p {
b"/proc/cpuinfo" => b"/proc/cpuinfo\0",
b"/proc/meminfo" => b"/proc/meminfo\0",
b"/proc/stat" => b"/proc/stat\0",
b"/proc/loadavg" => b"/proc/loadavg\0",
b"/proc/version" => b"/proc/version\0",
b"/proc/filesystems" => b"/proc/filesystems\0",
_ => return None,
})
}
const SYS_MEMFD_CREATE: i64 = 319;
fn guest_cmdline() -> &'static Mutex<Vec<u8>> {
static C: OnceLock<Mutex<Vec<u8>>> = OnceLock::new();
C.get_or_init(|| Mutex::new(Vec::new()))
}
fn guest_exe() -> &'static Mutex<Vec<u8>> {
static E: OnceLock<Mutex<Vec<u8>>> = OnceLock::new();
E.get_or_init(|| Mutex::new(Vec::new()))
}
fn parse_u32(b: &[u8]) -> Option<i32> {
if b.is_empty() {
return None;
}
let mut v: i64 = 0;
for &c in b {
if !c.is_ascii_digit() {
return None;
}
v = v * 10 + (c - b'0') as i64;
if v > i32::MAX as i64 {
return None;
}
}
Some(v as i32)
}
fn proc_self_rel<'a>(pid: i32, bare: &'a [u8]) -> Option<&'a [u8]> {
if let Some(rest) = bare.strip_prefix(b"/proc/self/") {
return Some(rest);
}
if let Some(rest) = bare.strip_prefix(b"/proc/thread-self/") {
return Some(rest);
}
let mut nb = [0u8; 24];
let i = fmt_i64(pid as i64, &mut nb);
let mut pfx = b"/proc/".to_vec();
pfx.extend_from_slice(&nb[i..]);
pfx.push(b'/');
bare.strip_prefix(pfx.as_slice())
}
fn proc_pid_rel(bare: &[u8]) -> Option<(i32, &[u8])> {
let rest = bare.strip_prefix(b"/proc/")?;
let slash = rest.iter().position(|&b| b == b'/')?;
let (num, tail) = rest.split_at(slash);
let pid = parse_u32(num)? as i32;
Some((pid, &tail[1..]))
}
fn proc_bare_pid(bare: &[u8]) -> Option<i32> {
let rest = bare.strip_prefix(b"/proc/")?;
if rest.is_empty() || rest.contains(&b'/') {
return None;
}
parse_u32(rest)
}
fn memfd_with(content: &[u8]) -> i64 {
let name = b"sentry-proc\0";
let fd = unsafe { host(SYS_MEMFD_CREATE, name.as_ptr() as u64, 0, 0, 0, 0, 0) };
if fd < 0 {
return fd;
}
let mut off = 0usize;
while off < content.len() {
let n = unsafe {
host(
SYS_WRITE,
fd as u64,
content[off..].as_ptr() as u64,
(content.len() - off) as u64,
0,
0,
0,
)
};
if n <= 0 {
break;
}
off += n as usize;
}
unsafe {
host(SYS_LSEEK, fd as u64, 0, 0 , 0, 0, 0)
};
fd
}
fn open_proc_self(pid: i32, rel: &[u8], flags: u64) -> i64 {
let f = flags & !(O_CREAT | O_TRUNC);
match rel {
b"cmdline" => {
let c = cmdline_for(pid);
if c.is_empty() {
memfd_with(&guest_cmdline().lock().unwrap())
} else {
memfd_with(&c)
}
}
b"cgroup" => memfd_with(&synth_self_cgroup()),
b"mountinfo" => memfd_with(&synth_self_mountinfo()),
b"mounts" => memfd_with(&synth_mounts()),
b"status" | b"stat" | b"statm" | b"comm" | b"task" | b"oom_score_adj" => {
let mut nb = [0u8; 24];
let i = fmt_i64(pid as i64, &mut nb);
let mut path = b"/proc/".to_vec();
path.extend_from_slice(&nb[i..]);
path.push(b'/');
path.extend_from_slice(rel);
path.push(0);
unsafe { host(SYS_OPEN, path.as_ptr() as u64, f, 0, 0, 0, 0) }
}
_ if rel.starts_with(b"task/") => {
let mut nb = [0u8; 24];
let i = fmt_i64(pid as i64, &mut nb);
let mut path = b"/proc/".to_vec();
path.extend_from_slice(&nb[i..]);
path.push(b'/');
path.extend_from_slice(rel);
path.push(0);
unsafe { host(SYS_OPEN, path.as_ptr() as u64, f, 0, 0, 0, 0) }
}
b"exe" => {
let mut exe = exe_for(pid);
if exe.is_empty() {
return -2;
}
exe.push(0);
let root = ROOT_FD.load(Ordering::Relaxed);
let how = OpenHow {
flags: f,
mode: 0,
resolve: RESOLVE_IN_ROOT,
};
let r = unsafe {
host(
SYS_OPENAT2,
root as u64,
exe.as_ptr() as u64,
&how as *const _ as u64,
std::mem::size_of::<OpenHow>() as u64,
0,
0,
)
};
if r < 0 {
let bare = exe
.iter()
.position(|&b| b == 0)
.map(|n| &exe[..n])
.unwrap_or(exe.as_slice());
ipc_logf_raw(
&[
b"PROCEXE_OPEN_FAIL pid=".as_slice(),
pid.to_string().as_bytes(),
b" errno=".as_slice(),
r.to_string().as_bytes(),
b" exe=".as_slice(),
bare,
]
.concat(),
);
}
r
}
b"cwd" => {
let mut cwd = cwd_of(pid);
cwd.push(0);
open_path(pid, &cwd, f | O_DIRECTORY, 0)
}
_ if rel.starts_with(b"fd/") && !rel[3..].iter().all(|&c| c == b'/') => {
let num = &rel[3..];
let num = &num[..num.iter().position(|&c| c == b'/').unwrap_or(num.len())];
let n = match parse_u32(num) {
Some(n) => n,
None => return -2,
};
let h = match fd_host(pid, n) {
Some(h) => h,
None => return -9, };
let mut nb = [0u8; 24];
let i = fmt_i64(h as i64, &mut nb);
let mut path = b"/proc/self/fd/".to_vec();
path.extend_from_slice(&nb[i..]);
path.push(0);
unsafe { host(SYS_OPEN, path.as_ptr() as u64, f, 0, 0, 0, 0) }
}
_ if rel == b"fd" || (rel.starts_with(b"fd/") && rel[3..].iter().all(|&c| c == b'/')) => {
let ph = b"/proc/self/fd\0";
let fd = unsafe {
host(
SYS_OPEN,
ph.as_ptr() as u64,
(libc::O_RDONLY | libc::O_DIRECTORY) as u64,
0,
0,
0,
0,
)
};
if fd >= 0 {
synth_fd_dirs().lock().unwrap().insert(fd as i32, (pid, 0));
}
fd
}
b"maps" => {
let raw = std::fs::read(format!("/proc/{pid}/maps")).unwrap_or_default();
let mut filtered = Vec::with_capacity(raw.len());
for line in raw.split_inclusive(|&b| b == b'\n') {
let dash = line.iter().position(|&b| b == b'-').unwrap_or(0);
let start = std::str::from_utf8(&line[..dash])
.ok()
.and_then(|s| u64::from_str_radix(s, 16).ok())
.unwrap_or(u64::MAX);
if start < WINDOW_FLOOR {
filtered.extend_from_slice(line);
}
}
memfd_with(&filtered)
}
_ => -2,
}
}
fn synth_fd_dirs() -> &'static Mutex<HashMap<i32, (i32, usize)>> {
static T: OnceLock<Mutex<HashMap<i32, (i32, usize)>>> = OnceLock::new();
T.get_or_init(|| Mutex::new(HashMap::new()))
}
fn synth_fd_dirents(pid: i32, cursor: usize, cap: usize) -> (Vec<u8>, usize) {
let fds: Vec<i32> = {
let t = fdt().lock().unwrap();
let mut v: Vec<i32> = t
.get(&pid)
.map(|m| m.keys().copied().collect())
.unwrap_or_default();
v.sort_unstable();
v
};
let mut names: Vec<(Vec<u8>, u8)> = Vec::with_capacity(fds.len() + 2);
names.push((b".".to_vec(), 4 ));
names.push((b"..".to_vec(), 4));
for fd in fds {
let mut nb = [0u8; 24];
let i = fmt_i64(fd as i64, &mut nb);
names.push((nb[i..].to_vec(), 10 ));
}
let mut out = Vec::new();
let mut idx = cursor;
while idx < names.len() {
let (name, dtype) = &names[idx];
let reclen = ((19 + name.len() + 1 + 7) & !7) as u16; if out.len() + reclen as usize > cap {
break; }
let start = out.len();
out.extend_from_slice(&((idx as u64) + 1).to_le_bytes()); out.extend_from_slice(&((idx as i64) + 1).to_le_bytes()); out.extend_from_slice(&reclen.to_le_bytes());
out.push(*dtype);
out.extend_from_slice(name);
out.push(0);
out.resize(start + reclen as usize, 0); idx += 1;
}
(out, idx)
}
fn synth_proc_dirs() -> &'static Mutex<HashMap<i32, usize>> {
static T: OnceLock<Mutex<HashMap<i32, usize>>> = OnceLock::new();
T.get_or_init(|| Mutex::new(HashMap::new()))
}
fn synth_proc_dirents(cursor: usize, cap: usize) -> (Vec<u8>, usize) {
let mut names: Vec<(Vec<u8>, u8)> = Vec::new();
names.push((b".".to_vec(), 4 ));
names.push((b"..".to_vec(), 4));
names.push((b"self".to_vec(), 10 ));
for (n, d) in [
(b"cpuinfo".as_slice(), 8u8),
(b"meminfo".as_slice(), 8),
(b"mounts".as_slice(), 8),
(b"uptime".as_slice(), 8),
(b"stat".as_slice(), 8),
(b"net".as_slice(), 4),
(b"sys".as_slice(), 4),
] {
names.push((n.to_vec(), d));
}
for h in proctree::live_host_pids() {
let mut nb = [0u8; 24];
let i = fmt_i64(h as i64, &mut nb);
names.push((nb[i..].to_vec(), 4 ));
}
let mut out = Vec::new();
let mut idx = cursor;
while idx < names.len() {
let (name, dtype) = &names[idx];
let reclen = ((19 + name.len() + 1 + 7) & !7) as u16; if out.len() + reclen as usize > cap {
break; }
let start = out.len();
out.extend_from_slice(&((idx as u64) + 1).to_le_bytes()); out.extend_from_slice(&((idx as i64) + 1).to_le_bytes()); out.extend_from_slice(&reclen.to_le_bytes());
out.push(*dtype);
out.extend_from_slice(name);
out.push(0);
out.resize(start + reclen as usize, 0); idx += 1;
}
(out, idx)
}
fn proc_exe() -> &'static Mutex<HashMap<i32, Vec<u8>>> {
static S: OnceLock<Mutex<HashMap<i32, Vec<u8>>>> = OnceLock::new();
S.get_or_init(|| Mutex::new(HashMap::new()))
}
fn proc_cmdline() -> &'static Mutex<HashMap<i32, Vec<u8>>> {
static S: OnceLock<Mutex<HashMap<i32, Vec<u8>>>> = OnceLock::new();
S.get_or_init(|| Mutex::new(HashMap::new()))
}
fn cmdline_for(pid: i32) -> Vec<u8> {
proc_cmdline()
.lock()
.unwrap()
.get(&pid)
.cloned()
.unwrap_or_default()
}
fn fdtrace_cmd_interesting(cmd: &[u8]) -> bool {
cmd.windows(b"pseudonymization-salt-handle".len())
.any(|w| w == b"pseudonymization-salt-handle")
|| cmd.windows(b"zygote".len()).any(|w| w == b"zygote")
}
fn host_fd_size(hfd: i32) -> Option<i64> {
let mut st: libc::stat = unsafe { std::mem::zeroed() };
let r = unsafe {
host(
SYS_FSTAT,
hfd as u64,
&mut st as *mut libc::stat as u64,
0,
0,
0,
0,
)
};
(r == 0).then_some(st.st_size)
}
fn repair_chromium_zygote_salt_fd(pid: i32) {
let cmd = cmdline_for(pid);
if !cmd
.windows(b"--type=zygote".len())
.any(|w| w == b"--type=zygote")
{
return;
}
if fd_host(pid, 10).and_then(host_fd_size) == Some(4) {
return;
}
let entries: Vec<(i32, FdVal)> = {
let t = fdt().lock().unwrap();
t.get(&pid)
.map(|m| m.iter().map(|(&g, &v)| (g, v)).collect())
.unwrap_or_default()
};
let mut candidate: Option<(i32, i32)> = None;
for (g, v) in entries {
if g == 10 {
continue;
}
let FdVal::Host(h) = v else {
continue;
};
if host_fd_size(h) == Some(4) {
candidate = Some((g, h));
}
}
let Some((src_g, src_h)) = candidate else {
return;
};
let dup = unsafe { host(SYS_DUP, src_h as u64, 0, 0, 0, 0, 0) };
if dup < 0 {
return;
}
fd_install_val_at_with_flags(pid, 10, FdVal::Host(dup as i32), 0);
ipc_logf(
&[
(b"FDSALTFIX pid=", pid as i64),
(b" src=", src_g as i64),
(b" h=", src_h as i64),
(b" dup=", dup),
],
b"",
);
}
fn fdtrace_dump_table(pid: i32, label: &[u8]) {
if !unsafe { FDTRACE } {
return;
}
let cmd = cmdline_for(pid);
let entries: Vec<(i32, FdVal)> = {
let t = fdt().lock().unwrap();
t.get(&pid)
.map(|m| m.iter().map(|(&g, &v)| (g, v)).collect())
.unwrap_or_default()
};
if !fdtrace_cmd_interesting(&cmd) && !entries.iter().any(|(g, _)| *g == 10 || *g == 11) {
return;
}
let mut line = Vec::new();
line.extend_from_slice(b"FDTRACE ");
line.extend_from_slice(label);
line.extend_from_slice(b" pid=");
line.extend_from_slice(pid.to_string().as_bytes());
line.extend_from_slice(b" cmd=");
for &b in cmd.iter().take(220) {
line.push(if b == 0 { b' ' } else { b });
}
ipc_logf_raw(&line);
for (g, v) in entries {
if g > 64 && g != 10 && g != 11 {
continue;
}
let mut row = Vec::new();
row.extend_from_slice(b"FDTRACE_FD pid=");
row.extend_from_slice(pid.to_string().as_bytes());
row.extend_from_slice(b" g=");
row.extend_from_slice(g.to_string().as_bytes());
row.extend_from_slice(b" flags=");
row.extend_from_slice(fd_get_desc_flags(pid, g, v).to_string().as_bytes());
match v {
FdVal::Host(h) => {
row.extend_from_slice(b" h=");
row.extend_from_slice(h.to_string().as_bytes());
if let Some((dev, ino)) = stat_dev_ino(h) {
row.extend_from_slice(b" dev=");
row.extend_from_slice(dev.to_string().as_bytes());
row.extend_from_slice(b" ino=");
row.extend_from_slice(ino.to_string().as_bytes());
}
}
FdVal::Loop(s) => {
row.extend_from_slice(b" loop=");
row.extend_from_slice((s as i64).to_string().as_bytes());
}
}
ipc_logf_raw(&row);
}
}
fn exe_for(pid: i32) -> Vec<u8> {
if let Some(e) = proc_exe().lock().unwrap().get(&pid) {
if !e.is_empty() {
return e.clone();
}
}
guest_exe().lock().unwrap().clone()
}
fn proc_self_readlink(pid: i32, bare: &[u8]) -> Option<Vec<u8>> {
let rel = proc_self_rel(pid, bare)?;
if rel == b"exe" {
let e = exe_for(pid);
return if e.is_empty() { None } else { Some(e) };
}
if rel == b"cwd" {
return Some(cwd_of(pid));
}
if let Some(num) = rel.strip_prefix(b"fd/") {
let n = parse_u32(num)?;
let h = fd_host(pid, n)?;
let mut nb = [0u8; 24];
let i = fmt_i64(h as i64, &mut nb);
let mut p = b"/proc/self/fd/".to_vec();
p.extend_from_slice(&nb[i..]);
p.push(0);
let mut buf = [0u8; 4096];
let r = unsafe {
host(
SYS_READLINK,
p.as_ptr() as u64,
buf.as_mut_ptr() as u64,
buf.len() as u64,
0,
0,
0,
)
};
if r <= 0 {
return None;
}
let target = &buf[..r as usize];
if target.first() != Some(&b'/') {
return Some(target.to_vec());
}
return host_to_guest_path(target);
}
None
}
fn open_path(pid: i32, path: &[u8], flags: u64, mode: u64) -> i64 {
let root = ROOT_FD.load(Ordering::Relaxed);
if root < 0 {
return unsafe { host(SYS_OPEN, path.as_ptr() as u64, flags, mode, 0, 0, 0) };
}
let bare = match path.split_last() {
Some((0, head)) => head,
_ => path,
};
if let Some(rel) = proc_self_rel(pid, bare) {
return open_proc_self(pid, rel, flags);
}
if let Some((tpid, rel)) = proc_pid_rel(bare) {
if proctree::vpid_for(tpid).is_some() {
return open_proc_self(tpid, rel, flags);
}
}
{
let tpid = if bare == b"/proc/self" || bare == b"/proc/thread-self" {
Some(pid)
} else {
proc_bare_pid(bare).filter(|&p| p == pid || proctree::vpid_for(p).is_some())
};
if let Some(tpid) = tpid {
let mut nb = [0u8; 24];
let i = fmt_i64(tpid as i64, &mut nb);
let mut p = b"/proc/".to_vec();
p.extend_from_slice(&nb[i..]);
p.push(0);
let f = flags & !(O_CREAT | O_TRUNC);
return unsafe { host(SYS_OPEN, p.as_ptr() as u64, f, 0, 0, 0, 0) };
}
}
if bare == b"/proc" || (bare.starts_with(b"/proc/") && bare[6..].iter().all(|&c| c == b'/')) {
let ph = b"/proc\0";
let fd = unsafe {
host(
SYS_OPEN,
ph.as_ptr() as u64,
(libc::O_RDONLY | libc::O_DIRECTORY) as u64,
0,
0,
0,
0,
)
};
if fd >= 0 {
synth_proc_dirs().lock().unwrap().insert(fd as i32, 0);
}
return fd;
}
if let Some(rel) = dev_fd_rel(bare) {
return open_proc_self(pid, &rel, flags);
}
if let Some(hostdev) = host_device_for(bare) {
let f = flags & !(O_CREAT | O_TRUNC);
let r = unsafe { host(SYS_OPEN, hostdev.as_ptr() as u64, f, 0, 0, 0, 0) };
if r >= 0 || !ends_with(bare, b"random") {
return r;
}
let mut buf = vec![0u8; 1 << 20];
let mut off = 0usize;
while off < buf.len() {
let n = unsafe {
host(
SYS_GETRANDOM,
buf[off..].as_mut_ptr() as u64,
(buf.len() - off) as u64,
0,
0,
0,
0,
)
};
if n <= 0 {
break;
}
off += n as usize;
}
if off == 0 {
return r;
}
return memfd_with(&buf[..off]);
}
if let Some(content) = proc_synth(bare) {
let f = flags & !(O_CREAT | O_TRUNC);
if is_write_open(f) {
return -30; }
return memfd_with(&content);
}
if let Some(procfile) = proc_redirect(bare) {
let f = flags & !(O_CREAT | O_TRUNC);
return unsafe { host(SYS_OPEN, procfile.as_ptr() as u64, f, 0, 0, 0, 0) };
}
let (dirfd, resolved, readonly) = resolve_mount(bare);
if readonly && is_write_open(flags) {
return -30; }
let created_candidate = (flags & O_CREAT) != 0;
let existed_before = if created_candidate {
let fd = opath(pid, path, false);
if fd >= 0 {
unsafe {
host(SYS_CLOSE, fd as u64, 0, 0, 0, 0, 0);
}
true
} else {
false
}
} else {
true
};
let how = OpenHow {
flags,
mode: mode_for_open_flags(pid, flags, mode),
resolve: RESOLVE_IN_ROOT,
};
let r = unsafe {
host(
SYS_OPENAT2,
dirfd as u64,
resolved.as_ptr() as u64,
&how as *const _ as u64,
std::mem::size_of::<OpenHow>() as u64,
0,
0,
)
};
if r >= 0 && created_candidate && !existed_before {
let exact_mode = mode_for_open_flags(pid, flags, mode);
unsafe { host(SYS_FCHMOD, r as u64, exact_mode, 0, 0, 0, 0) };
chown_created_fd_to_virtual(pid, r as i32);
}
if r < 0 && unsafe { IPCTRACE } {
let mut line: Vec<u8> = b"OPENFAIL errno=".to_vec();
let mut nb = [0u8; 24];
let i = fmt_i64(r, &mut nb);
line.extend_from_slice(&nb[i..]);
line.extend_from_slice(b" flags=0x");
let mut hb = [0u8; 16];
let mut fv = flags;
let mut hi = 16;
if fv == 0 {
hi -= 1;
hb[hi] = b'0';
}
while fv > 0 {
hi -= 1;
hb[hi] = b"0123456789abcdef"[(fv & 0xf) as usize];
fv >>= 4;
}
line.extend_from_slice(&hb[hi..]);
line.extend_from_slice(b" path=");
line.extend_from_slice(bare);
ipc_logf_raw(&line);
}
r
}
fn opath(pid: i32, path: &[u8], nofollow: bool) -> i64 {
let flags = O_PATH | if nofollow { O_NOFOLLOW } else { 0 };
open_path(pid, path, flags, 0)
}
fn split_parent_base(path_nul: &[u8]) -> (Vec<u8>, Vec<u8>) {
let bare = match path_nul.split_last() {
Some((0, head)) => head,
_ => path_nul,
};
let mut end = bare.len();
while end > 1 && bare[end - 1] == b'/' {
end -= 1;
}
let b = &bare[..end];
let (parent, base): (&[u8], &[u8]) = match b.iter().rposition(|&c| c == b'/') {
Some(0) => (b"/", &b[1..]),
Some(i) => (&b[..i], &b[i + 1..]),
None => (b".", b),
};
let mut pn = parent.to_vec();
pn.push(0);
let mut bn = base.to_vec();
bn.push(0);
(pn, bn)
}
fn confined_parent(pid: i32, dirfd: i32, path_ptr: u64) -> Result<(i32, Vec<u8>), i64> {
let raw = pull_path(pid, path_ptr); let raw_bare = match raw.split_last() {
Some((0, head)) => head,
_ => raw.as_slice(),
};
let absolute = raw_bare.first() == Some(&b'/');
if dirfd == AT_FDCWD || absolute {
let abs = if absolute {
raw.clone()
} else {
pull_cwd_path(pid, path_ptr)
};
let abs_bare = match abs.split_last() {
Some((0, head)) => head,
_ => abs.as_slice(),
};
if abs_bare.is_empty() {
return Err(-2); }
if abs_bare == b"/" {
return Err(-17); }
let (_, _, readonly) = resolve_mount(abs_bare);
if readonly {
return Err(-30); }
let (parent, base) = split_parent_base(&abs);
let pfd = open_path(pid, &parent, (O_PATH | O_DIRECTORY) as u64, 0);
if pfd < 0 {
return Err(pfd);
}
Ok((pfd as i32, base))
} else {
let hostdir = match fd_host(pid, dirfd) {
Some(h) => h,
None => return Err(-9), };
if raw_bare.is_empty() {
return Err(-2); }
let (parent_rel, base) = split_parent_base(&raw);
let how = OpenHow {
flags: O_PATH | O_DIRECTORY,
mode: 0,
resolve: RESOLVE_BENEATH,
};
let pfd = unsafe {
host(
SYS_OPENAT2,
hostdir as u64,
parent_rel.as_ptr() as u64,
&how as *const _ as u64,
std::mem::size_of::<OpenHow>() as u64,
0,
0,
)
};
if pfd < 0 {
return Err(pfd);
}
Ok((pfd as i32, base))
}
}
fn netns_bring_up_lo() {
#[repr(C)]
struct IfReq {
name: [u8; 16],
flags: i16,
_pad: [u8; 22],
}
const SIOCGIFFLAGS: u64 = 0x8913;
const SIOCSIFFLAGS: u64 = 0x8914;
const IFF_UP: i16 = 0x1;
unsafe {
let s = host(
SYS_SOCKET,
libc::AF_INET as u64,
libc::SOCK_DGRAM as u64,
0,
0,
0,
0,
);
if s < 0 {
return;
}
let mut req = IfReq {
name: [0; 16],
flags: 0,
_pad: [0; 22],
};
req.name[0] = b'l';
req.name[1] = b'o';
host(
SYS_IOCTL,
s as u64,
SIOCGIFFLAGS,
&mut req as *mut _ as u64,
0,
0,
0,
);
req.flags |= IFF_UP;
host(
SYS_IOCTL,
s as u64,
SIOCSIFFLAGS,
&mut req as *mut _ as u64,
0,
0,
0,
);
host(SYS_CLOSE, s as u64, 0, 0, 0, 0, 0);
}
}
static HOST_NETNS_FD: std::sync::atomic::AtomicI32 = std::sync::atomic::AtomicI32::new(-1);
static SANDBOX_NETNS_FD: std::sync::atomic::AtomicI32 = std::sync::atomic::AtomicI32::new(-1);
static OWN_LOOPBACK: AtomicBool = AtomicBool::new(false);
fn own_loopback() -> bool {
OWN_LOOPBACK.load(Ordering::Relaxed)
}
fn setup_netns() {
let nspath = b"/proc/self/ns/net\0";
let host_fd = unsafe {
host(
SYS_OPEN,
nspath.as_ptr() as u64,
libc::O_RDONLY as u64,
0,
0,
0,
0,
)
};
let r = unsafe { host(SYS_UNSHARE, libc::CLONE_NEWNET as u64, 0, 0, 0, 0, 0) };
if r < 0 {
if host_fd >= 0 {
unsafe { host(SYS_CLOSE, host_fd as u64, 0, 0, 0, 0, 0) };
}
if unsafe { TRACE } {
log(b"[supervisor] netns unshare failed; running in the shared host netns\n");
}
return;
}
netns_bring_up_lo();
let sb_fd = unsafe {
host(
SYS_OPEN,
nspath.as_ptr() as u64,
libc::O_RDONLY as u64,
0,
0,
0,
0,
)
};
if host_fd >= 0 && sb_fd >= 0 {
HOST_NETNS_FD.store(host_fd as i32, Ordering::Relaxed);
SANDBOX_NETNS_FD.store(sb_fd as i32, Ordering::Relaxed);
} else {
if host_fd >= 0 {
unsafe { host(SYS_CLOSE, host_fd as u64, 0, 0, 0, 0, 0) };
}
if sb_fd >= 0 {
unsafe { host(SYS_CLOSE, sb_fd as u64, 0, 0, 0, 0, 0) };
}
}
}
fn rehome_to_host_netns(old: i32) {
let host_ns = HOST_NETNS_FD.load(Ordering::Relaxed);
let sb_ns = SANDBOX_NETNS_FD.load(Ordering::Relaxed);
if host_ns < 0 || sb_ns < 0 {
return;
}
if socket_in_host_netns(old) {
return;
}
let dom = getsockopt_int(old, 39); let typ = getsockopt_int(old, 3); if dom < 0 || typ < 0 {
return;
}
let proto = getsockopt_int(old, 38).max(0); let fl = unsafe { host(SYS_FCNTL, old as u64, libc::F_GETFL as u64, 0, 0, 0, 0) };
unsafe {
if host(
SYS_SETNS,
host_ns as u64,
libc::CLONE_NEWNET as u64,
0,
0,
0,
0,
) < 0
{
return;
}
let new = host(SYS_SOCKET, dom as u64, typ as u64, proto as u64, 0, 0, 0);
host(
SYS_SETNS,
sb_ns as u64,
libc::CLONE_NEWNET as u64,
0,
0,
0,
0,
);
if new < 0 {
return;
}
if fl >= 0 {
host(
SYS_FCNTL,
new as u64,
libc::F_SETFL as u64,
fl as u64,
0,
0,
0,
); }
host(SYS_DUP2, new as u64, old as u64, 0, 0, 0, 0); host(SYS_CLOSE, new as u64, 0, 0, 0, 0, 0);
}
}
fn socket_in_host_netns(fd: i32) -> bool {
let host_ns = HOST_NETNS_FD.load(Ordering::Relaxed);
if host_ns < 0 {
return false;
}
const SIOCGSKNS: u64 = 0x894C;
unsafe {
let nsfd = host(SYS_IOCTL, fd as u64, SIOCGSKNS, 0, 0, 0, 0);
if nsfd < 0 {
return false;
}
let sock_ns = stat_dev_ino(nsfd as i32);
host(SYS_CLOSE, nsfd as u64, 0, 0, 0, 0, 0);
let host_id = stat_dev_ino(host_ns);
sock_ns.is_some() && sock_ns == host_id
}
}
fn stat_dev_ino(fd: i32) -> Option<(u64, u64)> {
let mut st = [0u8; 144];
let r = unsafe { host(SYS_FSTAT, fd as u64, st.as_mut_ptr() as u64, 0, 0, 0, 0) };
if r < 0 {
return None;
}
let dev = u64::from_le_bytes(st[0..8].try_into().unwrap());
let ino = u64::from_le_bytes(st[8..16].try_into().unwrap());
Some((dev, ino))
}
fn getsockopt_int(fd: i32, optname: i32) -> i32 {
let mut val: i32 = -1;
let mut len: u32 = 4;
let r = unsafe {
host(
SYS_GETSOCKOPT,
fd as u64,
1, optname as u64,
&mut val as *mut i32 as u64,
&mut len as *mut u32 as u64,
0,
)
};
if r < 0 {
-1
} else {
val
}
}
fn parse_connect_addr(addr: &[u8]) -> Option<std::net::SocketAddr> {
if addr.len() < 2 {
return None;
}
let family = u16::from_ne_bytes([addr[0], addr[1]]);
match family {
2 => {
if addr.len() < 8 {
return None;
}
let port = u16::from_be_bytes([addr[2], addr[3]]);
let ip = std::net::Ipv4Addr::new(addr[4], addr[5], addr[6], addr[7]);
Some(std::net::SocketAddr::from((ip, port)))
}
10 => {
if addr.len() < 24 {
return None;
}
let port = u16::from_be_bytes([addr[2], addr[3]]);
let mut o = [0u8; 16];
o.copy_from_slice(&addr[8..24]);
Some(std::net::SocketAddr::from((
std::net::Ipv6Addr::from(o),
port,
)))
}
_ => None,
}
}
#[repr(C)]
struct LocalIov {
base: u64,
len: u64,
}
#[repr(C)]
struct LocalMsghdr {
name: u64,
namelen: u32,
_p: u32,
iov: u64,
iovlen: u64,
control: u64,
controllen: u64,
flags: i32,
_p2: i32,
}
const SCM_SOL_SOCKET: i32 = 1;
const SCM_RIGHTS_TYPE: i32 = 1;
const SCM_CREDENTIALS_TYPE: i32 = 2;
const SCM_CTL_CAP: usize = 8192;
#[inline]
fn cmsg_align(n: usize) -> usize {
(n + 7) & !7
}
fn scm_count_fds(ctl: &[u8]) -> usize {
let mut n = 0usize;
let mut off = 0usize;
while off + 16 <= ctl.len() {
let clen = u64::from_le_bytes(ctl[off..off + 8].try_into().unwrap()) as usize;
if clen < 16 || off + clen > ctl.len() {
break;
}
let level = i32::from_le_bytes(ctl[off + 8..off + 12].try_into().unwrap());
let ctype = i32::from_le_bytes(ctl[off + 12..off + 16].try_into().unwrap());
if level == SCM_SOL_SOCKET && ctype == SCM_RIGHTS_TYPE {
n += (clen - 16) / 4;
}
off += cmsg_align(clen);
}
n
}
fn scm_send_control(pid: i32, gctl: &[u8]) -> Result<Vec<u8>, i64> {
let mut out: Vec<u8> = Vec::new();
let mut off = 0usize;
while off + 16 <= gctl.len() {
let clen = u64::from_le_bytes(gctl[off..off + 8].try_into().unwrap()) as usize;
if clen < 16 || off + clen > gctl.len() {
break;
}
let level = i32::from_le_bytes(gctl[off + 8..off + 12].try_into().unwrap());
let ctype = i32::from_le_bytes(gctl[off + 12..off + 16].try_into().unwrap());
if level == SCM_SOL_SOCKET && ctype == SCM_RIGHTS_TYPE {
let data = &gctl[off + 16..off + clen];
let nfds = data.len() / 4;
let hdr = out.len();
out.extend_from_slice(&0u64.to_le_bytes()); out.extend_from_slice(&SCM_SOL_SOCKET.to_le_bytes());
out.extend_from_slice(&SCM_RIGHTS_TYPE.to_le_bytes());
for i in 0..nfds {
let gfd = i32::from_le_bytes(data[i * 4..i * 4 + 4].try_into().unwrap());
match fd_host(pid, gfd) {
Some(h) => out.extend_from_slice(&h.to_le_bytes()),
None => return Err(-9), }
}
let clen_h = (16 + nfds * 4) as u64;
out[hdr..hdr + 8].copy_from_slice(&clen_h.to_le_bytes());
while out.len() % 8 != 0 {
out.push(0);
}
}
off += cmsg_align(clen);
}
Ok(out)
}
fn append_scm_credentials(ctl: &mut Vec<u8>, pid: i32) {
let hdr = ctl.len();
ctl.extend_from_slice(&0u64.to_le_bytes());
ctl.extend_from_slice(&SCM_SOL_SOCKET.to_le_bytes());
ctl.extend_from_slice(&SCM_CREDENTIALS_TYPE.to_le_bytes());
ctl.extend_from_slice(&pid.to_le_bytes());
ctl.extend_from_slice(&0u32.to_le_bytes()); ctl.extend_from_slice(&0u32.to_le_bytes()); let clen = 28u64; ctl[hdr..hdr + 8].copy_from_slice(&clen.to_le_bytes());
while ctl.len() % 8 != 0 {
ctl.push(0);
}
}
struct ScmRecv {
gctl: Vec<u8>,
installed: usize,
truncated: bool,
}
fn scm_recv_control(pid: i32, hctl: &[u8], guest_cap: usize) -> ScmRecv {
let mut out: Vec<u8> = Vec::new();
let mut installed = 0usize;
let mut truncated = false;
let mut off = 0usize;
while off + 16 <= hctl.len() {
let clen = u64::from_le_bytes(hctl[off..off + 8].try_into().unwrap()) as usize;
if clen < 16 || off + clen > hctl.len() {
break;
}
let level = i32::from_le_bytes(hctl[off + 8..off + 12].try_into().unwrap());
let ctype = i32::from_le_bytes(hctl[off + 12..off + 16].try_into().unwrap());
if level == SCM_SOL_SOCKET && ctype == SCM_RIGHTS_TYPE {
let data = &hctl[off + 16..off + clen];
let nfds = data.len() / 4;
let start = out.len();
let hdr = out.len();
out.extend_from_slice(&0u64.to_le_bytes());
out.extend_from_slice(&SCM_SOL_SOCKET.to_le_bytes());
out.extend_from_slice(&SCM_RIGHTS_TYPE.to_le_bytes());
let mut here = 0usize;
for i in 0..nfds {
let hfd = i32::from_le_bytes(data[i * 4..i * 4 + 4].try_into().unwrap());
let projected = start + cmsg_align(16 + (here + 1) * 4);
if projected > guest_cap {
truncated = true;
unsafe {
host(SYS_CLOSE, hfd as u64, 0, 0, 0, 0, 0);
}
continue;
}
let g = fd_install(pid, hfd, 0) as i32;
if unsafe { IPCTRACE } {
let mut st = [0u8; 144];
let ok =
unsafe { host(SYS_FSTAT, hfd as u64, st.as_mut_ptr() as u64, 0, 0, 0, 0) };
let (dev, ino, size) = if ok == 0 {
(
u64::from_le_bytes(st[0..8].try_into().unwrap()),
u64::from_le_bytes(st[8..16].try_into().unwrap()),
u64::from_le_bytes(st[48..56].try_into().unwrap()),
)
} else {
(0, 0, 0)
};
ipc_logf(
&[
(b"SCMRECV_FD pid=", pid as i64),
(b"hfd=", hfd as i64),
(b"gfd=", g as i64),
(b"dev=", dev as i64),
(b"ino=", ino as i64),
(b"sz=", size as i64),
],
&[],
);
}
out.extend_from_slice(&g.to_le_bytes());
here += 1;
installed += 1;
}
if here == 0 {
out.truncate(start);
} else {
let clen_h = (16 + here * 4) as u64;
out[hdr..hdr + 8].copy_from_slice(&clen_h.to_le_bytes());
while out.len() % 8 != 0 {
out.push(0);
}
}
} else if level == SCM_SOL_SOCKET && ctype == SCM_CREDENTIALS_TYPE && clen >= 28 {
let aligned = cmsg_align(28);
if out.len() + aligned > guest_cap {
truncated = true;
} else {
let hdr = out.len();
out.extend_from_slice(&0u64.to_le_bytes());
out.extend_from_slice(&SCM_SOL_SOCKET.to_le_bytes());
out.extend_from_slice(&SCM_CREDENTIALS_TYPE.to_le_bytes());
out.extend_from_slice(&hctl[off + 16..off + 28]);
let clen_h = 28u64;
out[hdr..hdr + 8].copy_from_slice(&clen_h.to_le_bytes());
while out.len() % 8 != 0 {
out.push(0);
}
}
}
off += cmsg_align(clen);
}
ScmRecv {
gctl: out,
installed,
truncated,
}
}
#[inline]
fn pollfd_bytes(fd: i32, events: i16) -> [u8; 8] {
let mut b = [0u8; 8];
b[0..4].copy_from_slice(&fd.to_le_bytes());
b[4..6].copy_from_slice(&events.to_le_bytes());
b
}
unsafe fn wait_or_cancel(fd: i32, efd: i32, intr_efd: i32, timeout_ms: i64) -> i32 {
unsafe {
let mut pf = [0u8; 24];
pf[0..8].copy_from_slice(&pollfd_bytes(fd, POLLIN_BIT | POLLPRI_BIT));
pf[8..16].copy_from_slice(&pollfd_bytes(efd, POLLIN_BIT));
pf[16..24].copy_from_slice(&pollfd_bytes(intr_efd, POLLIN_BIT));
let r = host(
SYS_POLL,
pf.as_mut_ptr() as u64,
3,
timeout_ms as u64,
0,
0,
0,
);
if r == EINTR {
return -2;
}
if r == 0 {
return 0;
}
if i16::from_le_bytes([pf[14], pf[15]]) != 0 {
return -1;
}
if i16::from_le_bytes([pf[22], pf[23]]) != 0 {
let mut drain = [0u8; 8];
host(
SYS_READ,
intr_efd as u64,
drain.as_mut_ptr() as u64,
8,
0,
0,
0,
);
let sr = SERVICER_RING.with(|cell| cell.get());
if !sr.is_null() {
std::ptr::write_volatile(std::ptr::addr_of_mut!((*sr).intr_pending), 0);
}
return -3;
}
1
}
}
unsafe fn host_cancellable(nr: i64, a: u64, b: u64, c: u64, d: u64, e: u64, f: u64) -> i64 {
unsafe {
let r = SERVICER_RING.with(|cell| cell.get());
let (efd, intr_efd) = if r.is_null() {
(-1, -1)
} else {
(
std::ptr::read_volatile(std::ptr::addr_of!((*r).cancel_efd)),
std::ptr::read_volatile(std::ptr::addr_of!((*r).intr_efd)),
)
};
if efd < 0 {
loop {
let n = host(nr, a, b, c, d, e, f);
if n != EINTR {
return n;
}
}
}
match nr {
SYS_POLL => {
let nfds = b as usize;
let mut local = vec![0u8; (nfds + 2) * 8];
if nfds > 0 {
std::ptr::copy_nonoverlapping(a as *const u8, local.as_mut_ptr(), nfds * 8);
}
local[nfds * 8..nfds * 8 + 8].copy_from_slice(&pollfd_bytes(efd, POLLIN_BIT));
local[nfds * 8 + 8..nfds * 8 + 16]
.copy_from_slice(&pollfd_bytes(intr_efd, POLLIN_BIT));
loop {
let rr = host(
SYS_POLL,
local.as_mut_ptr() as u64,
(nfds + 2) as u64,
c,
0,
0,
0,
);
if rr == EINTR {
continue;
}
let cancel = (nfds + 1) * 8;
let intr = (nfds + 2) * 8;
if i16::from_le_bytes([local[cancel - 2], local[cancel - 1]]) != 0 {
return CANCEL_SENTINEL;
}
if i16::from_le_bytes([local[intr - 2], local[intr - 1]]) != 0 {
let mut drain = [0u8; 8];
host(
SYS_READ,
intr_efd as u64,
drain.as_mut_ptr() as u64,
8,
0,
0,
0,
);
std::ptr::write_volatile(std::ptr::addr_of_mut!((*r).intr_pending), 0);
return EINTR;
}
if nfds > 0 {
std::ptr::copy_nonoverlapping(local.as_ptr(), a as *mut u8, nfds * 8);
}
return rr; }
}
SYS_NANOSLEEP | SYS_CLOCK_NANOSLEEP => {
let (clk, flags, req_ptr, rem_ptr) = if nr == SYS_NANOSLEEP {
(0u64, 0u64, a, b)
} else {
(a, b, c, d)
};
let secs = i64::from_le_bytes(*(req_ptr as *const [u8; 8]));
let nsecs = i64::from_le_bytes(*((req_ptr + 8) as *const [u8; 8]));
let (mut tsec, mut tnsec) = (secs, nsecs);
if nr == SYS_CLOCK_NANOSLEEP && (flags & 1) != 0 {
let (nsec0, nnsec0) = if clk as i32 == libc::CLOCK_MONOTONIC
|| clk as i32 == libc::CLOCK_BOOTTIME
{
let ts = guest_monotonic_timespec();
(
i64::from_le_bytes(ts[0..8].try_into().unwrap()),
i64::from_le_bytes(ts[8..16].try_into().unwrap()),
)
} else {
let mut now = [0u8; 16];
host(SYS_CLOCK_GETTIME, clk, now.as_mut_ptr() as u64, 0, 0, 0, 0);
(
i64::from_le_bytes(now[0..8].try_into().unwrap()),
i64::from_le_bytes(now[8..16].try_into().unwrap()),
)
};
tsec = secs - nsec0;
tnsec = nsecs - nnsec0;
}
let mut tmo_ms = tsec.saturating_mul(1000).saturating_add(tnsec / 1_000_000);
if tmo_ms < 0 {
tmo_ms = 0;
} else if tmo_ms == 0 && (tsec > 0 || tnsec > 0) {
tmo_ms = 1;
}
let tmo_ms = tmo_ms.clamp(0, i32::MAX as i64);
loop {
match wait_or_cancel(-1, efd, intr_efd, tmo_ms) {
-1 => return CANCEL_SENTINEL,
-3 => return EINTR,
-2 => continue,
_ => {
if rem_ptr != 0 {
std::ptr::write_bytes(rem_ptr as *mut u8, 0, 16);
}
return 0;
}
}
}
}
SYS_EPOLL_WAIT | SYS_EPOLL_PWAIT => {
let tmo = d as i64;
loop {
match wait_or_cancel(a as i32, efd, intr_efd, tmo) {
-1 => return CANCEL_SENTINEL,
-3 => return EINTR,
0 => return 0,
-2 => continue,
_ => {
let n = host(SYS_EPOLL_WAIT, a, b, c, 0, 0, 0);
if n != 0 {
return n;
}
}
}
}
}
SYS_FCNTL => {
let nb_cmd = match b {
F_SETLKW => F_SETLK,
F_OFD_SETLKW => F_OFD_SETLK,
_ => return host(SYS_FCNTL, a, b, c, d, e, f),
};
loop {
let n = host(SYS_FCNTL, a, nb_cmd, c, 0, 0, 0);
if n != EAGAIN && n != -13 {
return n; }
match wait_or_cancel(-1, efd, intr_efd, 20) {
-1 => return CANCEL_SENTINEL,
-3 => return EINTR,
_ => continue,
}
}
}
_ => {
let fd = a as i32;
let saved = host(SYS_FCNTL, fd as u64, libc::F_GETFL as u64, 0, 0, 0, 0);
let guest_nonblock = saved >= 0 && (saved as u64 & libc::O_NONBLOCK as u64) != 0;
const MSG_DONTWAIT: u64 = 0x40;
let percall_nonblock = match nr {
SYS_RECVMSG => (c & MSG_DONTWAIT) != 0,
SYS_RECVFROM | SYS_RECVMMSG => (d & MSG_DONTWAIT) != 0,
_ => false,
};
let nonblock = guest_nonblock || percall_nonblock;
if saved >= 0 && !nonblock {
host(
SYS_FCNTL,
fd as u64,
libc::F_SETFL as u64,
(saved as u64) | (libc::O_NONBLOCK as u64),
0,
0,
0,
);
}
let res = loop {
let n = host(nr, a, b, c, d, e, f);
if n != EAGAIN && n != EINTR {
break n;
}
if n == EINTR {
continue;
}
if nonblock {
break n;
}
match wait_or_cancel(fd, efd, intr_efd, -1) {
-1 => break CANCEL_SENTINEL,
-3 => break EINTR,
_ => continue,
}
};
if saved >= 0 && !nonblock {
host(
SYS_FCNTL,
fd as u64,
libc::F_SETFL as u64,
saved as u64,
0,
0,
0,
);
}
res
}
}
}
}
const POLLIN_B: u16 = 0x001;
const POLLPRI_B: u16 = 0x002;
const POLLOUT_B: u16 = 0x004;
const POLLNVAL_B: u16 = 0x020;
enum PollKind {
Loop(netstack::SockId),
Host(i32),
Nval,
Ignore,
}
fn loop_revents(ls: &LoopState, sid: netstack::SockId, events: u16) -> u16 {
let mut r = 0u16;
if (events & POLLIN_B) != 0 && ls.net.readable(sid) {
r |= POLLIN_B;
}
if (events & POLLOUT_B) != 0 && ls.net.writable(sid) {
r |= POLLOUT_B;
}
r
}
fn poll_merge_entries(pid: i32, gfds: &[(i32, u16)], timeout_ms: i64) -> (i64, Vec<u16>) {
let n = gfds.len();
let mut ents: Vec<(usize, PollKind, u16)> = Vec::with_capacity(n);
for (i, &(g, ev)) in gfds.iter().enumerate() {
let k = if g < 0 {
PollKind::Ignore
} else if let Some(sid) = fd_loop(pid, g) {
PollKind::Loop(sid)
} else if let Some(h) = fd_host(pid, g) {
PollKind::Host(h)
} else {
PollKind::Nval
};
ents.push((i, k, ev));
}
let has_loop = ents.iter().any(|(_, k, _)| matches!(k, PollKind::Loop(_)));
let deadline = if timeout_ms < 0 {
None
} else {
Some(now_ns().saturating_add((timeout_ms as u64).saturating_mul(1_000_000)))
};
loop {
let mut rev = vec![0u16; n];
let mut any_loop_ready = false;
{
let ls = loop_state().lock().unwrap();
for (i, k, ev) in &ents {
match k {
PollKind::Loop(sid) => {
let r = loop_revents(&ls, *sid, *ev);
if r != 0 {
any_loop_ready = true;
}
rev[*i] = r;
}
PollKind::Nval => rev[*i] = POLLNVAL_B,
_ => {}
}
}
}
let mut tp: Vec<u8> = Vec::new();
let mut host_map: Vec<(usize, usize)> = Vec::new();
let mut efds: Vec<(usize, i32)> = Vec::new();
{
let ls = loop_state().lock().unwrap();
for (i, k, ev) in &ents {
match k {
PollKind::Host(h) => {
host_map.push((tp.len() / 8, *i));
tp.extend_from_slice(&h.to_le_bytes());
tp.extend_from_slice(&ev.to_le_bytes());
tp.extend_from_slice(&0u16.to_le_bytes());
}
PollKind::Loop(sid) => {
if let Some(&efd) = ls.efds.get(sid) {
let slot = tp.len() / 8;
tp.extend_from_slice(&efd.to_le_bytes());
tp.extend_from_slice(&POLLIN_B.to_le_bytes());
tp.extend_from_slice(&0u16.to_le_bytes());
efds.push((slot, efd));
}
}
_ => {}
}
}
}
let nt = (tp.len() / 8) as u64;
let t: i64 = if any_loop_ready {
0
} else {
let base = match deadline {
None => -1,
Some(d) => {
let now = now_ns();
if now >= d {
0
} else {
((d - now) / 1_000_000).min(i32::MAX as u64) as i64
}
}
};
if has_loop && (base < 0 || base > LOOP_WAIT_TICK_MS) {
LOOP_WAIT_TICK_MS
} else {
base
}
};
let r =
unsafe { host_cancellable(SYS_POLL, tp.as_mut_ptr() as u64, nt, t as u64, 0, 0, 0) };
if r == CANCEL_SENTINEL {
return (CANCEL_SENTINEL, rev);
}
for &(slot, efd) in &efds {
let revents = u16::from_le_bytes(tp[slot * 8 + 6..slot * 8 + 8].try_into().unwrap());
if (revents & POLLIN_B) != 0 {
let mut sink = [0u8; 8];
unsafe { host(SYS_READ, efd as u64, sink.as_mut_ptr() as u64, 8, 0, 0, 0) };
}
}
for &(ts, oi) in &host_map {
rev[oi] |= u16::from_le_bytes(tp[ts * 8 + 6..ts * 8 + 8].try_into().unwrap());
}
{
let ls = loop_state().lock().unwrap();
for (i, k, ev) in &ents {
if let PollKind::Loop(sid) = k {
rev[*i] = loop_revents(&ls, *sid, *ev);
}
}
}
let mut ready = 0i64;
for &v in &rev {
if v != 0 {
ready += 1;
}
}
let elapsed = matches!(deadline, Some(d) if now_ns() >= d);
if ready > 0 || elapsed || r < 0 {
return (if r < 0 { r } else { ready }, rev);
}
}
}
fn poll_merge(pid: i32, pfds_ptr: u64, nfds: u64, timeout_ms: i64) -> i64 {
let n = (nfds.min(1024)) as usize;
let mut fds = vec![0u8; n * 8];
vm_read(pid, pfds_ptr, &mut fds);
let mut ents: Vec<(usize, PollKind, u16)> = Vec::with_capacity(n);
for i in 0..n {
let g = i32::from_le_bytes(fds[i * 8..i * 8 + 4].try_into().unwrap());
let ev = u16::from_le_bytes(fds[i * 8 + 4..i * 8 + 6].try_into().unwrap());
let k = if g < 0 {
PollKind::Ignore
} else if let Some(sid) = fd_loop(pid, g) {
PollKind::Loop(sid)
} else if let Some(h) = fd_host(pid, g) {
PollKind::Host(h)
} else {
PollKind::Nval
};
ents.push((i, k, ev));
}
let has_loop = ents.iter().any(|(_, k, _)| matches!(k, PollKind::Loop(_)));
let deadline = if timeout_ms < 0 {
None
} else {
Some(now_ns().saturating_add((timeout_ms as u64).saturating_mul(1_000_000)))
};
loop {
let mut rev = vec![0u16; n];
let mut any_loop_ready = false;
{
let ls = loop_state().lock().unwrap();
for (i, k, ev) in &ents {
match k {
PollKind::Loop(sid) => {
let r = loop_revents(&ls, *sid, *ev);
if r != 0 {
any_loop_ready = true;
}
rev[*i] = r;
}
PollKind::Nval => rev[*i] = POLLNVAL_B,
_ => {}
}
}
}
let mut tp: Vec<u8> = Vec::new();
let mut host_map: Vec<(usize, usize)> = Vec::new();
let mut efds: Vec<(usize, i32)> = Vec::new();
{
let ls = loop_state().lock().unwrap();
for (i, k, ev) in &ents {
match k {
PollKind::Host(h) => {
host_map.push((tp.len() / 8, *i));
tp.extend_from_slice(&h.to_le_bytes());
tp.extend_from_slice(&ev.to_le_bytes());
tp.extend_from_slice(&0u16.to_le_bytes());
}
PollKind::Loop(sid) => {
if let Some(&efd) = ls.efds.get(sid) {
let slot = tp.len() / 8;
tp.extend_from_slice(&efd.to_le_bytes());
tp.extend_from_slice(&POLLIN_B.to_le_bytes());
tp.extend_from_slice(&0u16.to_le_bytes());
efds.push((slot, efd));
}
}
_ => {}
}
}
}
let nt = (tp.len() / 8) as u64;
let t: i64 = if any_loop_ready {
0
} else {
let base = match deadline {
None => -1,
Some(d) => {
let now = now_ns();
if now >= d {
0
} else {
((d - now) / 1_000_000).min(i32::MAX as u64) as i64
}
}
};
if has_loop && (base < 0 || base > LOOP_WAIT_TICK_MS) {
LOOP_WAIT_TICK_MS
} else {
base
}
};
let r =
unsafe { host_cancellable(SYS_POLL, tp.as_mut_ptr() as u64, nt, t as u64, 0, 0, 0) };
if r == CANCEL_SENTINEL {
return CANCEL_SENTINEL;
}
for &(slot, efd) in &efds {
let revents = u16::from_le_bytes(tp[slot * 8 + 6..slot * 8 + 8].try_into().unwrap());
if (revents & POLLIN_B) != 0 {
let mut sink = [0u8; 8];
unsafe { host(SYS_READ, efd as u64, sink.as_mut_ptr() as u64, 8, 0, 0, 0) };
}
}
for &(ts, oi) in &host_map {
rev[oi] |= u16::from_le_bytes(tp[ts * 8 + 6..ts * 8 + 8].try_into().unwrap());
}
{
let ls = loop_state().lock().unwrap();
for (i, k, ev) in &ents {
if let PollKind::Loop(sid) = k {
rev[*i] = loop_revents(&ls, *sid, *ev);
}
}
}
let mut ready = 0i64;
for &v in &rev {
if v != 0 {
ready += 1;
}
}
let elapsed = matches!(deadline, Some(d) if now_ns() >= d);
if ready > 0 || elapsed || r < 0 {
for i in 0..n {
fds[i * 8 + 6..i * 8 + 8].copy_from_slice(&rev[i].to_le_bytes());
}
vm_write(pid, pfds_ptr, &fds);
return if r < 0 { r } else { ready };
}
}
}
fn epoll_loops() -> &'static Mutex<HashMap<i32, HashMap<netstack::SockId, (u32, u64, u32)>>> {
static T: OnceLock<Mutex<HashMap<i32, HashMap<netstack::SockId, (u32, u64, u32)>>>> =
OnceLock::new();
T.get_or_init(|| Mutex::new(HashMap::new()))
}
const EPOLLIN_B: u32 = 0x001;
const EPOLLOUT_B: u32 = 0x004;
const EPOLLET_B: u32 = 0x8000_0000;
fn et_filter(events: u32, ready: u32, et: &mut u32) -> u32 {
if events & EPOLLET_B == 0 {
*et = 0; return ready;
}
*et &= ready; let report = ready & !*et; *et |= report; report
}
fn epdbg_on() -> bool {
static O: std::sync::OnceLock<bool> = std::sync::OnceLock::new();
*O.get_or_init(|| std::env::var("SENTRY_EPDBG").is_ok())
}
const EPOLL_LOOP_TAG: u64 = 1 << 63;
fn epoll_drop_sid(sid: netstack::SockId) {
for (epfd, m) in epoll_loops().lock().unwrap().iter_mut() {
if m.remove(&sid).is_some() && epdbg_on() {
ipc_logf(
&[(b"DROPSID epfd=", *epfd as i64), (b" sid=", sid as i64)],
b"",
);
}
}
}
fn restore_loop_state_from_snapshot(snap: &netstack::LoopSnapshot) {
let mut ls = loop_state().lock().unwrap();
for (_, efd) in ls.efds.drain() {
unsafe { host(SYS_CLOSE, efd as u64, 0, 0, 0, 0, 0) };
}
ls.net = netstack::LoopNet::restore(snap);
for sock in &snap.socks {
ls.efds.insert(sock.id, loop_new_efd());
}
ls.nonblock = snap.nonblock.iter().copied().collect();
}
fn restore_epoll_loop_fd(watches: &[state_snap::EpollLoopWatch]) -> std::io::Result<i32> {
let epfd = unsafe { host(SYS_EPOLL_CREATE1, 0, 0, 0, 0, 0, 0) };
if epfd < 0 {
return Err(std::io::Error::from_raw_os_error((-epfd) as i32));
}
let epfd = epfd as i32;
for watch in watches {
let sid = watch.socket.0;
let efd = match loop_state().lock().unwrap().efds.get(&sid).copied() {
Some(efd) => efd,
None => {
unsafe { host(SYS_CLOSE, epfd as u64, 0, 0, 0, 0, 0) };
return Err(std::io::Error::new(
std::io::ErrorKind::InvalidData,
format!("snapshot: epoll watch references missing loop socket {sid}"),
));
}
};
let mut hev = [0u8; 12];
hev[0..4].copy_from_slice(&EPOLLIN_B.to_le_bytes());
hev[4..12].copy_from_slice(&(EPOLL_LOOP_TAG | sid as u64).to_le_bytes());
let r = unsafe {
host(
SYS_EPOLL_CTL,
epfd as u64,
1,
efd as u64,
hev.as_ptr() as u64,
0,
0,
)
};
if r < 0 && r != -17 {
unsafe { host(SYS_CLOSE, epfd as u64, 0, 0, 0, 0, 0) };
return Err(std::io::Error::from_raw_os_error((-r) as i32));
}
epoll_loops()
.lock()
.unwrap()
.entry(epfd)
.or_default()
.insert(sid, (watch.events, watch.data, 0));
}
Ok(epfd)
}
fn epoll_ctl_loop(pid: i32, host_epfd: i32, op: u64, sid: netstack::SockId, ev_ptr: u64) -> i64 {
let (events, data) = if ev_ptr != 0 {
let mut ev = [0u8; 12];
vm_read(pid, ev_ptr, &mut ev);
(
u32::from_le_bytes(ev[0..4].try_into().unwrap()),
u64::from_le_bytes(ev[4..12].try_into().unwrap()),
)
} else {
(0, 0)
};
let efd = match loop_state().lock().unwrap().efds.get(&sid).copied() {
Some(e) => e,
None => return -9, };
match op {
1 | 3 => {
epoll_loops()
.lock()
.unwrap()
.entry(host_epfd)
.or_default()
.insert(sid, (events, data, 0));
if epdbg_on() {
ipc_logf(
&[
(b"CTLADD epfd=", host_epfd as i64),
(b" sid=", sid as i64),
(b" op=", op as i64),
],
b"",
);
}
let mut hev = [0u8; 12];
hev[0..4].copy_from_slice(&EPOLLIN_B.to_le_bytes());
hev[4..12].copy_from_slice(&(EPOLL_LOOP_TAG | sid as u64).to_le_bytes());
let r = unsafe {
host(
SYS_EPOLL_CTL,
host_epfd as u64,
1,
efd as u64,
hev.as_ptr() as u64,
0,
0,
)
};
if r == -17 {
unsafe {
host(
SYS_EPOLL_CTL,
host_epfd as u64,
3,
efd as u64,
hev.as_ptr() as u64,
0,
0,
)
};
}
let ready_now = {
let ls = loop_state().lock().unwrap();
((events & EPOLLIN_B) != 0 && ls.net.readable(sid))
|| ((events & EPOLLOUT_B) != 0 && ls.net.writable(sid))
};
if ready_now {
let one = 1u64.to_ne_bytes();
unsafe { host(SYS_WRITE, efd as u64, one.as_ptr() as u64, 8, 0, 0, 0) };
}
0
}
2 => {
if let Some(m) = epoll_loops().lock().unwrap().get_mut(&host_epfd) {
if m.remove(&sid).is_some() && epdbg_on() {
ipc_logf(
&[(b"CTLDEL epfd=", host_epfd as i64), (b" sid=", sid as i64)],
b"",
);
}
}
unsafe { host(SYS_EPOLL_CTL, host_epfd as u64, 2, efd as u64, 0, 0, 0) };
0
}
_ => -22, }
}
fn epoll_wait_merge(pid: i32, host_epfd: i32, events_out: u64, max: usize, timeout_ms: i64) -> i64 {
let deadline = if timeout_ms < 0 {
None
} else {
Some(now_ns().saturating_add((timeout_ms as u64).saturating_mul(1_000_000)))
};
loop {
let mut out: Vec<(u32, u64)> = Vec::new();
let mut seen: std::collections::HashSet<u64> = std::collections::HashSet::new();
let mut has_loop_watch = false;
{
let mut lt = epoll_loops().lock().unwrap();
let ls = loop_state().lock().unwrap();
if let Some(m) = lt.get_mut(&host_epfd) {
has_loop_watch = !m.is_empty();
for (&sid, v) in m.iter_mut() {
let (events, data) = (v.0, v.1);
let mut ready = 0u32;
if (events & EPOLLIN_B) != 0 && ls.net.readable(sid) {
ready |= EPOLLIN_B;
}
if (events & EPOLLOUT_B) != 0 && ls.net.writable(sid) {
ready |= EPOLLOUT_B;
}
let re = et_filter(events, ready, &mut v.2);
if re != 0 && seen.insert(data) {
out.push((re, data));
}
}
}
}
let any_loop = !out.is_empty();
let t: i64 = if any_loop {
0
} else {
let base = match deadline {
None => -1,
Some(d) => {
let now = now_ns();
if now >= d {
0
} else {
((d - now) / 1_000_000).min(i32::MAX as u64) as i64
}
}
};
if has_loop_watch && (base < 0 || base > LOOP_WAIT_TICK_MS) {
LOOP_WAIT_TICK_MS
} else if own_loopback() && base < 0 {
LOOP_INSURANCE_TICK_MS
} else {
base
}
};
let mut hbuf = vec![0u8; max * 12];
let hn = unsafe {
host_cancellable(
SYS_EPOLL_WAIT,
host_epfd as u64,
hbuf.as_mut_ptr() as u64,
max as u64,
t as u64,
0,
0,
)
};
if hn == CANCEL_SENTINEL {
return CANCEL_SENTINEL;
}
if hn > 0 {
for i in 0..(hn as usize) {
let ev = u32::from_le_bytes(hbuf[i * 12..i * 12 + 4].try_into().unwrap());
let data = u64::from_le_bytes(hbuf[i * 12 + 4..i * 12 + 12].try_into().unwrap());
if data & EPOLL_LOOP_TAG != 0 {
let sid = (data & !EPOLL_LOOP_TAG) as u32;
if let Some(&efd) = loop_state().lock().unwrap().efds.get(&sid) {
let mut sink = [0u8; 8];
unsafe { host(SYS_READ, efd as u64, sink.as_mut_ptr() as u64, 8, 0, 0, 0) };
}
let mut lt = epoll_loops().lock().unwrap();
if let Some(v) = lt.get_mut(&host_epfd).and_then(|m| m.get_mut(&sid)) {
let (wev, gdata) = (v.0, v.1);
let ls = loop_state().lock().unwrap();
let mut ready = 0u32;
if (wev & EPOLLIN_B) != 0 && ls.net.readable(sid) {
ready |= EPOLLIN_B;
}
if (wev & EPOLLOUT_B) != 0 && ls.net.writable(sid) {
ready |= EPOLLOUT_B;
}
v.2 = ready;
if ready != 0 && seen.insert(gdata) {
out.push((ready, gdata));
}
} else {
let sib_watch = lt.iter().find_map(|(ep2, m)| {
if *ep2 != host_epfd {
m.get(&sid).map(|w| (w.0, w.1))
} else {
None
}
});
if let Some((wev, gdata)) = sib_watch {
let v = lt
.entry(host_epfd)
.or_default()
.entry(sid)
.or_insert((wev, gdata, 0));
let ls = loop_state().lock().unwrap();
let mut ready = 0u32;
if (wev & EPOLLIN_B) != 0 && ls.net.readable(sid) {
ready |= EPOLLIN_B;
}
if (wev & EPOLLOUT_B) != 0 && ls.net.writable(sid) {
ready |= EPOLLOUT_B;
}
v.2 = ready;
if ready != 0 && seen.insert(gdata) {
out.push((ready, gdata));
}
}
}
} else {
out.push((ev, data)); }
}
}
let elapsed = matches!(deadline, Some(d) if now_ns() >= d);
if !out.is_empty() || elapsed || hn < 0 {
let cnt = out.len().min(max);
if epdbg_on() {
ipc_logf(
&[
(b"EPMERGE epfd=", host_epfd as i64),
(b" tmo=", timeout_ms),
(b" anyloop=", any_loop as i64),
(b" hn=", hn),
(b" out=", out.len() as i64),
(b" ev0=", out.first().map(|x| x.0 as i64).unwrap_or(-1)),
],
b"",
);
}
let mut wbuf = vec![0u8; cnt * 12];
for (i, (ev, data)) in out.iter().take(cnt).enumerate() {
wbuf[i * 12..i * 12 + 4].copy_from_slice(&ev.to_le_bytes());
wbuf[i * 12 + 4..i * 12 + 12].copy_from_slice(&data.to_le_bytes());
}
if cnt > 0 {
vm_write(pid, events_out, &wbuf);
}
return if hn < 0 { hn } else { cnt as i64 };
}
}
}
fn ends_with(s: &[u8], suf: &[u8]) -> bool {
s.len() >= suf.len() && &s[s.len() - suf.len()..] == suf
}
#[derive(Clone, Debug)]
struct PipeFdInfo {
id: u64,
end: state_snap::PipeEnd,
flags: u32,
}
#[derive(Clone, Debug, Default)]
struct PipeCaptureGroup {
has_read: bool,
has_write: bool,
buffered: Vec<u8>,
}
fn fd_status_flags(h: i32) -> u32 {
let flags = unsafe { host(SYS_FCNTL, h as u64, libc::F_GETFL as u64, 0, 0, 0, 0) };
if flags >= 0 {
flags as u32
} else {
0
}
}
fn host_fd_stat_kind(h: i32) -> Option<(u32, libc::ino_t)> {
let mut st: libc::stat = unsafe { std::mem::zeroed() };
let r = unsafe {
host(
SYS_FSTAT,
h as u64,
&mut st as *mut libc::stat as u64,
0,
0,
0,
0,
)
};
(r == 0).then(|| (((st.st_mode as u32) & S_IFMT), st.st_ino))
}
fn pipe_fd_info(h: i32) -> Option<PipeFdInfo> {
let (kind, ino) = host_fd_stat_kind(h)?;
if kind != S_IFIFO {
return None;
}
let flags = fd_status_flags(h);
let end = match (flags as u64) & O_ACCMODE {
O_RDONLY => state_snap::PipeEnd::Read,
O_WRONLY => state_snap::PipeEnd::Write,
_ => return None,
};
Some(PipeFdInfo {
id: ino as u64,
end,
flags,
})
}
fn pipe_buffered_bytes(read_h: i32) -> std::io::Result<Vec<u8>> {
const FIONREAD: u64 = 0x541B;
const SPLICE_F_NONBLOCK: u64 = 0x02;
let mut avail: i32 = 0;
let r = unsafe {
host(
SYS_IOCTL,
read_h as u64,
FIONREAD,
&mut avail as *mut i32 as u64,
0,
0,
0,
)
};
if r < 0 {
return Err(std::io::Error::from_raw_os_error((-r) as i32));
}
if avail <= 0 {
return Ok(Vec::new());
}
let mut tmp = [0i32; 2];
let r = unsafe {
host(
SYS_PIPE2,
tmp.as_mut_ptr() as u64,
libc::O_CLOEXEC as u64,
0,
0,
0,
0,
)
};
if r < 0 {
return Err(std::io::Error::from_raw_os_error((-r) as i32));
}
let (tmp_r, tmp_w) = (tmp[0], tmp[1]);
let mut out = Vec::with_capacity(avail as usize);
let mut remaining = avail as usize;
let result = (|| {
while remaining > 0 {
let n = unsafe {
host(
SYS_TEE,
read_h as u64,
tmp_w as u64,
remaining as u64,
SPLICE_F_NONBLOCK,
0,
0,
)
};
if n < 0 {
return Err(std::io::Error::from_raw_os_error((-n) as i32));
}
if n == 0 {
return Err(std::io::Error::new(
std::io::ErrorKind::UnexpectedEof,
"snapshot: pipe buffer changed while copying",
));
}
let mut copied = 0usize;
let want = n as usize;
while copied < want {
let mut buf = [0u8; 8192];
let take = buf.len().min(want - copied);
let got = unsafe {
host(
SYS_READ,
tmp_r as u64,
buf.as_mut_ptr() as u64,
take as u64,
0,
0,
0,
)
};
if got <= 0 {
return Err(if got < 0 {
std::io::Error::from_raw_os_error((-got) as i32)
} else {
std::io::Error::new(
std::io::ErrorKind::UnexpectedEof,
"snapshot: pipe tee drained short",
)
});
}
out.extend_from_slice(&buf[..got as usize]);
copied += got as usize;
}
remaining -= want;
}
Ok(out)
})();
unsafe {
host(SYS_CLOSE, tmp_r as u64, 0, 0, 0, 0, 0);
host(SYS_CLOSE, tmp_w as u64, 0, 0, 0, 0, 0);
}
result
}
#[allow(dead_code)] fn host_fd_to_target(
h: i32,
pipe_groups: &HashMap<u64, PipeCaptureGroup>,
) -> std::io::Result<Option<state_snap::FdTarget>> {
if let Some(loop_watches) = epoll_loops().lock().unwrap().get(&h).map(|m| {
let mut watches = m
.iter()
.map(|(&sid, &(events, data, _))| state_snap::EpollLoopWatch {
socket: state_snap::SocketHandle(sid),
events,
data,
})
.collect::<Vec<_>>();
watches.sort_by_key(|w| w.socket.0);
watches
}) {
return Ok(Some(state_snap::FdTarget::Epoll { loop_watches }));
}
let Some((kind, _ino)) = host_fd_stat_kind(h) else {
return Ok(None);
};
match kind {
S_IFREG => {
let offset = unsafe {
host(SYS_LSEEK, h as u64, 0, 1 , 0, 0, 0)
};
let Some(path) = readlink_fd(h).and_then(|hp| host_to_guest_path(&hp)) else {
return Ok(None);
};
Ok(Some(state_snap::FdTarget::RootRelFile {
path,
offset: if offset >= 0 { offset as u64 } else { 0 },
flags: fd_status_flags(h),
}))
}
S_IFCHR => {
let p = readlink_fd(h).unwrap_or_default();
let kind = if ends_with(&p, b"null") {
state_snap::DevKind::Null
} else if ends_with(&p, b"zero") {
state_snap::DevKind::Zero
} else if ends_with(&p, b"full") {
state_snap::DevKind::Full
} else if ends_with(&p, b"random") || ends_with(&p, b"urandom") {
state_snap::DevKind::Random
} else {
state_snap::DevKind::Tty
};
if kind == state_snap::DevKind::Tty {
return Ok(None);
}
Ok(Some(state_snap::FdTarget::DevNode {
kind,
flags: fd_status_flags(h),
}))
}
S_IFIFO => {
let Some(info) = pipe_fd_info(h) else {
return Ok(None);
};
let Some(group) = pipe_groups.get(&info.id) else {
return Ok(None);
};
match info.end {
state_snap::PipeEnd::Read => {
if !group.has_read {
return Ok(None);
}
}
state_snap::PipeEnd::Write => {
if !group.has_read || !group.has_write {
return Ok(None);
}
}
}
Ok(Some(state_snap::FdTarget::Pipe {
id: info.id,
end: info.end,
buffered: group.buffered.clone(),
flags: info.flags,
}))
}
_ => Ok(None),
}
}
#[allow(dead_code)] fn capture_fdtable() -> std::io::Result<state_snap::FdTableSnapshot> {
let entries: Vec<(state_snap::Vpid, i32, FdVal)> = {
let table = fdt().lock().unwrap();
let mut entries = Vec::new();
for (&host_pid, fds) in table.iter() {
let Some(vpid) = proctree::vpid_for(host_pid) else {
continue;
};
for (&gfd, &val) in fds.iter() {
entries.push((vpid, gfd, val));
}
}
entries
};
let mut pipe_groups: HashMap<u64, PipeCaptureGroup> = HashMap::new();
for (_, _, val) in &entries {
let FdVal::Host(h) = *val else {
continue;
};
let Some(info) = pipe_fd_info(h) else {
continue;
};
let group = pipe_groups.entry(info.id).or_default();
match info.end {
state_snap::PipeEnd::Read => {
group.has_read = true;
if group.buffered.is_empty() {
group.buffered = pipe_buffered_bytes(h)?;
}
}
state_snap::PipeEnd::Write => {
group.has_write = true;
}
}
}
let mut by_vpid: BTreeMap<state_snap::Vpid, Vec<(i32, state_snap::FdTarget)>> = BTreeMap::new();
for (vpid, gfd, val) in entries {
let target = match val {
FdVal::Loop(sid) => Some(state_snap::FdTarget::LoopSocket(state_snap::SocketHandle(
sid,
))),
FdVal::Host(h) => host_fd_to_target(h, &pipe_groups)?,
};
if let Some(target) = target {
by_vpid.entry(vpid).or_default().push((gfd, target));
}
}
let mut per_vpid = Vec::with_capacity(by_vpid.len());
for (vpid, mut fds) in by_vpid {
fds.sort_by_key(|(gfd, _)| *gfd);
per_vpid.push((vpid, fds));
}
Ok(state_snap::FdTableSnapshot { per_vpid })
}
fn capture_cwds() -> Vec<(state_snap::Vpid, Vec<u8>)> {
let table = cwds().lock().unwrap();
let mut out = Vec::new();
for (&host_pid, cwd) in table.iter() {
if let Some(vpid) = proctree::vpid_for(host_pid) {
out.push((vpid, cwd.clone()));
}
}
out.sort_by_key(|(vpid, _)| *vpid);
out
}
fn capture_cmdlines() -> Vec<(state_snap::Vpid, Vec<u8>)> {
let table = proc_cmdline().lock().unwrap();
let mut out = Vec::new();
for (&host_pid, cmdline) in table.iter() {
if let Some(vpid) = proctree::vpid_for(host_pid) {
out.push((vpid, cmdline.clone()));
}
}
out.sort_by_key(|(vpid, _)| *vpid);
out
}
fn capture_exes() -> Vec<(state_snap::Vpid, Vec<u8>)> {
let table = proc_exe().lock().unwrap();
let mut out = Vec::new();
for (&host_pid, exe) in table.iter() {
if let Some(vpid) = proctree::vpid_for(host_pid) {
out.push((vpid, exe.clone()));
}
}
out.sort_by_key(|(vpid, _)| *vpid);
out
}
fn rekey_fdtable(
fdtable: state_snap::FdTableSnapshot,
vpid_map: &BTreeMap<state_snap::Vpid, state_snap::Vpid>,
) -> state_snap::FdTableSnapshot {
let mut per_vpid = Vec::new();
for (old_vpid, fds) in fdtable.per_vpid {
if let Some(new_vpid) = vpid_map.get(&old_vpid) {
per_vpid.push((*new_vpid, fds));
}
}
per_vpid.sort_by_key(|(vpid, _)| *vpid);
state_snap::FdTableSnapshot { per_vpid }
}
fn rekey_cwds(
cwds_in: Vec<(state_snap::Vpid, Vec<u8>)>,
vpid_map: &BTreeMap<state_snap::Vpid, state_snap::Vpid>,
) -> Vec<(state_snap::Vpid, Vec<u8>)> {
rekey_vpid_bytes(cwds_in, vpid_map)
}
fn rekey_vpid_bytes(
entries: Vec<(state_snap::Vpid, Vec<u8>)>,
vpid_map: &BTreeMap<state_snap::Vpid, state_snap::Vpid>,
) -> Vec<(state_snap::Vpid, Vec<u8>)> {
let mut out = Vec::new();
for (old_vpid, bytes) in entries {
if let Some(new_vpid) = vpid_map.get(&old_vpid) {
out.push((*new_vpid, bytes));
}
}
out.sort_by_key(|(vpid, _)| *vpid);
out
}
#[cfg(all(target_os = "linux", target_arch = "x86_64"))]
fn snapshot_tmp_suffix() -> u64 {
static SEQ: AtomicU64 = AtomicU64::new(0);
let nanos = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|d| d.as_nanos() as u64)
.unwrap_or(0);
nanos ^ SEQ.fetch_add(1, Ordering::Relaxed)
}
#[cfg(all(target_os = "linux", target_arch = "x86_64"))]
fn read_exact_process_mem(mem: &std::fs::File, addr: u64, out: &mut [u8]) -> bool {
use std::os::unix::fs::FileExt;
let mut done = 0usize;
while done < out.len() {
match mem.read_at(&mut out[done..], addr + done as u64) {
Ok(0) => return false,
Ok(n) => done += n,
Err(e) if e.kind() == std::io::ErrorKind::Interrupted => {}
Err(_) => return false,
}
}
true
}
#[cfg(all(target_os = "linux", target_arch = "x86_64"))]
fn read_process_u32(mem: &std::fs::File, addr: u64) -> Option<u32> {
let mut buf = [0u8; 4];
read_exact_process_mem(mem, addr, &mut buf).then(|| u32::from_le_bytes(buf))
}
#[cfg(all(target_os = "linux", target_arch = "x86_64"))]
fn read_process_u64(mem: &std::fs::File, addr: u64) -> Option<u64> {
let mut buf = [0u8; 8];
read_exact_process_mem(mem, addr, &mut buf).then(|| u64::from_le_bytes(buf))
}
#[cfg(all(target_os = "linux", target_arch = "x86_64"))]
fn capture_guest_vma_ranges(mem: &std::fs::File) -> Vec<(u64, u64)> {
let n_addr = std::ptr::addr_of!(GUEST_VMA_NEXT) as u64;
let n = read_process_u32(mem, n_addr)
.unwrap_or(0)
.min(GUEST_VMA_MAX as u32) as usize;
let base = std::ptr::addr_of!(GUEST_VMAS) as u64;
let stride = std::mem::size_of::<GuestVmaSlot>() as u64;
let mut ranges = Vec::new();
for i in 0..n {
let slot = base + i as u64 * stride;
let lo = read_process_u64(mem, slot).unwrap_or(0);
let hi = read_process_u64(mem, slot + 8).unwrap_or(0);
if lo >= WINDOW_FLOOR && hi > lo && hi <= USER_TOP {
ranges.push((lo, hi));
}
}
ranges.sort_unstable();
ranges.dedup();
ranges
}
#[cfg(all(target_os = "linux", target_arch = "x86_64"))]
fn select_guest_map_entries(
maps: &[u8],
guest_ranges: &[(u64, u64)],
extra_addrs: &[u64],
) -> Vec<memimage::MapEntry> {
let mut out = Vec::new();
for e in memimage::parse_maps(maps, USER_TOP) {
let end = e.va.saturating_add(e.len);
if e.va < WINDOW_FLOOR {
let low_end = end.min(WINDOW_FLOOR);
if low_end > e.va {
out.push(memimage::MapEntry {
va: e.va,
len: low_end - e.va,
prot: e.prot,
});
}
continue;
}
if extra_addrs.iter().any(|&a| a >= e.va && a < end) {
out.push(memimage::MapEntry {
va: e.va,
len: e.len,
prot: e.prot,
});
continue;
}
for &(lo, hi) in guest_ranges {
let s = e.va.max(lo);
let t = end.min(hi);
if t > s {
out.push(memimage::MapEntry {
va: s,
len: t - s,
prot: e.prot,
});
}
}
}
out.sort_by_key(|e| e.va);
out.dedup_by(|a, b| a.va == b.va && a.len == b.len && a.prot == b.prot);
out
}
#[cfg(all(target_os = "linux", target_arch = "x86_64"))]
fn capture_warm_thread_regs(
mem: &std::fs::File,
vtid: state_snap::Vpid,
) -> std::io::Result<state_snap::ThreadRegs> {
let mut gregs = [0i64; 18];
let mut gbuf = [0u8; 18 * std::mem::size_of::<i64>()];
let gaddr = std::ptr::addr_of!(CAPTURED) as u64;
if !read_exact_process_mem(mem, gaddr, &mut gbuf) {
return Err(std::io::Error::new(
std::io::ErrorKind::Other,
"snapshot: read warm-cell captured gregs failed",
));
}
for (i, chunk) in gbuf.chunks_exact(8).enumerate() {
gregs[i] = i64::from_le_bytes(chunk.try_into().unwrap());
}
gregs[REG_RAX] = 0;
let mut fsbuf = [0u8; std::mem::size_of::<u64>()];
let fsaddr = std::ptr::addr_of!(GUEST_FS) as u64;
if !read_exact_process_mem(mem, fsaddr, &mut fsbuf) {
return Err(std::io::Error::new(
std::io::ErrorKind::Other,
"snapshot: read warm-cell guest fs failed",
));
}
Ok(state_snap::ThreadRegs {
vtid,
gregs,
fs: u64::from_le_bytes(fsbuf),
xsave: Vec::new(),
})
}
#[cfg(all(target_os = "linux", target_arch = "x86_64"))]
fn read_mem_meta(dest_dir: &std::path::Path) -> std::io::Result<state_snap::MemImage> {
let mut f = std::fs::File::open(dest_dir.join("mem.meta"))?;
state_snap::MemImage::read_from(&mut f)
}
#[cfg(all(target_os = "linux", target_arch = "x86_64"))]
fn launcher_capture_live_mem(
launch_fd: c_int,
dest_dir: &std::path::Path,
targets: Vec<LauncherCaptureTarget>,
) -> std::io::Result<state_snap::MemImage> {
let req = LaunchReq {
path: Some("/.supermachine/capture".to_string()),
argv: vec!["/.supermachine/capture".to_string()],
warm: false,
env: Vec::new(),
cwd: None,
restore_dir: None,
restore_vpid: 1,
restore_warm: true,
detached: false,
slot: None,
stop_before_resume: false,
stop_before_start: false,
capture: Some(LauncherCaptureReq {
dest_dir: dest_dir.to_string_lossy().into_owned(),
targets,
}),
};
let json = serde_json::to_vec(&req).map_err(|e| {
std::io::Error::new(
std::io::ErrorKind::InvalidData,
format!("snapshot: launcher capture request serialize: {e}"),
)
})?;
unsafe { libc::send(launch_fd, json.as_ptr() as *const c_void, json.len(), 0) };
let mut code: i32 = -1;
let n = unsafe { libc::recv(launch_fd, &mut code as *mut i32 as *mut c_void, 4, 0) };
if n != 4 {
return Err(std::io::Error::new(
std::io::ErrorKind::UnexpectedEof,
"snapshot: launcher capture response missing",
));
}
if code != 0 {
return Err(std::io::Error::from_raw_os_error((-code).max(1)));
}
let mem = read_mem_meta(dest_dir)?;
let _ = std::fs::remove_file(dest_dir.join("mem.meta"));
Ok(mem)
}
#[cfg(all(target_os = "linux", target_arch = "x86_64"))]
fn capture_state_snapshot(
launch_fd: c_int,
warm_pid: i32,
dest_dir: &std::path::Path,
) -> std::io::Result<()> {
use std::io::Write as _;
std::fs::create_dir_all(dest_dir)?;
let live_instances = live_running_instances_snapshot();
if !live_instances.is_empty() {
let mut subtree_by_vpid: BTreeMap<state_snap::Vpid, (state_snap::Vpid, i32)> =
BTreeMap::new();
for root_host in &live_instances {
for (old_vpid, old_parent, host_pid) in proctree::live_subtree(*root_host) {
subtree_by_vpid
.entry(old_vpid)
.or_insert((old_parent, host_pid));
}
}
if subtree_by_vpid.is_empty() {
return Err(std::io::Error::new(
std::io::ErrorKind::NotFound,
"snapshot: preserved live instance is not in the owned process tree",
));
}
let mut vpid_map = BTreeMap::new();
let mut next_vpid: state_snap::Vpid = 1;
for old_vpid in subtree_by_vpid.keys() {
vpid_map.insert(*old_vpid, next_vpid);
next_vpid = next_vpid.saturating_add(1);
}
let targets: Vec<LauncherCaptureTarget> = subtree_by_vpid
.iter()
.filter_map(|(old_vpid, (_, host_pid))| {
vpid_map
.get(old_vpid)
.map(|new_vpid| LauncherCaptureTarget {
vpid: *new_vpid,
host_pid: *host_pid,
})
})
.collect();
eprintln!(
"supermachine(sentry): snapshot live mem capture ({} procs)…",
targets.len()
);
let mem = launcher_capture_live_mem(launch_fd, dest_dir, targets)?;
eprintln!("supermachine(sentry): snapshot live mem capture done");
let mut procs = Vec::with_capacity(subtree_by_vpid.len());
for (old_vpid, (old_parent, host_pid)) in &subtree_by_vpid {
let new_vpid = vpid_map[old_vpid];
let new_parent = vpid_map.get(old_parent).copied().unwrap_or(0);
let pgid = unsafe { libc::getpgid(*host_pid) };
let sid = unsafe { libc::getsid(*host_pid) };
let threads = mem
.proc_image(new_vpid)
.map(|image| image.threads.to_vec())
.unwrap_or_default();
procs.push(state_snap::GuestProcRecord {
vpid: new_vpid,
parent_vpid: new_parent,
pgid: if pgid > 0 { pgid as u32 } else { 0 },
sid: if sid > 0 { sid as u32 } else { 0 },
threads,
state: state_snap::ProcState::Running,
});
}
procs.sort_by_key(|p| p.vpid);
eprintln!("supermachine(sentry): snapshot netstack capture…");
let netstack = {
let ls = loop_state().lock().unwrap();
let mut snap = ls.net.capture();
snap.nonblock = ls.nonblock.iter().copied().collect();
snap.nonblock.sort_unstable();
snap
};
eprintln!("supermachine(sentry): snapshot fdtable capture…");
let fdtable = rekey_fdtable(capture_fdtable()?, &vpid_map);
eprintln!("supermachine(sentry): snapshot cwd/cmdline/exe capture…");
let state = state_snap::StateSnapshot {
version: state_snap::CURRENT_VERSION,
cpu_feature_baseline: state_snap::CpuFeatures::current(),
entry_kind: state_snap::SnapshotEntryKind::LiveTree,
mem,
proctree: state_snap::ProcTreeSnapshot { procs },
netstack,
fdtable,
cwds: rekey_cwds(capture_cwds(), &vpid_map),
cmdlines: rekey_vpid_bytes(capture_cmdlines(), &vpid_map),
exes: rekey_vpid_bytes(capture_exes(), &vpid_map),
};
eprintln!("supermachine(sentry): snapshot state serialize…");
let tmp_suffix = format!("{}-{}", std::process::id(), snapshot_tmp_suffix());
let snap_tmp = dest_dir.join(format!(".restore.snap.tmp-{tmp_suffix}"));
let mut snap_buf = Vec::new();
state.write_to(&mut snap_buf)?;
std::fs::write(&snap_tmp, snap_buf)?;
std::fs::rename(&snap_tmp, dest_dir.join("restore.snap"))?;
eprintln!("supermachine(sentry): snapshot restore.snap written");
return Ok(());
}
if warm_pid <= 1 {
return Err(std::io::Error::new(
std::io::ErrorKind::InvalidInput,
"snapshot: no warm cell or preserved live tree is parked",
));
}
let maps = std::fs::read(format!("/proc/{warm_pid}/maps"))?;
let mem = std::fs::File::open(format!("/proc/{warm_pid}/mem"))?;
let warm_vpid = proctree::vpid_for(warm_pid).ok_or_else(|| {
std::io::Error::new(
std::io::ErrorKind::Other,
format!("snapshot: warm pid {warm_pid} has no vpid"),
)
})?;
let thread = capture_warm_thread_regs(&mem, warm_vpid)?;
let guest_ranges = capture_guest_vma_ranges(&mem);
let entries = select_guest_map_entries(&maps, &guest_ranges, &[thread.fs]);
let tmp_suffix = format!("{}-{}", std::process::id(), snapshot_tmp_suffix());
let mem_tmp = dest_dir.join(format!(".mem.blob.tmp-{tmp_suffix}"));
let snap_tmp = dest_dir.join(format!(".restore.snap.tmp-{tmp_suffix}"));
let mut ram = std::fs::File::create(&mem_tmp)?;
let regions = memimage::capture_regions(
&entries,
|va, dst| read_exact_process_mem(&mem, va, dst),
&mut ram,
)?;
ram.flush()?;
let mut procs = proctree::capture(|host_pid| unsafe {
let pgid = libc::getpgid(host_pid);
let sid = libc::getsid(host_pid);
(
if pgid > 0 { pgid as u32 } else { 0 },
if sid > 0 { sid as u32 } else { 0 },
)
});
if let Some(proc_rec) = procs.procs.iter_mut().find(|p| p.vpid == warm_vpid) {
proc_rec.threads.push(thread.clone());
}
let state = state_snap::StateSnapshot {
version: state_snap::CURRENT_VERSION,
cpu_feature_baseline: state_snap::CpuFeatures::current(),
entry_kind: state_snap::SnapshotEntryKind::WarmZygote,
mem: state_snap::MemImage {
regions: regions.clone(),
threads: vec![thread.clone()],
procs: vec![state_snap::ProcMemImage {
vpid: warm_vpid,
blob_off: 0,
regions,
threads: vec![thread],
}],
},
proctree: procs,
netstack: {
let ls = loop_state().lock().unwrap();
let mut snap = ls.net.capture();
snap.nonblock = ls.nonblock.iter().copied().collect();
snap.nonblock.sort_unstable();
snap
},
fdtable: capture_fdtable()?,
cwds: capture_cwds(),
cmdlines: capture_cmdlines(),
exes: capture_exes(),
};
let mut snap_buf = Vec::new();
state.write_to(&mut snap_buf)?;
std::fs::write(&snap_tmp, snap_buf)?;
std::fs::rename(&mem_tmp, dest_dir.join("mem.blob"))?;
std::fs::rename(&snap_tmp, dest_dir.join("restore.snap"))?;
Ok(())
}
#[cfg(all(target_os = "linux", target_arch = "x86_64"))]
fn read_state_snapshot_dir(dir: &std::path::Path) -> std::io::Result<state_snap::StateSnapshot> {
let mut f = std::fs::File::open(dir.join("restore.snap"))?;
state_snap::StateSnapshot::read_from(&mut f)
}
fn io_err_from_neg(n: i64) -> std::io::Error {
std::io::Error::from_raw_os_error((-n) as i32)
}
fn snapshot_restore_open_flags(flags: u32) -> u64 {
(flags as u64) & !(O_CREAT | O_TRUNC | O_EXCL)
}
fn nul_path(path: &[u8]) -> std::io::Result<Vec<u8>> {
if path.is_empty() || path[0] != b'/' || path.iter().any(|&b| b == 0) {
return Err(std::io::Error::new(
std::io::ErrorKind::InvalidData,
"snapshot: fd path is not a clean absolute guest path",
));
}
let mut out = path.to_vec();
out.push(0);
Ok(out)
}
fn restore_root_rel_file_fd(
warm_pid: i32,
path: &[u8],
offset: u64,
flags: u32,
) -> std::io::Result<i32> {
let p = nul_path(path)?;
let fd = open_path(warm_pid, &p, snapshot_restore_open_flags(flags), 0);
if fd < 0 {
return Err(io_err_from_neg(fd));
}
if offset > 0 {
let r = unsafe {
host(SYS_LSEEK, fd as u64, offset, 0 , 0, 0, 0)
};
if r < 0 {
unsafe { host(SYS_CLOSE, fd as u64, 0, 0, 0, 0, 0) };
return Err(io_err_from_neg(r));
}
}
Ok(fd as i32)
}
fn dev_kind_path(kind: state_snap::DevKind) -> Option<&'static [u8]> {
match kind {
state_snap::DevKind::Null => Some(b"/dev/null"),
state_snap::DevKind::Zero => Some(b"/dev/zero"),
state_snap::DevKind::Full => Some(b"/dev/full"),
state_snap::DevKind::Random => Some(b"/dev/urandom"),
state_snap::DevKind::Tty => None,
}
}
fn restore_dev_node_fd(
warm_pid: i32,
kind: state_snap::DevKind,
flags: u32,
) -> std::io::Result<i32> {
let Some(path) = dev_kind_path(kind) else {
return Err(std::io::Error::new(
std::io::ErrorKind::InvalidData,
"snapshot: /dev/tty cannot be restored without a terminal peer",
));
};
let p = nul_path(path)?;
let fd = open_path(warm_pid, &p, snapshot_restore_open_flags(flags), 0);
if fd < 0 {
return Err(io_err_from_neg(fd));
}
Ok(fd as i32)
}
#[derive(Clone, Debug, Default)]
struct PipeRestoreGroup {
buffered: Vec<u8>,
entries: Vec<(i32, state_snap::PipeEnd, u32)>,
}
fn set_fd_status(fd: i32, flags: u32) -> std::io::Result<()> {
let r = unsafe {
host(
SYS_FCNTL,
fd as u64,
libc::F_SETFL as u64,
flags as u64,
0,
0,
0,
)
};
if r < 0 {
Err(io_err_from_neg(r))
} else {
Ok(())
}
}
fn write_all_to_fd(fd: i32, data: &[u8]) -> std::io::Result<()> {
let mut off = 0usize;
while off < data.len() {
let n = unsafe {
host(
SYS_WRITE,
fd as u64,
data[off..].as_ptr() as u64,
(data.len() - off) as u64,
0,
0,
0,
)
};
if n < 0 {
return Err(io_err_from_neg(n));
}
if n == 0 {
return Err(std::io::Error::new(
std::io::ErrorKind::WriteZero,
"snapshot: pipe restore wrote zero bytes",
));
}
off += n as usize;
}
Ok(())
}
fn restore_pipe_group(group: &PipeRestoreGroup) -> std::io::Result<Vec<(i32, FdVal)>> {
const F_SETPIPE_SZ: u64 = 1031;
let mut fds = [0i32; 2];
let r = unsafe {
host(
SYS_PIPE2,
fds.as_mut_ptr() as u64,
libc::O_CLOEXEC as u64,
0,
0,
0,
0,
)
};
if r < 0 {
return Err(io_err_from_neg(r));
}
let (read_fd, write_fd) = (fds[0], fds[1]);
let result = (|| {
if !group.buffered.is_empty() {
unsafe {
host(
SYS_FCNTL,
write_fd as u64,
F_SETPIPE_SZ,
group.buffered.len() as u64,
0,
0,
0,
);
}
let saved = fd_status_flags(write_fd);
set_fd_status(write_fd, saved | libc::O_NONBLOCK as u32)?;
let write_res = write_all_to_fd(write_fd, &group.buffered);
let _ = set_fd_status(write_fd, saved);
write_res?;
}
let mut restored = Vec::with_capacity(group.entries.len());
for &(gfd, end, flags) in &group.entries {
let base = match end {
state_snap::PipeEnd::Read => read_fd,
state_snap::PipeEnd::Write => write_fd,
};
let dup = unsafe { host(SYS_DUP, base as u64, 0, 0, 0, 0, 0) };
if dup < 0 {
for (_, val) in restored.drain(..) {
fd_close_val(val);
}
return Err(io_err_from_neg(dup));
}
let dup = dup as i32;
if let Err(e) = set_fd_status(dup, flags) {
unsafe { host(SYS_CLOSE, dup as u64, 0, 0, 0, 0, 0) };
for (_, val) in restored.drain(..) {
fd_close_val(val);
}
return Err(e);
}
restored.push((gfd, FdVal::Host(dup)));
}
Ok(restored)
})();
unsafe {
host(SYS_CLOSE, read_fd as u64, 0, 0, 0, 0, 0);
host(SYS_CLOSE, write_fd as u64, 0, 0, 0, 0, 0);
}
result
}
#[cfg(all(target_os = "linux", target_arch = "x86_64"))]
fn restore_supervisor_state_for_vpid(
host_pid: i32,
vpid: state_snap::Vpid,
state: &state_snap::StateSnapshot,
restore_loop: bool,
) -> std::io::Result<()> {
if !state
.cpu_feature_baseline
.satisfied_by(&state_snap::CpuFeatures::current())
{
return Err(std::io::Error::new(
std::io::ErrorKind::InvalidData,
"snapshot: CPU feature baseline is not satisfied by this host",
));
}
if restore_loop {
restore_loop_state_from_snapshot(&state.netstack);
}
if let Some((_, cwd)) = state.cwds.iter().find(|(rec_vpid, _)| *rec_vpid == vpid) {
cwds().lock().unwrap().insert(host_pid, cwd.clone());
}
if let Some((_, cmdline)) = state
.cmdlines
.iter()
.find(|(rec_vpid, _)| *rec_vpid == vpid)
{
proc_cmdline()
.lock()
.unwrap()
.insert(host_pid, cmdline.clone());
}
if let Some((_, exe)) = state.exes.iter().find(|(rec_vpid, _)| *rec_vpid == vpid) {
proc_exe().lock().unwrap().insert(host_pid, exe.clone());
}
let mut restored = Vec::new();
let mut pipe_groups: BTreeMap<u64, PipeRestoreGroup> = BTreeMap::new();
if let Some((_, fds)) = state
.fdtable
.per_vpid
.iter()
.find(|(rec_vpid, _)| *rec_vpid == vpid)
{
for (gfd, target) in fds {
match target {
state_snap::FdTarget::LoopSocket(state_snap::SocketHandle(sid)) => {
restored.push((*gfd, FdVal::Loop(*sid)));
}
state_snap::FdTarget::Epoll { loop_watches } => {
restored.push((*gfd, FdVal::Host(restore_epoll_loop_fd(loop_watches)?)));
}
state_snap::FdTarget::RootRelFile {
path,
offset,
flags,
} => restored.push((
*gfd,
FdVal::Host(restore_root_rel_file_fd(host_pid, path, *offset, *flags)?),
)),
state_snap::FdTarget::DevNode { kind, flags } => restored.push((
*gfd,
FdVal::Host(restore_dev_node_fd(host_pid, *kind, *flags)?),
)),
state_snap::FdTarget::Pipe {
id,
end,
buffered,
flags,
} => {
let group = pipe_groups.entry(*id).or_default();
if group.buffered.is_empty() {
group.buffered = buffered.clone();
} else if !buffered.is_empty() && group.buffered != *buffered {
return Err(std::io::Error::new(
std::io::ErrorKind::InvalidData,
"snapshot: inconsistent pipe buffers for one pipe id",
));
}
group.entries.push((*gfd, *end, *flags));
}
state_snap::FdTarget::HostEgressSocket { .. } => {
return Err(std::io::Error::new(
std::io::ErrorKind::Unsupported,
"snapshot: host-egress socket restore is not implemented",
));
}
}
}
}
for group in pipe_groups.values() {
restored.extend(restore_pipe_group(group)?);
}
if !restored.is_empty() {
let mut table = fdt().lock().unwrap();
let fds = table.entry(host_pid).or_default();
for (gfd, val) in restored {
fds.insert(gfd, val);
}
}
Ok(())
}
#[cfg(all(target_os = "linux", target_arch = "x86_64"))]
fn restore_supervisor_state_for_warm(
warm_pid: i32,
state: &state_snap::StateSnapshot,
) -> std::io::Result<()> {
restore_supervisor_state_for_vpid(warm_pid, 1, state, true)
}
#[cfg(test)]
mod c4_capture_tests {
use super::*;
use std::io::{Seek, SeekFrom, Write as _};
use std::os::fd::AsRawFd;
#[test]
fn host_fd_to_target_regular_file_carries_offset_and_flags() {
let path = std::env::temp_dir().join(format!("nev-c4-{}.bin", std::process::id()));
let mut f = std::fs::OpenOptions::new()
.read(true)
.write(true)
.create(true)
.truncate(true)
.open(&path)
.unwrap();
f.write_all(b"0123456789").unwrap();
f.seek(SeekFrom::Start(4)).unwrap();
match host_fd_to_target(f.as_raw_fd(), &HashMap::new()).unwrap() {
Some(state_snap::FdTarget::RootRelFile { offset, flags, .. }) => {
assert_eq!(offset, 4, "current seek offset captured");
assert_eq!(flags & 0x3, 2, "O_RDWR access mode captured");
}
other => panic!("expected RootRelFile, got {other:?}"),
}
let _ = std::fs::remove_file(&path);
}
#[test]
fn host_fd_to_target_external_pipe_is_omitted() {
let mut fds = [0i32; 2];
let r = unsafe {
host(
SYS_PIPE2,
fds.as_mut_ptr() as u64,
libc::O_CLOEXEC as u64,
0,
0,
0,
0,
)
};
assert_eq!(r, 0);
let target = host_fd_to_target(fds[1], &HashMap::new()).unwrap();
unsafe {
host(SYS_CLOSE, fds[0] as u64, 0, 0, 0, 0, 0);
host(SYS_CLOSE, fds[1] as u64, 0, 0, 0, 0, 0);
}
assert!(
target.is_none(),
"a pipe without its peer inside the captured fd graph is external"
);
}
#[test]
fn host_fd_to_target_closed_writer_pipe_preserves_read_buffer() {
let mut fds = [0i32; 2];
let r = unsafe {
host(
SYS_PIPE2,
fds.as_mut_ptr() as u64,
libc::O_CLOEXEC as u64,
0,
0,
0,
0,
)
};
assert_eq!(r, 0);
write_all_to_fd(fds[1], b"reply").unwrap();
unsafe { host(SYS_CLOSE, fds[1] as u64, 0, 0, 0, 0, 0) };
let read_info = pipe_fd_info(fds[0]).unwrap();
let mut groups = HashMap::new();
groups.insert(
read_info.id,
PipeCaptureGroup {
has_read: true,
has_write: false,
buffered: pipe_buffered_bytes(fds[0]).unwrap(),
},
);
let target = host_fd_to_target(fds[0], &groups).unwrap();
unsafe { host(SYS_CLOSE, fds[0] as u64, 0, 0, 0, 0, 0) };
match target {
Some(state_snap::FdTarget::Pipe { end, buffered, .. }) => {
assert_eq!(end, state_snap::PipeEnd::Read);
assert_eq!(buffered, b"reply");
}
other => panic!("expected read Pipe with buffered bytes, got {other:?}"),
}
}
#[test]
fn host_fd_to_target_internal_pipe_carries_id_and_buffer() {
let mut fds = [0i32; 2];
let r = unsafe {
host(
SYS_PIPE2,
fds.as_mut_ptr() as u64,
libc::O_CLOEXEC as u64,
0,
0,
0,
0,
)
};
assert_eq!(r, 0);
write_all_to_fd(fds[1], b"abc").unwrap();
let read_info = pipe_fd_info(fds[0]).unwrap();
let mut groups = HashMap::new();
groups.insert(
read_info.id,
PipeCaptureGroup {
has_read: true,
has_write: true,
buffered: pipe_buffered_bytes(fds[0]).unwrap(),
},
);
let target = host_fd_to_target(fds[0], &groups).unwrap();
unsafe {
host(SYS_CLOSE, fds[0] as u64, 0, 0, 0, 0, 0);
host(SYS_CLOSE, fds[1] as u64, 0, 0, 0, 0, 0);
}
match target {
Some(state_snap::FdTarget::Pipe {
id, end, buffered, ..
}) => {
assert_eq!(id, read_info.id);
assert_eq!(end, state_snap::PipeEnd::Read);
assert_eq!(buffered, b"abc");
}
other => panic!("expected Pipe, got {other:?}"),
}
}
#[test]
fn restore_pipe_group_preserves_buffered_bytes() {
let group = PipeRestoreGroup {
buffered: b"xyz".to_vec(),
entries: vec![
(3, state_snap::PipeEnd::Read, O_RDONLY as u32),
(4, state_snap::PipeEnd::Write, O_WRONLY as u32),
],
};
let restored = restore_pipe_group(&group).unwrap();
let read_fd = match restored.iter().find(|(gfd, _)| *gfd == 3).unwrap().1 {
FdVal::Host(h) => h,
FdVal::Loop(_) => panic!("pipe restore returned loop fd"),
};
let mut buf = [0u8; 3];
let n = unsafe {
host(
SYS_READ,
read_fd as u64,
buf.as_mut_ptr() as u64,
buf.len() as u64,
0,
0,
0,
)
};
assert_eq!(n, 3);
assert_eq!(&buf, b"xyz");
for (_, val) in restored {
fd_close_val(val);
}
}
}
fn service(slot: u64, pid: i32, nr: i64, a: u64, b: u64, c: u64, d: u64, e: u64, f: u64) -> i64 {
if let Some(sid) = fd_loop(pid, a as i32) {
match nr {
SYS_READ | SYS_WRITE | SYS_WRITEV | SYS_READV | SYS_SENDMSG | SYS_RECVMSG
| SYS_RECVFROM | SYS_SENDTO | SYS_LISTEN | SYS_ACCEPT | SYS_ACCEPT4 | SYS_SHUTDOWN
| SYS_GETSOCKNAME | SYS_GETPEERNAME | SYS_SETSOCKOPT | SYS_GETSOCKOPT | SYS_FCNTL
| SYS_DUP | SYS_DUP2 | SYS_DUP3 | SYS_SENDFILE => {
return loop_service(pid, nr, a as i32, sid, b, c, d, e, f);
}
_ => {}
}
}
let a = match nr {
SYS_READ | SYS_WRITE | SYS_PREAD64 | SYS_PWRITE64 | SYS_GETDENTS64 | SYS_WRITEV | SYS_READV
| SYS_FSTAT | SYS_LSEEK | SYS_IOCTL | SYS_FTRUNCATE | SYS_FSYNC | SYS_FDATASYNC
| SYS_FCHMOD | SYS_FCHOWN | SYS_FADVISE64 | SYS_SENDFILE | SYS_LISTEN
| SYS_SENDTO | SYS_RECVFROM | SYS_SENDMSG | SYS_RECVMSG
| SYS_SENDMMSG | SYS_RECVMMSG
| SYS_SHUTDOWN | SYS_ACCEPT | SYS_ACCEPT4 | SYS_GETSOCKNAME | SYS_GETPEERNAME
| SYS_SETSOCKOPT | SYS_GETSOCKOPT | SYS_EPOLL_CTL | SYS_EPOLL_WAIT
| SYS_EPOLL_PWAIT | SYS_TIMERFD_SETTIME | SYS_TIMERFD_GETTIME | SYS_DUP
| SYS_FALLOCATE | SYS_PREADV | SYS_PWRITEV | SYS_PREADV2 | SYS_PWRITEV2
| SYS_FSTATFS | SYS_COPY_FILE_RANGE | SYS_SPLICE | SYS_TEE | SYS_VMSPLICE
| SYS_FLOCK | SYS_FCNTL
| SYS_INOTIFY_ADD_WATCH | SYS_INOTIFY_RM_WATCH => match fd_host(pid, a as i32) {
Some(h) => h as u64,
None => return -9, },
SYS_OPENAT | SYS_NEWFSTATAT if a as i32 != -100 => match fd_host(pid, a as i32) {
Some(h) => h as u64,
None => return -9,
},
_ => a,
};
let b = match nr {
SYS_SENDFILE | SYS_TEE => match fd_host(pid, b as i32) {
Some(h) => h as u64,
None => return -9,
},
_ => b,
};
let c = match nr {
SYS_COPY_FILE_RANGE | SYS_SPLICE => match fd_host(pid, c as i32) {
Some(h) => h as u64,
None => return -9,
},
_ => c,
};
unsafe {
match nr {
SYS_WRITE => {
let len = (c.min(CAP)) as usize;
with_scratch(len, |buf| {
vm_read(pid, b, buf);
host(SYS_WRITE, a, buf.as_ptr() as u64, len as u64, 0, 0, 0)
})
}
SYS_PWRITE64 => {
let len = (c.min(CAP)) as usize;
with_scratch(len, |buf| {
vm_read(pid, b, buf);
host(SYS_PWRITE64, a, buf.as_ptr() as u64, len as u64, d, 0, 0)
})
}
SYS_WRITEV => {
let cnt = (c.min(1024)) as usize;
let mut iovs = vec![0u8; cnt * 16];
vm_read(pid, b, &mut iovs);
let mut data = Vec::new();
for i in 0..cnt {
let base = u64::from_le_bytes(iovs[i * 16..i * 16 + 8].try_into().unwrap());
let len = u64::from_le_bytes(iovs[i * 16 + 8..i * 16 + 16].try_into().unwrap());
let len = len.min(CAP) as usize;
let mut seg = vec![0u8; len];
if len > 0 {
vm_read(pid, base, &mut seg);
}
data.extend_from_slice(&seg);
}
host(
SYS_WRITE,
a,
data.as_ptr() as u64,
data.len() as u64,
0,
0,
0,
)
}
SYS_GETDENTS64 if synth_fd_dirs().lock().unwrap().contains_key(&(a as i32)) => {
let (owner, cursor) = *synth_fd_dirs().lock().unwrap().get(&(a as i32)).unwrap();
let (recs, next) = synth_fd_dirents(owner, cursor, (c.min(CAP)) as usize);
if !recs.is_empty() {
vm_write(pid, b, &recs);
synth_fd_dirs()
.lock()
.unwrap()
.insert(a as i32, (owner, next));
}
recs.len() as i64
}
SYS_GETDENTS64 if synth_proc_dirs().lock().unwrap().contains_key(&(a as i32)) => {
let cursor = *synth_proc_dirs().lock().unwrap().get(&(a as i32)).unwrap();
let (recs, next) = synth_proc_dirents(cursor, (c.min(CAP)) as usize);
if !recs.is_empty() {
vm_write(pid, b, &recs);
synth_proc_dirs().lock().unwrap().insert(a as i32, next);
}
recs.len() as i64
}
SYS_READ | SYS_PREAD64 | SYS_GETDENTS64 => {
let len = (c.min(CAP)) as usize;
with_scratch(len, |buf| {
let n = host_cancellable(nr, a, buf.as_mut_ptr() as u64, len as u64, d, e, f);
if n == CANCEL_SENTINEL {
return CANCEL_SENTINEL;
}
if n > 0 {
let w = vm_write(pid, b, &buf[..n as usize]);
if w != n {
return -14; }
}
n
})
}
SYS_GETRANDOM => {
let len = (b.min(CAP)) as usize;
with_scratch(len, |buf| {
let n = host(
SYS_GETRANDOM,
buf.as_mut_ptr() as u64,
len as u64,
c,
0,
0,
0,
);
if n > 0 {
vm_write(pid, a, &buf[..n as usize]);
}
n
})
}
SYS_CAPGET | SYS_CAPSET => {
const LINUX_CAPABILITY_VERSION_1: u32 = 0x1998_0330;
const LINUX_CAPABILITY_VERSION_2: u32 = 0x2007_1026;
const LINUX_CAPABILITY_VERSION_3: u32 = 0x2008_0522;
if a == 0 {
return -14; }
let mut hdr = [0u8; 8];
if vm_read(pid, a, &mut hdr) != hdr.len() as i64 {
return -14; }
let version = u32::from_ne_bytes([hdr[0], hdr[1], hdr[2], hdr[3]]);
let words = match version {
LINUX_CAPABILITY_VERSION_1 => 1usize,
LINUX_CAPABILITY_VERSION_2 | LINUX_CAPABILITY_VERSION_3 => 2usize,
_ => {
vm_write(pid, a, &LINUX_CAPABILITY_VERSION_3.to_ne_bytes());
return -22; }
};
if nr == SYS_CAPGET && b != 0 {
let zero = vec![0u8; words * 12];
if vm_write(pid, b, &zero) != zero.len() as i64 {
return -14; }
}
0
}
SYS_TGKILL => {
let tgid = a as i32;
let tid = b as i32;
let mine = tgid > 0
&& tid > 0
&& pid_in_sandbox(tgid)
&& std::path::Path::new(&format!("/proc/{}/task/{}", tgid, tid)).exists();
if mine {
let r = host(SYS_TGKILL, a, b, c, 0, 0, 0);
if r == 0 && c != 0 {
interrupt_slots_for_pid(tgid);
}
r
} else {
-1 }
}
SYS_TKILL => {
let tid = a as i32;
let mine = tid > 0
&& std::path::Path::new(&format!("/proc/{}/task/{}", pid, tid)).exists();
if mine {
let r = host(SYS_TKILL, a, b, 0, 0, 0, 0);
if r == 0 && b != 0 {
interrupt_slots_for_pid(pid);
}
r
} else {
-1 }
}
SYS_KILL => {
let target = a as i32;
if pid_in_sandbox(target) {
let r = host(SYS_KILL, a, b, 0, 0, 0, 0);
if r == 0 && b != 0 {
interrupt_slots_for_pid(target);
}
r
} else {
-3 }
}
SYS_OPEN => {
let p = pull_cwd_path(pid, a);
let n = open_path(pid, &p, b, c);
if n < 0 {
if n == -24 {
log_open_emfile(pid, nr, &p);
}
n
} else {
fd_install(pid, n as i32, 0)
}
}
SYS_CREAT => {
let p = pull_cwd_path(pid, a);
let flags = (libc::O_WRONLY | libc::O_CREAT | libc::O_TRUNC) as u64;
let n = open_path(pid, &p, flags, b);
if n < 0 {
if n == -24 {
log_open_emfile(pid, nr, &p);
}
n
} else {
fd_install(pid, n as i32, 0)
}
}
SYS_OPENAT => {
let raw = pull_path(pid, b);
let raw_bare = match raw.split_last() {
Some((0, head)) => head,
_ => raw.as_slice(),
};
let n = if a as i32 == AT_FDCWD || raw_bare.first() == Some(&b'/') {
let p = if raw_bare.first() == Some(&b'/') {
raw.clone()
} else {
pull_cwd_path(pid, b)
};
open_path(pid, &p, c, d)
} else {
match openat_abs_guest_path(a as i32, &raw) {
Some(abs) if abs.starts_with(b"/proc/") || abs.starts_with(b"/dev/fd") => {
open_path(pid, &abs, c, d)
}
_ => {
let created_candidate = (c & O_CREAT) != 0;
let existed_before = if created_candidate {
let how = OpenHow {
flags: O_PATH,
mode: 0,
resolve: RESOLVE_BENEATH,
};
let fd = host(
SYS_OPENAT2,
a,
raw.as_ptr() as u64,
&how as *const _ as u64,
std::mem::size_of::<OpenHow>() as u64,
0,
0,
);
if fd >= 0 {
host(SYS_CLOSE, fd as u64, 0, 0, 0, 0, 0);
true
} else {
false
}
} else {
true
};
let fd = host(SYS_OPENAT, a, raw.as_ptr() as u64, c, d, 0, 0);
if fd >= 0 && created_candidate && !existed_before {
chown_created_fd_to_virtual(pid, fd as i32);
}
fd
}
}
};
if n < 0 {
if n == -24 {
let p = if a as i32 == AT_FDCWD {
pull_cwd_path(pid, b)
} else {
raw.clone()
};
log_open_emfile(pid, nr, &p);
}
n
} else {
fd_install(pid, n as i32, 0)
}
}
SYS_OPENAT2 => {
let how = match pull_open_how(pid, c, d) {
Ok(how) => how,
Err(errno) => return errno,
};
let raw = pull_path(pid, b);
let raw_bare = match raw.split_last() {
Some((0, head)) => head,
_ => raw.as_slice(),
};
let n = if a as i32 == AT_FDCWD || raw_bare.first() == Some(&b'/') {
let p = if raw_bare.first() == Some(&b'/') {
raw.clone()
} else {
pull_cwd_path(pid, b)
};
open_path(pid, &p, how.flags, how.mode)
} else {
let hostdir = match fd_host(pid, a as i32) {
Some(h) => h,
None => return -9, };
match openat_abs_guest_path(hostdir, &raw) {
Some(abs) if abs.starts_with(b"/proc/") || abs.starts_with(b"/dev/fd") => {
open_path(pid, &abs, how.flags, how.mode)
}
_ => {
let resolve = if (how.resolve & RESOLVE_IN_ROOT) != 0 {
how.resolve
} else {
how.resolve | RESOLVE_BENEATH
};
let host_how = OpenHow {
flags: how.flags,
mode: mode_for_open_flags(pid, how.flags, how.mode),
resolve,
};
let created_candidate = (how.flags & O_CREAT) != 0;
let existed_before = if created_candidate {
let exists_how = OpenHow {
flags: O_PATH,
mode: 0,
resolve,
};
let fd = host(
SYS_OPENAT2,
hostdir as u64,
raw.as_ptr() as u64,
&exists_how as *const _ as u64,
std::mem::size_of::<OpenHow>() as u64,
0,
0,
);
if fd >= 0 {
host(SYS_CLOSE, fd as u64, 0, 0, 0, 0, 0);
true
} else {
false
}
} else {
true
};
let fd = host(
SYS_OPENAT2,
hostdir as u64,
raw.as_ptr() as u64,
&host_how as *const _ as u64,
std::mem::size_of::<OpenHow>() as u64,
0,
0,
);
if fd >= 0 && created_candidate && !existed_before {
let exact_mode = mode_for_open_flags(pid, how.flags, how.mode);
host(SYS_FCHMOD, fd as u64, exact_mode, 0, 0, 0, 0);
chown_created_fd_to_virtual(pid, fd as i32);
}
fd
}
}
};
if n < 0 {
if n == -24 {
log_open_emfile(pid, nr, &raw);
}
n
} else {
fd_install(pid, n as i32, 0)
}
}
SYS_ACCESS | SYS_FACCESSAT | SYS_FACCESSAT2 => {
let (dirfd, addr, mode, flags) = if nr == SYS_ACCESS {
(AT_FDCWD, a, b, 0)
} else {
(a as i32, b, c, d)
};
if (mode & !(R_OK | W_OK | X_OK)) != 0 {
return -22; }
let raw = pull_path(pid, addr);
let raw_bare = match raw.split_last() {
Some((0, head)) => head,
_ => raw.as_slice(),
};
let mut close_after = true;
let fd = if raw_bare.is_empty() {
if nr == SYS_ACCESS || (flags & AT_EMPTY_PATH) == 0 {
return -2; }
if dirfd == AT_FDCWD {
let cwd = cwd_of(pid);
opath(pid, &cwd, false)
} else {
close_after = false;
match fd_host(pid, dirfd) {
Some(h) => h as i64,
None => return -9, }
}
} else if nr == SYS_ACCESS || dirfd == AT_FDCWD || raw_bare.first() == Some(&b'/') {
let p = if raw_bare.first() == Some(&b'/') {
raw.clone()
} else {
pull_cwd_path(pid, addr)
};
opath(pid, &p, false)
} else {
let hostdir = match fd_host(pid, dirfd) {
Some(h) => h,
None => return -9, };
if let Some(abs) = openat_abs_guest_path(hostdir, &raw) {
opath(pid, &abs, false)
} else {
let how = OpenHow {
flags: O_PATH,
mode: 0,
resolve: RESOLVE_BENEATH,
};
host(
SYS_OPENAT2,
hostdir as u64,
raw.as_ptr() as u64,
&how as *const _ as u64,
std::mem::size_of::<OpenHow>() as u64,
0,
0,
)
}
};
if fd < 0 {
return fd;
}
let use_effective = nr != SYS_ACCESS && (flags & AT_EACCESS) != 0;
let r = faccess_fd(pid, fd as i32, mode, use_effective);
if close_after {
host(SYS_CLOSE, fd as u64, 0, 0, 0, 0, 0);
}
r
}
SYS_STATFS => {
let p = pull_cwd_path(pid, a);
let fd = opath(pid, &p, false);
if fd < 0 {
return fd;
}
let mut sb = [0u8; 120];
let n = host(SYS_FSTATFS, fd as u64, sb.as_mut_ptr() as u64, 0, 0, 0, 0);
host(SYS_CLOSE, fd as u64, 0, 0, 0, 0, 0);
if n == 0 {
vm_write(pid, b, &sb);
}
n
}
SYS_STAT | SYS_LSTAT => {
let p = pull_cwd_path(pid, a);
let fd = opath(pid, &p, nr == SYS_LSTAT);
if fd < 0 {
return fd;
}
let mut sb = [0u8; 144];
let n = host(SYS_FSTAT, fd as u64, sb.as_mut_ptr() as u64, 0, 0, 0, 0);
host(SYS_CLOSE, fd as u64, 0, 0, 0, 0, 0);
if n == 0 {
vm_write(pid, b, &sb);
}
n
}
SYS_FSTAT => {
let mut sb = [0u8; 144];
let n = host(SYS_FSTAT, a, sb.as_mut_ptr() as u64, 0, 0, 0, 0);
if n == 0 {
vm_write(pid, b, &sb);
}
n
}
SYS_NEWFSTATAT => {
if a as i32 == -100 {
let p = pull_cwd_path(pid, b);
let fd = opath(pid, &p, (d & 0x100) != 0); if fd < 0 {
return fd;
}
let mut sb = [0u8; 144];
let n = host(SYS_FSTAT, fd as u64, sb.as_mut_ptr() as u64, 0, 0, 0, 0);
host(SYS_CLOSE, fd as u64, 0, 0, 0, 0, 0);
if n == 0 {
vm_write(pid, c, &sb);
}
n
} else {
let p = pull_path(pid, b);
if synth_fd_dirs().lock().unwrap().contains_key(&(a as i32)) {
let stop = p.iter().position(|&c| c == b'/' || c == 0).unwrap_or(p.len());
return match parse_u32(&p[..stop]) {
Some(n) => match fd_host(pid, n as i32) {
Some(h) => {
let mut sb = [0u8; 144];
let n2 =
host(SYS_FSTAT, h as u64, sb.as_mut_ptr() as u64, 0, 0, 0, 0);
if n2 == 0 {
vm_write(pid, c, &sb);
}
n2
}
None => -9, },
None => {
let mut sb = [0u8; 144];
let n2 = host(SYS_FSTAT, a, sb.as_mut_ptr() as u64, 0, 0, 0, 0);
if n2 == 0 {
vm_write(pid, c, &sb);
}
n2
}
};
}
match openat_abs_guest_path(a as i32, &p) {
Some(abs) if abs.starts_with(b"/proc/") || abs.starts_with(b"/dev/fd") => {
let fd = opath(pid, &abs, (d & 0x100) != 0);
if fd < 0 {
fd
} else {
let mut sb = [0u8; 144];
let n =
host(SYS_FSTAT, fd as u64, sb.as_mut_ptr() as u64, 0, 0, 0, 0);
host(SYS_CLOSE, fd as u64, 0, 0, 0, 0, 0);
if n == 0 {
vm_write(pid, c, &sb);
}
n
}
}
_ => {
let mut sb = [0u8; 144];
let n = host(
SYS_NEWFSTATAT,
a,
p.as_ptr() as u64,
sb.as_mut_ptr() as u64,
d,
0,
0,
);
if n == 0 {
vm_write(pid, c, &sb);
}
n
}
}
}
}
SYS_STATX => {
let mut sb = [0u8; 256];
let empty = [0u8; 1];
let raw = pull_path(pid, b);
let raw_bare = match raw.split_last() {
Some((0, head)) => head,
_ => raw.as_slice(),
};
let n = if a as i32 != AT_FDCWD && raw_bare.is_empty() && (c & AT_EMPTY_PATH) != 0 {
let hostfd = match fd_host(pid, a as i32) {
Some(h) => h,
None => return -9,
};
host(
SYS_STATX,
hostfd as u64,
empty.as_ptr() as u64,
c,
d,
sb.as_mut_ptr() as u64,
0,
)
} else if a as i32 == AT_FDCWD || raw_bare.first() == Some(&b'/') {
let p = if raw_bare.first() == Some(&b'/') {
raw
} else {
pull_cwd_path(pid, b)
};
let fd = opath(pid, &p, (c & 0x100) != 0);
if fd < 0 {
return fd;
}
let n = host(
SYS_STATX,
fd as u64,
empty.as_ptr() as u64,
AT_EMPTY_PATH,
d,
sb.as_mut_ptr() as u64,
0,
);
host(SYS_CLOSE, fd as u64, 0, 0, 0, 0, 0);
n
} else {
let hostdir = match fd_host(pid, a as i32) {
Some(h) => h,
None => return -9,
};
match openat_abs_guest_path(hostdir, &raw) {
Some(abs) if abs.starts_with(b"/proc/") || abs.starts_with(b"/dev/fd") => {
let fd = opath(pid, &abs, (c & 0x100) != 0);
if fd < 0 {
fd
} else {
let n = host(
SYS_STATX,
fd as u64,
empty.as_ptr() as u64,
AT_EMPTY_PATH,
d,
sb.as_mut_ptr() as u64,
0,
);
host(SYS_CLOSE, fd as u64, 0, 0, 0, 0, 0);
n
}
}
_ => host(
SYS_STATX,
hostdir as u64,
raw.as_ptr() as u64,
c,
d,
sb.as_mut_ptr() as u64,
0,
),
}
};
if n == 0 {
vm_write(pid, e, &sb);
}
n
}
SYS_READLINK | SYS_READLINKAT => {
let (path_ptr, out_ptr, sz) = if nr == SYS_READLINK {
(a, b, c)
} else {
(b, c, d)
};
let p = if nr == SYS_READLINKAT && a as i32 != -100 {
fd_host(pid, a as i32)
.and_then(|h| openat_abs_guest_path(h, &pull_path(pid, path_ptr)))
.unwrap_or_else(|| pull_cwd_path(pid, path_ptr))
} else {
pull_cwd_path(pid, path_ptr)
};
let bare = match p.split_last() {
Some((0, head)) => head,
_ => p.as_slice(),
};
if let Some(target) = proc_self_readlink(pid, bare) {
let w = target.len().min(sz as usize);
if w > 0 {
vm_write(pid, out_ptr, &target[..w]);
}
return w as i64;
}
if bare == b"/dev/fd" {
let target = b"/proc/self/fd";
let w = target.len().min(sz as usize);
if w > 0 {
vm_write(pid, out_ptr, &target[..w]);
}
return w as i64;
}
let fd = opath(pid, &p, true); if fd < 0 {
return fd; }
let len = (sz.min(CAP)) as usize;
let mut buf = vec![0u8; len];
let empty = [0u8; 1];
let n = host(
SYS_READLINKAT,
fd as u64,
empty.as_ptr() as u64,
buf.as_mut_ptr() as u64,
len as u64,
0,
0,
);
host(SYS_CLOSE, fd as u64, 0, 0, 0, 0, 0);
if n > 0 {
vm_write(pid, out_ptr, &buf[..n as usize]);
n
} else {
-22 }
}
SYS_MKDIR | SYS_MKDIRAT => {
let (dirfd, path_ptr, mode) = if nr == SYS_MKDIR {
(AT_FDCWD, a, b)
} else {
(a as i32, b, c)
};
match confined_parent(pid, dirfd, path_ptr) {
Ok((pfd, base)) => {
let mode = apply_umask(pid, mode & 0o7777);
let r = host(SYS_MKDIRAT, pfd as u64, base.as_ptr() as u64, mode, 0, 0, 0);
if r == 0 {
host(SYS_FCHMODAT, pfd as u64, base.as_ptr() as u64, mode, 0, 0, 0);
chown_created_at_to_virtual(pid, pfd, &base, 0);
}
host(SYS_CLOSE, pfd as u64, 0, 0, 0, 0, 0);
r
}
Err(e) => e,
}
}
SYS_RMDIR | SYS_UNLINK | SYS_UNLINKAT => {
let (dirfd, path_ptr, flags) = match nr {
SYS_RMDIR => (AT_FDCWD, a, AT_REMOVEDIR),
SYS_UNLINK => (AT_FDCWD, a, 0),
_ => (a as i32, b, c),
};
if flags & AT_REMOVEDIR == 0 {
let raw = pull_path(pid, path_ptr);
let raw_bare = match raw.split_last() {
Some((0, head)) => head,
_ => raw.as_slice(),
};
let abs = if raw_bare.first() == Some(&b'/') {
raw.clone()
} else if dirfd == AT_FDCWD {
pull_cwd_path(pid, path_ptr)
} else {
fd_host(pid, dirfd)
.and_then(|h| openat_abs_guest_path(h, &raw))
.unwrap_or_else(|| raw.clone())
};
if let Some(mut alias) = unix_socket_alias_for_guest_path(&abs) {
alias.push(0);
return host(
SYS_UNLINKAT,
AT_FDCWD as u64,
alias.as_ptr() as u64,
0,
0,
0,
0,
);
}
}
match confined_parent(pid, dirfd, path_ptr) {
Ok((pfd, base)) => {
let r = host(
SYS_UNLINKAT,
pfd as u64,
base.as_ptr() as u64,
flags,
0,
0,
0,
);
host(SYS_CLOSE, pfd as u64, 0, 0, 0, 0, 0);
r
}
Err(e) => e,
}
}
SYS_SYMLINK | SYS_SYMLINKAT => {
let (target_ptr, dirfd, link_ptr) = if nr == SYS_SYMLINK {
(a, AT_FDCWD, b)
} else {
(a, b as i32, c)
};
let target = pull_path(pid, target_ptr);
match confined_parent(pid, dirfd, link_ptr) {
Ok((pfd, base)) => {
let r = host(
SYS_SYMLINKAT,
target.as_ptr() as u64,
pfd as u64,
base.as_ptr() as u64,
0,
0,
0,
);
if r == 0 {
chown_created_at_to_virtual(pid, pfd, &base, AT_SYMLINK_NOFOLLOW);
}
host(SYS_CLOSE, pfd as u64, 0, 0, 0, 0, 0);
r
}
Err(e) => e,
}
}
SYS_CHMOD | SYS_FCHMODAT => {
let (dirfd, path_ptr, mode, flags) = if nr == SYS_CHMOD {
(AT_FDCWD, a, b, 0)
} else {
(a as i32, b, c, d)
};
let raw = pull_path(pid, path_ptr);
let raw_bare = match raw.split_last() {
Some((0, head)) => head,
_ => raw.as_slice(),
};
let abs = if raw_bare.first() == Some(&b'/') {
raw.clone()
} else if dirfd == AT_FDCWD {
pull_cwd_path(pid, path_ptr)
} else {
fd_host(pid, dirfd)
.and_then(|h| openat_abs_guest_path(h, &raw))
.unwrap_or_else(|| raw.clone())
};
if let Some(mut alias) = unix_socket_alias_for_guest_path(&abs) {
alias.push(0);
return host(
SYS_FCHMODAT,
AT_FDCWD as u64,
alias.as_ptr() as u64,
mode,
flags,
0,
0,
);
}
match confined_parent(pid, dirfd, path_ptr) {
Ok((pfd, base)) => {
let r = host(
SYS_FCHMODAT,
pfd as u64,
base.as_ptr() as u64,
mode,
flags,
0,
0,
);
host(SYS_CLOSE, pfd as u64, 0, 0, 0, 0, 0);
r
}
Err(e) => e,
}
}
SYS_CHOWN | SYS_LCHOWN | SYS_FCHOWNAT => {
let (dirfd, path_ptr, owner, group, flags) = match nr {
SYS_CHOWN => (AT_FDCWD, a, b, c, 0),
SYS_LCHOWN => (AT_FDCWD, a, b, c, 0x100),
_ => (a as i32, b, c, d, e),
};
match confined_parent(pid, dirfd, path_ptr) {
Ok((pfd, base)) => {
let r = host(
SYS_FCHOWNAT,
pfd as u64,
base.as_ptr() as u64,
owner,
group,
flags,
0,
);
host(SYS_CLOSE, pfd as u64, 0, 0, 0, 0, 0);
r
}
Err(e) => e,
}
}
SYS_UTIMENSAT => {
let times = if c != 0 {
let mut t = [0u8; 32];
vm_read(pid, c, &mut t);
Some(t)
} else {
None
};
if b == 0 {
let hostfd = match fd_host(pid, a as i32) {
Some(h) => h as u64,
None => return -9,
};
let tp = times.as_ref().map(|t| t.as_ptr() as u64).unwrap_or(0);
host(SYS_UTIMENSAT, hostfd, 0, tp, d, 0, 0)
} else {
match confined_parent(pid, a as i32, b) {
Ok((pfd, base)) => {
let tp = times.as_ref().map(|t| t.as_ptr() as u64).unwrap_or(0);
let r =
host(SYS_UTIMENSAT, pfd as u64, base.as_ptr() as u64, tp, d, 0, 0);
host(SYS_CLOSE, pfd as u64, 0, 0, 0, 0, 0);
r
}
Err(e) => e,
}
}
}
SYS_TRUNCATE => {
let p = pull_cwd_path(pid, a);
let fd = open_path(pid, &p, libc::O_WRONLY as u64, 0);
if fd < 0 {
fd
} else {
let r = host(SYS_FTRUNCATE, fd as u64, b, 0, 0, 0, 0);
host(SYS_CLOSE, fd as u64, 0, 0, 0, 0, 0);
r
}
}
SYS_RENAME | SYS_RENAMEAT | SYS_RENAMEAT2 => {
let (od, op, nd, np, flags) = match nr {
SYS_RENAME => (AT_FDCWD, a, AT_FDCWD, b, 0),
SYS_RENAMEAT => (a as i32, b, c as i32, d, 0),
_ => (a as i32, b, c as i32, d, e),
};
match confined_parent(pid, od, op) {
Ok((opfd, obase)) => match confined_parent(pid, nd, np) {
Ok((npfd, nbase)) => {
let r = host(
SYS_RENAMEAT2,
opfd as u64,
obase.as_ptr() as u64,
npfd as u64,
nbase.as_ptr() as u64,
flags,
0,
);
host(SYS_CLOSE, opfd as u64, 0, 0, 0, 0, 0);
host(SYS_CLOSE, npfd as u64, 0, 0, 0, 0, 0);
r
}
Err(e) => {
host(SYS_CLOSE, opfd as u64, 0, 0, 0, 0, 0);
e
}
},
Err(e) => e,
}
}
SYS_LINK | SYS_LINKAT => {
let (od, op, nd, np, flags) = if nr == SYS_LINK {
(AT_FDCWD, a, AT_FDCWD, b, 0)
} else {
(a as i32, b, c as i32, d, e)
};
let tmpfile_old_fd: Option<i32> = if nr == SYS_LINKAT {
if (flags & AT_EMPTY_PATH) != 0 {
Some(od)
} else {
let raw = pull_path(pid, op);
let bare = match raw.split_last() {
Some((0, head)) => head,
_ => raw.as_slice(),
};
proc_self_rel(pid, bare)
.and_then(|rest| rest.strip_prefix(b"fd/"))
.and_then(parse_u32)
}
} else {
None
};
if let Some(gfd) = tmpfile_old_fd {
let oldh = match fd_host(pid, gfd) {
Some(h) => h,
None => return -9, };
return match confined_parent(pid, nd, np) {
Ok((npfd, nbase)) => {
let r = host(
SYS_LINKAT,
oldh as u64,
b"\0".as_ptr() as u64,
npfd as u64,
nbase.as_ptr() as u64,
AT_EMPTY_PATH,
0,
);
host(SYS_CLOSE, npfd as u64, 0, 0, 0, 0, 0);
r
}
Err(e) => e,
};
}
match confined_parent(pid, od, op) {
Ok((opfd, obase)) => match confined_parent(pid, nd, np) {
Ok((npfd, nbase)) => {
let r = host(
SYS_LINKAT,
opfd as u64,
obase.as_ptr() as u64,
npfd as u64,
nbase.as_ptr() as u64,
flags,
0,
);
host(SYS_CLOSE, opfd as u64, 0, 0, 0, 0, 0);
host(SYS_CLOSE, npfd as u64, 0, 0, 0, 0, 0);
r
}
Err(e) => {
host(SYS_CLOSE, opfd as u64, 0, 0, 0, 0, 0);
e
}
},
Err(e) => e,
}
}
SYS_GETCWD => {
let mut cwd = cwd_of(pid);
cwd.push(0);
if cwd.len() > b as usize {
-34 } else {
vm_write(pid, a, &cwd);
cwd.len() as i64
}
}
SYS_UNAME => {
let mut sb = [0u8; 390];
let n = host(SYS_UNAME, sb.as_mut_ptr() as u64, 0, 0, 0, 0, 0);
if n == 0 {
vm_write(pid, a, &sb);
}
n
}
SYS_SYSINFO => {
let mut sb = [0u8; 112];
let n = host(SYS_SYSINFO, sb.as_mut_ptr() as u64, 0, 0, 0, 0, 0);
if n == 0 {
vm_write(pid, a, &sb);
}
n
}
SYS_GETRUSAGE => {
let mut rb = [0u8; RUSAGE_SZ];
let n = host(SYS_GETRUSAGE, a, rb.as_mut_ptr() as u64, 0, 0, 0, 0);
if n == 0 {
vm_write(pid, b, &rb);
}
n
}
SYS_TIMES => {
let mut tb = [0u8; 32];
let n = host(SYS_TIMES, tb.as_mut_ptr() as u64, 0, 0, 0, 0, 0);
if n >= 0 && a != 0 {
vm_write(pid, a, &tb);
}
n
}
SYS_CLOCK_GETTIME => {
if a as i32 == libc::CLOCK_MONOTONIC || a as i32 == libc::CLOCK_BOOTTIME {
if b == 0 {
-14
} else {
let ts = guest_monotonic_timespec();
vm_write(pid, b, &ts);
0
}
} else {
let mut ts = [0u8; 16];
let n = host(SYS_CLOCK_GETTIME, a, ts.as_mut_ptr() as u64, 0, 0, 0, 0);
if n == 0 {
vm_write(pid, b, &ts);
}
n
}
}
SYS_GETTIMEOFDAY => {
let mut tv = [0u8; 16];
let n = host(SYS_GETTIMEOFDAY, tv.as_mut_ptr() as u64, 0, 0, 0, 0, 0);
if n == 0 {
vm_write(pid, a, &tv);
}
n
}
SYS_PRLIMIT64 => {
let mut nbuf = [0u8; 16];
let np = if c != 0 {
if vm_read(pid, c, &mut nbuf) != nbuf.len() as i64 {
return -14; }
nbuf.as_ptr() as u64
} else {
0
};
let mut obuf = [0u8; 16];
let op = if d != 0 { obuf.as_mut_ptr() as u64 } else { 0 };
if b == libc::RLIMIT_NOFILE as u64 {
let target = if a == 0 { pid } else { a as i32 };
if target != pid && proctree::vpid_for(target).is_none() {
return -1; }
let (old_cur, old_max) = guest_nofile_limit(target);
if c != 0 {
let new_cur = u64::from_le_bytes(nbuf[0..8].try_into().unwrap());
let new_max = u64::from_le_bytes(nbuf[8..16].try_into().unwrap());
if new_cur > new_max {
return -22; }
if new_max > old_max {
return -1; }
set_guest_nofile_limit(target, new_cur, new_max);
}
if d != 0 {
obuf[0..8].copy_from_slice(&old_cur.to_le_bytes());
obuf[8..16].copy_from_slice(&old_max.to_le_bytes());
if vm_write(pid, d, &obuf) != obuf.len() as i64 {
return -14; }
}
return 0;
}
let n = host(SYS_PRLIMIT64, a, b, np, op, 0, 0);
if n == 0 && d != 0 {
if vm_write(pid, d, &obuf) != obuf.len() as i64 {
return -14; }
}
n
}
SYS_SENDFILE => {
if c == 0 {
host(SYS_SENDFILE, a, b, 0, d, 0, 0)
} else {
let mut ob = [0u8; 8];
vm_read(pid, c, &mut ob);
let n = host(SYS_SENDFILE, a, b, ob.as_mut_ptr() as u64, d, 0, 0);
vm_write(pid, c, &ob);
n
}
}
SYS_SCHED_YIELD | SYS_FADVISE64 | SYS_FSYNC | SYS_FDATASYNC | SYS_FTRUNCATE
| SYS_FCHMOD | SYS_FCHOWN => host(nr, a, b, c, d, e, f),
SYS_FLOCK => host(SYS_FLOCK, a, b, 0, 0, 0, 0),
SYS_EPOLL_CREATE1 | SYS_EVENTFD2 | SYS_TIMERFD_CREATE | SYS_DUP | SYS_INOTIFY_INIT
| SYS_INOTIFY_INIT1 => {
let n = host(nr, a, b, c, d, e, f);
if n < 0 {
n
} else {
fd_install(pid, n as i32, 0)
}
}
SYS_EVENTFD | SYS_EPOLL_CREATE => {
let n = if nr == SYS_EVENTFD {
host(SYS_EVENTFD2, a, 0, 0, 0, 0, 0)
} else {
host(SYS_EPOLL_CREATE1, 0, 0, 0, 0, 0, 0)
};
if n < 0 {
n
} else {
fd_install(pid, n as i32, 0)
}
}
SYS_SIGNALFD | SYS_SIGNALFD4 => {
let gfd = a as i32;
let flags = if nr == SYS_SIGNALFD4 { d } else { 0 };
let mut mask = vec![0u8; c.min(1024) as usize];
if b != 0 {
if !mask.is_empty() && vm_read(pid, b, &mut mask) != mask.len() as i64 {
return -14; }
}
if gfd != -1 {
match fd_host(pid, gfd) {
Some(h) if signalfd_writers().lock().unwrap().contains_key(&h) => {
return gfd as i64;
}
Some(_) => return -22, None => return -9, }
}
let mut fds = [0i32; 2];
let n = host(SYS_PIPE2, fds.as_mut_ptr() as u64, flags, 0, 0, 0, 0);
if n < 0 {
n
} else {
signalfd_writers().lock().unwrap().insert(fds[0], fds[1]);
fd_install(pid, fds[0], 0)
}
}
SYS_SHMGET => sysv_shmget(a, b, c),
SYS_SHMAT => sysv_shmat(a as i32, b, c),
SYS_SHMDT => sysv_shmdt(a),
SYS_SHMCTL => sysv_shmctl(pid, a as i32, b, c),
SYS_MEMFD_CREATE => {
let name = pull_path(pid, a); let n = host(SYS_MEMFD_CREATE, name.as_ptr() as u64, b, 0, 0, 0, 0);
if n < 0 {
ipc_logf(
&[(b"MEMFD-FAIL pid=", pid as i64), (b"err=", n)],
&[],
);
n
} else {
fd_install(pid, n as i32, 0)
}
}
SYS_MMAP => match fd_host(pid, e as i32) {
Some(h) => {
let r = shm_pool_map(pid, h, b, c, f);
if IPCTRACE && r < 0 {
let mut st: libc::stat = std::mem::zeroed();
let sr = host(SYS_FSTAT, h as u64, &mut st as *mut libc::stat as u64, 0, 0, 0, 0);
ipc_logf(
&[
(b"MMAPERR pid=", pid as i64),
(b"gfd=", e as i64),
(b"hfd=", h as i64),
(b"len=", b as i64),
(b"prot=", c as i64),
(b"flags=", d as i64),
(b"off=", f as i64),
(b"fstat=", sr),
(b"dev=", st.st_dev as i64),
(b"ino=", st.st_ino as i64),
(b"size=", st.st_size),
(b"ret=", r),
],
&[],
);
}
r
}
None => {
let (tblsize, near) = {
let t = fdt().lock().unwrap();
match t.get(&pid) {
Some(m) => {
let g = e as i32;
let below = m.range(..g).next_back().map(|(k, _)| *k).unwrap_or(-1);
let above = m.range(g + 1..).next().map(|(k, _)| *k).unwrap_or(-1);
(m.len() as i64, (below as i64) << 32 | (above as i64 & 0xffff_ffff))
}
None => (-1, 0),
}
};
ipc_logf(
&[
(b"SHMPOOL-DECLINE reason=nofd pid=", pid as i64),
(b"gfd=", e as i64),
(b"len=", b as i64),
(b"prot=", c as i64),
(b"tblsize=", tblsize),
(b"near=", near),
],
&[],
);
if IPCTRACE {
ipc_logf(
&[
(b"MMAPERR pid=", pid as i64),
(b"gfd=", e as i64),
(b"hfd=", -1),
(b"len=", b as i64),
(b"prot=", c as i64),
(b"flags=", d as i64),
(b"off=", f as i64),
(b"ret=", -1),
],
&[],
);
}
-1
}
},
SYS_INOTIFY_ADD_WATCH => {
let cand = pull_cwd_path(pid, b);
let pfd = opath(pid, &cand, false);
if pfd < 0 {
pfd
} else {
let mut nb = [0u8; 24];
let i = fmt_i64(pfd, &mut nb);
let mut proc_path = b"/proc/self/fd/".to_vec();
proc_path.extend_from_slice(&nb[i..]);
proc_path.push(0);
let r = host(
SYS_INOTIFY_ADD_WATCH,
a,
proc_path.as_ptr() as u64,
c,
0,
0,
0,
);
host(SYS_CLOSE, pfd as u64, 0, 0, 0, 0, 0);
r
}
}
SYS_INOTIFY_RM_WATCH => host(SYS_INOTIFY_RM_WATCH, a, b, 0, 0, 0, 0),
SYS_MKNOD | SYS_MKNODAT => {
let (dirfd, path_ptr, mode, dev) = if nr == SYS_MKNOD {
(AT_FDCWD, a, b, c)
} else {
(a as i32, b, c, d)
};
let typ = (mode as u32) & S_IFMT;
if typ == S_IFCHR || typ == S_IFBLK {
-1 } else if typ != 0 && typ != S_IFREG && typ != S_IFIFO && typ != S_IFSOCK {
-22 } else {
match confined_parent(pid, dirfd, path_ptr) {
Ok((pfd, base)) => {
let mode = (mode & (S_IFMT as u64)) | apply_umask(pid, mode & 0o7777);
let r = host(
SYS_MKNODAT,
pfd as u64,
base.as_ptr() as u64,
mode,
dev,
0,
0,
);
if r == 0 {
host(SYS_FCHMODAT, pfd as u64, base.as_ptr() as u64, mode & 0o7777, 0, 0, 0);
chown_created_at_to_virtual(pid, pfd, &base, 0);
}
host(SYS_CLOSE, pfd as u64, 0, 0, 0, 0, 0);
r
}
Err(e) => e,
}
}
}
SYS_CLOSE_RANGE => {
let first = a as u32;
let last = b as u32;
let flags = c;
if first > last || (flags & !(CLOSE_RANGE_UNSHARE | CLOSE_RANGE_CLOEXEC)) != 0 {
-22 } else if (flags & CLOSE_RANGE_CLOEXEC) != 0 {
if FDTRACE && first <= 11 && last >= 10 {
ipc_logf(
&[
(b"FDTRACE_CLOSE_RANGE_CLOEXEC pid=", pid as i64),
(b" first=", first as i64),
(b" last=", last as i64),
],
b"",
);
}
let targets: Vec<(i32, FdVal)> = {
let t = fdt().lock().unwrap();
t.get(&pid)
.map(|m| {
m.iter()
.filter(|(&g, _)| (g as u32) >= first && (g as u32) <= last)
.map(|(&g, &v)| (g, v))
.collect()
})
.unwrap_or_default()
};
for (g, v) in targets {
let cur = fd_get_desc_flags(pid, g, v);
fd_set_desc_flags(pid, g, cur | FD_CLOEXEC as i32);
}
0
} else {
if FDTRACE && first <= 11 && last >= 10 {
ipc_logf(
&[
(b"FDTRACE_CLOSE_RANGE pid=", pid as i64),
(b" first=", first as i64),
(b" last=", last as i64),
],
b"",
);
}
let victims: Vec<i32> = {
let t = fdt().lock().unwrap();
t.get(&pid)
.map(|m| {
m.keys()
.copied()
.filter(|&g| (g as u32) >= first && (g as u32) <= last)
.collect()
})
.unwrap_or_default()
};
for g in victims {
match fd_remove(pid, g) {
Some(FdVal::Host(h)) => {
fd_forget_host_side_tables(h);
host(SYS_CLOSE, h as u64, 0, 0, 0, 0, 0);
}
Some(FdVal::Loop(s)) => loop_close(s),
None => {}
}
}
0
}
}
SYS_DUP2 | SYS_DUP3 => {
let (gold, gnew) = (a as i32, b as i32);
if nr == SYS_DUP3 && (c & !(libc::O_CLOEXEC as u64)) != 0 {
return -22; }
match fd_host(pid, gold) {
None => -9, Some(_) if gold == gnew => {
if nr == SYS_DUP2 {
gnew as i64
} else {
-22 }
}
Some(h) => {
let n = host(SYS_DUP, h as u64, 0, 0, 0, 0, 0);
if n < 0 {
n
} else {
let flags = if nr == SYS_DUP3 && (c & libc::O_CLOEXEC as u64) != 0 {
FD_CLOEXEC as i32
} else {
0
};
fd_install_val_at_with_flags(pid, gnew, FdVal::Host(n as i32), flags);
if FDTRACE && (gnew == 10 || gnew == 11) {
ipc_logf(
&[
(b"FDTRACE_DUP pid=", pid as i64),
(b" old=", gold as i64),
(b" new=", gnew as i64),
(b" hnew=", n),
(b" flags=", flags as i64),
],
b"",
);
fdtrace_dump_table(pid, b"POST_DUP");
}
gnew as i64
}
}
}
}
SYS_PKEY_ALLOC | SYS_PKEY_FREE | SYS_PKEY_MPROTECT => -38,
SYS_EPOLL_CTL => {
if let Some(sid) = fd_loop(pid, c as i32) {
return epoll_ctl_loop(pid, a as i32, b, sid, d);
}
let h = match fd_host(pid, c as i32) {
Some(h) => h,
None => return -9, };
let mut ev = [0u8; 12];
let ep = if d != 0 {
vm_read(pid, d, &mut ev);
ev.as_ptr() as u64
} else {
0
};
host(SYS_EPOLL_CTL, a, b, h as u64, ep, 0, 0)
}
SYS_EPOLL_WAIT | SYS_EPOLL_PWAIT => {
let max = (c.min(4096)) as usize;
let hasloop = epoll_loops()
.lock()
.unwrap()
.get(&(a as i32))
.is_some_and(|m| !m.is_empty());
if epdbg_on() {
ipc_logf(
&[
(b"EPDISP pid=", pid as i64),
(b" epfd=", a as i64),
(b" tmo=", (d as i32) as i64),
(b" hasloop=", hasloop as i64),
],
b"",
);
}
if hasloop {
return epoll_wait_merge(pid, a as i32, b, max, (d as i32) as i64);
}
let mut buf = vec![0u8; max * 12];
let n = host_cancellable(SYS_EPOLL_WAIT, a, buf.as_mut_ptr() as u64, c, d, 0, 0);
if n == CANCEL_SENTINEL {
return CANCEL_SENTINEL;
}
if n > 0 {
if epdbg_on() {
let e0 = u32::from_le_bytes(buf[0..4].try_into().unwrap());
let d0 = u64::from_le_bytes(buf[4..12].try_into().unwrap());
let e1 = if n >= 2 {
u32::from_le_bytes(buf[12..16].try_into().unwrap())
} else {
0
};
let d1 = if n >= 2 {
u64::from_le_bytes(buf[16..24].try_into().unwrap())
} else {
0
};
ipc_logf(
&[
(b"EPHOST epfd=", a as i64),
(b" tmo=", (d as i32) as i64),
(b" n=", n),
(b" ev0=", e0 as i64),
(b" data0=", d0 as i64),
(b" ev1=", e1 as i64),
(b" data1=", d1 as i64),
],
b"",
);
}
let has_tag = (0..n as usize).any(|i| {
let data =
u64::from_le_bytes(buf[i * 12 + 4..i * 12 + 12].try_into().unwrap());
data & EPOLL_LOOP_TAG != 0
});
if has_tag {
let mut out: Vec<u8> = Vec::with_capacity(n as usize * 12);
let mut lt = epoll_loops().lock().unwrap();
let ls = loop_state().lock().unwrap();
for i in 0..n as usize {
let ev =
u32::from_le_bytes(buf[i * 12..i * 12 + 4].try_into().unwrap());
let data = u64::from_le_bytes(
buf[i * 12 + 4..i * 12 + 12].try_into().unwrap(),
);
if data & EPOLL_LOOP_TAG != 0 {
let sid = (data & !EPOLL_LOOP_TAG) as u32;
if let Some(v) =
lt.get_mut(&(a as i32)).and_then(|m| m.get_mut(&sid))
{
let (wev, gdata) = (v.0, v.1);
let mut ready = 0u32;
if (wev & EPOLLIN_B) != 0 && ls.net.readable(sid) {
ready |= EPOLLIN_B;
}
if (wev & EPOLLOUT_B) != 0 && ls.net.writable(sid) {
ready |= EPOLLOUT_B;
}
let re = et_filter(wev, ready, &mut v.2);
if re != 0 {
out.extend_from_slice(&re.to_le_bytes());
out.extend_from_slice(&gdata.to_le_bytes());
}
} else {
let sib_watch = lt.iter().find_map(|(ep2, m)| {
if *ep2 != a as i32 {
m.get(&sid).map(|w| (w.0, w.1))
} else {
None
}
});
if let Some((wev, gdata)) = sib_watch {
let v = lt
.entry(a as i32)
.or_default()
.entry(sid)
.or_insert((wev, gdata, 0));
let mut ready = 0u32;
if (wev & EPOLLIN_B) != 0 && ls.net.readable(sid) {
ready |= EPOLLIN_B;
}
if (wev & EPOLLOUT_B) != 0 && ls.net.writable(sid) {
ready |= EPOLLOUT_B;
}
let re = et_filter(wev, ready, &mut v.2);
if re != 0 {
out.extend_from_slice(&re.to_le_bytes());
out.extend_from_slice(&gdata.to_le_bytes());
}
} else if let Some(&efd) = ls.efds.get(&sid) {
host(SYS_EPOLL_CTL, a, 2, efd as u64, 0, 0, 0);
}
}
} else {
out.extend_from_slice(&ev.to_le_bytes());
out.extend_from_slice(&data.to_le_bytes());
}
}
drop(ls);
drop(lt);
let cnt = out.len() / 12;
if cnt > 0 {
vm_write(pid, b, &out);
}
return cnt as i64;
}
vm_write(pid, b, &buf[..(n as usize) * 12]);
}
if WAITTRACE {
let (ev0, dat0) = if n > 0 {
(
u32::from_le_bytes(buf[0..4].try_into().unwrap()) as i64,
u64::from_le_bytes(buf[4..12].try_into().unwrap()) as i64,
)
} else {
(0, 0)
};
ipc_logf(
&[
(b"EPWAIT pid=", pid as i64),
(b"epfd=", a as i64),
(b"tmo=", (d as i32) as i64),
(b"ret=", n),
(b"ev0=", ev0),
(b"dat0=", dat0),
],
&[],
);
}
n
}
SYS_TIMERFD_SETTIME => {
let mut nb = [0u8; 32];
let np = if c != 0 {
vm_read(pid, c, &mut nb);
nb.as_ptr() as u64
} else {
0
};
let mut ob = [0u8; 32];
let op = if d != 0 { ob.as_mut_ptr() as u64 } else { 0 };
let n = host(SYS_TIMERFD_SETTIME, a, b, np, op, 0, 0);
if n == 0 && d != 0 {
vm_write(pid, d, &ob);
}
n
}
SYS_TIMERFD_GETTIME => {
let mut ob = [0u8; 32];
let n = host(SYS_TIMERFD_GETTIME, a, ob.as_mut_ptr() as u64, 0, 0, 0, 0);
if n == 0 {
vm_write(pid, b, &ob);
}
n
}
SYS_PIPE | SYS_PIPE2 => {
let mut fds = [0u8; 8];
let n = host(nr, fds.as_mut_ptr() as u64, b, 0, 0, 0, 0);
if n != 0 {
n
} else {
let h0 = i32::from_le_bytes(fds[0..4].try_into().unwrap());
let h1 = i32::from_le_bytes(fds[4..8].try_into().unwrap());
let g0 = fd_install(pid, h0, 0) as i32;
let g1 = fd_install(pid, h1, 0) as i32;
let mut out = [0u8; 8];
out[0..4].copy_from_slice(&g0.to_le_bytes());
out[4..8].copy_from_slice(&g1.to_le_bytes());
if vm_write(pid, a, &out) < 0 {
let _ = fd_remove(pid, g0);
let _ = fd_remove(pid, g1);
host(SYS_CLOSE, h0 as u64, 0, 0, 0, 0, 0);
host(SYS_CLOSE, h1 as u64, 0, 0, 0, 0, 0);
-14 } else {
n }
}
}
SYS_SOCKETPAIR => {
let mut fds = [0u8; 8];
let n = host(SYS_SOCKETPAIR, a, b, c, fds.as_mut_ptr() as u64, 0, 0);
if n != 0 {
n
} else {
let h0 = i32::from_le_bytes(fds[0..4].try_into().unwrap());
let h1 = i32::from_le_bytes(fds[4..8].try_into().unwrap());
let g0 = fd_install(pid, h0, 0) as i32;
let g1 = fd_install(pid, h1, 0) as i32;
let mut out = [0u8; 8];
out[0..4].copy_from_slice(&g0.to_le_bytes());
out[4..8].copy_from_slice(&g1.to_le_bytes());
if vm_write(pid, d, &out) < 0 {
let _ = fd_remove(pid, g0);
let _ = fd_remove(pid, g1);
host(SYS_CLOSE, h0 as u64, 0, 0, 0, 0, 0);
host(SYS_CLOSE, h1 as u64, 0, 0, 0, 0, 0);
-14 } else {
n }
}
}
SYS_TIME => {
let mut t = [0u8; 8];
let r = host(SYS_TIME, t.as_mut_ptr() as u64, 0, 0, 0, 0, 0);
if a != 0 {
vm_write(pid, a, &t);
}
r
}
SYS_CLOCK_GETRES => {
let mut ts = [0u8; 16];
let n = host(SYS_CLOCK_GETRES, a, ts.as_mut_ptr() as u64, 0, 0, 0, 0);
if n == 0 && b != 0 {
vm_write(pid, b, &ts);
}
n
}
SYS_SCHED_GETPARAM => {
if b == 0 {
-14 } else {
vm_write(pid, b, &0i32.to_le_bytes());
0
}
}
SYS_SCHED_GETSCHEDULER => libc::SCHED_OTHER as i64,
SYS_SCHED_GET_PRIORITY_MAX | SYS_SCHED_GET_PRIORITY_MIN => match a as i32 {
x if x == libc::SCHED_OTHER => 0,
x if x == libc::SCHED_FIFO || x == libc::SCHED_RR => {
if nr == SYS_SCHED_GET_PRIORITY_MAX {
99
} else {
1
}
}
#[cfg(target_os = "linux")]
x if x == libc::SCHED_BATCH || x == libc::SCHED_IDLE => 0,
_ => -22, },
SYS_SCHED_RR_GET_INTERVAL => {
if b == 0 {
-14
} else {
vm_write(pid, b, &[0u8; 16]);
0
}
}
SYS_SCHED_SETPARAM => {
if b == 0 {
-14
} else {
let mut param = [0u8; 4];
vm_read(pid, b, &mut param);
if i32::from_le_bytes(param) == 0 {
0
} else {
-1 }
}
}
SYS_SCHED_SETSCHEDULER => {
if c == 0 {
-14
} else {
let mut param = [0u8; 4];
vm_read(pid, c, &mut param);
if b as i32 == libc::SCHED_OTHER && i32::from_le_bytes(param) == 0 {
0
} else {
-1 }
}
}
SYS_GETPRIORITY => match a as i32 {
0 => {
let target = if b == 0 { pid } else { b as i32 };
let nice = nice_values()
.lock()
.unwrap()
.get(&target)
.copied()
.unwrap_or(0);
(20 - nice) as i64
}
1 | 2 => 20, _ => -22, },
SYS_SETPRIORITY => match a as i32 {
0 => {
let nice = c as i32;
if !(-20..=19).contains(&nice) {
-22 } else {
let target = if b == 0 { pid } else { b as i32 };
nice_values().lock().unwrap().insert(target, nice);
0
}
}
1 | 2 => {
let nice = c as i32;
if !(-20..=19).contains(&nice) {
-22
} else {
0
}
}
_ => -22,
},
SYS_SCHED_SETAFFINITY => {
0
}
SYS_SCHED_GETAFFINITY => {
let sz = (b.min(1024)) as usize;
let mut m = vec![0u8; sz];
let n = host(SYS_SCHED_GETAFFINITY, a, b, m.as_mut_ptr() as u64, 0, 0, 0);
if n > 0 {
let used = n as usize;
if let Some(count) = cgroup_cpu_count() {
for (i, byte) in m[..used].iter_mut().enumerate() {
let mut v = 0u8;
for bit in 0..8u32 {
if ((i * 8 + bit as usize) as u64) < count {
v |= 1 << bit;
}
}
*byte = v;
}
}
vm_write(pid, c, &m[..used]);
}
n
}
SYS_GETCPU => {
let mut cpu = [0u8; 4];
let mut node = [0u8; 4];
let cp = if a != 0 { cpu.as_mut_ptr() as u64 } else { 0 };
let np = if b != 0 { node.as_mut_ptr() as u64 } else { 0 };
let n = host(SYS_GETCPU, cp, np, 0, 0, 0, 0);
if a != 0 {
vm_write(pid, a, &cpu);
}
if b != 0 {
vm_write(pid, b, &node);
}
n
}
SYS_NANOSLEEP => {
let mut req = [0u8; 16];
vm_read(pid, a, &mut req);
let mut rem = [0u8; 16];
let rp = if b != 0 { rem.as_mut_ptr() as u64 } else { 0 };
let n = host_cancellable(SYS_NANOSLEEP, req.as_ptr() as u64, rp, 0, 0, 0, 0);
if n == CANCEL_SENTINEL {
return CANCEL_SENTINEL;
}
if b != 0 {
vm_write(pid, b, &rem);
}
n
}
SYS_CLOCK_NANOSLEEP => {
let mut req = [0u8; 16];
vm_read(pid, c, &mut req);
let mut rem = [0u8; 16];
let rp = if d != 0 { rem.as_mut_ptr() as u64 } else { 0 };
let n = host_cancellable(SYS_CLOCK_NANOSLEEP, a, b, req.as_ptr() as u64, rp, 0, 0);
if n == CANCEL_SENTINEL {
return CANCEL_SENTINEL;
}
if d != 0 {
vm_write(pid, d, &rem);
}
n
}
SYS_POLL => {
let n = (b.min(1024)) as usize;
let mut fds = vec![0u8; n * 8];
vm_read(pid, a, &mut fds);
if (0..n).any(|i| {
let g = i32::from_le_bytes(fds[i * 8..i * 8 + 4].try_into().unwrap());
g >= 0 && fd_loop(pid, g).is_some()
}) {
return poll_merge(pid, a, b, (c as i32) as i64);
}
let mut gf = vec![0i32; n];
for i in 0..n {
let g = i32::from_le_bytes(fds[i * 8..i * 8 + 4].try_into().unwrap());
gf[i] = g;
let h = if g < 0 {
g
} else {
fd_host(pid, g).unwrap_or(0x3FFF_FFFF)
};
fds[i * 8..i * 8 + 4].copy_from_slice(&h.to_le_bytes());
}
let r = host_cancellable(SYS_POLL, fds.as_mut_ptr() as u64, b, c, 0, 0, 0);
if r == CANCEL_SENTINEL {
return CANCEL_SENTINEL;
}
for i in 0..n {
fds[i * 8..i * 8 + 4].copy_from_slice(&gf[i].to_le_bytes());
}
vm_write(pid, a, &fds);
r
}
SYS_PPOLL => {
let n = (b.min(1024)) as usize;
let mut fds = vec![0u8; n * 8];
vm_read(pid, a, &mut fds);
if (0..n).any(|i| {
let g = i32::from_le_bytes(fds[i * 8..i * 8 + 4].try_into().unwrap());
g >= 0 && fd_loop(pid, g).is_some()
}) {
let tmo_ms: i64 = if c == 0 {
-1
} else {
let mut ts = [0u8; 16];
vm_read(pid, c, &mut ts);
let secs = i64::from_le_bytes(ts[0..8].try_into().unwrap());
let nsecs = i64::from_le_bytes(ts[8..16].try_into().unwrap());
let ms = secs.saturating_mul(1000).saturating_add(nsecs / 1_000_000);
let ms = if ms == 0 && (secs > 0 || nsecs > 0) {
1
} else {
ms
};
ms.clamp(0, i32::MAX as i64)
};
return poll_merge(pid, a, b, tmo_ms);
}
let mut gf = vec![0i32; n];
for i in 0..n {
let g = i32::from_le_bytes(fds[i * 8..i * 8 + 4].try_into().unwrap());
gf[i] = g;
let h = if g < 0 {
g
} else {
fd_host(pid, g).unwrap_or(0x3FFF_FFFF)
};
fds[i * 8..i * 8 + 4].copy_from_slice(&h.to_le_bytes());
}
let tmo_ms: u64 = if c == 0 {
(-1i64) as u64
} else {
let mut ts = [0u8; 16];
vm_read(pid, c, &mut ts);
let secs = i64::from_le_bytes(ts[0..8].try_into().unwrap());
let nsecs = i64::from_le_bytes(ts[8..16].try_into().unwrap());
let ms = secs.saturating_mul(1000).saturating_add(nsecs / 1_000_000);
let ms = if ms == 0 && (secs > 0 || nsecs > 0) {
1
} else {
ms
};
ms.clamp(0, i32::MAX as i64) as u64
};
let r = host_cancellable(SYS_POLL, fds.as_mut_ptr() as u64, b, tmo_ms, 0, 0, 0);
if r == CANCEL_SENTINEL {
return CANCEL_SENTINEL;
}
for i in 0..n {
fds[i * 8..i * 8 + 4].copy_from_slice(&gf[i].to_le_bytes());
}
vm_write(pid, a, &fds);
r
}
SYS_SELECT | SYS_PSELECT6 => {
let nfds = (a as i32).clamp(0, 1024) as usize;
let words = nfds.div_ceil(8); let read_set = |ptr: u64| -> Vec<u8> {
if ptr == 0 || words == 0 {
return Vec::new();
}
let mut s = vec![0u8; words];
vm_read(pid, ptr, &mut s);
s
};
let rd = read_set(b);
let wr = read_set(c);
let ex = read_set(d);
let is_set = |s: &[u8], fd: usize| -> bool {
!s.is_empty() && (s[fd / 8] >> (fd % 8)) & 1 == 1
};
let (pollin, pollout, pollpri) = (0x001u16, 0x004u16, 0x002u16);
let (pollerr, pollhup, pollnval) = (0x008u16, 0x010u16, 0x020u16);
let mut gfd_ev: Vec<(i32, u16)> = Vec::new();
for fd in 0..nfds {
let mut ev: u16 = 0;
if is_set(&rd, fd) {
ev |= pollin;
}
if is_set(&wr, fd) {
ev |= pollout;
}
if is_set(&ex, fd) {
ev |= pollpri;
}
if ev == 0 {
continue;
}
gfd_ev.push((fd as i32, ev));
}
let tmo_ms: u64 = if e == 0 {
(-1i64) as u64
} else {
let mut t = [0u8; 16];
vm_read(pid, e, &mut t);
let secs = i64::from_le_bytes(t[0..8].try_into().unwrap());
let sub = i64::from_le_bytes(t[8..16].try_into().unwrap());
let sub_ms = if nr == SYS_PSELECT6 {
sub / 1_000_000
} else {
sub / 1_000
};
let ms = secs.saturating_mul(1000).saturating_add(sub_ms);
let ms = if ms == 0 && (secs > 0 || sub > 0) {
1
} else {
ms
};
ms.clamp(0, i32::MAX as i64) as u64
};
let (r, rev) = poll_merge_entries(pid, &gfd_ev, tmo_ms as i64);
if r == CANCEL_SENTINEL {
return CANCEL_SENTINEL;
}
let mut out_rd = vec![0u8; words];
let mut out_wr = vec![0u8; words];
let mut out_ex = vec![0u8; words];
let mut count: i64 = 0;
if r >= 0 {
for (i, (fd, _ev)) in gfd_ev.iter().enumerate() {
let revents = rev[i];
let fd = *fd as usize;
let err = revents & (pollerr | pollhup | pollnval) != 0;
if is_set(&rd, fd) && (revents & (pollin | pollhup) != 0 || err) {
out_rd[fd / 8] |= 1 << (fd % 8);
count += 1;
}
if is_set(&wr, fd) && (revents & pollout != 0 || err) {
out_wr[fd / 8] |= 1 << (fd % 8);
count += 1;
}
if is_set(&ex, fd) && (revents & pollpri != 0) {
out_ex[fd / 8] |= 1 << (fd % 8);
count += 1;
}
}
}
if b != 0 && words > 0 {
vm_write(pid, b, &out_rd);
}
if c != 0 && words > 0 {
vm_write(pid, c, &out_wr);
}
if d != 0 && words > 0 {
vm_write(pid, d, &out_ex);
}
if r < 0 {
r
} else {
count
}
}
SYS_SIGALTSTACK => {
let old = guest_sigaltstack(slot);
if b != 0 {
if vm_write(pid, b, &old) != old.len() as i64 {
return -14; }
}
if a != 0 {
let mut next = [0u8; 24];
if vm_read(pid, a, &mut next) != next.len() as i64 {
return -14; }
if (slot as usize) < MAX_SLOTS {
GUEST_SIGALTSTACKS[slot as usize] = next;
}
}
0
}
SYS_LISTEN | SYS_SHUTDOWN => host(nr, a, b, c, d, e, f),
SYS_SOCKET => {
if a == 10 && !host_v6_route() {
return -97; }
let n = host(SYS_SOCKET, a, b, c, 0, 0, 0);
if n < 0 {
n
} else {
fd_install(pid, n as i32, 0)
}
}
SYS_BIND => {
let len = (c.min(128)) as usize;
let mut addr = vec![0u8; len];
vm_read(pid, b, &mut addr);
if let Some(ep) = ep_from_sockaddr(&addr) {
let wildcard_local =
ep.ip == [0u8; 16] && !published_ports().lock().unwrap().contains(&ep.port);
if own_loopback()
&& (ep_is_loopback(&ep) || wildcard_local)
&& fd_host(pid, a as i32)
.map(host_sock_is_stream)
.unwrap_or(false)
{
let sid = loop_socket_for(pid, a as i32);
return loop_state()
.lock()
.unwrap()
.net
.bind(sid, ep)
.map(|_| 0)
.unwrap_or_else(|e| -(e as i64));
}
}
let h = match fd_host(pid, a as i32) {
Some(h) => h,
None => return -9,
};
if let Some(sa) = parse_connect_addr(&addr) {
if published_ports().lock().unwrap().contains(&sa.port()) {
rehome_to_host_netns(h);
}
}
if let Some(rw) = rewrite_unix_sun_path(&addr) {
let orig = unix_sockaddr_host_path(&addr);
let rewritten = unix_sockaddr_path(&rw).map(|p| p.to_vec());
let r = host(
SYS_BIND,
h as u64,
rw.as_ptr() as u64,
rw.len() as u64,
0,
0,
0,
);
if r == 0 {
if let (Some(orig), Some(rewritten)) = (orig.as_deref(), rewritten.as_deref()) {
if orig != rewritten {
publish_unix_socket_alias_placeholder(orig);
unix_alias_placeholder_record(h, orig);
}
}
}
return r;
}
host(
SYS_BIND,
h as u64,
addr.as_ptr() as u64,
len as u64,
0,
0,
0,
)
}
SYS_CONNECT => {
let mut len = (c.min(128)) as usize;
let mut addr = vec![0u8; len];
vm_read(pid, b, &mut addr);
if let Some(nl) = unwrap_v4mapped_sockaddr(&mut addr) {
len = nl;
}
if let Some(ep) = ep_from_sockaddr(&addr) {
if own_loopback()
&& ep_is_loopback(&ep)
&& fd_host(pid, a as i32)
.map(host_sock_is_stream)
.unwrap_or(false)
{
let sid = loop_socket_for(pid, a as i32);
let ls = loop_state().lock().unwrap();
let mut ls = ls;
let r = match ls.net.connect(sid, ep) {
Ok(listener) => {
loop_wake(&ls, listener);
0
}
Err(e) => -(e as i64),
};
drop(ls);
loopring_ev(3, sid, r);
return r;
}
}
let h = match fd_host(pid, a as i32) {
Some(h) => h,
None => return -9,
};
if let Some(sa) = parse_connect_addr(&addr) {
if let Err(reason) = crate::vmm::egress_policy::check_addr(sa) {
if TRACE {
log(b"[supervisor] egress DENY: ");
log(reason.as_bytes());
log(b"\n");
}
return -13; }
if !sa.ip().is_loopback() {
rehome_to_host_netns(h);
}
note_dns_dest(h, &sa);
}
if let Some(rw) = rewrite_unix_sun_path(&addr) {
return host(
SYS_CONNECT,
h as u64,
rw.as_ptr() as u64,
rw.len() as u64,
0,
0,
0,
);
}
let r = host(
SYS_CONNECT,
h as u64,
addr.as_ptr() as u64,
len as u64,
0,
0,
0,
);
if IPCTRACE {
log_connect_debug(pid, h, &addr[..len], r);
}
r
}
SYS_SENDTO => {
let len = (c as usize).min(1 << 20);
let mut buf = vec![0u8; len];
vm_read(pid, b, &mut buf);
let mut addr: Vec<u8> = Vec::new();
let mut addr_len = 0usize;
if e != 0 && f != 0 {
addr.resize((f as usize).min(128), 0);
addr_len = addr.len();
vm_read(pid, e, &mut addr);
if let Some(nl) = unwrap_v4mapped_sockaddr(&mut addr) {
addr_len = nl;
}
if let Some(sa) = parse_connect_addr(&addr) {
if let Err(reason) = crate::vmm::egress_policy::check_addr(sa) {
if TRACE {
log(b"[supervisor] egress DENY (sendto): ");
log(reason.as_bytes());
log(b"\n");
}
return -13; }
if !sa.ip().is_loopback() {
rehome_to_host_netns(a as i32);
}
note_dns_dest(a as i32, &sa);
}
}
let (aptr, alen) = if addr.is_empty() {
(e, f)
} else {
(addr.as_ptr() as u64, addr_len as u64)
};
let r = host(
SYS_SENDTO,
a,
buf.as_ptr() as u64,
len as u64,
d,
aptr,
alen,
);
if IPCTRACE && addr.is_empty() {
ipc_logf(
&[
(b"SENDTO pid=", pid as i64),
(b"hfd=", a as i64),
(b"len=", len as i64),
(b"ret=", r),
],
&buf[..len.min(72)],
);
}
r
}
SYS_RECVFROM => {
let len = (c as usize).min(1 << 20);
let mut buf = vec![0u8; len];
if e != 0 && f != 0 {
let mut lenbuf = [0u8; 4];
vm_read(pid, f, &mut lenbuf);
let cap = (u32::from_le_bytes(lenbuf) as usize).min(128);
let mut addr = vec![0u8; cap];
let mut outlen = cap as u32;
let n = host_cancellable(
SYS_RECVFROM,
a,
buf.as_mut_ptr() as u64,
len as u64,
d,
addr.as_mut_ptr() as u64,
&mut outlen as *mut u32 as u64,
);
if n == CANCEL_SENTINEL {
return CANCEL_SENTINEL;
}
if n >= 0 {
let n = maybe_strip_aaaa(a as i32, &mut buf, n);
vm_write(pid, b, &buf[..n as usize]);
let w = (outlen as usize).min(cap);
vm_write(pid, e, &addr[..w]);
vm_write(pid, f, &outlen.to_le_bytes());
return n;
}
n
} else {
let n = host_cancellable(
SYS_RECVFROM,
a,
buf.as_mut_ptr() as u64,
len as u64,
d,
e,
f,
);
if n == CANCEL_SENTINEL {
return CANCEL_SENTINEL;
}
if n >= 0 {
let n = maybe_strip_aaaa(a as i32, &mut buf, n);
vm_write(pid, b, &buf[..n as usize]);
return n;
}
n
}
}
SYS_SENDMSG => {
let mut mh = [0u8; 56];
vm_read(pid, b, &mut mh);
let rd = |o: usize| u64::from_le_bytes(mh[o..o + 8].try_into().unwrap());
let name_ptr = rd(0);
let namelen = u32::from_le_bytes(mh[8..12].try_into().unwrap());
let (iov_ptr, iovlen) = (rd(16), (rd(24) as usize).min(1024));
let mut iovs = vec![0u8; iovlen * 16];
vm_read(pid, iov_ptr, &mut iovs);
let mut data = Vec::new();
for i in 0..iovlen {
let base = u64::from_le_bytes(iovs[i * 16..i * 16 + 8].try_into().unwrap());
let l = (u64::from_le_bytes(iovs[i * 16 + 8..i * 16 + 16].try_into().unwrap())
.min(CAP)) as usize;
let mut seg = vec![0u8; l];
if l > 0 {
vm_read(pid, base, &mut seg);
}
data.extend_from_slice(&seg);
}
let mut addr = Vec::new();
let mut addr_len = 0usize;
if name_ptr != 0 && namelen > 0 {
addr.resize((namelen as usize).min(128), 0);
addr_len = addr.len();
vm_read(pid, name_ptr, &mut addr);
if let Some(nl) = unwrap_v4mapped_sockaddr(&mut addr) {
addr_len = nl;
}
if let Some(sa) = parse_connect_addr(&addr) {
if let Err(reason) = crate::vmm::egress_policy::check_addr(sa) {
if TRACE {
log(b"[supervisor] egress DENY (sendmsg): ");
log(reason.as_bytes());
log(b"\n");
}
return -13;
}
if !sa.ip().is_loopback() {
rehome_to_host_netns(a as i32);
}
note_dns_dest(a as i32, &sa);
}
}
let ctl_ptr = rd(32); let ctl_len = rd(40) as usize; let mut sendctl: Vec<u8> = Vec::new();
let mut sendfds = 0usize; if ctl_ptr != 0 && ctl_len >= 16 {
let mut gctl = vec![0u8; ctl_len.min(SCM_CTL_CAP)];
vm_read(pid, ctl_ptr, &mut gctl);
match scm_send_control(pid, &gctl) {
Ok(c) => {
sendfds = scm_count_fds(&c);
sendctl = c;
}
Err(e) => return e,
}
}
if getsockopt_int(a as i32, 39) == libc::AF_UNIX {
append_scm_credentials(&mut sendctl, pid);
}
if FDTRACE
&& (data
.windows(b"pseudonymization-salt-handle".len())
.any(|w| w == b"pseudonymization-salt-handle")
|| data
.windows(b"--type=renderer".len())
.any(|w| w == b"--type=renderer")
|| data
.windows(b"--type=gpu-process".len())
.any(|w| w == b"--type=gpu-process"))
{
ipc_logf(
&[
(b"FDTRACE_ZYGMSG pid=", pid as i64),
(b"hfd=", a as i64),
(b"datalen=", data.len() as i64),
(b"scmfds=", sendfds as i64),
],
&data[..data.len().min(512)],
);
}
let iov = LocalIov {
base: data.as_ptr() as u64,
len: data.len() as u64,
};
let lmh = LocalMsghdr {
name: if addr.is_empty() {
0
} else {
addr.as_ptr() as u64
},
namelen: addr_len as u32,
_p: 0,
iov: &iov as *const _ as u64,
iovlen: 1,
control: if sendctl.is_empty() {
0
} else {
sendctl.as_ptr() as u64
},
controllen: sendctl.len() as u64,
flags: 0,
_p2: 0,
};
let r = host(SYS_SENDMSG, a, &lmh as *const _ as u64, c, 0, 0, 0);
if r >= 0 && sendfds > 0 {
SCM_FDS_SENT.fetch_add(sendfds as u64, Ordering::Relaxed);
}
if IPCTRACE {
ipc_logf(
&[
(b"SENDMSG pid=", pid as i64),
(b"hfd=", a as i64),
(b"datalen=", data.len() as i64),
(b"scmbytes=", sendctl.len() as i64),
(b"scmfds=", sendfds as i64),
(b"ret=", r),
(b"fds_sent_total=", SCM_FDS_SENT.load(Ordering::Relaxed) as i64),
],
&data[..data.len().min(72)],
);
if sendfds > 0 && r == 0 {
ipc_logf_raw(b"SENDMSG SCM anomaly: fds attached but 0 data bytes accepted");
}
}
r
}
SYS_RECVMSG => {
let mut mh = [0u8; 56];
vm_read(pid, b, &mut mh);
let rd = |o: usize| u64::from_le_bytes(mh[o..o + 8].try_into().unwrap());
let name_ptr = rd(0);
let namecap = u32::from_le_bytes(mh[8..12].try_into().unwrap());
let (iov_ptr, iovlen) = (rd(16), (rd(24) as usize).min(1024));
let mut iovs = vec![0u8; iovlen * 16];
vm_read(pid, iov_ptr, &mut iovs);
let mut segs: Vec<(u64, usize)> = Vec::new();
let mut total = 0usize;
for i in 0..iovlen {
let base = u64::from_le_bytes(iovs[i * 16..i * 16 + 8].try_into().unwrap());
let l = (u64::from_le_bytes(iovs[i * 16 + 8..i * 16 + 16].try_into().unwrap())
.min(CAP)) as usize;
segs.push((base, l));
total += l;
}
let mut bounce = vec![0u8; total];
let mut lname = vec![0u8; (namecap as usize).min(128)];
let ctl_ptr = rd(32); let ctl_cap = rd(40) as usize; let mut hctl: Vec<u8> = if ctl_ptr != 0 && ctl_cap >= 16 {
vec![0u8; SCM_CTL_CAP]
} else {
Vec::new()
};
let iov = LocalIov {
base: bounce.as_mut_ptr() as u64,
len: total as u64,
};
let mut lmh = LocalMsghdr {
name: if lname.is_empty() {
0
} else {
lname.as_mut_ptr() as u64
},
namelen: lname.len() as u32,
_p: 0,
iov: &iov as *const _ as u64,
iovlen: 1,
control: if hctl.is_empty() {
0
} else {
hctl.as_mut_ptr() as u64
},
controllen: hctl.len() as u64,
flags: 0,
_p2: 0,
};
let n = host_cancellable(SYS_RECVMSG, a, &mut lmh as *mut _ as u64, c, 0, 0, 0);
if n == CANCEL_SENTINEL {
return CANCEL_SENTINEL;
}
if n >= 0 {
let n = maybe_strip_aaaa(a as i32, &mut bounce, n);
let got = n as usize;
let mut off = 0usize;
for (base, l) in segs {
if off >= got {
break;
}
let take = l.min(got - off);
vm_write(pid, base, &bounce[off..off + take]);
off += take;
}
if name_ptr != 0 && namecap > 0 {
let w = (lmh.namelen as usize).min(lname.len());
vm_write(pid, name_ptr, &lname[..w]);
vm_write(pid, b + 8, &lmh.namelen.to_le_bytes());
}
let recvd = (lmh.controllen as usize).min(hctl.len());
let mut msg_flags = lmh.flags;
if ctl_ptr != 0 && recvd >= 16 {
let scm = scm_recv_control(pid, &hctl[..recvd], ctl_cap);
let w = scm.gctl.len().min(ctl_cap);
if w > 0 {
vm_write(pid, ctl_ptr, &scm.gctl[..w]);
}
vm_write(pid, b + 40, &(w as u64).to_le_bytes()); if scm.truncated {
msg_flags |= 0x8;
SCM_RECV_TRUNC.fetch_add(1, Ordering::Relaxed);
}
SCM_FDS_RECVD.fetch_add(scm.installed as u64, Ordering::Relaxed);
} else {
vm_write(pid, b + 40, &0u64.to_le_bytes()); }
vm_write(pid, b + 48, &msg_flags.to_le_bytes()); if IPCTRACE {
let hl = (n.max(0) as usize).min(bounce.len()).min(72);
ipc_logf(
&[
(b"RECVMSG pid=", pid as i64),
(b"hfd=", a as i64),
(b"got=", n),
(b"scmbytes=", lmh.controllen as i64),
(b"ctrunc=", ((msg_flags & 0x8) != 0) as i64),
(b"trunc_total=", SCM_RECV_TRUNC.load(Ordering::Relaxed) as i64),
(b"fds_recvd_total=", SCM_FDS_RECVD.load(Ordering::Relaxed) as i64),
(b"fds_sent_total=", SCM_FDS_SENT.load(Ordering::Relaxed) as i64),
],
&bounce[..hl],
);
if recvd >= 16 && ((msg_flags & 0x8) != 0) {
ipc_logf_raw(b"RECVMSG SCM_DROP: ancillary truncated, fd(s) lost (ipcz buffer would strand on is_pending)");
}
}
return n;
}
n
}
SYS_SENDMMSG => {
const MMH: u64 = 64;
let vlen = (c.min(1024)) as usize;
let flags = d;
let mut sent: i64 = 0;
let mut last_err: i64 = 0;
for m in 0..vlen {
let hp = b + (m as u64) * MMH; let mut mh = [0u8; 56];
vm_read(pid, hp, &mut mh);
let rd = |o: usize| u64::from_le_bytes(mh[o..o + 8].try_into().unwrap());
let name_ptr = rd(0);
let namelen = u32::from_le_bytes(mh[8..12].try_into().unwrap());
let (iov_ptr, iovlen) = (rd(16), (rd(24) as usize).min(1024));
let mut iovs = vec![0u8; iovlen * 16];
vm_read(pid, iov_ptr, &mut iovs);
let mut data = Vec::new();
for i in 0..iovlen {
let base = u64::from_le_bytes(iovs[i * 16..i * 16 + 8].try_into().unwrap());
let l =
(u64::from_le_bytes(iovs[i * 16 + 8..i * 16 + 16].try_into().unwrap())
.min(CAP)) as usize;
let mut seg = vec![0u8; l];
if l > 0 {
vm_read(pid, base, &mut seg);
}
data.extend_from_slice(&seg);
}
let mut addr = Vec::new();
let mut addr_len = 0usize;
if name_ptr != 0 && namelen > 0 {
addr.resize((namelen as usize).min(128), 0);
addr_len = addr.len();
vm_read(pid, name_ptr, &mut addr);
if let Some(nl) = unwrap_v4mapped_sockaddr(&mut addr) {
addr_len = nl;
}
if let Some(sa) = parse_connect_addr(&addr) {
if let Err(reason) = crate::vmm::egress_policy::check_addr(sa) {
if TRACE {
log(b"[supervisor] egress DENY (sendmmsg): ");
log(reason.as_bytes());
log(b"\n");
}
if sent == 0 {
return -13;
}
break;
}
if !sa.ip().is_loopback() {
rehome_to_host_netns(a as i32);
}
note_dns_dest(a as i32, &sa);
}
}
let iov = LocalIov {
base: data.as_ptr() as u64,
len: data.len() as u64,
};
let lmh = LocalMsghdr {
name: if addr.is_empty() {
0
} else {
addr.as_ptr() as u64
},
namelen: addr_len as u32,
_p: 0,
iov: &iov as *const _ as u64,
iovlen: 1,
control: 0,
controllen: 0,
flags: 0,
_p2: 0,
};
let r = host(SYS_SENDMSG, a, &lmh as *const _ as u64, flags, 0, 0, 0);
if r < 0 {
last_err = r;
break;
}
vm_write(pid, hp + 56, &(r as u32).to_le_bytes());
sent += 1;
}
if sent == 0 && last_err < 0 {
last_err
} else {
sent
}
}
SYS_RECVMMSG => {
const MMH: u64 = 64;
let vlen = (c.min(1024)) as usize;
let flags = d;
if e != 0 {
let mut ts = [0u8; 16];
vm_read(pid, e, &mut ts);
let secs = i64::from_le_bytes(ts[0..8].try_into().unwrap());
let nsecs = i64::from_le_bytes(ts[8..16].try_into().unwrap());
let ms = secs.saturating_mul(1000).saturating_add(nsecs / 1_000_000);
let ms = if ms == 0 && (secs > 0 || nsecs > 0) {
1
} else {
ms
};
let ms = ms.clamp(0, i32::MAX as i64);
let mut pfd = [0u8; 8];
pfd[0..4].copy_from_slice(&(a as i32).to_le_bytes());
pfd[4..6].copy_from_slice(&0x001u16.to_le_bytes()); let pr = host(SYS_POLL, pfd.as_mut_ptr() as u64, 1, ms as u64, 0, 0, 0);
if pr == 0 {
return 0; }
if pr < 0 {
return pr;
}
}
let mut got: i64 = 0;
for m in 0..vlen {
let hp = b + (m as u64) * MMH;
let mut mh = [0u8; 56];
vm_read(pid, hp, &mut mh);
let rd = |o: usize| u64::from_le_bytes(mh[o..o + 8].try_into().unwrap());
let name_ptr = rd(0);
let namecap = u32::from_le_bytes(mh[8..12].try_into().unwrap());
let (iov_ptr, iovlen) = (rd(16), (rd(24) as usize).min(1024));
let mut iovs = vec![0u8; iovlen * 16];
vm_read(pid, iov_ptr, &mut iovs);
let mut segs: Vec<(u64, usize)> = Vec::new();
let mut total = 0usize;
for i in 0..iovlen {
let base = u64::from_le_bytes(iovs[i * 16..i * 16 + 8].try_into().unwrap());
let l =
(u64::from_le_bytes(iovs[i * 16 + 8..i * 16 + 16].try_into().unwrap())
.min(CAP)) as usize;
segs.push((base, l));
total += l;
}
let mut bounce = vec![0u8; total];
let mut lname = vec![0u8; (namecap as usize).min(128)];
let iov = LocalIov {
base: bounce.as_mut_ptr() as u64,
len: total as u64,
};
let mut lmh = LocalMsghdr {
name: if lname.is_empty() {
0
} else {
lname.as_mut_ptr() as u64
},
namelen: lname.len() as u32,
_p: 0,
iov: &iov as *const _ as u64,
iovlen: 1,
control: 0,
controllen: 0,
flags: 0,
_p2: 0,
};
let mflags = if got > 0 { flags | 0x40 } else { flags };
let n = host_cancellable(
SYS_RECVMSG,
a,
&mut lmh as *mut _ as u64,
mflags,
0,
0,
0,
);
if n == CANCEL_SENTINEL {
return CANCEL_SENTINEL;
}
if n < 0 {
if got == 0 {
return n;
}
break;
}
let n = maybe_strip_aaaa(a as i32, &mut bounce, n);
let recvd = n as usize;
let mut off = 0usize;
for (base, l) in segs {
if off >= recvd {
break;
}
let take = l.min(recvd - off);
vm_write(pid, base, &bounce[off..off + take]);
off += take;
}
if name_ptr != 0 && namecap > 0 {
let w = (lmh.namelen as usize).min(lname.len());
vm_write(pid, name_ptr, &lname[..w]);
vm_write(pid, hp + 8, &lmh.namelen.to_le_bytes());
}
vm_write(pid, hp + 48, &lmh.flags.to_le_bytes()); vm_write(pid, hp + 40, &0u64.to_le_bytes()); vm_write(pid, hp + 56, &(n as u32).to_le_bytes()); got += 1;
}
got
}
SYS_ACCEPT | SYS_ACCEPT4 => {
let n = if b == 0 {
host_cancellable(nr, a, 0, 0, d, 0, 0)
} else {
let mut lenbuf = [0u8; 4];
vm_read(pid, c, &mut lenbuf);
let cap = (u32::from_le_bytes(lenbuf) as usize).min(128);
let mut addr = vec![0u8; cap];
let mut outlen = cap as u32;
let n = host_cancellable(
nr,
a,
addr.as_mut_ptr() as u64,
&mut outlen as *mut u32 as u64,
d,
0,
0,
);
if n >= 0 {
let w = (outlen as usize).min(cap);
vm_write(pid, b, &addr[..w]);
vm_write(pid, c, &outlen.to_le_bytes());
}
n
};
if n == CANCEL_SENTINEL {
CANCEL_SENTINEL
} else if n < 0 {
n
} else {
fd_install(pid, n as i32, 0)
}
}
SYS_GETSOCKNAME | SYS_GETPEERNAME => {
let mut lenbuf = [0u8; 4];
vm_read(pid, c, &mut lenbuf);
let cap = (u32::from_le_bytes(lenbuf) as usize).min(128);
let mut addr = vec![0u8; cap];
let mut outlen = cap as u32;
let n = host(
nr,
a,
addr.as_mut_ptr() as u64,
&mut outlen as *mut u32 as u64,
0,
0,
0,
);
if n == 0 {
let w = (outlen as usize).min(cap);
vm_write(pid, b, &addr[..w]);
vm_write(pid, c, &outlen.to_le_bytes());
}
n
}
SYS_SETSOCKOPT => {
let len = (e.min(256)) as usize;
let mut opt = vec![0u8; len];
vm_read(pid, d, &mut opt);
host(SYS_SETSOCKOPT, a, b, c, opt.as_ptr() as u64, len as u64, 0)
}
SYS_GETSOCKOPT => {
let mut lenbuf = [0u8; 4];
vm_read(pid, e, &mut lenbuf);
let cap = (u32::from_le_bytes(lenbuf) as usize).min(256);
let mut opt = vec![0u8; cap];
let mut outlen = cap as u32;
let n = host(
SYS_GETSOCKOPT,
a,
b,
c,
opt.as_mut_ptr() as u64,
&mut outlen as *mut u32 as u64,
0,
);
if n == 0 {
let w = (outlen as usize).min(cap);
vm_write(pid, d, &opt[..w]);
vm_write(pid, e, &outlen.to_le_bytes());
}
n
}
SYS_READV => {
let cnt = (c.min(1024)) as usize;
let mut iovs = vec![0u8; cnt * 16];
vm_read(pid, b, &mut iovs);
let mut segs: Vec<(u64, usize)> = Vec::new();
let mut total = 0usize;
for i in 0..cnt {
let base = u64::from_le_bytes(iovs[i * 16..i * 16 + 8].try_into().unwrap());
let len =
(u64::from_le_bytes(iovs[i * 16 + 8..i * 16 + 16].try_into().unwrap())
.min(CAP)) as usize;
segs.push((base, len));
total += len;
}
let mut bounce = vec![0u8; total];
let n = host(
SYS_READ,
a,
bounce.as_mut_ptr() as u64,
total as u64,
0,
0,
0,
);
if n > 0 {
let got = n as usize;
let mut off = 0usize;
for (base, len) in segs {
if off >= got {
break;
}
let take = len.min(got - off);
let w = vm_write(pid, base, &bounce[off..off + take]);
if w != take as i64 {
return -14; }
off += take;
}
}
n
}
SYS_LSEEK | SYS_GETUID | SYS_GETEUID | SYS_GETGID | SYS_GETEGID | SYS_GETPID
| SYS_GETTID => host(nr, a, b, c, d, e, f),
SYS_CLOSE => {
if FDTRACE && (a == 10 || a == 11) {
ipc_logf(
&[(b"FDTRACE_CLOSE pid=", pid as i64), (b" g=", a as i64)],
b"",
);
}
match fd_remove(pid, a as i32) {
Some(FdVal::Host(h)) => {
fd_forget_host_side_tables(h);
host(SYS_CLOSE, h as u64, 0, 0, 0, 0, 0)
}
Some(FdVal::Loop(s)) => {
loop_close(s);
0
}
None => -9, }
}
SYS_FCNTL => match b {
0 | 1030 => {
let n = host(SYS_FCNTL, a, b, 0, 0, 0, 0);
if n < 0 {
ipc_logf(
&[(b"DUP-FAIL pid=", pid as i64), (b"err=", n)],
&[],
);
n
} else {
let flags = if b == F_DUPFD_CLOEXEC {
FD_CLOEXEC as i32
} else {
0
};
fd_install_val_with_flags(pid, FdVal::Host(n as i32), c as i32, flags)
}
}
F_GETFD => {
if let Some(g) = fd_guest_for_host(pid, a as i32) {
fd_get_desc_flags(pid, g, FdVal::Host(a as i32)) as i64
} else {
host(SYS_FCNTL, a, b, c, 0, 0, 0)
}
}
F_SETFD => {
let r = host(SYS_FCNTL, a, b, c, 0, 0, 0);
if r == 0 {
if let Some(g) = fd_guest_for_host(pid, a as i32) {
fd_set_desc_flags(pid, g, c as i32);
if FDTRACE && (g == 10 || g == 11) {
ipc_logf(
&[
(b"FDTRACE_F_SETFD pid=", pid as i64),
(b" g=", g as i64),
(b" h=", a as i64),
(b" flags=", c as i64),
],
b"",
);
}
}
}
r
}
F_GETLK | F_SETLK | F_SETLKW | F_OFD_GETLK | F_OFD_SETLK | F_OFD_SETLKW => {
if c == 0 {
return -14; }
let mut fl = [0u8; FLOCK_SZ];
if vm_read(pid, c, &mut fl) != FLOCK_SZ as i64 {
return -14; }
let n = host_cancellable(SYS_FCNTL, a, b, fl.as_mut_ptr() as u64, 0, 0, 0);
if n == CANCEL_SENTINEL {
return CANCEL_SENTINEL;
}
if n >= 0 && (b == F_GETLK || b == F_OFD_GETLK) {
vm_write(pid, c, &fl);
}
n
}
_ => host(SYS_FCNTL, a, b, c, d, e, f),
},
SYS_IOCTL => match b {
0x5421 | 0x5452 => {
let mut v = [0u8; 4];
vm_read(pid, c, &mut v);
host(SYS_IOCTL, a, b, v.as_ptr() as u64, 0, 0, 0)
}
0x541B => {
let mut v = [0u8; 4];
let n = host(SYS_IOCTL, a, b, v.as_mut_ptr() as u64, 0, 0, 0);
if n >= 0 {
vm_write(pid, c, &v);
}
n
}
0x5413 => {
let mut v = [0u8; 8];
let n = host(SYS_IOCTL, a, b, v.as_mut_ptr() as u64, 0, 0, 0);
if n >= 0 {
vm_write(pid, c, &v);
}
n
}
0x5401 => {
let mut v = [0u8; 60];
let n = host(SYS_IOCTL, a, b, v.as_mut_ptr() as u64, 0, 0, 0);
if n >= 0 {
vm_write(pid, c, &v);
}
n
}
0x5402 | 0x5403 | 0x5404 => {
let mut v = [0u8; 60];
vm_read(pid, c, &mut v);
host(SYS_IOCTL, a, b, v.as_ptr() as u64, 0, 0, 0)
}
0x5414 => {
let mut v = [0u8; 8];
vm_read(pid, c, &mut v);
host(SYS_IOCTL, a, b, v.as_ptr() as u64, 0, 0, 0)
}
_ => -25, },
SYS_GETXATTR | SYS_LGETXATTR | SYS_FGETXATTR => -61,
SYS_CHDIR => {
let cand = pull_cwd_path(pid, a);
let fd = open_path(pid, &cand, O_PATH | O_DIRECTORY, 0);
if fd < 0 {
return fd; }
let guest_cwd = canonical_guest_cwd(fd as i32);
host(SYS_CLOSE, fd as u64, 0, 0, 0, 0, 0);
match guest_cwd {
Some(c) => {
cwds().lock().unwrap().insert(pid, c);
0
}
None => -2, }
}
SYS_FCHDIR => {
let h = match fd_host(pid, a as i32) {
Some(h) => h,
None => return -9, };
let mut cand = match canonical_guest_cwd(h) {
Some(g) => g,
None => return -2, };
cand.push(0); let fd = open_path(pid, &cand, O_PATH | O_DIRECTORY, 0);
if fd < 0 {
return fd; }
let guest_cwd = canonical_guest_cwd(fd as i32);
host(SYS_CLOSE, fd as u64, 0, 0, 0, 0, 0);
match guest_cwd {
Some(c) => {
cwds().lock().unwrap().insert(pid, c);
0
}
None => -2, }
}
CTL_REAP => {
signalfd_notify_sigchld(a as i32);
shm_reap_pid(a as i32);
fd_drop(a as i32);
0
}
CTL_SHM_FLUSH => {
if c == 2 {
shm_flush_pid_range_for_exec(pid, a, b)
} else {
shm_flush_pid_range(pid, a, b, c != 0)
}
}
CTL_SHM_ALIAS => shm_alias_pid_range(pid, a, b, c),
CTL_SHM_PROTECT => shm_protect_pid_range(pid, a, b, c),
CTL_SHM_MOVE => shm_move_pid_range(pid, a, b, c, d),
CTL_FORK_TABLE => {
if TRACE {
logn(b" [CTL_FORK_TABLE slot=", a as i64, b"");
logn(b" spawn_nr=", b as i64, b"");
logn(b" flags=", c as i64, b"]\n");
}
fd_snapshot(pid, a);
0
}
CTL_ENSURE_SERVICER => {
if ensure_servicer(a) {
0
} else {
-11 }
}
CTL_BIND_SLOT => {
proctree::bind_slot(a, b as i32);
0
}
CTL_SET_CREDS => {
let (ruid, euid) = unpack_u32_pair(a);
let (suid, fsuid) = unpack_u32_pair(b);
let (rgid, egid) = unpack_u32_pair(c);
let (sgid, fsgid) = unpack_u32_pair(d);
supervisor_creds().lock().unwrap().insert(
pid,
CellCredSnapshot {
ruid,
euid,
suid,
fsuid,
rgid,
egid,
sgid,
fsgid,
},
);
0
}
CTL_SET_UMASK => {
supervisor_umasks()
.lock()
.unwrap()
.insert(pid, (a as u32) & 0o777);
0
}
CTL_SET_EXE => {
let len = (b as usize).min(4096);
if a != 0 && len > 0 {
let mut buf = vec![0u8; len];
if vm_read(pid, a, &mut buf) == len as i64 {
if IPCTRACE {
ipc_logf_raw(
&[b"SETEXE pid=", pid.to_string().as_bytes(), b" exe=", &buf]
.concat(),
);
}
proc_exe().lock().unwrap().insert(pid, buf);
}
}
0
}
CTL_SET_CMDLINE => {
let len = (b as usize).min(8192);
if a != 0 && len > 0 {
let mut buf = vec![0u8; len];
if vm_read(pid, a, &mut buf) == len as i64 {
if IPCTRACE {
ipc_logf_raw(
&[b"SETCMD pid=", pid.to_string().as_bytes(), b" ", &buf].concat(),
);
}
proc_cmdline().lock().unwrap().insert(pid, buf);
repair_chromium_zygote_salt_fd(pid);
fdtrace_dump_table(pid, b"SETCMD");
}
}
0
}
CTL_ENVDBG => {
let len = (b as usize).min(8192);
if a != 0 && len > 0 {
let mut buf = vec![0u8; len];
if vm_read(pid, a, &mut buf) == len as i64 {
ipc_logf_raw(
&[b"ENVDBG_CTL pid=", pid.to_string().as_bytes(), b" ", &buf].concat(),
);
}
}
0
}
CTL_LOGLINE => {
let len = (b as usize).min(320);
if a != 0 && len > 0 {
let mut buf = vec![0u8; len];
if vm_read(pid, a, &mut buf) == len as i64 {
ipc_logf_raw(
&[b"CELL pid=", pid.to_string().as_bytes(), b" ", &buf].concat(),
);
}
}
0
}
CTL_PROCEXE_EXEC_FAIL => {
let len = (b as usize).min(1024);
if a != 0 && len > 0 {
let mut buf = vec![0u8; len];
if vm_read(pid, a, &mut buf) == len as i64 {
ipc_logf_raw(
&[
b"PROCEXE_EXEC_FAIL pid=".as_slice(),
pid.to_string().as_bytes(),
b" ".as_slice(),
&buf,
]
.concat(),
);
}
}
0
}
CTL_WATCHADDR => {
use std::sync::atomic::{AtomicU32, Ordering};
static WATCH_N: AtomicU32 = AtomicU32::new(0);
if a != 0 && WATCH_N.fetch_add(1, Ordering::Relaxed) < 12 {
let watch_pid = pid;
let addr = a;
let orig = (b & 0xff) as u8;
{
let mut arm: Vec<u8> = Vec::with_capacity(80);
arm.extend_from_slice(b"WATCHARM pid=");
arm.extend_from_slice(watch_pid.to_string().as_bytes());
arm.extend_from_slice(b" addr=0x");
const HX2: &[u8; 16] = b"0123456789abcdef";
for sh in (0..16u32).rev() {
arm.push(HX2[((addr >> (sh * 4)) & 0xf) as usize]);
}
arm.extend_from_slice(b" orig=");
arm.extend_from_slice(orig.to_string().as_bytes());
ipc_logf_raw(&arm);
}
let _ = orig;
std::thread::spawn(move || {
let fd = perf_open_wbp(watch_pid, addr);
if fd < 0 {
ipc_logf_raw(
&[b"WATCHBP_FAIL fd=".as_slice(), fd.to_string().as_bytes()]
.concat(),
);
return;
}
host(SYS_IOCTL, fd as u64, 0x2400, 0, 0, 0, 0); let ring = perf_mmap_ring(fd);
if ring.is_null() {
ipc_logf_raw(b"WATCHBP_NORING");
return;
}
const HX: &[u8; 16] = b"0123456789abcdef";
for _ in 0..3_000_000u64 {
if let Some(sample) = perf_read_sample(ring) {
let rip = sample.ip;
let mut out: Vec<u8> = Vec::with_capacity(320);
out.extend_from_slice(b"WATCHBP_RIP pid=");
out.extend_from_slice(watch_pid.to_string().as_bytes());
out.extend_from_slice(b" addr=0x");
for sh in (0..16u32).rev() {
out.push(HX[((addr >> (sh * 4)) & 0xf) as usize]);
}
out.extend_from_slice(b" rip=0x");
for sh in (0..16u32).rev() {
out.push(HX[((rip >> (sh * 4)) & 0xf) as usize]);
}
if let Some(regs) = sample.regs {
macro_rules! reg {
($name:expr, $value:expr) => {{
out.extend_from_slice($name);
out.extend_from_slice(b"=0x");
for sh in (0..16u32).rev() {
out.push(HX[((($value) >> (sh * 4)) & 0xf) as usize]);
}
}};
}
reg!(b" ax", regs.ax);
reg!(b" cx", regs.cx);
reg!(b" dx", regs.dx);
reg!(b" si", regs.si);
reg!(b" di", regs.di);
reg!(b" bp", regs.bp);
reg!(b" sp", regs.sp);
reg!(b" ipreg", regs.ip);
}
{
let mut ctx = [0u8; 96];
let cs = addr.wrapping_sub(72); if vm_read(watch_pid, cs, &mut ctx) == 96 {
let real = ctx.starts_with(b"DATABASE_URL=postgresql");
out.extend_from_slice(if real {
b" real=1 ctx=|"
} else {
b" real=0 ctx=|"
});
for &c in ctx.iter() {
out.push(if (0x20..0x7f).contains(&c) { c } else { b'.' });
}
out.extend_from_slice(b"|");
}
}
if let Ok(maps) =
std::fs::read_to_string(format!("/proc/{watch_pid}/maps"))
{
for line in maps.lines() {
let mut it = line.split([' ', '-']);
let s = it.next().and_then(|x| u64::from_str_radix(x, 16).ok());
let e = it.next().and_then(|x| u64::from_str_radix(x, 16).ok());
if let (Some(s), Some(e)) = (s, e) {
if rip >= s && rip < e {
out.extend_from_slice(b" map=");
out.extend_from_slice(line.as_bytes());
break;
}
}
}
}
{
let mut code = [0u8; 64];
let cstart = rip.wrapping_sub(48);
if vm_read(watch_pid, cstart, &mut code) == 64 {
out.extend_from_slice(b" code@rip-48=");
for &cb in code.iter() {
out.push(HX[(cb >> 4) as usize]);
out.push(HX[(cb & 0xf) as usize]);
}
}
}
ipc_logf_raw(&out);
break;
}
std::thread::sleep(std::time::Duration::from_micros(20));
}
});
}
0
}
CTL_COREDUMP => {
use std::sync::atomic::{AtomicU32, Ordering};
static COREDUMP_N: AtomicU32 = AtomicU32::new(0);
if COREDUMP_N.fetch_add(1, Ordering::Relaxed) == 0 {
ipc_logf_raw(
&[
b"COREDUMP_READY pid=",
pid.to_string().as_bytes(),
b" frozen 240s for external gdb",
]
.concat(),
);
std::thread::sleep(std::time::Duration::from_secs(240));
ipc_logf_raw(
&[b"COREDUMP_RESUME pid=", pid.to_string().as_bytes()].concat(),
);
}
0
}
CTL_CLOSE_CLOEXEC => {
fdtrace_dump_table(pid, b"PRE_CLOEXEC");
let victims: Vec<i32> = {
let t = fdt().lock().unwrap();
t.get(&pid)
.map(|m| {
m.iter()
.filter_map(|(&g, &v)| {
let flags = fd_get_desc_flags(pid, g, v);
((flags & FD_CLOEXEC as i32) != 0).then_some(g)
})
.collect()
})
.unwrap_or_default()
};
for g in victims {
match fd_remove(pid, g) {
Some(FdVal::Host(h)) => {
fd_forget_host_side_tables(h);
host(SYS_CLOSE, h as u64, 0, 0, 0, 0, 0);
}
Some(FdVal::Loop(s)) => {
loop_close(s)
}
None => {}
}
}
fdtrace_dump_table(pid, b"POST_CLOEXEC");
0
}
CTL_DETHREAD_FOR_EXEC => supervisor_dethread_for_exec(a, b as i32, c as i32),
CTL_FORK_CANCEL => {
if let Some(m) = pending().lock().unwrap().remove(&a) {
for (_, v) in m {
if let FdVal::Host(h) = v {
host(SYS_CLOSE, h as u64, 0, 0, 0, 0, 0);
}
}
}
pending_fd_flags().lock().unwrap().remove(&a);
pending_creds().lock().unwrap().remove(&a);
pending_umasks().lock().unwrap().remove(&a);
pending_cwd().lock().unwrap().remove(&a);
pending_exe().lock().unwrap().remove(&a);
pending_cmdline().lock().unwrap().remove(&a);
proctree::cancel_slot(a);
free_slot(a as u32);
0
}
SYS_FALLOCATE => host(SYS_FALLOCATE, a, b, c, d, 0, 0),
SYS_COPY_FILE_RANGE => {
let mut ib = [0u8; 8];
let ip = if b != 0 {
vm_read(pid, b, &mut ib);
ib.as_mut_ptr() as u64
} else {
0
};
let mut ob = [0u8; 8];
let op = if d != 0 {
vm_read(pid, d, &mut ob);
ob.as_mut_ptr() as u64
} else {
0
};
let n = host(SYS_COPY_FILE_RANGE, a, ip, c, op, e, f);
if b != 0 {
vm_write(pid, b, &ib);
}
if d != 0 {
vm_write(pid, d, &ob);
}
n
}
SYS_PREADV | SYS_PREADV2 => {
let cnt = (c.min(1024)) as usize;
let mut iovs = vec![0u8; cnt * 16];
vm_read(pid, b, &mut iovs);
let mut segs: Vec<(u64, usize)> = Vec::new();
let mut total = 0usize;
for i in 0..cnt {
let base = u64::from_le_bytes(iovs[i * 16..i * 16 + 8].try_into().unwrap());
let len =
(u64::from_le_bytes(iovs[i * 16 + 8..i * 16 + 16].try_into().unwrap())
.min(CAP)) as usize;
segs.push((base, len));
total += len;
}
let pos = ((e & 0xffff_ffff) << 32) | (d & 0xffff_ffff);
let mut bounce = vec![0u8; total];
let n = host(
SYS_PREAD64,
a,
bounce.as_mut_ptr() as u64,
total as u64,
pos,
0,
0,
);
if n > 0 {
let got = n as usize;
let mut off = 0usize;
for (base, len) in segs {
if off >= got {
break;
}
let take = len.min(got - off);
vm_write(pid, base, &bounce[off..off + take]);
off += take;
}
}
n
}
SYS_PWRITEV | SYS_PWRITEV2 => {
let cnt = (c.min(1024)) as usize;
let mut iovs = vec![0u8; cnt * 16];
vm_read(pid, b, &mut iovs);
let mut data = Vec::new();
for i in 0..cnt {
let base = u64::from_le_bytes(iovs[i * 16..i * 16 + 8].try_into().unwrap());
let len = u64::from_le_bytes(iovs[i * 16 + 8..i * 16 + 16].try_into().unwrap());
let len = len.min(CAP) as usize;
let mut seg = vec![0u8; len];
if len > 0 {
vm_read(pid, base, &mut seg);
}
data.extend_from_slice(&seg);
}
let pos = ((e & 0xffff_ffff) << 32) | (d & 0xffff_ffff);
host(
SYS_PWRITE64,
a,
data.as_ptr() as u64,
data.len() as u64,
pos,
0,
0,
)
}
SYS_FSTATFS => {
let mut sb = [0u8; 120];
let n = host(SYS_FSTATFS, a, sb.as_mut_ptr() as u64, 0, 0, 0, 0);
if n == 0 {
vm_write(pid, b, &sb);
}
n
}
SYS_SPLICE => {
let mut ib = [0u8; 8];
let ip = if b != 0 {
vm_read(pid, b, &mut ib);
ib.as_mut_ptr() as u64
} else {
0
};
let mut ob = [0u8; 8];
let op = if d != 0 {
vm_read(pid, d, &mut ob);
ob.as_mut_ptr() as u64
} else {
0
};
let n = host(SYS_SPLICE, a, ip, c, op, e, f);
if b != 0 {
vm_write(pid, b, &ib);
}
if d != 0 {
vm_write(pid, d, &ob);
}
n
}
SYS_TEE => host(SYS_TEE, a, b, c, d, 0, 0),
SYS_VMSPLICE => {
let cnt = (c.min(1024)) as usize;
let mut iovs = vec![0u8; cnt * 16];
vm_read(pid, b, &mut iovs);
let mut data = Vec::new();
for i in 0..cnt {
let base = u64::from_le_bytes(iovs[i * 16..i * 16 + 8].try_into().unwrap());
let len = u64::from_le_bytes(iovs[i * 16 + 8..i * 16 + 16].try_into().unwrap());
let len = len.min(CAP) as usize;
let mut seg = vec![0u8; len];
if len > 0 {
vm_read(pid, base, &mut seg);
}
data.extend_from_slice(&seg);
}
host(
SYS_WRITE,
a,
data.as_ptr() as u64,
data.len() as u64,
0,
0,
0,
)
}
other => {
if TRACE {
logn(
b"[supervisor] UNIMPLEMENTED delegated syscall nr=",
other,
b" -> ENOSYS\n",
);
}
-38
}
}
}
}
static MAIN_CELL_PID: AtomicU32 = AtomicU32::new(0);
fn running_instances() -> &'static std::sync::Mutex<Vec<i32>> {
static R: std::sync::OnceLock<std::sync::Mutex<Vec<i32>>> = std::sync::OnceLock::new();
R.get_or_init(|| std::sync::Mutex::new(Vec::new()))
}
fn live_running_instances_snapshot() -> Vec<i32> {
sweep_dead_cells();
let mut running = running_instances().lock().unwrap();
running.retain(|&pid| pid > 1 && !pid_is_dead(pid));
running.clone()
}
fn preserve_stragglers_as_running(foreground: i32) {
let protected = running_instances().lock().unwrap().clone();
let mut additions = Vec::new();
unsafe {
for i in 0..MAX_SLOTS as u64 {
let p = std::ptr::read_volatile(std::ptr::addr_of!((*ring_at(i)).pid));
if p > 1 && p != foreground && !protected.contains(&p) && !additions.contains(&p) {
additions.push(p);
}
}
}
if additions.is_empty() {
return;
}
let mut running = running_instances().lock().unwrap();
for pid in additions {
if pid > 1 && !pid_is_dead(pid) && !running.contains(&pid) {
fd_detach_stdio_to_dev_null(pid);
running.push(pid);
}
}
}
fn sweep_stragglers(foreground: i32) {
let protected = running_instances().lock().unwrap().clone();
unsafe {
let mut victims: [i32; MAX_SLOTS] = [0; MAX_SLOTS];
let mut n = 0usize;
for i in 0..MAX_SLOTS as u64 {
let p = std::ptr::read_volatile(std::ptr::addr_of!((*ring_at(i)).pid));
let protected_tree = proctree::slot_descends_from_any(i, p, &protected);
if p > 1 && p != foreground && !protected.contains(&p) && !protected_tree {
libc::kill(p, libc::SIGKILL);
if !victims[..n].contains(&p) {
victims[n] = p;
n += 1;
}
}
}
for &p in &victims[..n] {
let mut st: c_int = 0;
libc::waitpid(p, &mut st, 0);
reap_dead_pid(p);
}
}
}
extern "C" fn on_sigchld(_sig: c_int) {
loop {
let mut status: c_int = 0;
let r = unsafe { libc::waitpid(-1, &mut status, libc::WNOHANG) };
if r <= 0 {
return;
}
if r as u32 != MAIN_CELL_PID.load(Ordering::Relaxed) {
continue; }
unsafe {
for i in 0..MAX_SLOTS as u64 {
let p = std::ptr::read_volatile(std::ptr::addr_of!((*ring_at(i)).pid));
if p > 1 && p != r {
libc::kill(p, libc::SIGKILL);
}
}
}
if WARM_MODE.load(Ordering::Relaxed) {
unsafe {
if !ZYG.is_null() {
logn(b"[zygote] restores=", ZYG_RESTORES as i64, b"");
logn(b" first_exit_code=", (*ZYG).exit as i64, b"");
logn(b" mean_ns=", (*ZYG).mean_ns as i64, b"");
logn(b" min_ns=", (*ZYG).done as i64, b"\n");
}
}
}
let (code, err) = classify_status(status);
if matches!(err, Some(SentryError::SeccompViolation { .. })) {
log(b"[supervisor] cell was KILLED by the seccomp wall (SIGSYS) on a forbidden syscall.\n");
}
report_status(err);
loopring_dump();
unsafe { libc::_exit(code) };
}
}
thread_local! {
static SERVICER_RING: std::cell::Cell<*mut Ring> = const { std::cell::Cell::new(std::ptr::null_mut()) };
static SCRATCH: std::cell::RefCell<Vec<u8>> = const { std::cell::RefCell::new(Vec::new()) };
}
const SCRATCH_MAX: usize = 1024 * 1024;
fn with_scratch<R>(len: usize, f: impl FnOnce(&mut [u8]) -> R) -> R {
if len > SCRATCH_MAX {
let mut v = vec![0u8; len];
return f(&mut v);
}
SCRATCH.with(|s| {
let mut b = s.borrow_mut();
if b.len() < len {
b.resize(len, 0);
}
f(&mut b[..len])
})
}
fn pvm_read(pid: i32, addr: u64, buf: &mut [u8]) -> bool {
let local = libc::iovec {
iov_base: buf.as_mut_ptr() as *mut libc::c_void,
iov_len: buf.len(),
};
let remote = libc::iovec {
iov_base: addr as *mut libc::c_void,
iov_len: buf.len(),
};
let n = unsafe {
host(
SYS_PROCESS_VM_READV,
pid as u64,
&local as *const _ as u64,
1,
&remote as *const _ as u64,
1,
0,
)
};
n == buf.len() as i64
}
fn pvm_read_n(pid: i32, addr: u64, buf: &mut [u8]) -> i64 {
let local = libc::iovec {
iov_base: buf.as_mut_ptr() as *mut libc::c_void,
iov_len: buf.len(),
};
let remote = libc::iovec {
iov_base: addr as *mut libc::c_void,
iov_len: buf.len(),
};
unsafe {
host(
SYS_PROCESS_VM_READV,
pid as u64,
&local as *const _ as u64,
1,
&remote as *const _ as u64,
1,
0,
)
}
}
fn envscan_byte(pid: i32, addr: u64) -> Option<u8> {
let mut b = [0u8; 1];
if pvm_read(pid, addr, &mut b) {
Some(b[0])
} else {
None
}
}
fn envscan_find_all(pid: i32, needle: &[u8], out: &mut Vec<u64>) {
let maps = match std::fs::read_to_string(format!("/proc/{pid}/maps")) {
Ok(m) => m,
Err(_) => return,
};
let mut scanned: u64 = 0;
for line in maps.lines() {
let mut parts = line.split_whitespace();
let range = match parts.next() {
Some(r) => r,
None => continue,
};
let perms = parts.next().unwrap_or("");
if !perms.starts_with("rw") {
continue;
}
let mut rp = range.split('-');
let start = rp.next().and_then(|s| u64::from_str_radix(s, 16).ok());
let end = rp.next().and_then(|s| u64::from_str_radix(s, 16).ok());
let (start, end) = match (start, end) {
(Some(s), Some(e)) if e > s => (s, e),
_ => continue,
};
let len = (end - start) as usize;
let in_brk_stack = start >= 0x7000_0000_0000;
let in_arena = start >= 0x1000_0000_00 && start < 0x20_0000_0000;
if len == 0 || !(in_brk_stack || in_arena) {
continue;
}
if len > 96 * 1024 * 1024 {
if !in_arena {
continue;
}
let chunk = 4 * 1024 * 1024usize;
let mut buf = vec![0u8; chunk];
let mut off = 0u64;
let mut region_scanned = 0u64;
let region_cap = 3 * 1024 * 1024 * 1024u64; let span_cap = 96 * 1024 * 1024 * 1024u64; while off < (len as u64).min(span_cap) && region_scanned < region_cap {
let want = chunk.min((len as u64 - off) as usize);
let n = pvm_read_n(pid, start + off, &mut buf[..want]);
if n > 0 {
let nn = n as usize;
region_scanned += nn as u64;
let mut i = 0usize;
while i + needle.len() <= nn {
if &buf[i..i + needle.len()] == needle {
out.push(start + off + i as u64);
if out.len() >= 24 {
return;
}
i += needle.len();
} else {
i += 1;
}
}
}
off += (chunk - needle.len()) as u64;
}
continue;
}
if scanned > 1536 * 1024 * 1024 {
break;
}
scanned += len as u64;
let mut buf = vec![0u8; len];
if !pvm_read(pid, start, &mut buf) {
continue;
}
let mut i = 0usize;
while i + needle.len() <= buf.len() {
if &buf[i..i + needle.len()] == needle {
out.push(start + i as u64);
if out.len() >= 24 {
return;
}
i += needle.len();
} else {
i += 1;
}
}
}
}
fn envscan_comm(pid: i32) -> String {
std::fs::read_to_string(format!("/proc/{pid}/comm"))
.map(|c| c.trim().to_string())
.unwrap_or_default()
}
fn envscan_sym(pid: i32, rip: u64) -> String {
let maps = match std::fs::read_to_string(format!("/proc/{pid}/maps")) {
Ok(m) => m,
Err(_) => return "noproc".to_string(),
};
for line in maps.lines() {
let mut parts = line.split_whitespace();
let range = parts.next().unwrap_or("");
let perms = parts.next().unwrap_or("");
let mut rp = range.split('-');
let start = rp.next().and_then(|s| u64::from_str_radix(s, 16).ok());
let end = rp.next().and_then(|s| u64::from_str_radix(s, 16).ok());
if let (Some(s), Some(e)) = (start, end) {
if rip >= s && rip < e {
let off = rip - s;
let path = line.split_whitespace().nth(5).unwrap_or("");
let name = if path.is_empty() {
format!("[anon-{perms}]")
} else {
path.rsplit('/').next().unwrap_or(path).to_string()
};
return format!("{name}+0x{off:x}({perms})");
}
}
}
"unmapped".to_string()
}
fn perf_open_wbp(tid: i32, addr: u64) -> i32 {
const PERF_SAMPLE_IP: u64 = 1;
const PERF_SAMPLE_REGS_USER: u64 = 1 << 12;
const PERF_REG_X86_AX: u64 = 1 << 0;
const PERF_REG_X86_CX: u64 = 1 << 2;
const PERF_REG_X86_DX: u64 = 1 << 3;
const PERF_REG_X86_SI: u64 = 1 << 4;
const PERF_REG_X86_DI: u64 = 1 << 5;
const PERF_REG_X86_BP: u64 = 1 << 6;
const PERF_REG_X86_SP: u64 = 1 << 7;
const PERF_REG_X86_IP: u64 = 1 << 8;
const PERF_REGS: u64 = PERF_REG_X86_AX
| PERF_REG_X86_CX
| PERF_REG_X86_DX
| PERF_REG_X86_SI
| PERF_REG_X86_DI
| PERF_REG_X86_BP
| PERF_REG_X86_SP
| PERF_REG_X86_IP;
let mut attr = [0u8; 128];
attr[0..4].copy_from_slice(&5u32.to_le_bytes()); attr[4..8].copy_from_slice(&128u32.to_le_bytes()); attr[16..24].copy_from_slice(&1u64.to_le_bytes()); attr[24..32].copy_from_slice(&(PERF_SAMPLE_IP | PERF_SAMPLE_REGS_USER).to_le_bytes());
attr[40..48].copy_from_slice(&1u64.to_le_bytes()); attr[52..56].copy_from_slice(&2u32.to_le_bytes()); attr[56..64].copy_from_slice(&addr.to_le_bytes()); attr[64..72].copy_from_slice(&1u64.to_le_bytes()); attr[80..88].copy_from_slice(&PERF_REGS.to_le_bytes()); let fd = unsafe {
host(
298, attr.as_ptr() as u64,
tid as u64,
(-1i64) as u64, (-1i64) as u64, 0, 0,
)
};
fd as i32
}
fn perf_mmap_ring(fd: i32) -> *mut u8 {
let p = unsafe {
host(
SYS_MMAP,
0,
8192, (libc::PROT_READ | libc::PROT_WRITE) as u64,
libc::MAP_SHARED as u64,
fd as u64,
0,
)
};
if p < 0 {
std::ptr::null_mut()
} else {
p as *mut u8
}
}
#[derive(Clone, Copy)]
struct PerfRegs {
ax: u64,
cx: u64,
dx: u64,
si: u64,
di: u64,
bp: u64,
sp: u64,
ip: u64,
}
#[derive(Clone, Copy)]
struct PerfSample {
ip: u64,
regs: Option<PerfRegs>,
}
fn perf_read_sample(base: *mut u8) -> Option<PerfSample> {
if base.is_null() {
return None;
}
unsafe {
let head = std::ptr::read_volatile(base.add(1024) as *const u64);
let tail = std::ptr::read_volatile(base.add(1032) as *const u64);
if head <= tail {
return None;
}
let data_off = std::ptr::read_volatile(base.add(1040) as *const u64) as usize;
let data_size = std::ptr::read_volatile(base.add(1048) as *const u64) as usize;
if data_size == 0 {
return None;
}
let rec = base.add(data_off + (tail as usize % data_size));
let rtype = std::ptr::read_volatile(rec as *const u32);
let sample = if rtype == 9 {
let ip = std::ptr::read_volatile(rec.add(8) as *const u64);
let abi = std::ptr::read_volatile(rec.add(16) as *const u64);
let regs = if abi != 0 {
Some(PerfRegs {
ax: std::ptr::read_volatile(rec.add(24) as *const u64),
cx: std::ptr::read_volatile(rec.add(32) as *const u64),
dx: std::ptr::read_volatile(rec.add(40) as *const u64),
si: std::ptr::read_volatile(rec.add(48) as *const u64),
di: std::ptr::read_volatile(rec.add(56) as *const u64),
bp: std::ptr::read_volatile(rec.add(64) as *const u64),
sp: std::ptr::read_volatile(rec.add(72) as *const u64),
ip: std::ptr::read_volatile(rec.add(80) as *const u64),
})
} else {
None
};
Some(PerfSample { ip, regs })
} else {
None
};
std::ptr::write_volatile(base.add(1032) as *mut u64, head); sample
}
}
fn perf_read_ip(base: *mut u8) -> Option<u64> {
perf_read_sample(base).map(|s| s.ip)
}
fn env_scan_watchdog() -> ! {
let needle: &[u8] = b"shopify_test_harness_self_";
let nl = needle.len() as u64;
let mut watched: std::collections::HashSet<(i32, u64)> = std::collections::HashSet::new();
let mut tries: std::collections::HashMap<i32, u32> = std::collections::HashMap::new();
let mut bps: Vec<(*mut u8, u64, i32)> = Vec::new();
let mut tick: u64 = 0;
loop {
tick += 1;
let mut slots_set = 0u32;
let mut pids = 0u32;
let mut nodes: Vec<i32> = Vec::new();
for i in 0..(MAX_SLOTS as u64) {
if !slot_is_set(i) {
continue;
}
slots_set += 1;
let pid = unsafe { std::ptr::read_volatile(std::ptr::addr_of!((*ring_at(i)).pid)) };
if pid <= 1 {
continue;
}
pids += 1;
let c = envscan_comm(pid);
if c == "node" || c.starts_with("npm") || c == "npx" {
nodes.push(pid);
}
}
if tick <= 12 || tick % 120 == 1 {
let mut t: Vec<u8> = Vec::with_capacity(96);
t.extend_from_slice(b"ENVSCAN_TICK tick=");
t.extend_from_slice(tick.to_string().as_bytes());
t.extend_from_slice(b" slots=");
t.extend_from_slice(slots_set.to_string().as_bytes());
t.extend_from_slice(b" pids=");
t.extend_from_slice(pids.to_string().as_bytes());
t.extend_from_slice(b" nodes=");
t.extend_from_slice((nodes.len() as u32).to_string().as_bytes());
t.extend_from_slice(b" watched=");
t.extend_from_slice((watched.len() as u32).to_string().as_bytes());
ipc_logf_raw(&t);
}
for &pid in &nodes {
let t = tries.entry(pid).or_insert(0);
if *t >= 60 {
continue;
}
*t += 1;
let mut addrs: Vec<u64> = Vec::new();
envscan_find_all(pid, needle, &mut addrs);
for a in addrs {
let wb = a + nl;
if watched.contains(&(pid, wb)) {
continue;
}
if envscan_byte(pid, wb) == Some(b't') {
let mut sizeaddr = wb;
let mut is_len = false;
if a < 0x20_0000_0000 {
let deltas = [37u64, 41, 45, 33, 4, 8];
let mut vals = [0u32; 6];
for (k, &delta) in deltas.iter().enumerate() {
if a >= delta {
let mut b4 = [0u8; 4];
if pvm_read(pid, a - delta, &mut b4) {
vals[k] = u32::from_le_bytes(b4);
}
}
}
let mut m: Vec<u8> = Vec::with_capacity(112);
m.extend_from_slice(b"ENVSCAN_ARENA a=0x");
m.extend_from_slice(format!("{a:x}").as_bytes());
for (k, &delta) in deltas.iter().enumerate() {
m.extend_from_slice(b" l-");
m.extend_from_slice(delta.to_string().as_bytes());
m.extend_from_slice(b"=");
m.extend_from_slice(vals[k].to_string().as_bytes());
}
ipc_logf_raw(&m);
let mut found = false;
for (k, &delta) in deltas.iter().enumerate() {
if vals[k] == 108 {
sizeaddr = a - delta;
is_len = true;
found = true;
break;
}
}
if !found {
continue;
}
}
watched.insert((pid, wb));
ipc_logf_raw(
&[
b"ENVSCAN_ARM pid=".as_slice(),
pid.to_string().as_bytes(),
b" comm=".as_slice(),
envscan_comm(pid).as_bytes(),
b" addr=0x".as_slice(),
format!("{sizeaddr:x}").as_bytes(),
if is_len {
b" LENFIELD".as_slice()
} else {
b" valbyte".as_slice()
},
]
.concat(),
);
if bps.len() < 4 {
let fd = perf_open_wbp(pid, sizeaddr);
if fd >= 0 {
unsafe { host(SYS_IOCTL, fd as u64, 0x2400, 0, 0, 0, 0) }; let ring = perf_mmap_ring(fd);
ipc_logf_raw(
&[
b"ENVSCAN_BPARM pid=".as_slice(),
pid.to_string().as_bytes(),
b" sizeaddr=0x".as_slice(),
format!("{sizeaddr:x}").as_bytes(),
b" fd=".as_slice(),
fd.to_string().as_bytes(),
b" ring=".as_slice(),
if ring.is_null() { b"0" } else { b"1" },
]
.concat(),
);
if !ring.is_null() {
bps.push((ring, sizeaddr, pid));
}
} else {
ipc_logf_raw(
&[b"ENVSCAN_BPFAIL fd=".as_slice(), fd.to_string().as_bytes()]
.concat(),
);
}
}
}
}
}
for &(ring, sizeaddr, bppid) in &bps {
if let Some(rip) = perf_read_ip(ring) {
let region = envscan_sym(bppid, rip);
ipc_logf_raw(
&[
b"ENVSCAN_BPRIP rip=0x".as_slice(),
format!("{rip:x}").as_bytes(),
b" sizeaddr=0x".as_slice(),
format!("{sizeaddr:x}").as_bytes(),
b" in=".as_slice(),
region.as_bytes(),
]
.concat(),
);
}
}
let mut hit: Option<(i32, u64, u8)> = None;
for &(pid, wb) in watched.iter() {
match envscan_byte(pid, wb) {
Some(b't') => {}
Some(other) => {
hit = Some((pid, wb, other));
break;
}
None => {}
}
}
if let Some((pid, wb, val)) = hit {
let mut out: Vec<u8> = Vec::with_capacity(640);
out.extend_from_slice(b"ENVSCAN_HIT pid=");
out.extend_from_slice(pid.to_string().as_bytes());
out.extend_from_slice(b" comm=");
out.extend_from_slice(envscan_comm(pid).as_bytes());
out.extend_from_slice(b" newbyte=");
out.extend_from_slice((val as u32).to_string().as_bytes());
out.extend_from_slice(b" addr=0x");
for sh in (0..16u32).rev() {
out.push(b"0123456789abcdef"[((wb >> (sh * 4)) & 0xf) as usize]);
}
{
let mut ctx = [0u8; 128];
let cs = wb.saturating_sub(72);
if pvm_read(pid, cs, &mut ctx) {
let real = ctx.starts_with(b"DATABASE_URL=postgresql");
out.extend_from_slice(if real {
b" real=1 ctx=|"
} else {
b" real=0 ctx=|"
});
for &c in ctx.iter() {
out.push(if (0x20..0x7f).contains(&c) { c } else { b'.' });
}
out.extend_from_slice(b"|");
}
}
if let Ok(rd) = std::fs::read_dir(format!("/proc/{pid}/task")) {
for e in rd.flatten() {
let tid = e.file_name();
let tids = tid.to_string_lossy();
if let Ok(sc) =
std::fs::read_to_string(format!("/proc/{pid}/task/{tids}/syscall"))
{
out.extend_from_slice(b" t");
out.extend_from_slice(tids.as_bytes());
out.push(b'=');
out.extend_from_slice(sc.trim().as_bytes());
}
}
}
ipc_logf_raw(&out);
watched.remove(&(pid, wb));
}
std::thread::sleep(std::time::Duration::from_micros(120));
}
}
fn servicer_loop(slot: u64) -> ! {
if ENVSCAN.load(Ordering::Relaxed) {
static SCAN_ONCE: std::sync::Once = std::sync::Once::new();
SCAN_ONCE.call_once(|| {
ipc_logf_raw(b"ENVSCAN_SPAWN_LAZY");
let _ = std::thread::Builder::new().spawn(env_scan_watchdog);
});
}
let r = ring_at(slot);
let req = ring_word(unsafe { std::ptr::addr_of_mut!((*r).request) });
let resp = ring_word(unsafe { std::ptr::addr_of_mut!((*r).response) });
SERVICER_RING.with(|c| c.set(r));
let efd = unsafe { host(SYS_EVENTFD2, 0, EFD_NONBLOCK_CLOEXEC, 0, 0, 0, 0) } as i32;
let intr_efd = unsafe { host(SYS_EVENTFD2, 0, EFD_NONBLOCK_CLOEXEC, 0, 0, 0, 0) } as i32;
unsafe { std::ptr::write_volatile(std::ptr::addr_of_mut!((*r).cancel_efd), efd) };
unsafe { std::ptr::write_volatile(std::ptr::addr_of_mut!((*r).intr_efd), intr_efd) };
unsafe { std::ptr::write_volatile(std::ptr::addr_of_mut!((*r).cancel_pending), 0) };
unsafe { std::ptr::write_volatile(std::ptr::addr_of_mut!((*r).intr_pending), 0) };
let mut last = 0u32;
loop {
let mut spins = 0u32;
loop {
let cur = req.load(Ordering::Acquire);
if cur != last {
break;
}
if spins < RING_SPIN {
spins += 1;
std::hint::spin_loop();
continue;
}
unsafe {
host(
SYS_FUTEX,
req.as_ptr() as u64,
FUTEX_WAIT,
last as u64,
0,
0,
0,
)
};
}
let k = req.load(Ordering::Acquire);
unsafe {
let mut drain = [0u8; 8];
if std::ptr::read_volatile(std::ptr::addr_of!((*r).cancel_pending)) != 0 {
host(SYS_READ, efd as u64, drain.as_mut_ptr() as u64, 8, 0, 0, 0);
std::ptr::write_volatile(std::ptr::addr_of_mut!((*r).cancel_pending), 0);
}
if std::ptr::read_volatile(std::ptr::addr_of!((*r).intr_pending)) != 0 {
host(
SYS_READ,
intr_efd as u64,
drain.as_mut_ptr() as u64,
8,
0,
0,
0,
);
std::ptr::write_volatile(std::ptr::addr_of_mut!((*r).intr_pending), 0);
}
std::ptr::write_volatile(
std::ptr::addr_of_mut!((*r).cancel_mode),
CANCEL_REAP_PROCESS,
);
}
let (nr, args) = unsafe {
let nr = std::ptr::read_volatile(std::ptr::addr_of!((*r).nr));
let ap = std::ptr::addr_of!((*r).args) as *const u64;
(
nr,
[
ap.read_volatile(),
ap.add(1).read_volatile(),
ap.add(2).read_volatile(),
ap.add(3).read_volatile(),
ap.add(4).read_volatile(),
ap.add(5).read_volatile(),
],
)
};
let pid = unsafe { std::ptr::read_volatile(std::ptr::addr_of!((*r).pid)) };
if unsafe { TRACE } {
logn(b" [s", slot as i64, b"");
logn(b" pid=", pid as i64, b"");
logn(b"] nr=", nr, b"");
logn(b" a=", args[0] as i64, b"");
logn(b" b=", args[1] as i64, b"\n");
}
let fp = unsafe { std::ptr::read_volatile(std::ptr::addr_of!((*r).fork_parent)) };
if fp != 0 {
fd_adopt(slot, pid);
unsafe { std::ptr::write_volatile(std::ptr::addr_of_mut!((*r).fork_parent), 0) };
}
let ret = service(
slot, pid, nr, args[0], args[1], args[2], args[3], args[4], args[5],
);
let noisy = matches!(nr, 3 | 302 | 257 | 5 | 17 | 262 | 11 | 10 | 8);
if unsafe { SYSCALLTRACE } && ret != CANCEL_SENTINEL && !noisy {
let rip = unsafe { std::ptr::read_volatile(std::ptr::addr_of!((*r).dbg_rip)) };
let caller = unsafe { std::ptr::read_volatile(std::ptr::addr_of!((*r).dbg_caller)) };
ipc_logf(
&[
(b"DSYS pid=", pid as i64),
(b"slot=", slot as i64),
(b"nr=", nr),
(b"a=", args[0] as i64),
(b"b=", args[1] as i64),
(b"c=", args[2] as i64),
(b"ret=", ret),
(b"rip=", rip as i64),
(b"caller=", caller as i64),
],
&[],
);
}
if unsafe { IPCTRACE } && ret < 0 && ret != CANCEL_SENTINEL {
let e = -ret;
if !matches!(e, 11 | 4 | 115 | 2 | 17 | 25 | 10 | 3 | 110) {
ipc_logf(
&[
(b"DELGERR pid=", pid as i64),
(b"nr=", nr),
(b"a=", args[0] as i64),
(b"b=", args[1] as i64),
(b"c=", args[2] as i64),
(b"d=", args[3] as i64),
(b"e=", args[4] as i64),
(b"f=", args[5] as i64),
(b"ret=", ret),
],
&[],
);
}
}
if ret == CANCEL_SENTINEL {
let cancel_mode =
unsafe { std::ptr::read_volatile(std::ptr::addr_of!((*r).cancel_mode)) };
if cancel_mode != CANCEL_SLOT_ONLY {
shm_reap_pid(pid);
fd_drop(pid);
}
unsafe {
let mut drain = [0u8; 8];
host(SYS_READ, efd as u64, drain.as_mut_ptr() as u64, 8, 0, 0, 0);
std::ptr::write_volatile(std::ptr::addr_of_mut!((*r).cancel_pending), 0);
clear_ring_owner(slot);
std::ptr::write_volatile(
std::ptr::addr_of_mut!((*r).cancel_mode),
CANCEL_REAP_PROCESS,
);
}
if slot != 0 {
free_slot(slot as u32);
}
last = k;
continue;
}
unsafe { std::ptr::write_volatile(std::ptr::addr_of_mut!((*r).ret), ret) };
resp.store(k, Ordering::Release);
unsafe { host(SYS_FUTEX, resp.as_ptr() as u64, FUTEX_WAKE, 1, 0, 0, 0) };
last = k;
if nr == CTL_REAP {
free_slots_of(args[0] as i32);
}
}
}
fn servicer_spawned() -> &'static Mutex<[bool; MAX_SLOTS]> {
static S: OnceLock<Mutex<[bool; MAX_SLOTS]>> = OnceLock::new();
S.get_or_init(|| {
let mut a = [false; MAX_SLOTS];
a[0] = true; Mutex::new(a)
})
}
fn ensure_servicer(slot: u64) -> bool {
let i = slot as usize;
if i >= MAX_SLOTS {
return false; }
let mut bm = servicer_spawned().lock().unwrap();
if bm[i] {
return true;
}
match std::thread::Builder::new().spawn(move || servicer_loop(slot)) {
Ok(_) => {
bm[i] = true;
true
}
Err(_) => false,
}
}
fn spawn_deleg_servicers() {
for slot in PROCESS_SLOTS as u64..MAX_SLOTS as u64 {
let _ = ensure_servicer(slot);
}
}
fn seed_cell_fds(pid: i32) {
let mut t = fdt().lock().unwrap();
let m = t.entry(pid).or_default();
for fd in 0..3i32 {
let d = unsafe { host(SYS_DUP, fd as u64, 0, 0, 0, 0, 0) };
m.insert(fd, FdVal::Host(if d >= 0 { d as i32 } else { fd }));
}
}
fn seed_cell_fds_capture(pid: i32, cap_fd: c_int) {
let mut t = fdt().lock().unwrap();
let m = t.entry(pid).or_default();
let d0 = unsafe { host(SYS_DUP, 0, 0, 0, 0, 0, 0) };
m.insert(0, FdVal::Host(if d0 >= 0 { d0 as i32 } else { 0 }));
for fd in 1..3i32 {
let d = unsafe { host(SYS_DUP, cap_fd as u64, 0, 0, 0, 0, 0) };
m.insert(fd, FdVal::Host(if d >= 0 { d as i32 } else { cap_fd }));
}
}
fn seed_cell_fds_streaming(pid: i32, in_r: c_int, out_w: c_int, err_w: c_int) {
let mut t = fdt().lock().unwrap();
let m = t.entry(pid).or_default();
for (gfd, host_fd) in [(0i32, in_r), (1, out_w), (2, err_w)] {
let d = unsafe { host(SYS_DUP, host_fd as u64, 0, 0, 0, 0, 0) };
m.insert(gfd, FdVal::Host(if d >= 0 { d as i32 } else { host_fd }));
}
}
fn warm_capture_redirect(warm_pid: i32, cap_fd: c_int) -> (i32, i32) {
let mut t = fdt().lock().unwrap();
let m = t.entry(warm_pid).or_default();
let old1 = match m.get(&1) {
Some(FdVal::Host(h)) => *h,
_ => 1,
};
let old2 = match m.get(&2) {
Some(FdVal::Host(h)) => *h,
_ => 2,
};
let d1 = unsafe { host(SYS_DUP, cap_fd as u64, 0, 0, 0, 0, 0) };
let d2 = unsafe { host(SYS_DUP, cap_fd as u64, 0, 0, 0, 0, 0) };
m.insert(1, FdVal::Host(if d1 >= 0 { d1 as i32 } else { old1 }));
m.insert(2, FdVal::Host(if d2 >= 0 { d2 as i32 } else { old2 }));
(old1, old2)
}
fn warm_capture_restore(warm_pid: i32, old1: i32, old2: i32) {
let mut t = fdt().lock().unwrap();
let m = t.entry(warm_pid).or_default();
if let Some(FdVal::Host(h)) = m.get(&1) {
if *h != old1 {
unsafe { host(SYS_CLOSE, *h as u64, 0, 0, 0, 0, 0) };
}
}
if let Some(FdVal::Host(h)) = m.get(&2) {
if *h != old2 {
unsafe { host(SYS_CLOSE, *h as u64, 0, 0, 0, 0, 0) };
}
}
m.insert(1, FdVal::Host(old1));
m.insert(2, FdVal::Host(old2));
}
fn slot_dump_watchdog() {
let rings = unsafe { RINGS };
if rings.is_null() {
return;
}
loop {
unsafe { host(SYS_POLL, 0, 0, 3000, 0, 0, 0) };
let mut any = false;
for s in 0..MAX_SLOTS {
let r = unsafe { rings.add(s) };
let req = unsafe { std::ptr::read_volatile(std::ptr::addr_of!((*r).request)) };
let resp = unsafe { std::ptr::read_volatile(std::ptr::addr_of!((*r).response)) };
if req == resp {
continue;
}
any = true;
let pid = unsafe { std::ptr::read_volatile(std::ptr::addr_of!((*r).pid)) };
let tid = unsafe { std::ptr::read_volatile(std::ptr::addr_of!((*r).tid)) };
let nr = unsafe { std::ptr::read_volatile(std::ptr::addr_of!((*r).nr)) };
let rip = unsafe { std::ptr::read_volatile(std::ptr::addr_of!((*r).dbg_rip)) };
let caller = unsafe { std::ptr::read_volatile(std::ptr::addr_of!((*r).dbg_caller)) };
ipc_logf(
&[
(b"SLOTDUMP slot=", s as i64),
(b" pid=", pid as i64),
(b" tid=", tid as i64),
(b" nr=", nr),
(b" rip=", rip as i64),
(b" caller=", caller as i64),
],
b"",
);
}
if any {
ipc_logf(&[(b"SLOTDUMP ---tick---", 0)], b"");
}
}
}
fn hang_dump_watchdog(delay_s: u64) {
ipc_logf(
&[
(b"WATCHDOG-START delay=", delay_s as i64),
(b" rings_null=", unsafe { RINGS.is_null() } as i64),
],
b"",
);
let rings = unsafe { RINGS };
if rings.is_null() {
return;
}
fn read_proc(pid: i32, leaf: &str, buf: &mut [u8]) -> usize {
let path = format!("/proc/{pid}/{leaf}\0");
let fd = unsafe {
host(
SYS_OPENAT,
(-100i64) as u64,
path.as_ptr() as u64,
libc::O_RDONLY as u64,
0,
0,
0,
)
};
if fd < 0 {
return 0;
}
let n = unsafe {
host(
SYS_READ,
fd as u64,
buf.as_mut_ptr() as u64,
buf.len() as u64,
0,
0,
0,
)
};
unsafe { host(SYS_CLOSE, fd as u64, 0, 0, 0, 0, 0) };
if n > 0 {
n as usize
} else {
0
}
}
let hexu = |s: &str| -> u64 {
u64::from_str_radix(s.trim().trim_start_matches("0x"), 16).unwrap_or(0)
};
for _ in 0..delay_s {
unsafe { host(SYS_POLL, 0, 0, 1000, 0, 0, 0) };
}
loop {
let mut any = false;
for s in 0..MAX_SLOTS {
let r = unsafe { rings.add(s) };
let pid = unsafe { std::ptr::read_volatile(std::ptr::addr_of!((*r).pid)) };
if pid <= 1 {
continue;
}
let mut sbuf = [0u8; 256];
let scn = read_proc(pid, "syscall", &mut sbuf);
if scn == 0 {
continue; }
let sc = std::str::from_utf8(&sbuf[..scn]).unwrap_or("");
let mut it = sc.split_whitespace();
let nr = it.next().and_then(|t| t.parse::<i64>().ok()).unwrap_or(-2);
let a0 = it.next().map(|t| hexu(t) as i64).unwrap_or(-1);
let _a1 = it.next().map(|t| hexu(t) as i64).unwrap_or(-1);
let mut stbuf = [0u8; 4096];
let stn = read_proc(pid, "status", &mut stbuf);
let st = std::str::from_utf8(&stbuf[..stn]).unwrap_or("");
let (mut blk, mut pnd, mut shd) = (0u64, 0u64, 0u64);
for line in st.lines() {
if let Some(v) = line.strip_prefix("SigBlk:") {
blk = hexu(v);
} else if let Some(v) = line.strip_prefix("SigPnd:") {
pnd = hexu(v);
} else if let Some(v) = line.strip_prefix("ShdPnd:") {
shd = hexu(v);
}
}
let chld_blk = ((blk >> 16) & 1) as i64;
let chld_pnd = (((pnd | shd) >> 16) & 1) as i64;
let ringnr = unsafe { std::ptr::read_volatile(std::ptr::addr_of!((*r).nr)) };
let req = unsafe { std::ptr::read_volatile(std::ptr::addr_of!((*r).request)) };
let resp = unsafe { std::ptr::read_volatile(std::ptr::addr_of!((*r).response)) };
let caller = unsafe { std::ptr::read_volatile(std::ptr::addr_of!((*r).dbg_caller)) };
any = true;
ipc_logf(
&[
(b"HANGDUMP slot=", s as i64),
(b" pid=", pid as i64),
(b" procnr=", nr),
(b" ringnr=", ringnr),
(b" pend=", (req != resp) as i64),
(b" a0=", a0),
(b" sigblk=", blk as i64),
(b" shdpnd=", shd as i64),
(b" chldBLK=", chld_blk),
(b" chldPND=", chld_pnd),
(b" caller=", caller as i64),
],
b"",
);
}
if any {
ipc_logf(&[(b"HANGDUMP ---tick---", 0)], b"");
for (v, pv, hp) in proctree::dump_all() {
ipc_logf(
&[
(b"PROCTREE vpid=", v),
(b" parent=", pv),
(b" hostpid=", hp),
],
b"",
);
}
}
unsafe { host(SYS_POLL, 0, 0, 3000, 0, 0, 0) };
}
}
fn supervisor_main(pid: i32) -> ! {
MAIN_CELL_PID.store(pid as u32, Ordering::Relaxed);
seed_cell_fds(pid);
proctree::register_main(pid);
if let Some(c) = guest_cwd_init().lock().unwrap().clone() {
cwds().lock().unwrap().insert(pid, c);
}
supervisor_umasks()
.lock()
.unwrap()
.insert(pid, DEFAULT_UMASK);
unsafe {
let null = libc::open(b"/dev/null\0".as_ptr() as *const c_char, libc::O_RDWR);
if null >= 0 {
libc::dup2(null, 0);
libc::dup2(null, 1);
libc::dup2(null, 2);
if null > 2 {
libc::close(null);
}
}
}
unsafe {
for i in 0..MAX_SLOTS as u64 {
set_ring_owner(i, pid, pid);
}
let mut sa: libc::sigaction = std::mem::zeroed();
sa.sa_sigaction = on_sigchld as *const () as usize;
sa.sa_flags = libc::SA_RESTART | libc::SA_NOCLDSTOP;
libc::sigemptyset(&mut sa.sa_mask);
libc::sigaction(libc::SIGCHLD, &sa, std::ptr::null_mut());
}
let _ = std::thread::Builder::new().spawn(fd_leak_sweeper);
if std::env::var("SENTRY_SLOTDUMP").is_ok() {
let _ = std::thread::Builder::new().spawn(slot_dump_watchdog);
}
if let Ok(v) = std::env::var("SENTRY_HANGDUMP") {
let delay = v.parse::<u64>().unwrap_or(8);
let _ = std::thread::Builder::new().spawn(move || hang_dump_watchdog(delay));
}
drop_supervisor_caps();
harden_supervisor();
spawn_deleg_servicers();
servicer_loop(0); }
fn rd_u16(b: &[u8], o: usize) -> u16 {
u16::from_le_bytes(b[o..o + 2].try_into().unwrap())
}
fn rd_u32(b: &[u8], o: usize) -> u32 {
u32::from_le_bytes(b[o..o + 4].try_into().unwrap())
}
fn rd_u64(b: &[u8], o: usize) -> u64 {
u64::from_le_bytes(b[o..o + 8].try_into().unwrap())
}
const PT_LOAD: u32 = 1;
const PT_INTERP: u32 = 3;
const PF_X: u32 = 1;
const PF_W: u32 = 2;
const PF_R: u32 = 4;
struct Loaded {
entry: u64,
phdr_vaddr: u64,
phent: u16,
phnum: u16,
interp: Option<Vec<u8>>,
}
fn read_in_root(root: &Option<std::path::PathBuf>, guest: &str) -> std::io::Result<Vec<u8>> {
use std::os::unix::ffi::OsStrExt;
let r = match root {
Some(r) => r,
None => return std::fs::read(guest),
};
let mut rootb = r.as_os_str().as_bytes().to_vec();
rootb.push(0);
let rootfd = unsafe {
host(
SYS_OPENAT,
(-100i64) as u64,
rootb.as_ptr() as u64,
O_PATH | O_DIRECTORY,
0,
0,
0,
)
};
if rootfd < 0 {
return Err(std::io::Error::from_raw_os_error(-rootfd as i32));
}
let mut gp = guest.as_bytes().to_vec();
gp.push(0);
let how = OpenHow {
flags: 0,
mode: 0,
resolve: RESOLVE_IN_ROOT,
};
let fd = unsafe {
host(
SYS_OPENAT2,
rootfd as u64,
gp.as_ptr() as u64,
std::ptr::addr_of!(how) as u64,
std::mem::size_of::<OpenHow>() as u64,
0,
0,
)
};
unsafe { host(SYS_CLOSE, rootfd as u64, 0, 0, 0, 0, 0) };
if fd < 0 {
return Err(std::io::Error::from_raw_os_error(-fd as i32));
}
let mut buf = Vec::new();
let mut chunk = [0u8; 65536];
loop {
let n = unsafe {
host(
SYS_READ,
fd as u64,
chunk.as_mut_ptr() as u64,
chunk.len() as u64,
0,
0,
0,
)
};
if n == 0 {
break; }
if n < 0 {
unsafe { host(SYS_CLOSE, fd as u64, 0, 0, 0, 0, 0) };
return Err(std::io::Error::from_raw_os_error(-n as i32));
}
buf.extend_from_slice(&chunk[..n as usize]);
}
unsafe { host(SYS_CLOSE, fd as u64, 0, 0, 0, 0, 0) };
Ok(buf)
}
const DEFAULT_PATH: &[u8] = b"/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin";
fn is_regular_in_root(root: &std::path::Path, guest: &str) -> bool {
use std::os::unix::ffi::OsStrExt;
let mut rootb = root.as_os_str().as_bytes().to_vec();
rootb.push(0);
let rootfd = unsafe {
host(
SYS_OPENAT,
(-100i64) as u64,
rootb.as_ptr() as u64,
O_PATH | O_DIRECTORY,
0,
0,
0,
)
};
if rootfd < 0 {
return false;
}
let mut gp = guest.as_bytes().to_vec();
gp.push(0);
let how = OpenHow {
flags: O_PATH,
mode: 0,
resolve: RESOLVE_IN_ROOT,
};
let fd = unsafe {
host(
SYS_OPENAT2,
rootfd as u64,
gp.as_ptr() as u64,
std::ptr::addr_of!(how) as u64,
std::mem::size_of::<OpenHow>() as u64,
0,
0,
)
};
unsafe { host(SYS_CLOSE, rootfd as u64, 0, 0, 0, 0, 0) };
if fd < 0 {
return false;
}
let mut sb = [0u8; 144];
let n = unsafe { host(SYS_FSTAT, fd as u64, sb.as_mut_ptr() as u64, 0, 0, 0, 0) };
unsafe { host(SYS_CLOSE, fd as u64, 0, 0, 0, 0, 0) };
if n != 0 {
return false;
}
let mode = u32::from_le_bytes([sb[24], sb[25], sb[26], sb[27]]);
(mode & S_IFMT) == S_IFREG
}
fn canonical_exe_in_root(root: &std::path::Path, guest: &str) -> Option<String> {
use std::os::unix::ffi::OsStrExt;
let mut rootb = root.as_os_str().as_bytes().to_vec();
rootb.push(0);
let rootfd = unsafe {
host(
SYS_OPENAT,
(-100i64) as u64,
rootb.as_ptr() as u64,
O_PATH | O_DIRECTORY,
0,
0,
0,
)
};
if rootfd < 0 {
return None;
}
let mut gp = guest.as_bytes().to_vec();
gp.push(0);
let how = OpenHow {
flags: O_PATH,
mode: 0,
resolve: RESOLVE_IN_ROOT,
};
let fd = unsafe {
host(
SYS_OPENAT2,
rootfd as u64,
gp.as_ptr() as u64,
std::ptr::addr_of!(how) as u64,
std::mem::size_of::<OpenHow>() as u64,
0,
0,
)
};
unsafe { host(SYS_CLOSE, rootfd as u64, 0, 0, 0, 0, 0) };
if fd < 0 {
return None;
}
let host_path = readlink_fd(fd as i32);
unsafe { host(SYS_CLOSE, fd as u64, 0, 0, 0, 0, 0) };
let guest_path = host_to_guest_path(&host_path?)?;
String::from_utf8(guest_path).ok()
}
fn dir_exists_in_root(root: &Option<std::path::PathBuf>, guest: &str) -> bool {
use std::os::unix::ffi::OsStrExt;
let r = match root {
Some(r) => r,
None => return std::path::Path::new(guest).is_dir(),
};
let mut rootb = r.as_os_str().as_bytes().to_vec();
rootb.push(0);
let rootfd = unsafe {
host(
SYS_OPENAT,
(-100i64) as u64,
rootb.as_ptr() as u64,
O_PATH | O_DIRECTORY,
0,
0,
0,
)
};
if rootfd < 0 {
return false;
}
let mut gp = guest.as_bytes().to_vec();
gp.push(0);
let how = OpenHow {
flags: O_PATH,
mode: 0,
resolve: RESOLVE_IN_ROOT,
};
let fd = unsafe {
host(
SYS_OPENAT2,
rootfd as u64,
gp.as_ptr() as u64,
std::ptr::addr_of!(how) as u64,
std::mem::size_of::<OpenHow>() as u64,
0,
0,
)
};
unsafe { host(SYS_CLOSE, rootfd as u64, 0, 0, 0, 0, 0) };
if fd < 0 {
return false;
}
let mut sb = [0u8; 144];
let n = unsafe { host(SYS_FSTAT, fd as u64, sb.as_mut_ptr() as u64, 0, 0, 0, 0) };
unsafe { host(SYS_CLOSE, fd as u64, 0, 0, 0, 0, 0) };
if n != 0 {
return false;
}
let mode = u32::from_le_bytes([sb[24], sb[25], sb[26], sb[27]]);
(mode & S_IFMT) == S_IFDIR
}
fn supermachine_checkpoint_elf() -> Vec<u8> {
let code: [u8; 53] = [
0x48, 0xc7, 0xc0, 0x59, 0x53, 0x00, 0x00, 0x0f, 0x05, 0x48, 0x8b, 0x7c, 0x24, 0x10, 0x48, 0x8d, 0x74, 0x24, 0x10, 0x48, 0x8b, 0x0c, 0x24, 0x48, 0x8d, 0x54, 0xcc, 0x10, 0x48, 0xc7, 0xc0, 0x3b, 0x00, 0x00, 0x00, 0x0f, 0x05, 0x48, 0xc7, 0xc7, 0x7f, 0x00, 0x00, 0x00, 0x48, 0xc7, 0xc0, 0x3c, 0x00, 0x00, 0x00, 0x0f, 0x05, ];
const BASE: u64 = 0x40_0000;
const EHSIZE: u64 = 64;
const PHSIZE: u64 = 56;
let entry = BASE + EHSIZE + PHSIZE;
let filesz = EHSIZE + PHSIZE + code.len() as u64;
let mut e = Vec::with_capacity(filesz as usize);
e.extend_from_slice(&[0x7f, b'E', b'L', b'F', 2, 1, 1, 0]);
e.extend_from_slice(&[0u8; 8]);
e.extend_from_slice(&2u16.to_le_bytes()); e.extend_from_slice(&0x3eu16.to_le_bytes()); e.extend_from_slice(&1u32.to_le_bytes()); e.extend_from_slice(&entry.to_le_bytes()); e.extend_from_slice(&EHSIZE.to_le_bytes()); e.extend_from_slice(&0u64.to_le_bytes()); e.extend_from_slice(&0u32.to_le_bytes()); e.extend_from_slice(&(EHSIZE as u16).to_le_bytes()); e.extend_from_slice(&(PHSIZE as u16).to_le_bytes()); e.extend_from_slice(&1u16.to_le_bytes()); e.extend_from_slice(&0u16.to_le_bytes()); e.extend_from_slice(&0u16.to_le_bytes()); e.extend_from_slice(&0u16.to_le_bytes()); e.extend_from_slice(&1u32.to_le_bytes()); e.extend_from_slice(&5u32.to_le_bytes()); e.extend_from_slice(&0u64.to_le_bytes()); e.extend_from_slice(&BASE.to_le_bytes()); e.extend_from_slice(&BASE.to_le_bytes()); e.extend_from_slice(&filesz.to_le_bytes()); e.extend_from_slice(&filesz.to_le_bytes()); e.extend_from_slice(&0x1000u64.to_le_bytes()); e.extend_from_slice(&code);
e
}
fn resolve_in_root_path(root: &std::path::Path, path_env: &[u8], argv0: &str) -> Option<String> {
if argv0.contains('/') {
return Some(argv0.to_string());
}
if argv0.is_empty() {
return None;
}
let env = if path_env.is_empty() {
DEFAULT_PATH
} else {
path_env
};
for dir in env.split(|&b| b == b':') {
let dir = if dir.is_empty() { b"." } else { dir };
let mut cand = String::from_utf8_lossy(dir).into_owned();
if !cand.ends_with('/') {
cand.push('/');
}
cand.push_str(argv0);
if is_regular_in_root(root, &cand) {
return Some(cand);
}
}
None
}
fn path_from_env(env: &[(String, String)]) -> Vec<u8> {
env.iter()
.find(|(k, _)| k == "PATH")
.map(|(_, v)| v.as_bytes().to_vec())
.unwrap_or_default()
}
fn load_elf(bytes: &[u8], base: u64) -> Loaded {
if bytes.len() < 64 || &bytes[0..4] != b"\x7fELF" || bytes[4] != 2 || bytes[5] != 1 {
die(b"not an ELF64-LE\n");
}
let e_type = rd_u16(bytes, 16);
if e_type != 2 && e_type != 3 {
die(b"not ET_EXEC/ET_DYN\n");
}
let entry = rd_u64(bytes, 24) + base;
let phoff = rd_u64(bytes, 32);
let phent = rd_u16(bytes, 54);
let phnum = rd_u16(bytes, 56);
let mut phdr_vaddr = 0u64;
let mut max_x = 0u64;
let mut interp: Option<Vec<u8>> = None;
let mut loads: Vec<(u64, u64, u64, u64, u32)> = Vec::new();
let mut lo = u64::MAX;
let mut hi = 0u64;
for i in 0..phnum as usize {
let ph = phoff as usize + i * phent as usize;
let pt = rd_u32(bytes, ph);
if pt == PT_INTERP {
let o = rd_u64(bytes, ph + 8) as usize;
let sz = rd_u64(bytes, ph + 32) as usize;
let mut s = bytes[o..o + sz].to_vec();
while s.last() == Some(&0) {
s.pop();
}
interp = Some(s);
continue;
}
if pt != PT_LOAD {
continue;
}
let p_flags = rd_u32(bytes, ph + 4);
let p_offset = rd_u64(bytes, ph + 8);
let p_vaddr = rd_u64(bytes, ph + 16) + base;
let p_filesz = rd_u64(bytes, ph + 32);
let p_memsz = rd_u64(bytes, ph + 40);
if phoff >= p_offset && phoff < p_offset + p_filesz {
phdr_vaddr = p_vaddr + (phoff - p_offset);
}
if (p_flags & PF_X) != 0 {
max_x = max_x.max(page_up(p_vaddr + p_memsz));
}
lo = lo.min(page_down(p_vaddr));
hi = hi.max(page_up(p_vaddr + p_memsz));
loads.push((p_vaddr, p_offset, p_filesz, p_memsz, p_flags));
}
if max_x == 0 || max_x >= WINDOW_FLOOR {
die(b"executable segments do not fit below the SUD floor\n");
}
let p = unsafe {
libc::mmap(
lo as *mut c_void,
(hi - lo) as usize,
libc::PROT_READ | libc::PROT_WRITE,
libc::MAP_PRIVATE | libc::MAP_ANONYMOUS | libc::MAP_FIXED,
-1,
0,
)
};
if p == libc::MAP_FAILED || p as u64 != lo {
die(b"image reservation failed\n");
}
for &(p_vaddr, p_offset, p_filesz, _, _) in &loads {
unsafe {
std::ptr::copy_nonoverlapping(
bytes.as_ptr().add(p_offset as usize),
p_vaddr as *mut u8,
p_filesz as usize,
)
};
}
for &(p_vaddr, _, _, p_memsz, p_flags) in &loads {
let mut prot = 0;
if p_flags & PF_R != 0 {
prot |= libc::PROT_READ;
}
if p_flags & PF_W != 0 {
prot |= libc::PROT_WRITE;
}
if p_flags & PF_X != 0 {
prot |= libc::PROT_EXEC;
}
let s = page_down(p_vaddr);
let ee = page_up(p_vaddr + p_memsz);
unsafe { libc::mprotect(s as *mut c_void, (ee - s) as usize, prot) };
}
record_loader_span(base, lo, hi);
Loaded {
entry,
phdr_vaddr,
phent,
phnum,
interp,
}
}
const AT_NULL: u64 = 0;
const AT_PHDR: u64 = 3;
const AT_PHENT: u64 = 4;
const AT_PHNUM: u64 = 5;
const AT_PAGESZ: u64 = 6;
const AT_BASE: u64 = 7;
const AT_FLAGS: u64 = 8;
const AT_ENTRY: u64 = 9;
const AT_UID: u64 = 11;
const AT_EUID: u64 = 12;
const AT_GID: u64 = 13;
const AT_EGID: u64 = 14;
const AT_PLATFORM: u64 = 15;
const AT_HWCAP: u64 = 16;
const AT_CLKTCK: u64 = 17;
const AT_SECURE: u64 = 23;
const AT_RANDOM: u64 = 25;
const AT_EXECFN: u64 = 31;
fn build_stack(l: &Loaded, interp_base: Option<u64>, argv: &[Vec<u8>], envp: &[Vec<u8>]) -> u64 {
const SZ: usize = 8 * 1024 * 1024;
let base = unsafe {
libc::mmap(
std::ptr::null_mut(),
SZ,
libc::PROT_READ | libc::PROT_WRITE,
libc::MAP_PRIVATE | libc::MAP_ANONYMOUS,
-1,
0,
)
};
if base == libc::MAP_FAILED {
die(b"stack mmap failed SMARK_RTRIP_7\n");
}
guest_vma_note_mapping(base as u64, SZ as u64);
arena_note_mapping(base as u64, SZ as u64);
arena_free_purge(base as u64, base as u64 + SZ as u64);
let top = base as u64 + SZ as u64;
let mut sp = top;
macro_rules! push {
($bytes:expr) => {{
let bb: &[u8] = $bytes;
sp -= bb.len() as u64;
unsafe { std::ptr::copy_nonoverlapping(bb.as_ptr(), sp as *mut u8, bb.len()) };
sp
}};
}
let mut rnd = [0u8; 16];
delegate(SYS_GETRANDOM, rnd.as_mut_ptr() as u64, 16, 0, 0, 0, 0);
{
let cell_guard: u64;
unsafe {
std::arch::asm!("mov {g}, fs:[0x28]", g = out(reg) cell_guard, options(nostack, preserves_flags))
};
if cell_guard != 0 {
rnd[0..8].copy_from_slice(&cell_guard.to_le_bytes());
}
}
let at_random = push!(&rnd);
let at_platform = push!(b"x86_64\0");
let execfn = [argv[0].as_slice(), b"\0"].concat();
let at_execfn = push!(&execfn);
let mut argv_p = Vec::new();
for s in argv {
let z = [s.as_slice(), b"\0"].concat();
argv_p.push(push!(&z));
}
let mut envp_p = Vec::new();
for s in envp {
let z = [s.as_slice(), b"\0"].concat();
let addr = push!(&z);
envp_p.push(addr);
}
let uid = cred_get(&CELL_CREDS.ruid) as u64;
let euid = cred_get(&CELL_CREDS.euid) as u64;
let gid = cred_get(&CELL_CREDS.rgid) as u64;
let egid = cred_get(&CELL_CREDS.egid) as u64;
let aux: [(u64, u64); 16] = [
(AT_PHDR, l.phdr_vaddr),
(AT_PHENT, l.phent as u64),
(AT_PHNUM, l.phnum as u64),
(AT_PAGESZ, 4096),
(AT_BASE, interp_base.unwrap_or(0)),
(AT_FLAGS, 0),
(AT_ENTRY, l.entry),
(AT_UID, uid),
(AT_EUID, euid),
(AT_GID, gid),
(AT_EGID, egid),
(AT_CLKTCK, 100),
(AT_HWCAP, 0),
(AT_SECURE, 0),
(AT_PLATFORM, at_platform),
(AT_RANDOM, at_random),
];
let n = 1 + (argv_p.len() + 1) + (envp_p.len() + 1) + (aux.len() + 1 + 1) * 2;
let mut cur = (sp - (n as u64) * 8) & !0xF;
let argc_addr = cur;
let mut put = |v: u64| {
unsafe { *(cur as *mut u64) = v };
cur += 8;
};
put(argv_p.len() as u64);
for p in &argv_p {
put(*p);
}
put(0);
for p in &envp_p {
put(*p);
}
put(0);
for (t, v) in &aux {
put(*t);
put(*v);
}
put(AT_EXECFN);
put(at_execfn);
put(AT_NULL);
put(0);
argc_addr
}
const AUDIT_ARCH_X86_64: u32 = 0xC000_003E;
const SECCOMP_RET_KILL_PROCESS: u32 = 0x8000_0000;
const SECCOMP_RET_ALLOW: u32 = 0x7fff_0000;
const BPF_LD_W_ABS: u16 = 0x20;
const BPF_JMP_JEQ_K: u16 = 0x15;
const BPF_RET_K: u16 = 0x06;
fn sf(code: u16, jt: u8, jf: u8, k: u32) -> libc::sock_filter {
libc::sock_filter { code, jt, jf, k }
}
fn install_wall() {
let allow: &[u32] = &[
SYS_FUTEX as u32,
SYS_MMAP as u32,
SYS_MPROTECT as u32,
SYS_MUNMAP as u32,
SYS_MREMAP as u32,
SYS_MADVISE as u32,
SYS_MINCORE as u32, SYS_PKEY_ALLOC as u32, SYS_PKEY_FREE as u32, SYS_PKEY_MPROTECT as u32, SYS_BRK as u32,
SYS_ARCH_PRCTL as u32,
SYS_EXIT as u32,
SYS_EXIT_GROUP as u32,
15, SYS_RT_SIGACTION as u32, SYS_RT_SIGPROCMASK as u32, SYS_RT_SIGPENDING as u32, SYS_SETITIMER as u32,
SYS_GETITIMER as u32,
SYS_ALARM as u32, SYS_CLONE as u32, SYS_FORK as u32, SYS_VFORK as u32,
SYS_WAIT4 as u32, SYS_WAITID as u32, SYS_SET_TID_ADDRESS as u32, SYS_SET_ROBUST_LIST as u32, SYS_RSEQ as u32, SYS_PAUSE as u32, SYS_RT_SIGTIMEDWAIT as u32, SYS_RT_SIGSUSPEND as u32, 219, SYS_MEMBARRIER as u32, SYS_EVENTFD2 as u32, SYS_READ as u32, SYS_WRITE as u32, SYS_CLOSE as u32, 131, 157, SYS_GETPID as u32,
SYS_GETTID as u32,
SYS_GETPPID as u32,
SYS_GETUID as u32,
SYS_GETEUID as u32,
SYS_GETGID as u32,
SYS_GETEGID as u32, SYS_SETSID as u32,
SYS_SETPGID as u32,
SYS_GETPGID as u32,
SYS_GETPGRP as u32,
SYS_GETSID as u32,
SYS_CLOCK_GETTIME as u32, SYS_GETRANDOM as u32, ];
let n = allow.len();
let mut prog: Vec<libc::sock_filter> = Vec::new();
prog.push(sf(BPF_LD_W_ABS, 0, 0, 4)); prog.push(sf(BPF_JMP_JEQ_K, 1, 0, AUDIT_ARCH_X86_64));
prog.push(sf(BPF_RET_K, 0, 0, SECCOMP_RET_KILL_PROCESS));
prog.push(sf(BPF_LD_W_ABS, 0, 0, 0)); for (p, &nr) in allow.iter().enumerate() {
prog.push(sf(BPF_JMP_JEQ_K, (n - p) as u8, 0, nr)); }
prog.push(sf(BPF_RET_K, 0, 0, SECCOMP_RET_KILL_PROCESS)); prog.push(sf(BPF_RET_K, 0, 0, SECCOMP_RET_ALLOW)); let mut fprog = libc::sock_fprog {
len: prog.len() as u16,
filter: prog.as_mut_ptr(),
};
unsafe {
if libc::prctl(libc::PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0) != 0 {
die(b"PR_SET_NO_NEW_PRIVS failed\n");
}
if libc::prctl(
libc::PR_SET_SECCOMP,
libc::SECCOMP_MODE_FILTER as libc::c_ulong,
&mut fprog as *mut _ as libc::c_ulong,
0,
0,
) != 0
{
die(b"PR_SET_SECCOMP failed\n");
}
}
}
#[doc(hidden)]
pub fn harden_self_test_signal() -> i32 {
let pid = unsafe { libc::fork() };
if pid == 0 {
harden_supervisor();
let p = b"/bin/true\0";
let argv = [p.as_ptr() as *const libc::c_char, std::ptr::null()];
unsafe {
libc::execve(
p.as_ptr() as *const libc::c_char,
argv.as_ptr(),
std::ptr::null(),
);
libc::_exit(0); }
}
let mut st = 0i32;
unsafe { libc::waitpid(pid, &mut st, 0) };
if libc::WIFSIGNALED(st) {
libc::WTERMSIG(st)
} else {
-1
}
}
#[doc(hidden)]
pub fn seccomp_wall_self_test() -> Option<SentryError> {
let pid = unsafe { libc::fork() };
if pid == 0 {
install_wall();
unsafe {
libc::syscall(libc::SYS_socket, libc::AF_INET, libc::SOCK_STREAM, 0);
libc::_exit(0); }
}
let mut st = 0i32;
unsafe { libc::waitpid(pid, &mut st, 0) };
classify_status(st).1
}
#[doc(hidden)]
pub fn caps_dropped_self_test() -> bool {
let pid = unsafe { libc::fork() };
if pid == 0 {
drop_supervisor_caps();
#[repr(C)]
struct Hdr {
version: u32,
pid: i32,
}
#[repr(C)]
struct Data {
effective: u32,
permitted: u32,
inheritable: u32,
}
let hdr = Hdr {
version: 0x2008_0522,
pid: 0,
};
let mut data = [
Data {
effective: 0,
permitted: 0,
inheritable: 0,
},
Data {
effective: 0,
permitted: 0,
inheritable: 0,
},
];
let r = unsafe { libc::syscall(libc::SYS_capget, &hdr as *const _, data.as_mut_ptr()) };
let eff = data[0].effective;
let ok = r == 0 && (eff & (1 << 22)) == 0 && (eff & (1 << 21)) != 0; unsafe { libc::_exit(if ok { 0 } else { 1 }) };
}
let mut st = 0i32;
unsafe { libc::waitpid(pid, &mut st, 0) };
libc::WIFEXITED(st) && libc::WEXITSTATUS(st) == 0
}
pub(crate) fn drop_supervisor_caps() {
const KEEP: u32 = (1 << 0) | (1 << 1) | (1 << 2) | (1 << 3) | (1 << 5) | (1 << 6) | (1 << 7) | (1 << 10) | (1 << 12) | (1 << 19) | (1 << 21); #[repr(C)]
struct Hdr {
version: u32,
pid: i32,
}
#[repr(C)]
struct Data {
effective: u32,
permitted: u32,
inheritable: u32,
}
let hdr = Hdr {
version: 0x2008_0522,
pid: 0,
}; let data = [
Data {
effective: KEEP,
permitted: KEEP,
inheritable: 0,
},
Data {
effective: 0,
permitted: 0,
inheritable: 0,
},
];
unsafe { libc::syscall(libc::SYS_capset, &hdr as *const _, data.as_ptr()) };
}
pub(crate) fn harden_supervisor() {
let deny: &[u32] = &[
59, 322, 101, 165, 166, 155, 161, 175, 313, 176, 246, 320, 169, 321, 167, 168, 250, 248, 249, 428, 429, 430, 432, 442, ];
let d = deny.len();
let mut prog: Vec<libc::sock_filter> = Vec::new();
prog.push(sf(BPF_LD_W_ABS, 0, 0, 4)); prog.push(sf(BPF_JMP_JEQ_K, 1, 0, AUDIT_ARCH_X86_64)); prog.push(sf(BPF_RET_K, 0, 0, SECCOMP_RET_KILL_PROCESS)); prog.push(sf(BPF_LD_W_ABS, 0, 0, 0)); for (i, &nr) in deny.iter().enumerate() {
prog.push(sf(BPF_JMP_JEQ_K, (d - i) as u8, 0, nr));
}
prog.push(sf(BPF_RET_K, 0, 0, SECCOMP_RET_ALLOW)); prog.push(sf(BPF_RET_K, 0, 0, SECCOMP_RET_KILL_PROCESS)); let mut fprog = libc::sock_fprog {
len: prog.len() as u16,
filter: prog.as_mut_ptr(),
};
unsafe {
libc::prctl(libc::PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0);
libc::prctl(
libc::PR_SET_SECCOMP,
libc::SECCOMP_MODE_FILTER as libc::c_ulong,
&mut fprog as *mut _ as libc::c_ulong,
0,
0,
);
}
}
const SIG_DETHREAD: i32 = 64;
extern "C" fn dethread_handler(_sig: c_int, _info: *mut libc::siginfo_t, _ctx: *mut c_void) {
unsafe {
host(SYS_EXIT, 0, 0, 0, 0, 0, 0);
}
}
fn install_handler() {
unsafe {
register_altstack();
let mut sa: libc::sigaction = std::mem::zeroed();
sa.sa_sigaction = cell_layer1 as *const () as usize;
sa.sa_flags = libc::SA_SIGINFO | libc::SA_NODEFER | libc::SA_ONSTACK;
libc::sigemptyset(&mut sa.sa_mask);
if libc::sigaction(libc::SIGSYS, &sa, std::ptr::null_mut()) != 0 {
die(b"sigaction(SIGSYS) failed\n");
}
let mut sd: libc::sigaction = std::mem::zeroed();
sd.sa_sigaction = dethread_handler as *const () as usize;
sd.sa_flags = libc::SA_SIGINFO | libc::SA_ONSTACK;
libc::sigemptyset(&mut sd.sa_mask);
libc::sigaction(SIG_DETHREAD, &sd, std::ptr::null_mut());
let mut reserved = sentry_reserved_signal_mask();
host(
SYS_RT_SIGPROCMASK,
1,
std::ptr::addr_of_mut!(reserved) as u64,
0,
8,
0,
0,
);
}
}
fn arm_sud() {
let rc = unsafe {
libc::prctl(
PR_SET_SYSCALL_USER_DISPATCH,
PR_SYS_DISPATCH_ON,
WINDOW_FLOOR as libc::c_ulong,
(USER_TOP - WINDOW_FLOOR) as libc::c_ulong,
std::ptr::addr_of!(SELECTOR) as libc::c_ulong,
)
};
if rc != 0 {
die(b"PR_SET_SYSCALL_USER_DISPATCH failed\n");
}
}
#[inline(never)]
unsafe fn enter_guest(entry: u64, sp: u64) -> ! {
unsafe {
std::arch::asm!(
"mov rsp, {sp}",
"xor rdx, rdx",
"xor rbp, rbp",
"jmp {entry}",
sp = in(reg) sp, entry = in(reg) entry,
options(noreturn, nostack),
);
}
}
#[inline(never)]
unsafe fn resume(g: *const i64) -> ! {
unsafe {
std::arch::asm!(
"mov rbx, [rax + 16*8]", "mov rsp, [rax + 15*8]", "push rbx", "push qword ptr [rax + 13*8]", "mov r8, [rax + 0*8]",
"mov r9, [rax + 1*8]",
"mov r10, [rax + 2*8]",
"mov r11, [rax + 3*8]",
"mov r12, [rax + 4*8]",
"mov r13, [rax + 5*8]",
"mov r14, [rax + 6*8]",
"mov r15, [rax + 7*8]",
"mov rdi, [rax + 8*8]",
"mov rsi, [rax + 9*8]",
"mov rbp, [rax + 10*8]",
"mov rdx, [rax + 12*8]",
"mov rcx, [rax + 14*8]",
"mov rbx, [rax + 11*8]", "pop rax",
"ret", in("rax") g,
options(noreturn),
);
}
}
unsafe fn resume_pending_syscall() -> ! {
unsafe {
RESTORE_PENDING_SYSCALL.store(false, Ordering::Release);
host(SYS_ARCH_PRCTL, ARCH_SET_FS, CELL_FS, 0, 0, 0, 0);
let nr = RESTORE_PENDING_NR;
let args = RESTORE_PENDING_ARGS;
let ret = dispatch_simple(
nr, args[0], args[1], args[2], args[3], args[4], args[5], None,
);
CAPTURED[REG_RAX] = ret;
host(SYS_ARCH_PRCTL, ARCH_SET_FS, GUEST_FS, 0, 0, 0, 0);
resume(std::ptr::addr_of!(CAPTURED) as *const i64);
}
}
unsafe fn resume_instance() -> ! {
unsafe {
host(
157,
PR_SET_SYSCALL_USER_DISPATCH as u64,
PR_SYS_DISPATCH_ON,
WINDOW_FLOOR,
USER_TOP - WINDOW_FLOOR,
std::ptr::addr_of!(SELECTOR) as u64,
0,
);
if RESTORE_PENDING_SYSCALL.load(Ordering::Acquire) {
resume_pending_syscall();
}
host(SYS_ARCH_PRCTL, ARCH_SET_FS, GUEST_FS, 0, 0, 0, 0);
resume(std::ptr::addr_of!(CAPTURED) as *const i64);
}
}
fn decode_exit(st: c_int) -> i32 {
if st & 0x7f == 0 {
(st >> 8) & 0xff
} else {
128 + (st & 0x7f)
}
}
fn now_ns() -> u64 {
let mut ts = [0i64; 2];
unsafe {
host(
SYS_CLOCK_GETTIME,
1,
ts.as_mut_ptr() as u64,
0,
0,
0,
0,
)
};
ts[0] as u64 * 1_000_000_000 + ts[1] as u64
}
fn zygote_loop() -> ! {
if ZYG_DRIVEN.load(Ordering::Relaxed) {
driven_zygote_loop()
} else {
bench_zygote_loop()
}
}
fn allow_ptrace_from(pid: i32) {
if pid > 1 {
unsafe {
let _ = host(SYS_PRCTL, PR_SET_PTRACER, pid as u64, 0, 0, 0, 0);
}
}
}
const ZYG_RESTORES: u32 = 2000;
fn bench_zygote_loop() -> ! {
let mut total = 0u64;
let mut min = u64::MAX;
let mut first_code = -1i32;
for i in 0..ZYG_RESTORES {
let t0 = now_ns();
let pid = unsafe { host(SYS_FORK, 0, 0, 0, 0, 0, 0) };
if pid == 0 {
unsafe { resume_instance() };
}
let mut st: c_int = 0;
unsafe {
host(
SYS_WAIT4,
pid as u64,
&mut st as *mut c_int as u64,
0,
0,
0,
0,
)
};
if i == 0 {
first_code = decode_exit(st);
}
let dt = now_ns() - t0;
total += dt;
min = min.min(dt);
}
unsafe {
if !ZYG.is_null() {
(*ZYG).exit = first_code;
(*ZYG).mean_ns = total / ZYG_RESTORES as u64;
(*ZYG).done = min as u32; (*ZYG).ready = 1;
}
host(SYS_EXIT_GROUP, 0, 0, 0, 0, 0, 0);
}
unreachable!()
}
fn driven_zygote_loop() -> ! {
let warm = unsafe { host(SYS_GETPID, 0, 0, 0, 0, 0, 0) as i32 };
let launcher = unsafe { host(SYS_GETPPID, 0, 0, 0, 0, 0, 0) as i32 };
let req = ring_word(unsafe { std::ptr::addr_of_mut!((*ZYG).req) });
let done = ring_word(unsafe { std::ptr::addr_of_mut!((*ZYG).done) });
unsafe {
let r = ring_word(std::ptr::addr_of_mut!((*ZYG).ready));
r.store(1, Ordering::Release);
host(SYS_FUTEX, r.as_ptr() as u64, FUTEX_WAKE, 1, 0, 0, 0);
}
let mut last = 0u32;
loop {
loop {
let cur = req.load(Ordering::Acquire);
if cur != last {
break;
}
unsafe {
host(
SYS_FUTEX,
req.as_ptr() as u64,
FUTEX_WAIT,
last as u64,
0,
0,
0,
)
};
}
let r = req.load(Ordering::Acquire);
last = r;
if r == u32::MAX {
unsafe { host(SYS_EXIT_GROUP, 0, 0, 0, 0, 0, 0) };
}
let mode = unsafe { std::ptr::read_volatile(std::ptr::addr_of!((*ZYG).run_mode)) };
if mode == 2 {
let victim = unsafe { std::ptr::read_volatile(std::ptr::addr_of!((*ZYG).inst_pid)) };
if victim > 1 {
let mut st: c_int = 0;
unsafe {
host(
SYS_WAIT4,
victim as u64,
&mut st as *mut c_int as u64,
0,
0,
0,
0,
)
};
}
unsafe {
(*ZYG).exit = 0;
done.store(r, Ordering::Release);
host(SYS_FUTEX, done.as_ptr() as u64, FUTEX_WAKE, 1, 0, 0, 0);
}
continue;
}
let slot = alloc_slot();
let code: i32 = if slot as usize >= MAX_SLOTS {
-11
} else {
delegate(CTL_ENSURE_SERVICER, slot as u64, 0, 0, 0, 0, 0);
delegate(CTL_FORK_TABLE, slot as u64, SYS_FORK as u64, 0, 0, 0, 0);
unsafe { (*ring_at(slot as u64)).fork_parent = warm };
let pid = unsafe { host(SYS_FORK, 0, 0, 0, 0, 0, 0) };
if pid == 0 {
set_slot(slot as u64);
unsafe {
allow_ptrace_from(launcher);
set_ring_owner(
slot as u64,
host(SYS_GETPID, 0, 0, 0, 0, 0, 0) as i32,
host(SYS_GETTID, 0, 0, 0, 0, 0, 0) as i32,
);
resume_instance();
}
} else if pid < 0 {
unsafe { (*ring_at(slot as u64)).fork_parent = 0 };
delegate(CTL_FORK_CANCEL, slot as u64, 0, 0, 0, 0, 0);
-11
} else if mode == 1 {
delegate(CTL_BIND_SLOT, slot as u64, pid as u64, 0, 0, 0, 0);
unsafe {
(*ZYG).inst_pid = pid as i32;
(*ZYG).exit = 0;
done.store(r, Ordering::Release);
host(SYS_FUTEX, done.as_ptr() as u64, FUTEX_WAKE, 1, 0, 0, 0);
}
continue; } else {
delegate(CTL_BIND_SLOT, slot as u64, pid as u64, 0, 0, 0, 0);
let mut st: c_int = 0;
unsafe {
host(
SYS_WAIT4,
pid as u64,
&mut st as *mut c_int as u64,
0,
0,
0,
0,
)
};
delegate(CTL_REAP, pid as u64, 0, 0, 0, 0, 0);
decode_exit(st)
}
};
unsafe {
(*ZYG).exit = code;
(*ZYG).inst_pid = 0; done.store(r, Ordering::Release);
host(SYS_FUTEX, done.as_ptr() as u64, FUTEX_WAKE, 1, 0, 0, 0);
}
}
}
fn resolve_shebang(
path: &str,
args: &[String],
read: impl Fn(&str) -> Option<Vec<u8>>,
) -> Option<(String, Vec<u8>, Vec<String>)> {
let mut path = path.to_string();
let mut argv: Vec<String> = args.to_vec();
let mut bytes = read(&path)?;
for _ in 0..4 {
if bytes.len() < 2 || &bytes[0..2] != b"#!" {
break;
}
let end = bytes
.iter()
.take(256)
.position(|&b| b == b'\n')
.unwrap_or_else(|| bytes.len().min(255));
let line = String::from_utf8_lossy(&bytes[2..end]);
let line = line.trim();
let (interp, optarg) = match line.find([' ', '\t']) {
Some(i) => {
let a = line[i..].trim();
(
line[..i].to_string(),
(!a.is_empty()).then(|| a.to_string()),
)
}
None => (line.to_string(), None),
};
if interp.is_empty() {
die_code(b"bad shebang interpreter\n", 127);
}
let mut next = Vec::with_capacity(argv.len() + 2);
next.push(interp.clone());
if let Some(a) = optarg {
next.push(a);
}
next.push(path.clone()); next.extend(argv.iter().skip(1).cloned());
argv = next;
path = interp;
bytes = read(&path)?;
}
Some((path, bytes, argv))
}
fn cell_main(path: &str, args: &[String], root: Option<std::path::PathBuf>) -> ! {
unsafe {
CELL_ROOT = root.clone();
host(
SYS_ARCH_PRCTL,
ARCH_GET_FS,
std::ptr::addr_of_mut!(CELL_FS) as u64,
0,
0,
0,
0,
);
}
let (path, bytes, args) = resolve_shebang(path, args, |p| read_in_root(&root, p).ok())
.unwrap_or_else(|| die_code(b"cannot read guest exe\n", 127));
let (_path, args) = (path.as_str(), args.as_slice());
let exe_base = if rd_u16(&bytes, 16) == 3 {
EXE_DYN_BASE
} else {
0
};
let exe = load_elf(&bytes, exe_base);
unsafe {
let h = libc::mmap(
std::ptr::null_mut(),
HEAP_SIZE as usize,
libc::PROT_READ | libc::PROT_WRITE,
libc::MAP_PRIVATE | libc::MAP_ANONYMOUS,
-1,
0,
);
HEAP_BASE = h as u64;
HEAP_CUR.store(h as u64, std::sync::atomic::Ordering::Release);
HEAP_END = h as u64 + HEAP_SIZE;
}
let (jump_to, interp_base) = if let Some(ref ip) = exe.interp {
let ipath = String::from_utf8_lossy(ip).to_string();
let ibytes =
read_in_root(&root, &ipath).unwrap_or_else(|_| die_code(b"cannot read ld.so\n", 127));
let interp = load_elf(&ibytes, INTERP_BASE);
(interp.entry, Some(INTERP_BASE))
} else {
(exe.entry, None)
};
let argv: Vec<Vec<u8>> = args.iter().map(|s| s.as_bytes().to_vec()).collect();
let envp: Vec<Vec<u8>> = {
let e = guest_env().lock().unwrap();
if e.is_empty() {
vec![
{
let mut v = b"PATH=".to_vec();
v.extend_from_slice(DEFAULT_PATH);
v
},
b"PWD=/".to_vec(),
b"LANG=C".to_vec(),
]
} else {
let mut out = e.clone();
fn has(out: &[Vec<u8>], k: &[u8]) -> bool {
out.iter()
.any(|kv| kv.starts_with(k) && kv.get(k.len()) == Some(&b'='))
}
if !has(&out, b"PATH") {
let mut v = b"PATH=".to_vec();
v.extend_from_slice(DEFAULT_PATH);
out.push(v);
}
if !has(&out, b"HOME") {
out.push(b"HOME=/".to_vec());
}
out
}
};
if let Some(c) = guest_cwd_init().lock().unwrap().clone() {
let cwd_s = String::from_utf8_lossy(&c).into_owned();
if !dir_exists_in_root(&root, &cwd_s) {
die_code(b"sentry: requested working directory does not exist\n", 127);
}
let me = unsafe { host(SYS_GETPID, 0, 0, 0, 0, 0, 0) } as i32;
cwds().lock().unwrap().insert(me, c);
}
let initial_creds = run_uid().lock().unwrap().unwrap_or((0, 0));
cred_reset(initial_creds.0, initial_creds.1);
cell_umask_reset();
let sp = build_stack(&exe, interp_base, &argv, &envp);
if let Some(dir) = cgroup_path().lock().unwrap().clone() {
let me = unsafe { host(SYS_GETPID, 0, 0, 0, 0, 0, 0) };
let _ = std::fs::write(dir.join("cgroup.procs"), me.to_string());
}
if let Some((uid, gid)) = *run_uid().lock().unwrap() {
unsafe {
host(SYS_SETGROUPS, 0, 0, 0, 0, 0, 0);
host(SYS_SETGID, gid as u64, 0, 0, 0, 0, 0);
host(SYS_SETUID, uid as u64, 0, 0, 0, 0, 0);
}
}
unregister_host_libc_rseq();
unsafe { reset_exec_signal_dispositions() };
install_handler();
arm_sud();
install_wall();
if WARM_MODE.load(Ordering::Relaxed) {
let from_checkpoint =
unsafe { __sigsetjmp(std::ptr::addr_of_mut!(JMPBUF) as *mut c_void, 0) };
if from_checkpoint == 0 {
unsafe { enter_guest(jump_to, sp) };
} else {
zygote_loop();
}
}
unsafe { enter_guest(jump_to, sp) };
}
#[cfg(all(target_os = "linux", target_arch = "x86_64"))]
fn cell_restore_main(
snapshot_dir: &str,
restore_vpid: state_snap::Vpid,
restore_warm: bool,
stop_before_resume: bool,
root: Option<std::path::PathBuf>,
) -> ! {
unsafe {
CELL_ROOT = root;
host(
SYS_ARCH_PRCTL,
ARCH_GET_FS,
std::ptr::addr_of_mut!(CELL_FS) as u64,
0,
0,
0,
0,
);
}
let dir = std::path::Path::new(snapshot_dir);
let state = read_state_snapshot_dir(dir).unwrap_or_else(|e| {
let msg = format!("cannot read sentry state snapshot: {e}\n");
die_code(msg.as_bytes(), 127)
});
if !state
.cpu_feature_baseline
.satisfied_by(&state_snap::CpuFeatures::current())
{
die_code(b"sentry restore: CPU feature baseline not satisfied\n", 127);
}
let ram = std::fs::File::open(dir.join("mem.blob"))
.unwrap_or_else(|_| die_code(b"cannot read sentry mem.blob\n", 127));
let image = state
.mem
.proc_image(restore_vpid)
.unwrap_or_else(|| die_code(b"sentry restore: missing requested memory image\n", 127));
for rec in image.regions {
unsafe {
memimage::restore_region(rec, &ram, image.blob_off)
.unwrap_or_else(|_| die_code(b"sentry restore: map memory failed\n", 127));
}
}
let thread = image
.threads
.first()
.unwrap_or_else(|| die_code(b"sentry restore: missing thread regs\n", 127));
unsafe {
CAPTURED = thread.gregs;
GUEST_FS = thread.fs;
if let Some((nr, args)) = decode_pending_syscall(&thread.xsave) {
RESTORE_PENDING_NR = nr;
RESTORE_PENDING_ARGS = args;
RESTORE_PENDING_SYSCALL.store(true, Ordering::Release);
} else {
RESTORE_PENDING_SYSCALL.store(false, Ordering::Release);
}
}
if let Some(dir) = cgroup_path().lock().unwrap().clone() {
let me = unsafe { host(SYS_GETPID, 0, 0, 0, 0, 0, 0) };
let _ = std::fs::write(dir.join("cgroup.procs"), me.to_string());
}
let initial_creds = run_uid().lock().unwrap().unwrap_or((0, 0));
cred_reset(initial_creds.0, initial_creds.1);
cell_umask_reset();
if let Some((uid, gid)) = *run_uid().lock().unwrap() {
unsafe {
host(SYS_SETGROUPS, 0, 0, 0, 0, 0, 0);
host(SYS_SETGID, gid as u64, 0, 0, 0, 0, 0);
host(SYS_SETUID, uid as u64, 0, 0, 0, 0, 0);
}
}
unregister_host_libc_rseq();
unsafe { reset_exec_signal_dispositions() };
install_handler();
arm_sud();
if !restore_warm && stop_before_resume {
let me = unsafe { host(SYS_GETPID, 0, 0, 0, 0, 0, 0) };
unsafe { host(SYS_KILL, me as u64, libc::SIGSTOP as u64, 0, 0, 0, 0) };
}
install_wall();
if restore_warm {
zygote_loop();
}
unsafe { resume_instance() };
}
pub fn run(
elf: &std::path::Path,
args: &[String],
root: Option<&std::path::Path>,
) -> std::io::Result<i32> {
run_in(elf, args, root, &SandboxCfg::default())
}
pub fn run_classified(
elf: &std::path::Path,
args: &[String],
root: Option<&std::path::Path>,
) -> std::io::Result<(i32, Option<SentryError>)> {
run_in_classified(elf, args, root, &SandboxCfg::default())
}
#[derive(Clone, serde::Serialize, serde::Deserialize)]
struct SandboxCfg {
egress: Option<String>,
mounts: Vec<(std::path::PathBuf, std::path::PathBuf, bool)>,
limits: Limits,
run_uid: Option<(u32, u32)>,
cgroup_name: Option<String>,
warm: bool,
netns: bool,
published_ports: Vec<u16>,
env: Vec<(String, String)>,
cwd: Option<String>,
exec_timeout_ms: u64,
preserve_stragglers: bool,
}
impl Default for SandboxCfg {
fn default() -> Self {
SandboxCfg {
egress: None,
mounts: Vec::new(),
limits: Limits::default(),
run_uid: None,
cgroup_name: None,
warm: false,
netns: true,
published_ports: Vec::new(),
env: Vec::new(),
cwd: None,
exec_timeout_ms: 0,
preserve_stragglers: false,
}
}
}
#[derive(Default, Clone, Copy)]
struct Stdio {
stdin: Option<c_int>,
stdout: Option<c_int>,
stderr: Option<c_int>,
ctty: bool,
status_wr: Option<c_int>,
}
#[derive(Default, Clone, serde::Serialize, serde::Deserialize)]
struct Limits {
memory_max: Option<u64>, pids_max: Option<u64>, cpu_max: Option<String>, }
impl Limits {
fn any(&self) -> bool {
self.memory_max.is_some() || self.pids_max.is_some() || self.cpu_max.is_some()
}
}
#[non_exhaustive]
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum SentryError {
SeccompViolation { signum: i32 },
Signaled { signum: i32 },
Timeout,
}
impl SentryError {
pub fn exit_code(self) -> i32 {
match self {
SentryError::SeccompViolation { signum } | SentryError::Signaled { signum } => {
128 + signum
}
SentryError::Timeout => 128 + libc::SIGKILL,
}
}
}
impl std::fmt::Display for SentryError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
SentryError::SeccompViolation { signum } => {
write!(
f,
"cell killed by the seccomp wall on a forbidden syscall (SIG{signum})"
)
}
SentryError::Signaled { signum } => write!(f, "cell terminated by signal {signum}"),
SentryError::Timeout => write!(f, "cell SIGKILL'd after exceeding the exec timeout"),
}
}
}
impl std::error::Error for SentryError {}
const ERR_NONE: u32 = 0;
const ERR_SECCOMP: u32 = 1;
const ERR_SIGNALED: u32 = 2;
const ERR_TIMEOUT: u32 = 3;
fn err_to_tag(e: Option<SentryError>) -> u32 {
match e {
None => ERR_NONE,
Some(SentryError::SeccompViolation { .. }) => ERR_SECCOMP,
Some(SentryError::Signaled { .. }) => ERR_SIGNALED,
Some(SentryError::Timeout) => ERR_TIMEOUT,
}
}
fn tag_to_err(code: i32, tag: u32) -> Option<SentryError> {
let signum = (code - 128).max(0);
match tag {
ERR_SECCOMP => Some(SentryError::SeccompViolation { signum }),
ERR_SIGNALED => Some(SentryError::Signaled { signum }),
ERR_TIMEOUT => Some(SentryError::Timeout),
_ => None,
}
}
fn classify_status(st: c_int) -> (i32, Option<SentryError>) {
if libc::WIFSIGNALED(st) {
let signum = libc::WTERMSIG(st);
let err = if signum == libc::SIGSYS {
SentryError::SeccompViolation { signum }
} else {
SentryError::Signaled { signum }
};
(128 + signum, Some(err))
} else if libc::WIFEXITED(st) {
(libc::WEXITSTATUS(st), None)
} else {
(-1, None)
}
}
fn run_in(
elf: &std::path::Path,
args: &[String],
root: Option<&std::path::Path>,
cfg: &SandboxCfg,
) -> std::io::Result<i32> {
run_in_classified(elf, args, root, cfg).map(|(c, _)| c)
}
fn run_in_classified(
elf: &std::path::Path,
args: &[String],
root: Option<&std::path::Path>,
cfg: &SandboxCfg,
) -> std::io::Result<(i32, Option<SentryError>)> {
let mut stp = [0 as c_int; 2];
if unsafe { libc::pipe2(stp.as_mut_ptr(), libc::O_CLOEXEC) } < 0 {
return Err(std::io::Error::last_os_error());
}
let (st_r, st_w) = (stp[0], stp[1]);
let stdio = Stdio {
status_wr: Some(st_w),
..Stdio::default()
};
let pid = match spawn_sandbox(elf, args, root, stdio, false, cfg) {
Ok(p) => p,
Err(e) => {
unsafe {
libc::close(st_r);
libc::close(st_w);
}
return Err(e);
}
};
unsafe { libc::close(st_w) };
let code = wait_exit(pid)?;
let err = read_status_reason(st_r, code);
Ok((code, err))
}
fn read_status_reason(st_r: c_int, code: i32) -> Option<SentryError> {
unsafe {
let fl = libc::fcntl(st_r, libc::F_GETFL);
if fl >= 0 {
libc::fcntl(st_r, libc::F_SETFL, fl | libc::O_NONBLOCK);
}
}
let mut tag = [0u8; 1];
let n = unsafe { libc::read(st_r, tag.as_mut_ptr() as *mut c_void, 1) };
unsafe { libc::close(st_r) };
if n == 1 {
tag_to_err(code, tag[0] as u32)
} else {
None
}
}
pub fn zygote_bench(
elf: &std::path::Path,
args: &[String],
root: Option<&std::path::Path>,
) -> std::io::Result<i32> {
let cfg = SandboxCfg {
warm: true,
..SandboxCfg::default()
};
let pid = spawn_sandbox(elf, args, root, Stdio::default(), false, &cfg)?;
wait_exit(pid)
}
fn wait_exit(pid: c_int) -> std::io::Result<i32> {
wait_exit_classified(pid).map(|(c, _)| c)
}
fn wait_exit_classified(pid: c_int) -> std::io::Result<(i32, Option<SentryError>)> {
let mut st: c_int = 0;
if unsafe { libc::waitpid(pid, &mut st, 0) } < 0 {
return Err(std::io::Error::last_os_error());
}
remove_cgroup(pid);
Ok(classify_status(st))
}
const SYS_PIDFD_OPEN: i64 = 434;
fn now_ms() -> i64 {
let mut ts = [0i64; 2];
unsafe {
host(
SYS_CLOCK_GETTIME,
libc::CLOCK_MONOTONIC as u64,
ts.as_mut_ptr() as u64,
0,
0,
0,
0,
)
};
ts[0].saturating_mul(1000) + ts[1] / 1_000_000
}
fn wait_exit_deadline(pid: c_int, timeout_ms: u64) -> std::io::Result<(i32, Option<SentryError>)> {
if timeout_ms == 0 {
return wait_exit_classified(pid);
}
let pidfd = unsafe { host(SYS_PIDFD_OPEN, pid as u64, 0, 0, 0, 0, 0) };
if pidfd < 0 {
return wait_exit_classified(pid);
}
let deadline = now_ms().saturating_add(timeout_ms as i64);
let timed_out = loop {
let remaining = deadline - now_ms();
if remaining <= 0 {
break true;
}
let mut pf = pollfd_bytes(pidfd as i32, POLLIN_BIT);
let r = unsafe {
host(
SYS_POLL,
pf.as_mut_ptr() as u64,
1,
remaining as u64,
0,
0,
0,
)
};
if r > 0 {
break false; }
if r == 0 {
break true; }
};
unsafe { host(SYS_CLOSE, pidfd as u64, 0, 0, 0, 0, 0) };
if timed_out {
unsafe {
if libc::getpgid(pid) == pid {
libc::kill(-pid, libc::SIGKILL);
} else {
libc::kill(pid, libc::SIGKILL);
}
let mut st: c_int = 0;
libc::waitpid(pid, &mut st, 0); }
remove_cgroup(pid);
return Ok((128 + libc::SIGKILL, Some(SentryError::Timeout)));
}
wait_exit_classified(pid)
}
fn wait_ready_deadline(ready: &AtomicU32, pid: c_int, timeout_ms: u64) -> bool {
let deadline = if timeout_ms == 0 {
None
} else {
Some(now_ms().saturating_add(timeout_ms as i64))
};
loop {
if ready.load(Ordering::Acquire) != 0 {
return true;
}
let mut st: c_int = 0;
if unsafe { libc::waitpid(pid, &mut st, libc::WNOHANG) } == pid {
return ready.load(Ordering::Acquire) != 0;
}
let mut slice_ms: i64 = 50;
if let Some(dl) = deadline {
let rem = dl - now_ms();
if rem <= 0 {
unsafe {
if libc::getpgid(pid) == pid {
libc::kill(-pid, libc::SIGKILL);
} else {
libc::kill(pid, libc::SIGKILL);
}
let mut s: c_int = 0;
libc::waitpid(pid, &mut s, 0);
}
remove_cgroup(pid);
return false;
}
slice_ms = slice_ms.min(rem);
}
let ts: [i64; 2] = [slice_ms / 1000, (slice_ms % 1000) * 1_000_000];
unsafe {
host(
SYS_FUTEX,
ready.as_ptr() as u64,
FUTEX_WAIT,
0,
ts.as_ptr() as u64,
0,
0,
)
};
}
}
fn spawn_sandbox(
elf: &std::path::Path,
args: &[String],
root: Option<&std::path::Path>,
stdio: Stdio,
new_group: bool,
cfg: &SandboxCfg,
) -> std::io::Result<c_int> {
let rc = ReexecConfig::Sandbox {
path: elf.to_path_buf(),
args: args.to_vec(),
root: root.map(|p| p.to_path_buf()),
cfg: cfg.clone(),
has_status: stdio.status_wr.is_some(),
};
let json = serde_json::to_vec(&rc)
.map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidInput, e))?;
let cfg_mfd =
unsafe { libc::memfd_create(b"sentry_reexec_cfg\0".as_ptr() as *const c_char, 0) };
if cfg_mfd < 0 {
return Err(std::io::Error::last_os_error());
}
{
let mut off = 0usize;
while off < json.len() {
let n = unsafe {
libc::write(
cfg_mfd,
json[off..].as_ptr() as *const c_void,
json.len() - off,
)
};
if n <= 0 {
unsafe { libc::close(cfg_mfd) };
return Err(std::io::Error::last_os_error());
}
off += n as usize;
}
}
let reexec_exe = reexec_exe_cstring()?;
let pid = unsafe { libc::fork() };
if pid < 0 {
unsafe { libc::close(cfg_mfd) };
return Err(std::io::Error::last_os_error());
}
if pid == 0 {
if new_group && !stdio.ctty {
unsafe { libc::setpgid(0, 0) };
}
unsafe {
let mut to_close = [false; 3]; let targets = [(stdio.stdin, 0), (stdio.stdout, 1), (stdio.stderr, 2)];
for (i, (src, tgt)) in targets.iter().enumerate() {
if let Some(fd) = *src {
libc::dup2(fd, *tgt);
if fd > 2 {
to_close[i] = true;
}
}
}
let mut closed: [c_int; 3] = [-1, -1, -1];
for (i, (src, _)) in targets.iter().enumerate() {
if let (true, Some(fd)) = (to_close[i], *src) {
if !closed.contains(&fd) {
libc::close(fd);
closed[i] = fd;
}
}
}
if stdio.ctty {
libc::setsid();
libc::ioctl(0, libc::TIOCSCTTY as _, 0);
}
let cfg_scratch = libc::fcntl(cfg_mfd, libc::F_DUPFD, 16);
let status_scratch = match stdio.status_wr {
Some(w) => libc::fcntl(w, libc::F_DUPFD, 16),
None => -1,
};
if cfg_scratch >= 0 {
libc::dup2(cfg_scratch, REEXEC_CONFIG_FD);
} else if cfg_mfd != REEXEC_CONFIG_FD {
libc::dup2(cfg_mfd, REEXEC_CONFIG_FD); }
let _status_fd: c_int = if status_scratch >= 0 {
libc::dup2(status_scratch, STATUS_PIPE_FD);
STATUS_PIPE_FD
} else if let Some(w) = stdio.status_wr {
if w != STATUS_PIPE_FD {
libc::dup2(w, STATUS_PIPE_FD); }
STATUS_PIPE_FD
} else {
libc::close(STATUS_PIPE_FD);
-1
};
let sweep_lo = REEXEC_CONFIG_FD as u64 + 1;
if host(SYS_CLOSE_RANGE, sweep_lo, u32::MAX as u64, 0, 0, 0, 0) < 0 {
let mut lim: libc::rlimit = std::mem::zeroed();
let max = if libc::getrlimit(libc::RLIMIT_NOFILE, &mut lim) == 0 {
(lim.rlim_cur as c_int).clamp(64, 4096)
} else {
1024
};
for fd in sweep_lo as c_int..max {
libc::close(fd);
}
}
}
unsafe {
let argv: [*const c_char; 3] = [
reexec_exe.as_ptr(),
REEXEC_MARKER.as_ptr() as *const c_char,
std::ptr::null(),
];
libc::execve(reexec_exe.as_ptr(), argv.as_ptr(), current_environ());
libc::_exit(127); }
}
unsafe { libc::close(cfg_mfd) }; Ok(pid)
}
const STATUS_PIPE_FD: c_int = 3;
static STATUS_PIPE_FD_ACTIVE: std::sync::atomic::AtomicI32 = std::sync::atomic::AtomicI32::new(-1);
const REEXEC_CONFIG_FD: c_int = 4; const REEXEC_CTRL_FD: c_int = 5; const REEXEC_MARKER: &[u8] = b"--__sentry_reexec_supervisor__\0";
#[derive(serde::Serialize, serde::Deserialize)]
enum ReexecConfig {
Sandbox {
path: std::path::PathBuf,
args: Vec<String>,
root: Option<std::path::PathBuf>,
cfg: SandboxCfg,
has_status: bool,
},
Persistent {
root: Option<std::path::PathBuf>,
cfg: SandboxCfg,
#[serde(default)]
exec_sock: Option<std::path::PathBuf>,
},
}
unsafe fn current_environ() -> *const *const c_char {
unsafe {
extern "C" {
static mut environ: *mut *mut c_char;
}
environ as *const *const c_char
}
}
extern "C" fn sentry_reexec_ctor(argc: c_int, argv: *const *const c_char) {
unsafe {
if argc < 2 || argv.is_null() {
return;
}
let a1 = *argv.add(1);
if a1.is_null() || std::ffi::CStr::from_ptr(a1).to_bytes_with_nul() != REEXEC_MARKER {
return; }
libc::lseek(REEXEC_CONFIG_FD, 0, libc::SEEK_SET);
let mut data = Vec::with_capacity(8192);
let mut tmp = [0u8; 8192];
loop {
let n = libc::read(REEXEC_CONFIG_FD, tmp.as_mut_ptr() as *mut c_void, tmp.len());
if n <= 0 {
break;
}
data.extend_from_slice(&tmp[..n as usize]);
}
libc::close(REEXEC_CONFIG_FD);
let rc: ReexecConfig = match serde_json::from_slice(&data) {
Ok(r) => r,
Err(_) => libc::_exit(127),
};
match rc {
ReexecConfig::Sandbox {
path,
args,
root,
cfg,
has_status,
} => {
if has_status {
STATUS_PIPE_FD_ACTIVE.store(STATUS_PIPE_FD, Ordering::Relaxed);
}
sandbox_main(&path, &args, root.as_deref(), &cfg);
}
ReexecConfig::Persistent {
root,
cfg,
exec_sock,
} => {
persistent_supervisor_main(REEXEC_CTRL_FD, root.as_deref(), &cfg, exec_sock);
}
}
}
}
#[used]
#[cfg_attr(
all(target_os = "linux", target_arch = "x86_64"),
link_section = ".init_array"
)]
static SENTRY_REEXEC_CTOR: extern "C" fn(c_int, *const *const c_char) = sentry_reexec_ctor;
fn write_reexec_config(rc: &ReexecConfig) -> std::io::Result<c_int> {
let json = serde_json::to_vec(rc)
.map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidInput, e))?;
let mfd = unsafe { libc::memfd_create(b"sentry_reexec_cfg\0".as_ptr() as *const c_char, 0) };
if mfd < 0 {
return Err(std::io::Error::last_os_error());
}
let mut off = 0usize;
while off < json.len() {
let n =
unsafe { libc::write(mfd, json[off..].as_ptr() as *const c_void, json.len() - off) };
if n <= 0 {
unsafe { libc::close(mfd) };
return Err(std::io::Error::last_os_error());
}
off += n as usize;
}
Ok(mfd)
}
fn reexec_exe_cstring() -> std::io::Result<std::ffi::CString> {
let exe = std::env::var("SUPERMACHINE_SENTRY_REEXEC_BIN")
.unwrap_or_else(|_| "/proc/self/exe".to_string());
std::ffi::CString::new(exe).map_err(|_| {
std::io::Error::new(std::io::ErrorKind::InvalidInput, "reexec path contains NUL")
})
}
fn report_status(err: Option<SentryError>) {
let fd = STATUS_PIPE_FD_ACTIVE.load(Ordering::Relaxed);
if fd < 0 {
return;
}
let tag = err_to_tag(err);
if tag == ERR_NONE {
return;
}
let b = [tag as u8];
unsafe { libc::write(fd, b.as_ptr() as *const c_void, 1) };
}
pub struct Child {
pid: c_int,
}
impl Child {
pub fn id(&self) -> i32 {
self.pid
}
pub fn wait(&self) -> std::io::Result<i32> {
wait_exit(self.pid)
}
pub fn try_wait(&self) -> std::io::Result<Option<i32>> {
let mut st: c_int = 0;
let r = unsafe { libc::waitpid(self.pid, &mut st, libc::WNOHANG) };
if r < 0 {
Err(std::io::Error::last_os_error())
} else if r == 0 {
Ok(None)
} else if libc::WIFEXITED(st) {
Ok(Some(libc::WEXITSTATUS(st)))
} else if libc::WIFSIGNALED(st) {
Ok(Some(128 + libc::WTERMSIG(st)))
} else {
Ok(None)
}
}
pub fn kill(&self) {
unsafe { libc::kill(-self.pid, libc::SIGKILL) };
}
}
pub fn spawn(
elf: &std::path::Path,
args: &[String],
root: Option<&std::path::Path>,
) -> std::io::Result<Child> {
Ok(Child {
pid: spawn_sandbox(
elf,
args,
root,
Stdio::default(),
true,
&SandboxCfg::default(),
)?,
})
}
pub struct Sandbox {
root: std::path::PathBuf,
egress: Option<String>,
mounts: Vec<(std::path::PathBuf, std::path::PathBuf, bool)>,
limits: Limits,
run_uid: Option<(u32, u32)>,
netns: bool,
published_ports: Vec<u16>,
env: Vec<(String, String)>,
cgroup_name: String,
exec_timeout_ms: u64,
preserve_stragglers: bool,
workload: std::sync::Mutex<Option<Child>>,
}
impl Sandbox {
pub fn new(root: impl Into<std::path::PathBuf>) -> Sandbox {
static N: AtomicU32 = AtomicU32::new(0);
let cgroup_name = format!(
"sentry-sb-{}-{}",
unsafe { libc::getpid() },
N.fetch_add(1, Ordering::Relaxed)
);
Sandbox {
root: root.into(),
egress: None,
mounts: Vec::new(),
limits: Limits::default(),
run_uid: None,
netns: true,
published_ports: Vec::new(),
env: Vec::new(),
cgroup_name,
exec_timeout_ms: 0,
preserve_stragglers: false,
workload: std::sync::Mutex::new(None),
}
}
pub fn with_root(mut self, root: impl Into<std::path::PathBuf>) -> Sandbox {
self.root = root.into();
self
}
pub fn clone_template(&self) -> Sandbox {
let mut next = Sandbox::new(self.root.clone());
next.egress = self.egress.clone();
next.mounts = self.mounts.clone();
next.limits = self.limits.clone();
next.run_uid = self.run_uid;
next.netns = self.netns;
next.published_ports = self.published_ports.clone();
next.env = self.env.clone();
next.exec_timeout_ms = self.exec_timeout_ms;
next.preserve_stragglers = self.preserve_stragglers;
next
}
pub fn with_uid(mut self, uid: u32, gid: u32) -> Sandbox {
self.run_uid = Some((uid, gid));
self
}
pub fn with_env(mut self, env: Vec<(String, String)>) -> Sandbox {
self.env = env;
self
}
pub fn with_netns(mut self) -> Sandbox {
self.netns = true;
self
}
pub fn without_netns(mut self) -> Sandbox {
self.netns = false;
self
}
pub fn with_published_port(mut self, port: u16) -> Sandbox {
self.netns = true;
self.published_ports.push(port);
self
}
pub fn with_memory_limit(mut self, bytes: u64) -> Sandbox {
self.limits.memory_max = Some(bytes);
self
}
pub fn with_exec_timeout(mut self, dur: std::time::Duration) -> Sandbox {
self.exec_timeout_ms = dur.as_millis().min(u64::MAX as u128) as u64;
self
}
pub fn with_pids_limit(mut self, n: u64) -> Sandbox {
self.limits.pids_max = Some(n);
self
}
pub fn with_cpu_limit(mut self, quota_us: u64, period_us: u64) -> Sandbox {
self.limits.cpu_max = Some(format!("{quota_us} {period_us}"));
self
}
pub fn with_egress(mut self, policy: impl Into<String>) -> Sandbox {
self.egress = Some(policy.into());
self
}
pub fn with_mount(
mut self,
guest_path: impl Into<std::path::PathBuf>,
host_path: impl Into<std::path::PathBuf>,
readonly: bool,
) -> Sandbox {
self.mounts
.push((guest_path.into(), host_path.into(), readonly));
self
}
pub(crate) fn with_preserve_stragglers(mut self, on: bool) -> Sandbox {
self.preserve_stragglers = on;
self
}
fn cfg(&self) -> SandboxCfg {
SandboxCfg {
egress: self.egress.clone(),
mounts: self.mounts.clone(),
limits: self.limits.clone(),
run_uid: self.run_uid,
netns: self.netns,
published_ports: self.published_ports.clone(),
cgroup_name: self.limits.any().then(|| self.cgroup_name.clone()),
warm: false,
env: self.env.clone(),
cwd: None,
exec_timeout_ms: self.exec_timeout_ms,
preserve_stragglers: self.preserve_stragglers,
}
}
pub fn root(&self) -> &std::path::Path {
&self.root
}
pub fn pool(&self) -> std::io::Result<Pool> {
Pool::start(Some(&self.root), &self.cfg(), None)
}
pub fn pool_with_exec(&self, exec_sock: std::path::PathBuf) -> std::io::Result<Pool> {
Pool::start(Some(&self.root), &self.cfg(), Some(exec_sock))
}
pub fn exec<I, S>(&self, argv: I) -> std::io::Result<i32>
where
I: IntoIterator<Item = S>,
S: Into<String>,
{
self.exec_classified(argv).map(|(c, _)| c)
}
pub fn exec_classified<I, S>(&self, argv: I) -> std::io::Result<(i32, Option<SentryError>)>
where
I: IntoIterator<Item = S>,
S: Into<String>,
{
let v: Vec<String> = argv.into_iter().map(Into::into).collect();
if v.is_empty() {
return Err(std::io::Error::from(std::io::ErrorKind::InvalidInput));
}
run_in_classified(
std::path::Path::new(&v[0]),
&v[1..],
Some(&self.root),
&self.cfg(),
)
}
pub fn exec_capture<I, S>(&self, argv: I) -> std::io::Result<(i32, Vec<u8>)>
where
I: IntoIterator<Item = S>,
S: Into<String>,
{
self.exec_capture_classified(argv).map(|(c, o, _)| (c, o))
}
pub fn exec_capture_classified<I, S>(
&self,
argv: I,
) -> std::io::Result<(i32, Vec<u8>, Option<SentryError>)>
where
I: IntoIterator<Item = S>,
S: Into<String>,
{
let v: Vec<String> = argv.into_iter().map(Into::into).collect();
if v.is_empty() {
return Err(std::io::Error::from(std::io::ErrorKind::InvalidInput));
}
let mut fds = [0 as c_int; 2];
if unsafe { libc::pipe(fds.as_mut_ptr()) } < 0 {
return Err(std::io::Error::last_os_error());
}
let (rd, wr) = (fds[0], fds[1]);
let mut stp = [0 as c_int; 2];
if unsafe { libc::pipe2(stp.as_mut_ptr(), libc::O_CLOEXEC) } < 0 {
let e = std::io::Error::last_os_error();
unsafe {
libc::close(rd);
libc::close(wr);
}
return Err(e);
}
let (st_r, st_w) = (stp[0], stp[1]);
let elf = std::path::PathBuf::from(&v[0]);
let cap = Stdio {
stdout: Some(wr),
stderr: Some(wr),
status_wr: Some(st_w),
..Stdio::default()
};
let pid = match spawn_sandbox(&elf, &v[1..], Some(&self.root), cap, true, &self.cfg()) {
Ok(p) => p,
Err(e) => {
unsafe {
libc::close(rd);
libc::close(wr);
libc::close(st_r);
libc::close(st_w);
}
return Err(e);
}
};
unsafe {
libc::close(wr);
libc::close(st_w);
}
let mut out = Vec::new();
let mut buf = [0u8; 16 * 1024];
loop {
let n = unsafe { libc::read(rd, buf.as_mut_ptr() as *mut c_void, buf.len()) };
if n <= 0 {
break;
}
out.extend_from_slice(&buf[..n as usize]);
}
unsafe { libc::close(rd) };
let code = wait_exit(pid)?;
let err = read_status_reason(st_r, code);
Ok((code, out, err))
}
pub fn start_workload(&self) -> std::io::Result<()> {
let script = self.root.join(".supermachine/run-workload");
if !script.is_file() {
return Err(std::io::Error::new(
std::io::ErrorKind::NotFound,
"no /.supermachine/run-workload staged in the rootfs",
));
}
let child = Child {
pid: spawn_sandbox(
std::path::Path::new("/bin/sh"),
&["/.supermachine/run-workload".to_string()],
Some(&self.root),
Stdio::default(),
true,
&self.cfg(),
)?,
};
let mut g = self.workload.lock().unwrap();
if let Some(old) = g.take() {
old.kill();
let _ = old.wait();
}
*g = Some(child);
Ok(())
}
pub fn workload_running(&self) -> bool {
let g = self.workload.lock().unwrap();
match g.as_ref() {
Some(c) => matches!(c.try_wait(), Ok(None)),
None => false,
}
}
pub fn stop(&self) -> std::io::Result<()> {
if let Some(c) = self.workload.lock().unwrap().take() {
c.kill();
let _ = c.wait();
}
remove_cgroup_named(&self.cgroup_name);
Ok(())
}
}
impl Drop for Sandbox {
fn drop(&mut self) {
let _ = self.stop();
}
}
const EW_REQUEST: u8 = 0xff;
const EW_CONTROL: u8 = 0xfe;
const EW_STDIN: u8 = 0;
const EW_STDOUT: u8 = 1;
const EW_STDERR: u8 = 2;
const EW_RESIZE: u8 = 3;
const EW_SIGNAL: u8 = 4;
const EW_EXIT: u8 = 5;
const EW_ERROR: u8 = 6;
const EW_BRIDGE: u8 = 0xfd;
const EW_PROTOCOL: u32 = 3;
const EW_MAX_FRAME: u32 = 16 * 1024 * 1024;
pub struct ExecServer {
path: std::path::PathBuf,
stop: std::sync::Arc<AtomicBool>,
join: Mutex<Option<std::thread::JoinHandle<()>>>,
workload: std::sync::Arc<Mutex<Option<Child>>>,
}
impl ExecServer {
fn start(
root: std::path::PathBuf,
cfg: SandboxCfg,
sock_path: std::path::PathBuf,
start_workload: bool,
) -> std::io::Result<ExecServer> {
use std::os::unix::net::UnixListener;
let _ = std::fs::remove_file(&sock_path);
let listener = UnixListener::bind(&sock_path)?;
listener.set_nonblocking(true)?;
let workload: std::sync::Arc<Mutex<Option<Child>>> = std::sync::Arc::new(Mutex::new(None));
if start_workload {
let script = root.join(".supermachine/run-workload");
if script.is_file() {
let pid = spawn_sandbox(
std::path::Path::new("/bin/sh"),
&["/.supermachine/run-workload".to_string()],
Some(&root),
Stdio::default(),
true,
&cfg,
)?;
*workload.lock().unwrap() = Some(Child { pid });
}
}
let stop = std::sync::Arc::new(AtomicBool::new(false));
let join = {
let (stop, root, cfg, workload) =
(stop.clone(), root.clone(), cfg.clone(), workload.clone());
std::thread::Builder::new()
.name("sentry-exec-accept".into())
.spawn(move || accept_loop(listener, stop, root, cfg, workload))?
};
Ok(ExecServer {
path: sock_path,
stop,
join: Mutex::new(Some(join)),
workload,
})
}
pub fn path(&self) -> &std::path::Path {
&self.path
}
pub fn stop(&self) {
self.stop.store(true, Ordering::Relaxed);
let join = self.join.lock().unwrap_or_else(|e| e.into_inner()).take();
if let Some(j) = join {
let _ = j.join();
}
if let Some(c) = self.workload.lock().unwrap().take() {
c.kill();
let _ = c.wait();
}
let _ = std::fs::remove_file(&self.path);
}
}
impl Drop for ExecServer {
fn drop(&mut self) {
self.stop();
}
}
fn accept_loop(
listener: std::os::unix::net::UnixListener,
stop: std::sync::Arc<AtomicBool>,
root: std::path::PathBuf,
cfg: SandboxCfg,
workload: std::sync::Arc<Mutex<Option<Child>>>,
) {
while !stop.load(Ordering::Relaxed) {
match listener.accept() {
Ok((stream, _)) => {
let (root, cfg, workload) = (root.clone(), cfg.clone(), workload.clone());
let _ = std::thread::Builder::new()
.name("sentry-exec-conn".into())
.spawn(move || {
let _ = handle_conn(stream, &root, &cfg, &workload, None);
});
}
Err(e) if e.kind() == std::io::ErrorKind::WouldBlock => {
std::thread::sleep(std::time::Duration::from_millis(20));
}
Err(_) => break,
}
}
}
fn ew_read_frame(s: &mut std::os::unix::net::UnixStream) -> std::io::Result<(u8, Vec<u8>)> {
use std::io::Read;
let mut hdr = [0u8; 5];
s.read_exact(&mut hdr)?;
let len = u32::from_be_bytes([hdr[1], hdr[2], hdr[3], hdr[4]]);
if len > EW_MAX_FRAME {
return Err(std::io::Error::new(
std::io::ErrorKind::InvalidData,
"frame too large",
));
}
let mut body = vec![0u8; len as usize];
if len > 0 {
s.read_exact(&mut body)?;
}
Ok((hdr[0], body))
}
fn ew_send_frame(
s: &Mutex<std::os::unix::net::UnixStream>,
kind: u8,
payload: &[u8],
) -> std::io::Result<()> {
use std::io::Write;
let mut g = s.lock().unwrap();
let mut hdr = [0u8; 5];
hdr[0] = kind;
hdr[1..5].copy_from_slice(&(payload.len() as u32).to_be_bytes());
g.write_all(&hdr)?;
if !payload.is_empty() {
g.write_all(payload)?;
}
g.flush()
}
fn handle_conn(
mut stream: std::os::unix::net::UnixStream,
root: &std::path::Path,
cfg: &SandboxCfg,
workload: &std::sync::Arc<Mutex<Option<Child>>>,
launch: Option<std::sync::Arc<Mutex<c_int>>>,
) -> std::io::Result<()> {
let (kind, body) = ew_read_frame(&mut stream)?;
match kind {
EW_CONTROL => handle_control(&mut stream, root, &body, workload),
EW_REQUEST => handle_request(stream, root, cfg, &body, launch),
EW_BRIDGE if launch.is_some() => {
std::thread::Builder::new()
.name("sentry-loop-bridge".into())
.spawn(move || {
let _ = handle_bridge(stream, &body);
})?;
Ok(())
}
_ => Ok(()),
}
}
fn handle_bridge(mut stream: std::os::unix::net::UnixStream, body: &[u8]) -> std::io::Result<()> {
use std::io::{Read, Write};
if body.len() != 2 {
stream.write_all(&[0])?;
return Ok(());
}
let port = u16::from_be_bytes([body[0], body[1]]);
let sid = loop_state().lock().unwrap().net.socket();
let efd = loop_new_efd();
loop_state().lock().unwrap().efds.insert(sid, efd);
let deadline = std::time::Instant::now() + std::time::Duration::from_secs(15);
let connected = loop {
let result = {
let mut ls = loop_state().lock().unwrap();
let result = if port == 0 {
ls.net.connect_first(sid)
} else {
ls.net
.connect(sid, netstack::Endpoint::v4([127, 0, 0, 1], port))
};
if let Ok(listener) = result {
loop_wake(&ls, listener);
}
result
};
match result {
Ok(_) => break true,
Err(netstack::ECONNREFUSED) if std::time::Instant::now() < deadline => {
std::thread::sleep(std::time::Duration::from_millis(10));
}
Err(_) => break false,
}
};
stream.write_all(&[u8::from(connected)])?;
if !connected {
loop_close(sid);
return Ok(());
}
let mut reader = stream.try_clone()?;
let writer = std::sync::Arc::new(Mutex::new(stream));
let writer2 = writer.clone();
let up = std::thread::Builder::new()
.name("sentry-bridge-host-to-loop".into())
.spawn(move || {
let mut buf = [0u8; 16 * 1024];
loop {
match reader.read(&mut buf) {
Ok(0) | Err(_) => break,
Ok(n) if loop_send_all(sid, &buf[..n], false) < (n as i64) => break,
Ok(_) => {}
}
}
let _ = loop_state()
.lock()
.unwrap()
.net
.shutdown(sid, netstack::SHUT_WR);
let _ = writer.lock().unwrap().shutdown(std::net::Shutdown::Write);
})?;
let down = std::thread::Builder::new()
.name("sentry-bridge-loop-to-host".into())
.spawn(move || {
loop {
let (n, data) = loop_recv_bytes(sid, 16 * 1024, false);
if n <= 0 {
break;
}
if writer2.lock().unwrap().write_all(&data).is_err() {
break;
}
}
let _ = writer2.lock().unwrap().shutdown(std::net::Shutdown::Both);
})?;
let _ = up.join();
let _ = down.join();
loop_close(sid);
Ok(())
}
fn handle_control(
stream: &mut std::os::unix::net::UnixStream,
root: &std::path::Path,
body: &[u8],
workload: &std::sync::Arc<Mutex<Option<Child>>>,
) -> std::io::Result<()> {
let v: serde_json::Value = serde_json::from_slice(body).unwrap_or(serde_json::Value::Null);
let action = v.get("action").and_then(|a| a.as_str()).unwrap_or("");
let ack: serde_json::Value = match action {
"probe" => serde_json::json!({ "ok": true, "protocol": EW_PROTOCOL }),
"write_file" => {
let path = v.get("path").and_then(|p| p.as_str()).unwrap_or("");
let data = v.get("data_b64").and_then(|d| d.as_str()).unwrap_or("");
let mode = v.get("mode").and_then(|m| m.as_u64());
match crate::api::b64_decode(data) {
Ok(bytes) => match write_in_root(root, path, &bytes, mode) {
Ok(()) => serde_json::json!({ "ok": true }),
Err(e) => serde_json::json!({ "ok": false, "error": e.to_string() }),
},
Err(e) => serde_json::json!({ "ok": false, "error": e }),
}
}
"read_file" => {
let path = v.get("path").and_then(|p| p.as_str()).unwrap_or("");
let max = v
.get("max_bytes")
.and_then(|m| m.as_u64())
.unwrap_or(4 * 1024 * 1024);
match read_in_root(&Some(root.to_path_buf()), path) {
Ok(bytes) if bytes.len() as u64 <= max => {
serde_json::json!({ "ok": true, "data_b64": crate::api::b64_encode(&bytes) })
}
Ok(_) => serde_json::json!({ "ok": false, "error": "file exceeds max_bytes" }),
Err(e) => serde_json::json!({ "ok": false, "error": e.to_string() }),
}
}
"signal" => {
let signum = v.get("signum").and_then(|s| s.as_i64()).unwrap_or(15) as i32;
if let Some(c) = workload.lock().unwrap().as_ref() {
unsafe { libc::kill(-c.pid, signum) };
}
serde_json::json!({ "ok": true })
}
"sync_time" | "sync" | "drop_vfs_caches" | "smpark_park" | "smpark_unpark" => {
serde_json::json!({ "ok": true })
}
other => serde_json::json!({ "ok": false, "error": format!("unknown action: {other}") }),
};
let m = Mutex::new(stream.try_clone()?);
let body = serde_json::to_vec(&ack).unwrap_or_default();
ew_send_frame(&m, EW_CONTROL, &body)
}
fn write_in_root(
root: &std::path::Path,
guest: &str,
bytes: &[u8],
mode: Option<u64>,
) -> std::io::Result<()> {
use std::os::unix::ffi::OsStrExt;
let mut rootb = root.as_os_str().as_bytes().to_vec();
rootb.push(0);
let rootfd = unsafe {
host(
SYS_OPENAT,
(-100i64) as u64,
rootb.as_ptr() as u64,
O_PATH | O_DIRECTORY,
0,
0,
0,
)
};
if rootfd < 0 {
return Err(std::io::Error::from_raw_os_error(-rootfd as i32));
}
let mut gp = guest.as_bytes().to_vec();
gp.push(0);
const O_WRONLY: u64 = 1;
const O_CREAT: u64 = 0o100;
const O_TRUNC: u64 = 0o1000;
let how = OpenHow {
flags: O_WRONLY | O_CREAT | O_TRUNC,
mode: mode.unwrap_or(0o644),
resolve: RESOLVE_IN_ROOT,
};
let fd = unsafe {
host(
SYS_OPENAT2,
rootfd as u64,
gp.as_ptr() as u64,
std::ptr::addr_of!(how) as u64,
std::mem::size_of::<OpenHow>() as u64,
0,
0,
)
};
unsafe { host(SYS_CLOSE, rootfd as u64, 0, 0, 0, 0, 0) };
if fd < 0 {
return Err(std::io::Error::from_raw_os_error(-fd as i32));
}
if let Some(m) = mode {
unsafe { host(SYS_FCHMOD, fd as u64, m, 0, 0, 0, 0) };
}
let mut off = 0usize;
while off < bytes.len() {
let n = unsafe {
host(
SYS_WRITE,
fd as u64,
bytes[off..].as_ptr() as u64,
(bytes.len() - off) as u64,
0,
0,
0,
)
};
if n <= 0 {
unsafe { host(SYS_CLOSE, fd as u64, 0, 0, 0, 0, 0) };
return Err(std::io::Error::last_os_error());
}
off += n as usize;
}
unsafe { host(SYS_CLOSE, fd as u64, 0, 0, 0, 0, 0) };
Ok(())
}
fn handle_request(
stream: std::os::unix::net::UnixStream,
root: &std::path::Path,
base_cfg: &SandboxCfg,
body: &[u8],
launch: Option<std::sync::Arc<Mutex<c_int>>>,
) -> std::io::Result<()> {
let _launch_guard = launch.as_ref().map(|l| l.lock().unwrap());
let launch = _launch_guard.as_ref().map(|g| **g);
let v: serde_json::Value = serde_json::from_slice(body)
.map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?;
let argv = json_str_vec(v.get("argv"));
let writer = std::sync::Arc::new(Mutex::new(stream.try_clone()?));
if argv.is_empty() {
let _ = ew_send_frame(&writer, EW_ERROR, b"REQUEST.argv is empty");
return Ok(());
}
let mut cfg = base_cfg.clone();
let req_env = json_env(v.get("env"));
cfg.env = merge_env(&base_cfg.env, &req_env);
cfg.cwd = v.get("cwd").and_then(|c| c.as_str()).map(|s| s.to_string());
let tty = v.get("tty").and_then(|t| t.as_bool()).unwrap_or(false);
let cols = v.get("cols").and_then(|c| c.as_u64()).map(|c| c as u16);
let rows = v.get("rows").and_then(|r| r.as_u64()).map(|r| r as u16);
if let Some(sf) = v.get("stage_files").and_then(|s| s.as_array()) {
for f in sf {
let path = f.get("path").and_then(|p| p.as_str()).unwrap_or("");
let data = f.get("data_b64").and_then(|d| d.as_str()).unwrap_or("");
let mode = f.get("mode").and_then(|m| m.as_u64());
let res = match crate::api::b64_decode(data) {
Ok(bytes) => write_in_root(root, path, &bytes, mode),
Err(e) => Err(std::io::Error::new(std::io::ErrorKind::InvalidData, e)),
};
if let Err(e) = res {
let _ = ew_send_frame(
&writer,
EW_ERROR,
format!("stage_file({path}): {e}").as_bytes(),
);
send_exit_with_rss(&writer, 127);
return Ok(());
}
}
}
let mut argvs: Vec<Vec<String>> = vec![argv];
for c in v
.get("chain")
.and_then(|c| c.as_array())
.into_iter()
.flatten()
{
argvs.push(json_str_vec(Some(c)));
}
let mut last = 0i32;
for (i, av) in argvs.iter().enumerate() {
if av.is_empty() {
continue;
}
last = match (launch, tty && i == 0) {
(Some(lfd), true) => {
run_one_tty_inproc(&stream, &writer, root, &cfg, av, cols, rows, lfd)?
}
(Some(lfd), false) => run_one_inproc(&stream, &writer, root, &cfg, av, i == 0, lfd)?,
(None, true) => run_one_tty(&stream, &writer, root, &cfg, av, cols, rows)?,
(None, false) => run_one(&stream, &writer, root, &cfg, av, i == 0)?,
};
if last != 0 {
break;
}
}
if launch.is_some() {
let _ = ew_send_frame(&writer, EW_EXIT, &(last as u32).to_be_bytes());
} else {
send_exit_with_rss(&writer, last);
}
Ok(())
}
fn send_exit_with_rss(writer: &std::sync::Arc<Mutex<std::os::unix::net::UnixStream>>, code: i32) {
let mut ru: libc::rusage = unsafe { std::mem::zeroed() };
let peak_rss_kib: u64 = unsafe {
if libc::getrusage(libc::RUSAGE_CHILDREN, &mut ru) == 0 {
ru.ru_maxrss as u64
} else {
0
}
};
let mut payload = [0u8; 12];
payload[0..4].copy_from_slice(&(code as u32).to_be_bytes());
payload[4..12].copy_from_slice(&peak_rss_kib.to_be_bytes());
let _ = ew_send_frame(writer, EW_EXIT, &payload);
}
fn spawn_lock() -> &'static Mutex<()> {
static L: OnceLock<Mutex<()>> = OnceLock::new();
L.get_or_init(|| Mutex::new(()))
}
fn run_one(
sock: &std::os::unix::net::UnixStream,
writer: &std::sync::Arc<Mutex<std::os::unix::net::UnixStream>>,
root: &std::path::Path,
cfg: &SandboxCfg,
argv: &[String],
consume_stdin: bool,
) -> std::io::Result<i32> {
let spawn_guard = spawn_lock().lock().unwrap();
let mut p_in = [0 as c_int; 2];
let mut p_out = [0 as c_int; 2];
let mut p_err = [0 as c_int; 2];
unsafe {
if libc::pipe(p_in.as_mut_ptr()) < 0
|| libc::pipe(p_out.as_mut_ptr()) < 0
|| libc::pipe(p_err.as_mut_ptr()) < 0
{
return Err(std::io::Error::last_os_error());
}
}
let (in_r, in_w) = (p_in[0], p_in[1]);
let (out_r, out_w) = (p_out[0], p_out[1]);
let (err_r, err_w) = (p_err[0], p_err[1]);
let stdio = Stdio {
stdin: Some(in_r),
stdout: Some(out_w),
stderr: Some(err_w),
ctty: false,
..Stdio::default()
};
let elf = match resolve_in_root_path(root, &path_from_env(&cfg.env), &argv[0]) {
Some(p) => std::path::PathBuf::from(p),
None => {
for fd in [in_r, in_w, out_r, out_w, err_r, err_w] {
unsafe { libc::close(fd) };
}
return Ok(127);
}
};
let pid = match spawn_sandbox(&elf, &argv[1..], Some(root), stdio, true, cfg) {
Ok(p) => p,
Err(_e) => {
for fd in [in_r, in_w, out_r, out_w, err_r, err_w] {
unsafe { libc::close(fd) };
}
return Ok(127);
}
};
unsafe {
libc::close(in_r);
libc::close(out_w);
libc::close(err_w);
}
if !consume_stdin {
unsafe { libc::close(in_w) };
}
drop(spawn_guard);
let pump = |fd: c_int, kind: u8, w: std::sync::Arc<Mutex<std::os::unix::net::UnixStream>>| {
std::thread::spawn(move || {
let mut buf = [0u8; 32 * 1024];
loop {
let n = unsafe { libc::read(fd, buf.as_mut_ptr() as *mut c_void, buf.len()) };
if n <= 0 {
break;
}
if ew_send_frame(&w, kind, &buf[..n as usize]).is_err() {
break;
}
}
unsafe { libc::close(fd) };
})
};
let t_out = pump(out_r, EW_STDOUT, writer.clone());
let t_err = pump(err_r, EW_STDERR, writer.clone());
let reader_done = std::sync::Arc::new(AtomicBool::new(false));
let t_in = {
let mut rs = sock.try_clone()?;
let reader_done = reader_done.clone();
std::thread::spawn(move || {
loop {
match ew_read_frame(&mut rs) {
Ok((EW_STDIN, data)) if consume_stdin => {
if data.is_empty() {
unsafe { libc::close(in_w) }; continue;
}
let mut off = 0usize;
while off < data.len() {
let n = unsafe {
libc::write(
in_w,
data[off..].as_ptr() as *const c_void,
data.len() - off,
)
};
if n <= 0 {
break;
}
off += n as usize;
}
}
Ok((EW_SIGNAL, data)) if !data.is_empty() => {
signal_cell_tree(pid, data[0] as c_int);
}
Ok(_) => {}
Err(_) => break,
}
if reader_done.load(Ordering::Relaxed) {
break;
}
}
})
};
let (code, _) = wait_exit_deadline(pid, cfg.exec_timeout_ms)?;
reader_done.store(true, Ordering::Relaxed);
let _ = sock.shutdown(std::net::Shutdown::Read);
unsafe { libc::close(in_w) };
let _ = t_in.join();
let _ = t_out.join();
let _ = t_err.join();
Ok(code)
}
fn signal_cell_tree(pid: i32, sig: c_int) {
unsafe {
if libc::getpgid(pid) == pid {
libc::kill(-pid, sig);
} else {
libc::kill(pid, sig);
}
}
}
#[cfg(all(target_os = "linux", target_arch = "x86_64"))]
fn run_one_inproc(
sock: &std::os::unix::net::UnixStream,
writer: &std::sync::Arc<Mutex<std::os::unix::net::UnixStream>>,
root: &std::path::Path,
cfg: &SandboxCfg,
argv: &[String],
consume_stdin: bool,
launch_fd: c_int,
) -> std::io::Result<i32> {
let elf = match resolve_in_root_path(root, &path_from_env(&cfg.env), &argv[0]) {
Some(p) => p,
None => return Ok(127),
};
let elf = canonical_exe_in_root(root, &elf).unwrap_or(elf);
let env_kv: Vec<String> = cfg.env.iter().map(|(k, v)| format!("{k}={v}")).collect();
let spawn_guard = spawn_lock().lock().unwrap();
let mut p_in = [0 as c_int; 2];
let mut p_out = [0 as c_int; 2];
let mut p_err = [0 as c_int; 2];
unsafe {
if libc::pipe(p_in.as_mut_ptr()) < 0
|| libc::pipe(p_out.as_mut_ptr()) < 0
|| libc::pipe(p_err.as_mut_ptr()) < 0
{
return Err(std::io::Error::last_os_error());
}
}
let (in_r, in_w) = (p_in[0], p_in[1]);
let (out_r, out_w) = (p_out[0], p_out[1]);
let (err_r, err_w) = (p_err[0], p_err[1]);
let pid = launch_cell_env(
launch_fd,
Some(&elf),
argv,
false,
&env_kv,
cfg.cwd.as_deref(),
);
if pid <= 0 {
for fd in [in_r, in_w, out_r, out_w, err_r, err_w] {
unsafe { libc::close(fd) };
}
return Ok(127);
}
seed_cell_fds_streaming(pid, in_r, out_w, err_w);
if let Some(c) = &cfg.cwd {
cwds().lock().unwrap().insert(pid, c.as_bytes().to_vec());
}
unsafe { set_ring_owner(0, pid, pid) };
continue_launched_cell(pid);
unsafe {
libc::close(in_r);
libc::close(out_w);
libc::close(err_w);
}
if !consume_stdin {
unsafe { libc::close(in_w) };
}
drop(spawn_guard);
let pump = |fd: c_int, kind: u8, w: std::sync::Arc<Mutex<std::os::unix::net::UnixStream>>| {
std::thread::spawn(move || {
let mut buf = [0u8; 32 * 1024];
loop {
let n = unsafe { libc::read(fd, buf.as_mut_ptr() as *mut c_void, buf.len()) };
if n <= 0 {
break;
}
if ew_send_frame(&w, kind, &buf[..n as usize]).is_err() {
break;
}
}
unsafe { libc::close(fd) };
})
};
let t_out = pump(out_r, EW_STDOUT, writer.clone());
let t_err = pump(err_r, EW_STDERR, writer.clone());
let reader_done = std::sync::Arc::new(AtomicBool::new(false));
let t_in = {
let mut rs = sock.try_clone()?;
let _ = rs.set_read_timeout(Some(std::time::Duration::from_millis(5)));
let reader_done = reader_done.clone();
std::thread::spawn(move || loop {
match ew_read_frame(&mut rs) {
Ok((EW_STDIN, data)) if consume_stdin => {
if data.is_empty() {
unsafe { libc::close(in_w) };
continue;
}
let mut off = 0usize;
while off < data.len() {
let n = unsafe {
libc::write(
in_w,
data[off..].as_ptr() as *const c_void,
data.len() - off,
)
};
if n <= 0 {
break;
}
off += n as usize;
}
}
Ok((EW_SIGNAL, data)) if !data.is_empty() => {
signal_cell_tree(pid, data[0] as c_int);
}
Ok(_) => {}
Err(ref e)
if e.kind() == std::io::ErrorKind::WouldBlock
|| e.kind() == std::io::ErrorKind::TimedOut =>
{
if reader_done.load(Ordering::Relaxed) {
break;
}
}
Err(_) => {
if !reader_done.load(Ordering::Relaxed) {
signal_cell_tree(pid, libc::SIGKILL);
}
break;
}
}
})
};
let (code, _) = await_launched_exit(launch_fd, pid, cfg.exec_timeout_ms);
shm_reap_pid(pid);
fd_drop(pid);
free_slots_of(pid);
let _ = cfg.preserve_stragglers;
preserve_stragglers_as_running(0);
reader_done.store(true, Ordering::Relaxed);
unsafe { libc::close(in_w) };
let _ = t_in.join();
let _ = t_out.join();
let _ = t_err.join();
Ok(code)
}
#[cfg(all(target_os = "linux", target_arch = "x86_64"))]
#[allow(clippy::too_many_arguments)]
fn run_one_tty_inproc(
sock: &std::os::unix::net::UnixStream,
writer: &std::sync::Arc<Mutex<std::os::unix::net::UnixStream>>,
root: &std::path::Path,
cfg: &SandboxCfg,
argv: &[String],
_cols: Option<u16>,
_rows: Option<u16>,
launch_fd: c_int,
) -> std::io::Result<i32> {
run_one_inproc(sock, writer, root, cfg, argv, true, launch_fd)
}
fn run_one_tty(
sock: &std::os::unix::net::UnixStream,
writer: &std::sync::Arc<Mutex<std::os::unix::net::UnixStream>>,
root: &std::path::Path,
cfg: &SandboxCfg,
argv: &[String],
cols: Option<u16>,
rows: Option<u16>,
) -> std::io::Result<i32> {
let spawn_guard = spawn_lock().lock().unwrap();
let pty = open_pty()?;
let master = pty.master;
if let (Some(c), Some(r)) = (cols, rows) {
let _ = set_winsize(master, c, r);
}
let slave_c = match std::ffi::CString::new(pty.slave_path.clone()) {
Ok(c) => c,
Err(_) => {
cleanup_private_devpts(pty.private_devpts.as_deref());
unsafe { libc::close(master) };
return Err(std::io::Error::new(
std::io::ErrorKind::InvalidData,
"pty slave path NUL",
));
}
};
let slave = unsafe { libc::open(slave_c.as_ptr(), libc::O_RDWR) };
if slave < 0 {
let e = std::io::Error::last_os_error();
cleanup_private_devpts(pty.private_devpts.as_deref());
unsafe { libc::close(master) };
return Err(e);
}
cleanup_private_devpts(pty.private_devpts.as_deref());
let stdio = Stdio {
stdin: Some(slave),
stdout: Some(slave),
stderr: Some(slave),
ctty: true,
..Stdio::default()
};
let elf = match resolve_in_root_path(root, &path_from_env(&cfg.env), &argv[0]) {
Some(p) => std::path::PathBuf::from(p),
None => {
unsafe {
libc::close(master);
libc::close(slave);
}
return Ok(127);
}
};
let pid = match spawn_sandbox(&elf, &argv[1..], Some(root), stdio, true, cfg) {
Ok(p) => p,
Err(_e) => {
unsafe {
libc::close(master);
libc::close(slave);
}
return Ok(127);
}
};
unsafe { libc::close(slave) };
drop(spawn_guard);
let (pump_done_tx, pump_done_rx) = std::sync::mpsc::channel::<()>();
let t_out = {
let w = writer.clone();
std::thread::spawn(move || {
let mut buf = [0u8; 32 * 1024];
loop {
let n = unsafe { libc::read(master, buf.as_mut_ptr() as *mut c_void, buf.len()) };
if n <= 0 {
break;
}
if ew_send_frame(&w, EW_STDOUT, &buf[..n as usize]).is_err() {
break;
}
}
let _ = pump_done_tx.send(());
})
};
let reader_done = std::sync::Arc::new(AtomicBool::new(false));
let t_in = {
let mut rs = sock.try_clone()?;
let reader_done = reader_done.clone();
std::thread::spawn(move || {
loop {
match ew_read_frame(&mut rs) {
Ok((EW_STDIN, data)) => {
if data.is_empty() {
let eof = [0x04u8];
unsafe {
libc::write(master, eof.as_ptr() as *const c_void, 1);
}
continue;
}
let mut off = 0usize;
while off < data.len() {
let n = unsafe {
libc::write(
master,
data[off..].as_ptr() as *const c_void,
data.len() - off,
)
};
if n <= 0 {
break;
}
off += n as usize;
}
}
Ok((EW_RESIZE, data)) if data.len() == 4 => {
let cols = u16::from_be_bytes([data[0], data[1]]);
let rows = u16::from_be_bytes([data[2], data[3]]);
let _ = set_winsize(master, cols, rows);
}
Ok((EW_SIGNAL, data)) if !data.is_empty() => {
signal_cell_tree(pid, data[0] as c_int);
}
Ok(_) => {}
Err(_) => break,
}
if reader_done.load(Ordering::Relaxed) {
break;
}
}
})
};
let (code, _) = wait_exit_deadline(pid, cfg.exec_timeout_ms)?;
let drained = pump_done_rx
.recv_timeout(std::time::Duration::from_secs(2))
.is_ok();
if !drained {
unsafe {
let null = libc::open(b"/dev/null\0".as_ptr() as *const c_char, libc::O_RDWR);
if null >= 0 {
libc::dup2(null, master);
libc::close(null);
}
}
}
reader_done.store(true, Ordering::Relaxed);
let _ = sock.shutdown(std::net::Shutdown::Read);
let _ = t_in.join();
let _ = t_out.join();
unsafe { libc::close(master) };
Ok(code)
}
struct OpenPty {
master: c_int,
slave_path: Vec<u8>,
private_devpts: Option<std::path::PathBuf>,
}
fn open_pty() -> std::io::Result<OpenPty> {
match open_pty_at(b"/dev/ptmx\0", None) {
Ok(pty) => Ok(pty),
Err(e) if e.raw_os_error() == Some(libc::ENODEV) => open_private_devpts_pty(),
Err(e) => Err(e),
}
}
fn open_pty_at(
ptmx_path: &[u8],
private_devpts: Option<std::path::PathBuf>,
) -> std::io::Result<OpenPty> {
let master = unsafe {
libc::open(
ptmx_path.as_ptr() as *const c_char,
libc::O_RDWR | libc::O_NOCTTY,
)
};
if master < 0 {
return Err(std::io::Error::last_os_error());
}
let fail = |m: c_int| {
let e = std::io::Error::last_os_error();
unsafe { libc::close(m) };
e
};
if unsafe { libc::grantpt(master) } < 0 {
return Err(fail(master));
}
if unsafe { libc::unlockpt(master) } < 0 {
return Err(fail(master));
}
let slave_path = if let Some(dir) = private_devpts.as_ref() {
let mut n: libc::c_uint = 0;
if unsafe { libc::ioctl(master, libc::TIOCGPTN as _, &mut n as *mut _) } < 0 {
return Err(fail(master));
}
dir.join(n.to_string())
.to_string_lossy()
.as_bytes()
.to_vec()
} else {
let mut buf = [0u8; 256];
let r = unsafe { libc::ptsname_r(master, buf.as_mut_ptr() as *mut c_char, buf.len()) };
if r != 0 {
unsafe { libc::close(master) };
return Err(std::io::Error::from_raw_os_error(r));
}
let len = buf.iter().position(|&b| b == 0).unwrap_or(buf.len());
buf[..len].to_vec()
};
Ok(OpenPty {
master,
slave_path,
private_devpts,
})
}
fn open_private_devpts_pty() -> std::io::Result<OpenPty> {
static PTY_MOUNT_SEQ: AtomicU64 = AtomicU64::new(0);
let seq = PTY_MOUNT_SEQ.fetch_add(1, Ordering::Relaxed);
let dir =
std::env::temp_dir().join(format!("sentry-devpts-{}-{seq}", unsafe { libc::getpid() }));
std::fs::create_dir_all(&dir)?;
let target = match std::ffi::CString::new(dir.to_string_lossy().as_bytes().to_vec()) {
Ok(c) => c,
Err(_) => {
let _ = std::fs::remove_dir_all(&dir);
return Err(std::io::Error::new(
std::io::ErrorKind::InvalidInput,
"private devpts path contains NUL",
));
}
};
let source = b"devpts\0";
let fstype = b"devpts\0";
let data = b"newinstance,ptmxmode=0666,mode=0600\0";
let mount_rc = unsafe {
libc::mount(
source.as_ptr() as *const c_char,
target.as_ptr(),
fstype.as_ptr() as *const c_char,
(libc::MS_NOSUID | libc::MS_NOEXEC) as libc::c_ulong,
data.as_ptr() as *const c_void,
)
};
if mount_rc < 0 {
let e = std::io::Error::last_os_error();
let _ = std::fs::remove_dir_all(&dir);
return Err(e);
}
let ptmx = dir.join("ptmx");
let ptmx_c = match std::ffi::CString::new(ptmx.to_string_lossy().as_bytes().to_vec()) {
Ok(c) => c,
Err(_) => {
cleanup_private_devpts(Some(&dir));
return Err(std::io::Error::new(
std::io::ErrorKind::InvalidInput,
"private ptmx path contains NUL",
));
}
};
match open_pty_at(ptmx_c.as_bytes_with_nul(), Some(dir.clone())) {
Ok(pty) => Ok(pty),
Err(e) => {
cleanup_private_devpts(Some(&dir));
Err(e)
}
}
}
fn cleanup_private_devpts(dir: Option<&std::path::Path>) {
if let Some(dir) = dir {
if let Ok(c) = std::ffi::CString::new(dir.to_string_lossy().as_bytes().to_vec()) {
unsafe {
libc::umount2(c.as_ptr(), libc::MNT_DETACH);
}
}
let _ = std::fs::remove_dir_all(dir);
}
}
fn set_winsize(fd: c_int, cols: u16, rows: u16) -> std::io::Result<()> {
let ws = libc::winsize {
ws_row: rows,
ws_col: cols,
ws_xpixel: 0,
ws_ypixel: 0,
};
let r = unsafe { libc::ioctl(fd, libc::TIOCSWINSZ as _, &ws as *const _) };
if r < 0 {
return Err(std::io::Error::last_os_error());
}
Ok(())
}
fn json_str_vec(v: Option<&serde_json::Value>) -> Vec<String> {
v.and_then(|x| x.as_array())
.map(|a| {
a.iter()
.filter_map(|s| s.as_str().map(|s| s.to_string()))
.collect()
})
.unwrap_or_default()
}
fn json_env(v: Option<&serde_json::Value>) -> Vec<(String, String)> {
v.and_then(|x| x.as_object())
.map(|o| {
o.iter()
.filter_map(|(k, val)| val.as_str().map(|s| (k.clone(), s.to_string())))
.collect()
})
.unwrap_or_default()
}
fn merge_env(base: &[(String, String)], overlay: &[(String, String)]) -> Vec<(String, String)> {
if overlay.is_empty() {
return base.to_vec();
}
if base.is_empty() {
return overlay.to_vec();
}
let mut out: Vec<(String, String)> = base.to_vec();
for (k, val) in overlay {
if let Some(slot) = out.iter_mut().find(|(ek, _)| ek == k) {
slot.1 = val.clone();
} else {
out.push((k.clone(), val.clone()));
}
}
out
}
impl Sandbox {
pub fn serve_exec(
&self,
sock_path: impl Into<std::path::PathBuf>,
start_workload: bool,
) -> std::io::Result<ExecServer> {
ExecServer::start(
self.root.clone(),
self.cfg(),
sock_path.into(),
start_workload,
)
}
}
enum Ctrl {
Exec(Vec<String>), ExecCapture(Vec<String>), ExecFull(Vec<u8>), Warm(Vec<String>), WarmRestore(String), Acquire, AcquireCapture, AcquireRunning, Release(i32), Snapshot(String), Quit, }
fn recv_ctrl(fd: c_int) -> (Ctrl, Option<c_int>) {
let mut buf = vec![0u8; 256 * 1024];
let mut passed: Option<c_int> = None;
let n = unsafe {
let mut iov = libc::iovec {
iov_base: buf.as_mut_ptr() as *mut c_void,
iov_len: buf.len(),
};
let mut cmsgbuf = [0u8; 64]; let mut msg: libc::msghdr = std::mem::zeroed();
msg.msg_iov = &mut iov;
msg.msg_iovlen = 1;
msg.msg_control = cmsgbuf.as_mut_ptr() as *mut c_void;
msg.msg_controllen = cmsgbuf.len() as _;
let n = libc::recvmsg(fd, &mut msg, 0);
if n > 0 {
let mut cmsg = libc::CMSG_FIRSTHDR(&msg);
while !cmsg.is_null() {
if (*cmsg).cmsg_level == libc::SOL_SOCKET && (*cmsg).cmsg_type == libc::SCM_RIGHTS {
let mut f: c_int = -1;
std::ptr::copy_nonoverlapping(
libc::CMSG_DATA(cmsg),
&mut f as *mut c_int as *mut u8,
std::mem::size_of::<c_int>(),
);
passed = Some(f);
}
cmsg = libc::CMSG_NXTHDR(&msg, cmsg);
}
}
n
};
if n <= 0 {
if let Some(f) = passed {
unsafe { libc::close(f) };
}
return (Ctrl::Quit, None);
}
let op = buf[0];
let argv: Vec<String> = buf[1..n as usize]
.split(|&b| b == 0)
.filter(|s| !s.is_empty())
.map(|s| String::from_utf8_lossy(s).into_owned())
.collect();
let ctrl = match op {
b'X' => Ctrl::ExecFull(buf[1..n as usize].to_vec()),
b'C' => Ctrl::ExecCapture(argv),
b'W' => Ctrl::Warm(argv),
b'Z' => Ctrl::WarmRestore(argv.first().cloned().unwrap_or_default()),
b'A' => Ctrl::Acquire,
b'a' => Ctrl::AcquireCapture,
b'R' => Ctrl::AcquireRunning,
b'r' => Ctrl::Release(argv.first().and_then(|s| s.parse().ok()).unwrap_or(-1)),
b'S' => Ctrl::Snapshot(argv.first().cloned().unwrap_or_default()),
b'Q' => Ctrl::Quit,
_ => Ctrl::Exec(argv), };
(ctrl, passed)
}
fn reply_status(fd: c_int, code: i32, err: Option<SentryError>) {
let mut bytes = [0u8; 8];
bytes[0..4].copy_from_slice(&code.to_le_bytes());
bytes[4..8].copy_from_slice(&err_to_tag(err).to_le_bytes());
unsafe { libc::send(fd, bytes.as_ptr() as *const c_void, 8, libc::MSG_NOSIGNAL) };
}
fn reply_code(fd: c_int, code: i32) {
reply_status(fd, code, None);
}
fn send_ctrl_with_fd(sock: c_int, payload: &[u8], passfd: c_int) -> std::io::Result<()> {
unsafe {
let mut iov = libc::iovec {
iov_base: payload.as_ptr() as *mut c_void,
iov_len: payload.len(),
};
let mut cmsgbuf = [0u8; 64];
let mut msg: libc::msghdr = std::mem::zeroed();
msg.msg_iov = &mut iov;
msg.msg_iovlen = 1;
msg.msg_control = cmsgbuf.as_mut_ptr() as *mut c_void;
msg.msg_controllen = libc::CMSG_SPACE(std::mem::size_of::<c_int>() as u32) as _;
let cmsg = libc::CMSG_FIRSTHDR(&msg);
(*cmsg).cmsg_level = libc::SOL_SOCKET;
(*cmsg).cmsg_type = libc::SCM_RIGHTS;
(*cmsg).cmsg_len = libc::CMSG_LEN(std::mem::size_of::<c_int>() as u32) as _;
std::ptr::copy_nonoverlapping(
&passfd as *const c_int as *const u8,
libc::CMSG_DATA(cmsg),
std::mem::size_of::<c_int>(),
);
if libc::sendmsg(sock, &msg, libc::MSG_NOSIGNAL) < 0 {
return Err(std::io::Error::last_os_error());
}
}
Ok(())
}
fn zygote_acquire(seq: u32) -> i32 {
let req = ring_word(unsafe { std::ptr::addr_of_mut!((*ZYG).req) });
let done = ring_word(unsafe { std::ptr::addr_of_mut!((*ZYG).done) });
req.store(seq, Ordering::Release);
unsafe { host(SYS_FUTEX, req.as_ptr() as u64, FUTEX_WAKE, 1, 0, 0, 0) };
while done.load(Ordering::Acquire) != seq {
unsafe {
host(
SYS_FUTEX,
done.as_ptr() as u64,
FUTEX_WAIT,
(seq - 1) as u64,
0,
0,
0,
)
};
}
unsafe { (*ZYG).exit }
}
#[derive(serde::Serialize, serde::Deserialize)]
struct LaunchReq {
#[serde(default)]
path: Option<String>,
argv: Vec<String>,
warm: bool,
#[serde(default)]
env: Vec<String>,
#[serde(default)]
cwd: Option<String>,
#[serde(default)]
restore_dir: Option<String>,
#[serde(default = "default_restore_vpid")]
restore_vpid: state_snap::Vpid,
#[serde(default = "default_restore_warm")]
restore_warm: bool,
#[serde(default)]
detached: bool,
#[serde(default)]
slot: Option<u64>,
#[serde(default)]
stop_before_resume: bool,
#[serde(default)]
stop_before_start: bool,
#[serde(default)]
capture: Option<LauncherCaptureReq>,
}
fn default_restore_vpid() -> state_snap::Vpid {
1
}
fn default_restore_warm() -> bool {
true
}
#[derive(serde::Serialize, serde::Deserialize)]
struct LauncherCaptureReq {
dest_dir: String,
targets: Vec<LauncherCaptureTarget>,
}
#[derive(Clone, serde::Serialize, serde::Deserialize)]
struct LauncherCaptureTarget {
vpid: state_snap::Vpid,
host_pid: i32,
}
#[cfg(all(target_os = "linux", target_arch = "x86_64"))]
fn ptrace_attach_stop(pid: i32) -> std::io::Result<()> {
let r = unsafe {
libc::ptrace(
libc::PTRACE_ATTACH,
pid,
std::ptr::null_mut::<c_void>(),
std::ptr::null_mut::<c_void>(),
)
};
if r < 0 {
let err = std::io::Error::last_os_error();
if let Ok(status) = std::fs::read_to_string(format!("/proc/{pid}/status")) {
for line in status.lines() {
if line.starts_with("Name:")
|| line.starts_with("State:")
|| line.starts_with("Tgid:")
|| line.starts_with("Pid:")
|| line.starts_with("PPid:")
|| line.starts_with("TracerPid:")
|| line.starts_with("Uid:")
|| line.starts_with("Gid:")
|| line.starts_with("CoreDumping:")
|| line.starts_with("NoNewPrivs:")
|| line.starts_with("Seccomp:")
{
eprintln!("sentry live capture: attach target {pid}: {line}");
}
}
}
return Err(err);
}
let mut st: c_int = 0;
loop {
let w = unsafe { libc::waitpid(pid, &mut st, 0) };
if w < 0 {
return Err(std::io::Error::last_os_error());
}
if w == pid && libc::WIFSTOPPED(st) {
return Ok(());
}
}
}
#[cfg(all(target_os = "linux", target_arch = "x86_64"))]
fn ptrace_detach(pid: i32) {
unsafe {
libc::ptrace(
libc::PTRACE_DETACH,
pid,
std::ptr::null_mut::<c_void>(),
std::ptr::null_mut::<c_void>(),
);
}
}
#[cfg(all(target_os = "linux", target_arch = "x86_64"))]
fn capture_ptrace_thread_regs(
pid: i32,
vtid: state_snap::Vpid,
) -> std::io::Result<state_snap::ThreadRegs> {
let mut regs: libc::user_regs_struct = unsafe { std::mem::zeroed() };
let r = unsafe {
libc::ptrace(
libc::PTRACE_GETREGS,
pid,
std::ptr::null_mut::<c_void>(),
&mut regs as *mut _ as *mut c_void,
)
};
if r < 0 {
return Err(std::io::Error::last_os_error());
}
if regs.rip >= WINDOW_FLOOR {
if let Some(mut thread) = ring_snapshot_regs_for_pid(pid, vtid) {
if thread.fs == 0 && regs.fs_base != 0 && regs.fs_base < WINDOW_FLOOR {
thread.fs = regs.fs_base;
}
return Ok(thread);
}
}
let mut gregs = [0i64; 18];
gregs[REG_R8] = regs.r8 as i64;
gregs[REG_R9] = regs.r9 as i64;
gregs[REG_R10] = regs.r10 as i64;
gregs[REG_R11] = regs.r11 as i64;
gregs[REG_R12] = regs.r12 as i64;
gregs[REG_R13] = regs.r13 as i64;
gregs[REG_R14] = regs.r14 as i64;
gregs[REG_R15] = regs.r15 as i64;
gregs[REG_RDI] = regs.rdi as i64;
gregs[REG_RSI] = regs.rsi as i64;
gregs[REG_RBP] = regs.rbp as i64;
gregs[REG_RBX] = regs.rbx as i64;
gregs[REG_RDX] = regs.rdx as i64;
gregs[REG_RAX] = regs.rax as i64;
gregs[REG_RCX] = regs.rcx as i64;
gregs[REG_RSP] = regs.rsp as i64;
gregs[REG_RIP] = regs.rip as i64;
Ok(state_snap::ThreadRegs {
vtid,
gregs,
fs: regs.fs_base,
xsave: Vec::new(),
})
}
#[cfg(all(target_os = "linux", target_arch = "x86_64"))]
fn launcher_capture_one_proc(
target: &LauncherCaptureTarget,
ram: &mut std::fs::File,
) -> std::io::Result<state_snap::ProcMemImage> {
use std::io::{Seek as _, SeekFrom, Write as _};
let mut blob_off = ram.seek(SeekFrom::End(0))?;
let misalign = blob_off % memimage::PAGE;
if misalign != 0 {
let pad = vec![0u8; (memimage::PAGE - misalign) as usize];
ram.write_all(&pad)?;
blob_off += memimage::PAGE - misalign;
}
let maps_path = format!("/proc/{}/maps", target.host_pid);
let maps = std::fs::read(&maps_path).map_err(|e| {
eprintln!("sentry live capture: read {maps_path} failed: {e}");
e
})?;
let mem_path = format!("/proc/{}/mem", target.host_pid);
let mem = std::fs::File::open(&mem_path).map_err(|e| {
eprintln!("sentry live capture: open {mem_path} failed: {e}");
e
})?;
let thread = capture_ptrace_thread_regs(target.host_pid, target.vpid).map_err(|e| {
eprintln!(
"sentry live capture: ptrace regs for pid {} vpid {} failed: {e}",
target.host_pid, target.vpid
);
e
})?;
let guest_ranges = capture_guest_vma_ranges(&mem);
let entries = select_guest_map_entries(&maps, &guest_ranges, &[thread.fs]);
let regions = memimage::capture_regions(
&entries,
|va, dst| read_exact_process_mem(&mem, va, dst),
ram,
)
.map_err(|e| {
eprintln!(
"sentry live capture: memory copy for pid {} vpid {} failed: {e}",
target.host_pid, target.vpid
);
e
})?;
Ok(state_snap::ProcMemImage {
vpid: target.vpid,
blob_off,
regions,
threads: vec![thread],
})
}
#[cfg(all(target_os = "linux", target_arch = "x86_64"))]
fn launcher_capture_mem_images(req: &LauncherCaptureReq) -> std::io::Result<()> {
use std::io::Write as _;
if req.targets.is_empty() {
return Err(std::io::Error::new(
std::io::ErrorKind::InvalidInput,
"snapshot: no live capture targets",
));
}
let dest = std::path::Path::new(&req.dest_dir);
std::fs::create_dir_all(dest)?;
let tmp_suffix = format!("{}-{}", std::process::id(), snapshot_tmp_suffix());
let mem_tmp = dest.join(format!(".mem.blob.live.tmp-{tmp_suffix}"));
let meta_tmp = dest.join(format!(".mem.meta.tmp-{tmp_suffix}"));
launcher_quiesce_live_targets(&req.targets)?;
let mut attached = Vec::new();
for target in &req.targets {
if let Err(e) = ptrace_attach_stop(target.host_pid) {
for pid in attached {
ptrace_detach(pid);
}
return Err(e);
}
attached.push(target.host_pid);
}
let result = (|| {
let mut ram = std::fs::File::create(&mem_tmp)?;
let mut procs = Vec::with_capacity(req.targets.len());
for target in &req.targets {
procs.push(launcher_capture_one_proc(target, &mut ram)?);
}
ram.flush()?;
let first = procs.first().cloned().ok_or_else(|| {
std::io::Error::new(std::io::ErrorKind::InvalidData, "snapshot: empty mem image")
})?;
let mem = state_snap::MemImage {
regions: first.regions.clone(),
threads: first.threads.clone(),
procs,
};
let mut meta = Vec::new();
mem.write_to(&mut meta)?;
std::fs::write(&meta_tmp, meta)?;
std::fs::rename(&mem_tmp, dest.join("mem.blob"))?;
std::fs::rename(&meta_tmp, dest.join("mem.meta"))?;
Ok(())
})();
for pid in attached {
ptrace_detach(pid);
}
if result.is_err() {
let _ = std::fs::remove_file(&mem_tmp);
let _ = std::fs::remove_file(&meta_tmp);
}
result
}
fn cell_fork_launcher(req_fd: c_int, root_owned: Option<std::path::PathBuf>) -> ! {
loop {
let mut buf = vec![0u8; 1 << 16];
let n = unsafe { libc::recv(req_fd, buf.as_mut_ptr() as *mut c_void, buf.len(), 0) };
if n <= 0 {
launcher_kill_owned_processes_on_eof();
unsafe { libc::_exit(0) }; }
let req: LaunchReq = match serde_json::from_slice::<LaunchReq>(&buf[..n as usize]) {
Ok(r) if !r.argv.is_empty() => r,
_ => {
let pid: i32 = -22; unsafe { libc::send(req_fd, &pid as *const i32 as *const c_void, 4, 0) };
continue;
}
};
if let Some(capture) = req.capture {
let code = match launcher_capture_mem_images(&capture) {
Ok(()) => 0,
Err(e) => -e.raw_os_error().unwrap_or(libc::EIO),
};
unsafe { libc::send(req_fd, &code as *const i32 as *const c_void, 4, 0) };
continue;
}
let path = req.path.as_deref().unwrap_or(&req.argv[0]);
set_guest_cmdline(&req.argv);
if !req.env.is_empty() {
*guest_env().lock().unwrap() = req.env.iter().map(|s| s.as_bytes().to_vec()).collect();
}
if let Some(cwd) = &req.cwd {
*guest_cwd_init().lock().unwrap() = Some(cwd.as_bytes().to_vec());
}
if req.warm {
WARM_MODE.store(true, Ordering::Relaxed);
ZYG_DRIVEN.store(true, Ordering::Relaxed);
OWN_LOOPBACK.store(true, Ordering::Relaxed);
}
let pid = unsafe { libc::fork() };
if pid == 0 {
unsafe { libc::close(req_fd) };
if let Some(slot) = req.slot {
set_slot(slot);
unsafe {
let me = host(SYS_GETPID, 0, 0, 0, 0, 0, 0) as i32;
let tid = host(SYS_GETTID, 0, 0, 0, 0, 0, 0) as i32;
set_ring_owner(slot, me, tid);
}
}
unsafe { libc::setpgid(0, 0) };
if req.stop_before_start {
unsafe {
let me = host(SYS_GETPID, 0, 0, 0, 0, 0, 0);
host(SYS_KILL, me as u64, libc::SIGSTOP as u64, 0, 0, 0, 0);
}
}
if let Some(dir) = req.restore_dir.as_deref() {
cell_restore_main(
dir,
req.restore_vpid,
req.restore_warm,
req.stop_before_resume,
root_owned.clone(),
); } else {
cell_main(path, &req.argv, root_owned.clone()); }
}
if req.warm {
WARM_MODE.store(false, Ordering::Relaxed); ZYG_DRIVEN.store(false, Ordering::Relaxed);
}
unsafe { libc::send(req_fd, &pid as *const i32 as *const c_void, 4, 0) };
if !req.warm && !req.detached && pid > 0 {
let mut st: c_int = 0;
unsafe { libc::waitpid(pid, &mut st, 0) };
unsafe { libc::send(req_fd, &st as *const i32 as *const c_void, 4, 0) };
}
}
}
fn launch_cell(launch_fd: c_int, argv: &[String], warm: bool) -> i32 {
launch_cell_env(launch_fd, None, argv, warm, &[], None)
}
fn launch_cell_env(
launch_fd: c_int,
path: Option<&str>,
argv: &[String],
warm: bool,
env: &[String],
cwd: Option<&str>,
) -> i32 {
let req = LaunchReq {
path: path.map(|s| s.to_string()),
argv: argv.to_vec(),
warm,
env: env.to_vec(),
cwd: cwd.map(|s| s.to_string()),
restore_dir: None,
restore_vpid: 1,
restore_warm: true,
detached: false,
slot: None,
stop_before_resume: false,
stop_before_start: true,
capture: None,
};
let json = match serde_json::to_vec(&req) {
Ok(j) => j,
Err(_) => return -1,
};
unsafe { libc::send(launch_fd, json.as_ptr() as *const c_void, json.len(), 0) };
let mut pid: i32 = -1;
let n = unsafe { libc::recv(launch_fd, &mut pid as *mut i32 as *mut c_void, 4, 0) };
if n != 4 {
return -1;
}
if pid > 0 && !argv.is_empty() {
let mut joined: Vec<u8> = Vec::with_capacity(256);
for a in argv {
joined.extend_from_slice(a.as_bytes());
joined.push(0);
if joined.len() >= 4096 {
break;
}
}
proc_cmdline().lock().unwrap().insert(pid, joined);
proc_exe()
.lock()
.unwrap()
.insert(pid, path.unwrap_or(&argv[0]).as_bytes().to_vec());
}
pid
}
fn launch_restore_cell_vpid(
launch_fd: c_int,
snapshot_dir: &std::path::Path,
restore_vpid: state_snap::Vpid,
restore_warm: bool,
detached: bool,
slot: Option<u64>,
stop_before_resume: bool,
) -> i32 {
let req = LaunchReq {
path: Some("/.supermachine/restore".to_string()),
argv: vec!["/.supermachine/restore".to_string()],
warm: restore_warm,
env: Vec::new(),
cwd: None,
restore_dir: Some(snapshot_dir.to_string_lossy().into_owned()),
restore_vpid,
restore_warm,
detached,
slot,
stop_before_resume,
stop_before_start: true,
capture: None,
};
let json = match serde_json::to_vec(&req) {
Ok(j) => j,
Err(_) => return -1,
};
unsafe { libc::send(launch_fd, json.as_ptr() as *const c_void, json.len(), 0) };
let mut pid: i32 = -1;
let n = unsafe { libc::recv(launch_fd, &mut pid as *mut i32 as *mut c_void, 4, 0) };
if n != 4 {
return -1;
}
if pid > 0 {
proc_cmdline()
.lock()
.unwrap()
.insert(pid, b"/.supermachine/restore\0".to_vec());
proc_exe()
.lock()
.unwrap()
.insert(pid, b"/.supermachine/restore".to_vec());
}
pid
}
fn launch_restore_cell(launch_fd: c_int, snapshot_dir: &std::path::Path) -> i32 {
launch_restore_cell_vpid(launch_fd, snapshot_dir, 1, true, false, None, false)
}
#[cfg(all(target_os = "linux", target_arch = "x86_64"))]
fn continue_launched_cell(pid: i32) {
if pid > 0 {
wait_pid_stopped(pid, 5_000);
unsafe {
libc::kill(pid, libc::SIGCONT);
}
}
}
#[cfg(all(target_os = "linux", target_arch = "x86_64"))]
fn wait_pid_stopped(pid: i32, timeout_ms: u64) -> bool {
let deadline = now_ms().saturating_add(timeout_ms as i64);
loop {
if let Ok(status) = std::fs::read_to_string(format!("/proc/{pid}/status")) {
if status
.lines()
.any(|line| line.starts_with("State:") && line.contains("stopped"))
{
return true;
}
}
if now_ms() >= deadline {
return false;
}
let ts = [0i64, 1_000_000]; unsafe {
host(SYS_NANOSLEEP, ts.as_ptr() as u64, 0, 0, 0, 0, 0);
}
}
}
fn await_launched_exit(launch_fd: c_int, pid: i32, timeout_ms: u64) -> (i32, Option<SentryError>) {
let deadline = if timeout_ms > 0 {
now_ms().saturating_add(timeout_ms as i64)
} else {
i64::MAX
};
loop {
let remaining: i64 = if timeout_ms > 0 {
(deadline - now_ms()).max(0)
} else {
-1
};
let mut pf = pollfd_bytes(launch_fd, POLLIN_BIT);
let r = unsafe {
host(
SYS_POLL,
pf.as_mut_ptr() as u64,
1,
remaining as u64,
0,
0,
0,
)
};
if r > 0 {
let mut st: c_int = 0;
let n = unsafe { libc::recv(launch_fd, &mut st as *mut c_int as *mut c_void, 4, 0) };
if n != 4 {
return (-1, None);
}
remove_cgroup(pid);
return classify_status(st);
}
if r == 0 && timeout_ms > 0 {
unsafe {
if libc::getpgid(pid) == pid {
libc::kill(-pid, libc::SIGKILL);
} else {
libc::kill(pid, libc::SIGKILL);
}
}
let mut st: c_int = 0;
let _ = unsafe { libc::recv(launch_fd, &mut st as *mut c_int as *mut c_void, 4, 0) };
remove_cgroup(pid);
return (128 + libc::SIGKILL, Some(SentryError::Timeout));
}
}
}
#[cfg(all(target_os = "linux", target_arch = "x86_64"))]
fn run_exec_full(
launch_fd: c_int,
root: Option<&std::path::Path>,
base_cfg: &SandboxCfg,
body: &[u8],
cap: c_int,
warm_pid: i32,
) -> i32 {
let close = |cap: c_int| unsafe { libc::close(cap) };
let v: serde_json::Value = match serde_json::from_slice(body) {
Ok(v) => v,
Err(_) => {
close(cap);
return 127;
}
};
if let Some(root) = root {
if let Some(sf) = v.get("stage_files").and_then(|s| s.as_array()) {
for f in sf {
let path = f.get("path").and_then(|p| p.as_str()).unwrap_or("");
let data = f.get("data_b64").and_then(|d| d.as_str()).unwrap_or("");
let mode = f.get("mode").and_then(|m| m.as_u64());
let bytes = match crate::api::b64_decode(data) {
Ok(b) => b,
Err(_) => {
close(cap);
return 127;
}
};
if write_in_root(root, path, &bytes, mode).is_err() {
close(cap);
return 127;
}
}
}
}
let req_env = json_env(v.get("env"));
let merged = merge_env(&base_cfg.env, &req_env);
let env_kv: Vec<String> = merged.iter().map(|(k, val)| format!("{k}={val}")).collect();
let cwd = v.get("cwd").and_then(|c| c.as_str()).map(|s| s.to_string());
let mut argvs: Vec<Vec<String>> = vec![json_str_vec(v.get("argv"))];
for c in v
.get("chain")
.and_then(|c| c.as_array())
.into_iter()
.flatten()
{
argvs.push(json_str_vec(Some(c)));
}
let mut last = 0i32;
for av in &argvs {
if av.is_empty() {
continue;
}
let argv = av.clone();
let path = if let Some(root) = root {
match resolve_in_root_path(root, &path_from_env(&merged), &argv[0]) {
Some(p) => Some(canonical_exe_in_root(root, &p).unwrap_or(p)),
None => {
last = 127; break;
}
}
} else {
None
};
let pid = launch_cell_env(
launch_fd,
path.as_deref(),
&argv,
false,
&env_kv,
cwd.as_deref(),
);
if pid <= 0 {
last = 127;
break;
}
if let Some(c) = &cwd {
cwds().lock().unwrap().insert(pid, c.as_bytes().to_vec());
}
seed_cell_fds_capture(pid, cap); unsafe { set_ring_owner(0, pid, pid) };
continue_launched_cell(pid);
let (c, _e) = await_launched_exit(launch_fd, pid, base_cfg.exec_timeout_ms);
shm_reap_pid(pid);
fd_drop(pid);
free_slots_of(pid);
if base_cfg.preserve_stragglers {
preserve_stragglers_as_running(if warm_pid > 0 { warm_pid } else { 0 });
} else {
sweep_stragglers(if warm_pid > 0 { warm_pid } else { 0 });
}
if warm_pid > 0 {
unsafe { set_ring_owner(0, warm_pid, warm_pid) };
}
last = c;
if last != 0 {
break;
}
}
close(cap); last
}
fn persistent_supervisor_main(
ctrl_fd: c_int,
root: Option<&std::path::Path>,
cfg: &SandboxCfg,
exec_sock: Option<std::path::PathBuf>,
) -> ! {
setup_sandbox_env(root, cfg);
install_child_subreaper(b"persistent");
if exec_sock.is_some() {
OWN_LOOPBACK.store(true, Ordering::Relaxed);
}
let root_owned = root.map(|r| r.to_path_buf());
let mut lsv = [0 as c_int; 2];
if unsafe { libc::socketpair(libc::AF_UNIX, libc::SOCK_SEQPACKET, 0, lsv.as_mut_ptr()) } < 0 {
die(b"sentry: launcher socketpair failed\n");
}
let (launch_fd, launch_child_fd) = (lsv[0], lsv[1]);
let lpid = unsafe { libc::fork() };
if lpid == 0 {
unsafe { libc::close(launch_fd) };
cell_fork_launcher(launch_child_fd, root_owned.clone()); }
unsafe { libc::close(launch_child_fd) };
drop_supervisor_caps();
harden_supervisor();
let _ = std::thread::Builder::new().spawn(|| servicer_loop(0));
spawn_deleg_servicers();
let _ = std::thread::Builder::new().spawn(fd_leak_sweeper);
let _ = std::thread::Builder::new().spawn(move || adopted_orphan_reaper(lpid));
{
let mut delay: Option<u64> = std::env::var("SENTRY_HANGDUMP")
.ok()
.map(|v| v.trim().parse().unwrap_or(8));
if delay.is_none() {
let mut buf = [0u8; 16];
let fd = unsafe {
host(
SYS_OPENAT,
(-100i64) as u64,
b"/tmp/sentry_hangdump\0".as_ptr() as u64,
libc::O_RDONLY as u64,
0,
0,
0,
)
};
if fd >= 0 {
let n = unsafe {
host(
SYS_READ,
fd as u64,
buf.as_mut_ptr() as u64,
buf.len() as u64,
0,
0,
0,
)
};
unsafe { host(SYS_CLOSE, fd as u64, 0, 0, 0, 0, 0) };
delay = Some(if n > 0 {
std::str::from_utf8(&buf[..n as usize])
.ok()
.and_then(|s| s.trim().parse().ok())
.unwrap_or(8)
} else {
8
});
}
}
ipc_logf(
&[(
b"HANGDUMP-GATE delay=",
delay.map(|d| d as i64).unwrap_or(-1),
)],
b"",
);
if let Some(d) = delay {
let _ = std::thread::Builder::new().spawn(move || hang_dump_watchdog(d));
}
}
if let Some(sock_path) = exec_sock {
use std::os::unix::net::UnixListener;
let _ = std::fs::remove_file(&sock_path);
match UnixListener::bind(&sock_path) {
Ok(listener) => {
let root_owned2 = root_owned.clone();
let cfg2 = cfg.clone();
let launch_mu = std::sync::Arc::new(Mutex::new(launch_fd));
let _ = std::thread::Builder::new()
.name("sentry-exec-accept-inproc".into())
.spawn(move || {
let workload: std::sync::Arc<Mutex<Option<Child>>> =
std::sync::Arc::new(Mutex::new(None));
for conn in listener.incoming() {
match conn {
Ok(stream) => {
let root_for_conn =
root_owned2.clone().unwrap_or_else(|| "/".into());
let cfg_for_conn = cfg2.clone();
let workload_for_conn = workload.clone();
let launch_for_conn = launch_mu.clone();
let _ = std::thread::Builder::new()
.name("sentry-exec-conn-inproc".into())
.spawn(move || {
let _ = handle_conn(
stream,
&root_for_conn,
&cfg_for_conn,
&workload_for_conn,
Some(launch_for_conn),
);
});
}
Err(_) => break,
}
}
});
}
Err(_) => die(b"sentry: unified exec socket bind failed\n"),
}
}
let mut warm_pid: i32 = 0; let mut pending_live_checkout: i32 = 0; let mut acq_seq: u32 = 0; loop {
let (ctrl, cap_fd) = recv_ctrl(ctrl_fd);
match ctrl {
Ctrl::Quit => unsafe {
loopring_dump();
if let Some(f) = cap_fd {
libc::close(f);
}
if warm_pid > 0 {
libc::kill(warm_pid, libc::SIGKILL);
}
if pending_live_checkout > 0 {
libc::kill(pending_live_checkout, libc::SIGKILL);
}
for pid in running_instances().lock().unwrap().drain(..) {
if pid > 1 {
libc::kill(pid, libc::SIGKILL);
}
}
libc::_exit(0);
},
Ctrl::Exec(argv) if !argv.is_empty() => {
let pid = launch_cell(launch_fd, &argv, false);
let (code, err) = if pid > 0 {
seed_cell_fds(pid);
unsafe { set_ring_owner(0, pid, pid) };
continue_launched_cell(pid);
let (c, e) = await_launched_exit(launch_fd, pid, cfg.exec_timeout_ms);
shm_reap_pid(pid);
fd_drop(pid);
free_slots_of(pid);
if cfg.preserve_stragglers {
preserve_stragglers_as_running(0);
} else {
sweep_stragglers(0);
}
if warm_pid > 0 {
unsafe { set_ring_owner(0, warm_pid, warm_pid) };
}
(c, e)
} else {
(-1, None)
};
reply_status(ctrl_fd, code, err);
}
Ctrl::ExecCapture(argv) if !argv.is_empty() && cap_fd.is_some() => {
let cap = cap_fd.unwrap();
let pid = launch_cell(launch_fd, &argv, false);
let (code, err) = if pid > 0 {
seed_cell_fds_capture(pid, cap);
unsafe { set_ring_owner(0, pid, pid) }; continue_launched_cell(pid);
unsafe { libc::close(cap) }; let (c, e) = await_launched_exit(launch_fd, pid, cfg.exec_timeout_ms);
shm_reap_pid(pid);
fd_drop(pid);
free_slots_of(pid);
if cfg.preserve_stragglers {
preserve_stragglers_as_running(0);
} else {
sweep_stragglers(0);
}
if warm_pid > 0 {
unsafe { set_ring_owner(0, warm_pid, warm_pid) };
}
(c, e)
} else {
unsafe { libc::close(cap) };
(-1, None)
};
reply_status(ctrl_fd, code, err);
}
Ctrl::ExecFull(body) if cap_fd.is_some() => {
let cap = cap_fd.unwrap();
let code =
run_exec_full(launch_fd, root_owned.as_deref(), cfg, &body, cap, warm_pid);
reply_status(ctrl_fd, code, None);
}
Ctrl::WarmRestore(snapshot_dir) if !snapshot_dir.is_empty() => {
OWN_LOOPBACK.store(true, Ordering::Relaxed);
let snapshot_dir = std::path::PathBuf::from(snapshot_dir);
let state = match read_state_snapshot_dir(&snapshot_dir) {
Ok(s) => s,
Err(e) => {
eprintln!(
"sentry snapshot restore failed reading {}: {e}",
snapshot_dir.display()
);
reply_code(ctrl_fd, -e.raw_os_error().unwrap_or(libc::EIO));
continue;
}
};
let mut bindings: Vec<(state_snap::Vpid, i32)> = Vec::new();
let mut restored_live: Vec<i32> = Vec::new();
let mut restored_live_root: i32 = 0;
let mut restore_err: Option<std::io::Error> = None;
let live_tree = state.entry_kind == state_snap::SnapshotEntryKind::LiveTree;
let mut procs = state.proctree.procs.clone();
if procs.is_empty() {
procs.push(state_snap::GuestProcRecord {
vpid: 1,
parent_vpid: 0,
pgid: 0,
sid: 0,
threads: Vec::new(),
state: state_snap::ProcState::Running,
});
}
procs.sort_by_key(|p| (p.vpid != 1, p.parent_vpid, p.vpid));
for rec in &procs {
if rec.vpid == 1 {
if live_tree {
let slot = alloc_slot();
let slot_u64 = slot as u64;
if slot as usize >= MAX_SLOTS || !ensure_servicer(slot_u64) {
restore_err = Some(std::io::Error::new(
std::io::ErrorKind::Other,
"snapshot: failed to allocate ring slot for restored root process",
));
break;
}
let pid = launch_restore_cell_vpid(
launch_fd,
&snapshot_dir,
rec.vpid,
false,
true,
Some(slot_u64),
true,
);
if pid <= 0 {
free_slot(slot as u32);
restore_err = Some(std::io::Error::new(
std::io::ErrorKind::Other,
"snapshot: failed to launch restored root process",
));
break;
}
unsafe { set_ring_owner(slot_u64, pid, pid) };
continue_launched_cell(pid);
if !wait_pid_stopped(pid, 5_000) {
restore_err = Some(std::io::Error::new(
std::io::ErrorKind::TimedOut,
"snapshot: restored root did not stop before resume",
));
break;
}
bindings.push((1, pid));
restored_live.push(pid);
restored_live_root = pid;
} else {
let pid = launch_restore_cell(launch_fd, &snapshot_dir);
if pid <= 0 {
restore_err = Some(std::io::Error::new(
std::io::ErrorKind::Other,
"snapshot: failed to launch restored root process",
));
break;
}
seed_cell_fds(pid);
unsafe { set_ring_owner(0, pid, pid) };
continue_launched_cell(pid);
bindings.push((1, pid));
warm_pid = pid;
}
} else {
let slot = alloc_slot();
let slot_u64 = slot as u64;
if slot as usize >= MAX_SLOTS || !ensure_servicer(slot_u64) {
restore_err = Some(std::io::Error::new(
std::io::ErrorKind::Other,
"snapshot: failed to allocate ring slot for restored process",
));
break;
}
let pid = launch_restore_cell_vpid(
launch_fd,
&snapshot_dir,
rec.vpid,
false,
true,
Some(slot_u64),
true,
);
if pid <= 0 {
free_slot(slot as u32);
restore_err = Some(std::io::Error::new(
std::io::ErrorKind::Other,
"snapshot: failed to launch restored child process",
));
break;
}
unsafe { set_ring_owner(slot_u64, pid, pid) };
continue_launched_cell(pid);
if !wait_pid_stopped(pid, 5_000) {
restore_err = Some(std::io::Error::new(
std::io::ErrorKind::TimedOut,
"snapshot: restored child did not stop before resume",
));
break;
}
bindings.push((rec.vpid, pid));
restored_live.push(pid);
}
}
if restore_err.is_none() {
proctree::restore_bindings(&state.proctree, &bindings);
for (idx, (vpid, pid)) in bindings.iter().copied().enumerate() {
if let Err(e) =
restore_supervisor_state_for_vpid(pid, vpid, &state, idx == 0)
{
restore_err = Some(e);
break;
}
}
}
if let Some(e) = restore_err {
eprintln!("sentry snapshot restore supervisor state failed: {e}");
for (_, pid) in &bindings {
proctree::mark_dead(*pid);
unsafe { libc::kill(*pid, libc::SIGKILL) };
}
warm_pid = 0;
reply_code(ctrl_fd, -e.raw_os_error().unwrap_or(libc::EIO));
continue;
}
for pid in &restored_live {
unsafe { libc::kill(*pid, libc::SIGCONT) };
}
running_instances().lock().unwrap().extend(restored_live);
if live_tree {
if restored_live_root > 0 {
pending_live_checkout = restored_live_root;
reply_code(ctrl_fd, 0);
} else {
reply_code(ctrl_fd, -1);
}
} else if warm_pid > 0 {
let ready = ring_word(unsafe { std::ptr::addr_of_mut!((*ZYG).ready) });
if wait_ready_deadline(ready, warm_pid, cfg.exec_timeout_ms) {
reply_code(ctrl_fd, 0);
} else {
proctree::mark_dead(warm_pid);
unsafe { libc::kill(warm_pid, libc::SIGKILL) };
warm_pid = 0;
reply_code(ctrl_fd, -1);
}
} else {
reply_code(ctrl_fd, -1);
}
}
Ctrl::Warm(argv) if !argv.is_empty() => {
OWN_LOOPBACK.store(true, Ordering::Relaxed);
let pid = launch_cell(launch_fd, &argv, true);
if pid > 0 {
seed_cell_fds(pid);
proctree::register_main(pid);
unsafe { set_ring_owner(0, pid, pid) };
continue_launched_cell(pid);
warm_pid = pid;
let ready = ring_word(unsafe { std::ptr::addr_of_mut!((*ZYG).ready) });
if wait_ready_deadline(ready, pid, cfg.exec_timeout_ms) {
reply_code(ctrl_fd, 0);
} else {
proctree::mark_dead(pid);
warm_pid = 0;
reply_code(ctrl_fd, -1);
}
} else {
reply_code(ctrl_fd, -1);
}
}
Ctrl::Acquire if warm_pid > 0 => {
acq_seq += 1;
let code = zygote_acquire(acq_seq);
sweep_stragglers(warm_pid);
reply_code(ctrl_fd, code);
}
Ctrl::AcquireCapture if warm_pid > 0 && cap_fd.is_some() => {
let cap = cap_fd.unwrap();
let (old1, old2) = warm_capture_redirect(warm_pid, cap);
unsafe { libc::close(cap) }; acq_seq += 1;
let code = zygote_acquire(acq_seq);
sweep_stragglers(warm_pid);
warm_capture_restore(warm_pid, old1, old2);
reply_code(ctrl_fd, code);
}
Ctrl::AcquireRunning if pending_live_checkout > 0 => {
let pid = pending_live_checkout;
pending_live_checkout = 0;
reply_code(ctrl_fd, pid);
}
Ctrl::AcquireRunning if warm_pid > 0 => {
acq_seq += 1;
unsafe { (*ZYG).run_mode = 1 };
let _ = zygote_acquire(acq_seq);
let inst_pid = unsafe { (*ZYG).inst_pid };
unsafe { (*ZYG).run_mode = 0 }; if inst_pid > 1 {
running_instances().lock().unwrap().push(inst_pid);
}
sweep_stragglers(warm_pid);
reply_code(ctrl_fd, inst_pid);
}
Ctrl::Release(victim) if warm_pid > 0 => {
running_instances().lock().unwrap().retain(|&p| p != victim);
if victim > 1 {
unsafe { libc::kill(victim, libc::SIGKILL) };
}
acq_seq += 1;
unsafe {
(*ZYG).inst_pid = victim;
(*ZYG).run_mode = 2;
}
let _ = zygote_acquire(acq_seq);
unsafe { (*ZYG).run_mode = 0 };
reap_dead_pid(victim);
sweep_stragglers(warm_pid);
reply_code(ctrl_fd, 0);
}
Ctrl::Release(victim) => {
running_instances().lock().unwrap().retain(|&p| p != victim);
if victim > 1 {
unsafe { libc::kill(victim, libc::SIGKILL) };
let mut st: c_int = 0;
unsafe { libc::waitpid(victim, &mut st, 0) };
reap_dead_pid(victim);
}
sweep_stragglers(0);
reply_code(ctrl_fd, 0);
}
Ctrl::Snapshot(dest) if !dest.is_empty() => {
let code = match capture_state_snapshot(
launch_fd,
warm_pid,
std::path::Path::new(&dest),
) {
Ok(()) => 0,
Err(e) => {
eprintln!("sentry snapshot capture failed for pid {warm_pid}: {e}");
-e.raw_os_error().unwrap_or(libc::EIO)
}
};
reply_code(ctrl_fd, code);
}
_ => {
if let Some(f) = cap_fd {
unsafe { libc::close(f) };
}
reply_code(ctrl_fd, -1);
}
}
}
}
#[derive(Default)]
struct PoolMetrics {
execs: std::sync::atomic::AtomicU64, ok: std::sync::atomic::AtomicU64, nonzero: std::sync::atomic::AtomicU64, seccomp: std::sync::atomic::AtomicU64, signaled: std::sync::atomic::AtomicU64, timeouts: std::sync::atomic::AtomicU64, errors: std::sync::atomic::AtomicU64, total_ns: std::sync::atomic::AtomicU64, max_ns: std::sync::atomic::AtomicU64, }
#[derive(Debug, Clone, Copy)]
pub struct SentryPoolStats {
pub execs: u64,
pub ok: u64,
pub nonzero: u64,
pub seccomp_kills: u64,
pub signaled: u64,
pub timeouts: u64,
pub errors: u64,
pub mean_exec_ms: f64,
pub max_exec_ms: f64,
}
fn live_ctrl_fds() -> &'static Mutex<Vec<c_int>> {
static R: std::sync::OnceLock<Mutex<Vec<c_int>>> = std::sync::OnceLock::new();
R.get_or_init(|| Mutex::new(Vec::new()))
}
pub struct Pool {
ctrl: std::sync::atomic::AtomicI32,
pid: c_int,
lock: Mutex<()>,
cgroup_name: Option<String>,
metrics: PoolMetrics,
exec_sock: Option<std::path::PathBuf>,
}
impl Pool {
pub fn new(root: Option<&std::path::Path>) -> std::io::Result<Pool> {
Pool::start(root, &SandboxCfg::default(), None)
}
pub fn exec_socket_path(&self) -> Option<&std::path::Path> {
self.exec_sock.as_deref()
}
fn start(
root: Option<&std::path::Path>,
cfg: &SandboxCfg,
exec_sock: Option<std::path::PathBuf>,
) -> std::io::Result<Pool> {
let mut sv = [0 as c_int; 2];
if unsafe { libc::socketpair(libc::AF_UNIX, libc::SOCK_SEQPACKET, 0, sv.as_mut_ptr()) } < 0
{
return Err(std::io::Error::last_os_error());
}
let (lib_fd, sup_fd) = (sv[0], sv[1]);
let cgroup_name = cfg.cgroup_name.clone();
let inherited_ctrl: Vec<c_int> = live_ctrl_fds().lock().unwrap().clone();
let cfg_mfd = write_reexec_config(&ReexecConfig::Persistent {
root: root.map(|p| p.to_path_buf()),
cfg: cfg.clone(),
exec_sock: exec_sock.clone(),
})?;
let reexec_exe = reexec_exe_cstring()?;
let pid = unsafe { libc::fork() };
if pid < 0 {
unsafe {
libc::close(lib_fd);
libc::close(sup_fd);
libc::close(cfg_mfd);
}
return Err(std::io::Error::last_os_error());
}
if pid == 0 {
unsafe {
libc::close(lib_fd);
for fd in inherited_ctrl {
libc::close(fd); }
let cfg_hi = libc::fcntl(cfg_mfd, libc::F_DUPFD, 20);
let sup_hi = libc::fcntl(sup_fd, libc::F_DUPFD, 20);
libc::dup2(cfg_hi, REEXEC_CONFIG_FD);
libc::dup2(sup_hi, REEXEC_CTRL_FD);
libc::close(STATUS_PIPE_FD); let lo = REEXEC_CTRL_FD as u64 + 1;
if host(SYS_CLOSE_RANGE, lo, u32::MAX as u64, 0, 0, 0, 0) < 0 {
let mut lim: libc::rlimit = std::mem::zeroed();
let max = if libc::getrlimit(libc::RLIMIT_NOFILE, &mut lim) == 0 {
(lim.rlim_cur as c_int).clamp(64, 4096)
} else {
1024
};
for fd in lo as c_int..max {
libc::close(fd);
}
}
let argv: [*const c_char; 3] = [
reexec_exe.as_ptr(),
REEXEC_MARKER.as_ptr() as *const c_char,
std::ptr::null(),
];
libc::execve(reexec_exe.as_ptr(), argv.as_ptr(), current_environ());
libc::_exit(127); }
}
unsafe {
libc::close(sup_fd);
libc::close(cfg_mfd);
};
live_ctrl_fds().lock().unwrap().push(lib_fd);
Ok(Pool {
ctrl: std::sync::atomic::AtomicI32::new(lib_fd),
pid,
lock: Mutex::new(()),
cgroup_name,
metrics: PoolMetrics::default(),
exec_sock,
})
}
fn recv_status(fd: c_int) -> std::io::Result<(i32, Option<SentryError>)> {
let mut buf = [0u8; 8];
let n = unsafe { libc::recv(fd, buf.as_mut_ptr() as *mut c_void, 8, 0) };
if n != 8 {
return Err(std::io::Error::new(
std::io::ErrorKind::BrokenPipe,
"supervisor closed",
));
}
let code = i32::from_le_bytes(buf[0..4].try_into().unwrap());
let tag = u32::from_le_bytes(buf[4..8].try_into().unwrap());
Ok((code, tag_to_err(code, tag)))
}
fn request(&self, op: u8, argv: &[String]) -> std::io::Result<(i32, Option<SentryError>)> {
let mut req = vec![op];
for a in argv {
req.extend_from_slice(a.as_bytes());
req.push(0);
}
let _g = self.lock.lock().unwrap();
let fd = self.ctrl.load(Ordering::Relaxed);
let sent = unsafe {
libc::send(
fd,
req.as_ptr() as *const c_void,
req.len(),
libc::MSG_NOSIGNAL,
)
};
if sent < 0 {
return Err(std::io::Error::last_os_error());
}
Self::recv_status(fd)
}
fn record_exec(
&self,
op: &'static str,
argv: &[String],
code: Option<i32>,
reason: Option<&SentryError>,
dur: std::time::Duration,
) {
use std::sync::atomic::Ordering::Relaxed;
let m = &self.metrics;
m.execs.fetch_add(1, Relaxed);
let ns = dur.as_nanos().min(u64::MAX as u128) as u64;
m.total_ns.fetch_add(ns, Relaxed);
m.max_ns.fetch_max(ns, Relaxed);
let outcome = match (code, reason) {
(None, _) => {
m.errors.fetch_add(1, Relaxed);
"error"
}
(Some(_), Some(SentryError::SeccompViolation { .. })) => {
m.seccomp.fetch_add(1, Relaxed);
"seccomp_kill"
}
(Some(_), Some(SentryError::Signaled { .. })) => {
m.signaled.fetch_add(1, Relaxed);
"signaled"
}
(Some(_), Some(SentryError::Timeout)) => {
m.timeouts.fetch_add(1, Relaxed);
"timeout"
}
(Some(0), None) => {
m.ok.fetch_add(1, Relaxed);
"ok"
}
(Some(_), None) => {
m.nonzero.fetch_add(1, Relaxed);
"nonzero"
}
};
tracing::info!(
target: "supermachine.sentry.exec",
op,
argv = ?argv,
code = code.unwrap_or(-1),
outcome,
dur_ms = dur.as_secs_f64() * 1e3,
"sentry exec"
);
}
pub fn stats(&self) -> SentryPoolStats {
use std::sync::atomic::Ordering::Relaxed;
let m = &self.metrics;
let execs = m.execs.load(Relaxed);
let total_ns = m.total_ns.load(Relaxed);
SentryPoolStats {
execs,
ok: m.ok.load(Relaxed),
nonzero: m.nonzero.load(Relaxed),
seccomp_kills: m.seccomp.load(Relaxed),
signaled: m.signaled.load(Relaxed),
timeouts: m.timeouts.load(Relaxed),
errors: m.errors.load(Relaxed),
mean_exec_ms: if execs > 0 {
(total_ns as f64 / execs as f64) / 1e6
} else {
0.0
},
max_exec_ms: m.max_ns.load(Relaxed) as f64 / 1e6,
}
}
pub fn exec<I, S>(&self, argv: I) -> std::io::Result<i32>
where
I: IntoIterator<Item = S>,
S: Into<String>,
{
self.exec_classified(argv).map(|(c, _)| c)
}
pub fn exec_classified<I, S>(&self, argv: I) -> std::io::Result<(i32, Option<SentryError>)>
where
I: IntoIterator<Item = S>,
S: Into<String>,
{
let v: Vec<String> = argv.into_iter().map(Into::into).collect();
if v.is_empty() {
return Err(std::io::Error::from(std::io::ErrorKind::InvalidInput));
}
let t0 = std::time::Instant::now();
let r = self.request(b'E', &v);
let (code, reason) = match &r {
Ok((c, rn)) => (Some(*c), rn.as_ref()),
Err(_) => (None, None),
};
self.record_exec("exec", &v, code, reason, t0.elapsed());
r
}
pub fn exec_capture<I, S>(&self, argv: I) -> std::io::Result<(i32, Vec<u8>)>
where
I: IntoIterator<Item = S>,
S: Into<String>,
{
self.exec_capture_classified(argv).map(|(c, o, _)| (c, o))
}
pub fn exec_capture_classified<I, S>(
&self,
argv: I,
) -> std::io::Result<(i32, Vec<u8>, Option<SentryError>)>
where
I: IntoIterator<Item = S>,
S: Into<String>,
{
let v: Vec<String> = argv.into_iter().map(Into::into).collect();
if v.is_empty() {
return Err(std::io::Error::from(std::io::ErrorKind::InvalidInput));
}
let mut fds = [0 as c_int; 2];
if unsafe { libc::pipe(fds.as_mut_ptr()) } < 0 {
return Err(std::io::Error::last_os_error());
}
let (r, w) = (fds[0], fds[1]);
let t0 = std::time::Instant::now();
let mut req = vec![b'C'];
for a in &v {
req.extend_from_slice(a.as_bytes());
req.push(0);
}
let _g = self.lock.lock().unwrap();
let fd = self.ctrl.load(Ordering::Relaxed);
if let Err(e) = send_ctrl_with_fd(fd, &req, w) {
unsafe {
libc::close(r);
libc::close(w);
}
return Err(e);
}
unsafe { libc::close(w) };
let mut out = Vec::new();
let mut buf = [0u8; 8192];
loop {
let n = unsafe { libc::read(r, buf.as_mut_ptr() as *mut c_void, buf.len()) };
if n > 0 {
out.extend_from_slice(&buf[..n as usize]);
} else if n == 0 {
break;
} else {
let e = std::io::Error::last_os_error();
if e.kind() == std::io::ErrorKind::Interrupted {
continue;
}
break;
}
}
unsafe { libc::close(r) };
let res = Self::recv_status(fd).map(|(code, err)| (code, out, err));
let (code, reason) = match &res {
Ok((c, _, rn)) => (Some(*c), rn.as_ref()),
Err(_) => (None, None),
};
self.record_exec("exec_capture", &v, code, reason, t0.elapsed());
res
}
pub fn exec_capture_full(&self, body: &[u8]) -> std::io::Result<(i32, Vec<u8>)> {
let mut fds = [0 as c_int; 2];
if unsafe { libc::pipe(fds.as_mut_ptr()) } < 0 {
return Err(std::io::Error::last_os_error());
}
let (r, w) = (fds[0], fds[1]);
let t0 = std::time::Instant::now();
let mut req = vec![b'X'];
req.extend_from_slice(body);
let _g = self.lock.lock().unwrap();
let fd = self.ctrl.load(Ordering::Relaxed);
if let Err(e) = send_ctrl_with_fd(fd, &req, w) {
unsafe {
libc::close(r);
libc::close(w);
}
return Err(e);
}
unsafe { libc::close(w) };
let mut out = Vec::new();
let mut buf = [0u8; 8192];
loop {
let n = unsafe { libc::read(r, buf.as_mut_ptr() as *mut c_void, buf.len()) };
if n > 0 {
out.extend_from_slice(&buf[..n as usize]);
} else if n == 0 {
break;
} else {
let e = std::io::Error::last_os_error();
if e.kind() == std::io::ErrorKind::Interrupted {
continue;
}
break;
}
}
unsafe { libc::close(r) };
let res = Self::recv_status(fd);
let (rcode, reason) = match &res {
Ok((c, rn)) => (Some(*c), rn.as_ref()),
Err(_) => (None, None),
};
self.record_exec("exec_capture_full", &[], rcode, reason, t0.elapsed());
let (code, _err) = res?;
Ok((code, out))
}
pub fn warm<I, S>(&self, argv: I) -> std::io::Result<()>
where
I: IntoIterator<Item = S>,
S: Into<String>,
{
let v: Vec<String> = argv.into_iter().map(Into::into).collect();
if v.is_empty() {
return Err(std::io::Error::from(std::io::ErrorKind::InvalidInput));
}
let (code, _) = self.request(b'W', &v)?;
if code < 0 {
return Err(std::io::Error::new(
std::io::ErrorKind::TimedOut,
"sentry warm: cell wedged or died before reaching the SENTINEL checkpoint",
));
}
Ok(())
}
pub fn warm_restore(&self, snapshot_dir: impl AsRef<std::path::Path>) -> std::io::Result<()> {
let path = snapshot_dir.as_ref().to_string_lossy().into_owned();
let (code, _) = self.request(b'Z', &[path])?;
if code == 0 {
Ok(())
} else if code < 0 {
Err(std::io::Error::from_raw_os_error((-code).max(1)))
} else {
Err(std::io::Error::new(
std::io::ErrorKind::Other,
format!("sentry warm_restore returned unexpected code {code}"),
))
}
}
pub fn acquire(&self) -> std::io::Result<i32> {
self.acquire_classified().map(|(c, _)| c)
}
pub fn acquire_classified(&self) -> std::io::Result<(i32, Option<SentryError>)> {
self.request(b'A', &[])
}
pub fn acquire_capture(&self) -> std::io::Result<(i32, Vec<u8>)> {
let mut fds = [0 as c_int; 2];
if unsafe { libc::pipe(fds.as_mut_ptr()) } < 0 {
return Err(std::io::Error::last_os_error());
}
let (r, w) = (fds[0], fds[1]);
let _g = self.lock.lock().unwrap();
let fd = self.ctrl.load(Ordering::Relaxed);
if let Err(e) = send_ctrl_with_fd(fd, b"a", w) {
unsafe {
libc::close(r);
libc::close(w);
}
return Err(e);
}
unsafe { libc::close(w) };
let mut out = Vec::new();
let mut buf = [0u8; 8192];
loop {
let n = unsafe { libc::read(r, buf.as_mut_ptr() as *mut c_void, buf.len()) };
if n > 0 {
out.extend_from_slice(&buf[..n as usize]);
} else if n == 0 {
break;
} else {
let e = std::io::Error::last_os_error();
if e.kind() == std::io::ErrorKind::Interrupted {
continue;
}
break;
}
}
unsafe { libc::close(r) };
let (code, _err) = Self::recv_status(fd)?;
Ok((code, out))
}
pub fn acquire_running(&self) -> std::io::Result<i32> {
let (pid, _) = self.request(b'R', &[])?;
if pid <= 1 {
return Err(std::io::Error::new(
std::io::ErrorKind::Other,
"acquire_running: warm fork failed",
));
}
Ok(pid)
}
pub fn release(&self, inst_pid: i32) -> std::io::Result<()> {
self.request(b'r', &[inst_pid.to_string()]).map(|_| ())
}
pub fn snapshot(&self, dest_dir: impl AsRef<std::path::Path>) -> std::io::Result<()> {
let path = dest_dir.as_ref().to_string_lossy().into_owned();
let (code, _) = self.request(b'S', &[path])?;
if code == 0 {
Ok(())
} else {
Err(std::io::Error::from_raw_os_error((-code).max(1)))
}
}
pub fn stop(&self) {
let fd = self.ctrl.swap(-1, Ordering::Relaxed);
if fd >= 0 {
live_ctrl_fds().lock().unwrap().retain(|&f| f != fd);
unsafe { libc::close(fd) };
let mut st: c_int = 0;
unsafe { libc::waitpid(self.pid, &mut st, 0) };
if let Some(name) = &self.cgroup_name {
remove_cgroup_named(name);
}
if let Some(p) = &self.exec_sock {
let _ = std::fs::remove_file(p);
}
}
}
}
impl Drop for Pool {
fn drop(&mut self) {
self.stop();
}
}
fn sandbox_main(
path: &std::path::Path,
args: &[String],
root: Option<&std::path::Path>,
cfg: &SandboxCfg,
) -> ! {
setup_sandbox_env(root, cfg);
let path_str = path.to_string_lossy().to_string();
let mut cell_args: Vec<String> = Vec::with_capacity(args.len() + 1);
cell_args.push(path_str.clone());
cell_args.extend(args.iter().cloned());
let root_owned = root.map(|r| r.to_path_buf());
set_guest_cmdline(&cell_args);
let pid = unsafe { libc::fork() };
if pid < 0 {
die(b"fork failed\n");
}
if pid == 0 {
cell_main(&path_str, &cell_args, root_owned);
} else {
supervisor_main(pid);
}
}
fn set_guest_cmdline(cell_args: &[String]) {
let mut cl = Vec::new();
for a in cell_args {
cl.extend_from_slice(a.as_bytes());
cl.push(0);
}
*guest_cmdline().lock().unwrap() = cl;
*guest_exe().lock().unwrap() = cell_args
.first()
.map(|s| s.as_bytes().to_vec())
.unwrap_or_default();
}
fn setup_sandbox_env(root: Option<&std::path::Path>, cfg: &SandboxCfg) {
seed_guest_monotonic_clock();
unsafe {
let mut lim: libc::rlimit = std::mem::zeroed();
if libc::getrlimit(libc::RLIMIT_NOFILE, &mut lim) == 0 {
GUEST_NOFILE_DEFAULT_CUR.store(lim.rlim_cur as u64, Ordering::Relaxed);
GUEST_NOFILE_DEFAULT_MAX.store(lim.rlim_max as u64, Ordering::Relaxed);
const WANT: u64 = 1_048_576;
if (lim.rlim_cur as u64) < WANT || (lim.rlim_max as u64) < WANT {
let raised = libc::rlimit {
rlim_cur: WANT as libc::rlim_t,
rlim_max: WANT.max(lim.rlim_max as u64) as libc::rlim_t,
};
if libc::setrlimit(libc::RLIMIT_NOFILE, &raised) != 0 {
let fallback = libc::rlimit {
rlim_cur: WANT.min(lim.rlim_max as u64) as libc::rlim_t,
rlim_max: lim.rlim_max,
};
let _ = libc::setrlimit(libc::RLIMIT_NOFILE, &fallback);
}
let _ = libc::getrlimit(libc::RLIMIT_NOFILE, &mut lim);
}
}
}
emfile_reserve_open();
let _ = host_v6_route();
if cfg.netns {
setup_netns();
}
if let Some(p) = &cfg.egress {
crate::vmm::egress_policy::set(p);
}
*run_uid().lock().unwrap() = cfg.run_uid;
if !cfg.env.is_empty() {
*guest_env().lock().unwrap() = cfg
.env
.iter()
.map(|(k, v)| {
let mut b = k.as_bytes().to_vec();
b.push(b'=');
b.extend_from_slice(v.as_bytes());
b
})
.collect();
}
if let Some(c) = &cfg.cwd {
*guest_cwd_init().lock().unwrap() = Some(c.as_bytes().to_vec());
}
if !cfg.published_ports.is_empty() {
*published_ports().lock().unwrap() = cfg.published_ports.clone();
}
if std::env::var("SENTRY_REWRITE")
.map(|v| v == "1")
.unwrap_or(false)
{
REWRITE_ON.store(true, Ordering::Relaxed);
setup_trampoline();
}
let open_dir = |p: &std::path::Path| -> i32 {
match std::ffi::CString::new(p.to_string_lossy().as_bytes().to_vec()) {
Ok(c) => unsafe { libc::open(c.as_ptr(), libc::O_PATH | libc::O_DIRECTORY) },
Err(_) => -1,
}
};
if let Some(r) = root {
let fd = open_dir(r);
if fd < 0 {
die(b"cannot open root directory\n");
}
ROOT_FD.store(fd, Ordering::Relaxed);
if let Some(canon) = readlink_fd(fd) {
*root_path().lock().unwrap() = canon;
}
ensure_runtime_dirs_in_rootfs(r);
{
let ns = host_resolv_nameserver();
use std::os::unix::fs::PermissionsExt;
let pid = unsafe { libc::getpid() };
let smdir = r.join(".supermachine");
let _ = std::fs::create_dir_all(&smdir);
let ckpt = smdir.join("supermachine-checkpoint");
if !ckpt.exists() {
let tmp = smdir.join(format!(".supermachine-checkpoint.{pid}.tmp"));
if std::fs::write(&tmp, supermachine_checkpoint_elf()).is_ok() {
let _ = std::fs::set_permissions(&tmp, std::fs::Permissions::from_mode(0o755));
if std::fs::rename(&tmp, &ckpt).is_err() {
let _ = std::fs::remove_file(&tmp);
}
} else {
let _ = std::fs::remove_file(&tmp);
}
}
let etc = r.join("etc");
let _ = std::fs::create_dir_all(&etc);
let resolv_tmp = etc.join(format!(".resolv.conf.{pid}.tmp"));
if std::fs::write(
&resolv_tmp,
format!("nameserver {ns}\noptions single-request no-aaaa\n"),
)
.is_ok()
{
if std::fs::rename(&resolv_tmp, etc.join("resolv.conf")).is_err() {
let _ = std::fs::remove_file(&resolv_tmp);
}
} else {
let _ = std::fs::remove_file(&resolv_tmp);
}
}
}
if !cfg.mounts.is_empty() {
let mut mt = mounts().lock().unwrap();
for (guest, host_dir, ro) in &cfg.mounts {
let fd = open_dir(host_dir);
if fd < 0 {
die(b"cannot open mount host directory\n");
}
let host_canonical = readlink_fd(fd).unwrap_or_default();
let mut gp = guest.to_string_lossy().as_bytes().to_vec();
while gp.last() == Some(&b'/') && gp.len() > 1 {
gp.pop();
}
if gp.first() != Some(&b'/') {
die(b"mount guest_path must be absolute\n");
}
mt.push(Mount {
guest_prefix: gp,
dirfd: fd,
host_canonical,
readonly: *ro,
});
}
}
if root.is_some() {
let shm = std::env::temp_dir().join(format!("sentry-shm-{}", unsafe { libc::getpid() }));
let _ = std::fs::remove_dir_all(&shm);
let _ = std::fs::create_dir_all(&shm);
let fd = open_dir(&shm);
if fd >= 0 {
let host_canonical = readlink_fd(fd).unwrap_or_default();
mounts().lock().unwrap().push(Mount {
guest_prefix: b"/dev/shm".to_vec(),
dirfd: fd,
host_canonical,
readonly: false,
});
}
}
unsafe { TRACE = std::env::var("SENTRY_TRACE").is_ok() };
unsafe { IPCTRACE = std::env::var("SENTRY_IPCTRACE").is_ok() };
unsafe { WAITTRACE = std::env::var("SENTRY_WAITTRACE").is_ok() };
unsafe { SYSCALLTRACE = std::env::var("SENTRY_SYSCALLTRACE").is_ok() };
unsafe { FDTRACE = std::env::var("SENTRY_FDTRACE").is_ok() };
unsafe { SLOTDUMP = std::env::var("SENTRY_SLOTDUMP").is_ok() };
REGDIFF.store(
std::env::var("SENTRY_REGDIFF").is_ok(),
std::sync::atomic::Ordering::Relaxed,
);
ENVDBG.store(
std::env::var("SENTRY_ENVDBG").is_ok(),
std::sync::atomic::Ordering::Relaxed,
);
ENVSCAN.store(
std::env::var("SENTRY_ENVSCAN").is_ok(),
std::sync::atomic::Ordering::Relaxed,
);
if cfg.limits.any() {
*cell_limits().lock().unwrap() = cfg.limits.clone();
let shared = cfg.cgroup_name.is_some();
let name = cfg
.cgroup_name
.clone()
.unwrap_or_else(|| format!("sentry-{}", unsafe { libc::getpid() }));
match setup_cgroup(&name, &cfg.limits, !shared) {
Some(dir) => *cgroup_path().lock().unwrap() = Some(dir),
None => {
if unsafe { TRACE } {
log(b"[supervisor] cgroup v2 unavailable; running uncapped\n");
}
}
}
}
let stacks_bytes = std::mem::size_of::<SlotStack>() * MAX_SLOTS;
let tail_bytes = SLOT_WORDS * 8 + 64;
let table_bytes = std::mem::size_of::<Ring>() * MAX_SLOTS + tail_bytes + stacks_bytes;
let rings = unsafe {
libc::mmap(
std::ptr::null_mut(),
table_bytes,
libc::PROT_READ | libc::PROT_WRITE,
libc::MAP_SHARED | libc::MAP_ANONYMOUS | libc::MAP_NORESERVE,
-1,
0,
)
};
if rings == libc::MAP_FAILED {
die(b"ring mmap failed\n");
}
unsafe {
libc::madvise(rings, table_bytes, libc::MADV_DONTDUMP);
}
unsafe {
RINGS = rings as *mut Ring;
std::ptr::write_bytes(rings as *mut u8, 0, table_bytes);
for i in 0..MAX_SLOTS as u64 {
(*ring_at(i)).cancel_efd = -1;
(*ring_at(i)).intr_efd = -1;
}
let tail = (rings as *mut u8).add(std::mem::size_of::<Ring>() * MAX_SLOTS);
SLOT_BM = tail as *mut u64;
*SLOT_BM = 1; ZYG = tail.add(SLOT_WORDS * 8) as *mut ZygShared;
SLOT_STACKS = tail.add(tail_bytes) as *mut SlotStack;
SHARED_ZONE_LO.store(rings as u64, Ordering::Relaxed);
SHARED_ZONE_HI.store(rings as u64 + table_bytes as u64, Ordering::Relaxed);
}
unsafe {
let pool = libc::mmap(
std::ptr::null_mut(),
SHM_POOL_SIZE as usize,
libc::PROT_READ | libc::PROT_WRITE,
libc::MAP_SHARED | libc::MAP_ANONYMOUS | libc::MAP_NORESERVE,
-1,
0,
);
if pool != libc::MAP_FAILED {
libc::madvise(pool, SHM_POOL_SIZE as usize, libc::MADV_DONTDUMP);
SHM_POOL_BASE.store(pool as u64, Ordering::Relaxed);
}
}
if cfg.warm {
WARM_MODE.store(true, Ordering::Relaxed);
}
}
fn ensure_runtime_dirs_in_rootfs(root: &std::path::Path) {
use std::os::unix::fs::PermissionsExt;
let ensure_dir = |rel: &str, mode: u32| {
let p = root.join(rel);
if std::fs::create_dir_all(&p).is_ok() {
let _ = std::fs::set_permissions(&p, std::fs::Permissions::from_mode(mode));
}
};
ensure_dir("tmp", 0o1777);
ensure_dir("tmp/.X11-unix", 0o1777);
if let Ok(entries) = std::fs::read_dir(root.join("tmp/.X11-unix")) {
for entry in entries.flatten() {
let _ = std::fs::remove_file(entry.path());
}
}
if let Ok(entries) = std::fs::read_dir(root.join("tmp")) {
for entry in entries.flatten() {
let name = entry.file_name();
let name = name.to_string_lossy();
if name.starts_with(".X") && name.ends_with("-lock") {
let _ = std::fs::remove_file(entry.path());
}
}
}
ensure_dir("run", 0o755);
ensure_dir("run/dbus", 0o755);
ensure_dir("var/run/dbus", 0o755);
}
pub fn kvm_usable() -> bool {
use std::os::unix::fs::FileTypeExt;
match std::fs::metadata("/dev/kvm") {
Ok(m) if m.file_type().is_char_device() => {}
_ => return false,
}
let c = match std::ffi::CString::new("/dev/kvm") {
Ok(c) => c,
Err(_) => return false,
};
let fd = unsafe { libc::open(c.as_ptr(), libc::O_RDWR | libc::O_CLOEXEC) };
if fd < 0 {
return false;
}
const KVM_CREATE_VM: libc::c_ulong = 0xAE01;
let vmfd = unsafe { libc::ioctl(fd, KVM_CREATE_VM, 0) };
let ok = vmfd >= 0;
if vmfd >= 0 {
unsafe { libc::close(vmfd) };
}
unsafe { libc::close(fd) };
ok
}