use std::collections::HashMap;
use std::collections::HashSet;
use std::fs;
use std::io::Read;
use std::ops::Range;
use std::os::unix::process::CommandExt;
use std::os::unix::process::ExitStatusExt;
use std::path::Path;
use std::path::PathBuf;
use std::process::ExitStatus;
use std::process::Stdio;
use std::sync::Arc;
use std::sync::atomic::AtomicBool;
use std::sync::atomic::Ordering;
use anyhow::Context;
use anyhow::Error;
use anyhow::anyhow;
pub use detcore_model::summary::PathEvidence;
use nix::sys::ptrace;
use nix::sys::signal::Signal;
use nix::sys::wait::WaitPidFlag;
use nix::sys::wait::WaitStatus;
use nix::sys::wait::waitpid;
use nix::unistd::Pid;
use reverie_sabre_stats::SabrePatchRoute;
use reverie_sabre_stats::SabreSlowPath;
use reverie_sabre_stats::SabreStats;
const SYSCALL_INSN: [u8; 2] = [0x0f, 0x05];
const SABRE_SYSCALL_MARKER: [u8; 2] = [0x0f, 0xff];
#[derive(Debug)]
pub struct Output {
pub status: ExitStatus,
pub stdout: Vec<u8>,
pub stderr: Vec<u8>,
pub path_evidence: PathEvidence,
}
#[derive(Clone, Debug, Eq, PartialEq)]
struct MappingClassification {
trusted: bool,
trusted_shared_object: Option<PathBuf>,
}
#[derive(Default)]
struct TraceeState {
pending_patch: Option<PendingPatch>,
pending_bootstrap: Option<i64>,
}
struct PendingPatch {
site: usize,
syscall: u64,
}
struct SignalDiagnostic {
signal: Signal,
si_code: Option<i32>,
si_errno: Option<i32>,
fault_address: Option<usize>,
mapping: Option<MappingDiagnostic>,
instruction_bytes: Option<[u8; 16]>,
registers: libc::user_regs_struct,
}
struct MappingDiagnostic {
line: String,
relative_offset: usize,
file_offset: usize,
}
fn should_replace_signal_diagnostic(
existing: Option<(Signal, Option<i32>, bool)>,
signal: Signal,
si_code: Option<i32>,
) -> bool {
if let Some((Signal::SIGILL, Some(existing_code), existing_is_marker)) = existing
&& signal == Signal::SIGILL
&& existing_code > 0
&& !existing_is_marker
&& si_code.is_none_or(|code| code <= 0)
{
return false;
}
true
}
fn is_sabre_sigill_marker(instruction_bytes: Option<&[u8; 16]>) -> bool {
instruction_bytes
.is_some_and(|bytes| matches!(&bytes[..2], [0x0f, 0xff] | [0x0f, 0x0b] | [0x0f, 0x0c]))
}
fn final_physical_exit(status: &WaitStatus) -> Option<(Pid, ExitStatus)> {
match *status {
WaitStatus::Exited(pid, code) => Some((pid, ExitStatus::from_raw(code << 8))),
WaitStatus::Signaled(pid, signal, core_dumped) => {
let raw = signal as i32 | if core_dumped { 0x80 } else { 0 };
Some((pid, ExitStatus::from_raw(raw)))
}
_ => None,
}
}
#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq, PartialOrd, Ord)]
struct AddressSpaceId(u64);
struct MappingCache {
entries: HashMap<(AddressSpaceId, usize), MappingClassification>,
spaces: HashMap<Pid, AddressSpaceId>,
provisional: HashSet<AddressSpaceId>,
next: u64,
}
impl MappingCache {
fn new(root: Pid) -> Self {
Self {
entries: HashMap::new(),
spaces: HashMap::from([(root, AddressSpaceId(0))]),
provisional: HashSet::new(),
next: 1,
}
}
fn fresh(&mut self) -> AddressSpaceId {
let id = AddressSpaceId(self.next);
self.next += 1;
id
}
fn space_of(&mut self, pid: Pid) -> AddressSpaceId {
if let Some(id) = self.spaces.get(&pid) {
return *id;
}
let id = self.fresh();
self.provisional.insert(id);
self.spaces.insert(pid, id);
id
}
fn get(&mut self, pid: Pid, page: usize) -> Option<&MappingClassification> {
let space = self.space_of(pid);
self.entries.get(&(space, page))
}
fn insert(&mut self, pid: Pid, page: usize, classification: MappingClassification) {
let space = self.space_of(pid);
self.entries.insert((space, page), classification);
}
fn invalidate_address_space(&mut self, pid: Pid) {
let space = self.space_of(pid);
self.entries.retain(|(cached, _), _| *cached != space);
}
fn evict(&mut self, pid: Pid, eviction: &Eviction) {
let space = self.space_of(pid);
if self.provisional.contains(&space) {
self.entries.clear();
return;
}
match eviction {
Eviction::WholeSpace => self.entries.retain(|(cached, _), _| *cached != space),
Eviction::Ranges(ranges) => self.entries.retain(|(cached, page), _| {
*cached != space || !ranges.iter().any(|range| range.contains(page))
}),
}
}
fn share_address_space(&mut self, parent: Pid, child: Pid) {
let space = self.space_of(parent);
self.spaces.insert(child, space);
}
fn new_address_space(&mut self, child: Pid) {
let space = self.fresh();
self.spaces.insert(child, space);
}
fn replace_address_space(&mut self, pid: Pid) {
self.invalidate_address_space(pid);
let space = self.fresh();
self.spaces.insert(pid, space);
}
fn forget(&mut self, pid: Pid) {
let Some(space) = self.spaces.remove(&pid) else {
return;
};
if !self.spaces.values().any(|other| *other == space) {
self.entries.retain(|(cached, _), _| *cached != space);
self.provisional.remove(&space);
}
}
fn admit_child(&mut self, parent: Pid, child: Pid, shares_mm: MmSharing) {
match shares_mm {
MmSharing::Shared => self.share_address_space(parent, child),
MmSharing::Private => self.new_address_space(child),
}
}
}
struct Supervisor {
root: Pid,
tracees: HashSet<Pid>,
states: HashMap<Pid, TraceeState>,
mapping_cache: MappingCache,
sabre_id: Option<FileId>,
plugin_id: Option<FileId>,
readiness: Arc<AtomicBool>,
ready_observed: bool,
patched_sites: HashSet<(Pid, usize)>,
trusted_shared_object_sites: HashSet<(Pid, usize)>,
trusted_shared_objects: HashSet<PathBuf>,
signal_diagnostics: HashMap<Pid, SignalDiagnostic>,
physical_exit_observer: Arc<detcore::GlobalState>,
bootstrap_launch: Option<super::sabre_bootstrap::Launch>,
bootstrap: Option<super::sabre_bootstrap::Bootstrap>,
stats: SabreStats,
}
impl Supervisor {
fn new(
root: Pid,
sabre: PathBuf,
plugin: PathBuf,
readiness: Arc<AtomicBool>,
physical_exit_observer: Arc<detcore::GlobalState>,
bootstrap_launch: super::sabre_bootstrap::Launch,
stats: SabreStats,
) -> Self {
Self {
root,
tracees: HashSet::from([root]),
states: HashMap::from([(root, TraceeState::default())]),
mapping_cache: MappingCache::new(root),
sabre_id: launched_file_id(&sabre),
plugin_id: launched_file_id(&plugin),
readiness,
ready_observed: false,
patched_sites: HashSet::new(),
trusted_shared_object_sites: HashSet::new(),
trusted_shared_objects: HashSet::new(),
signal_diagnostics: HashMap::new(),
physical_exit_observer,
bootstrap_launch: Some(bootstrap_launch),
bootstrap: None,
stats,
}
}
fn run(mut self) -> Result<(ExitStatus, PathEvidence), Error> {
let result = self.run_inner();
if result.is_err() {
for pid in &self.tracees {
let _ = nix::sys::signal::kill(*pid, Signal::SIGKILL);
}
let deadline = std::time::Instant::now() + std::time::Duration::from_secs(2);
while !self.tracees.is_empty() && std::time::Instant::now() < deadline {
let owned: Vec<_> = self.tracees.iter().copied().collect();
for pid in owned {
match waitpid(pid, Some(WaitPidFlag::__WALL | WaitPidFlag::WNOHANG)) {
Ok(status) if final_physical_exit(&status).is_some() => {
self.remove_tracee(pid);
self.physical_exit_observer
.complete_physical_process_exit(pid.as_raw());
}
Ok(
WaitStatus::Stopped(..)
| WaitStatus::PtraceEvent(..)
| WaitStatus::PtraceSyscall(..),
) => {
let _ = ptrace::cont(pid, Some(Signal::SIGKILL));
}
Ok(_) | Err(nix::errno::Errno::EINTR) => {}
Err(error) => {
return Err(anyhow!(
"SaBRe owned-child cleanup refused after {result:?}: {error}"
));
}
}
}
if !self.tracees.is_empty() {
std::thread::sleep(std::time::Duration::from_millis(10));
}
}
if !self.tracees.is_empty() {
return Err(anyhow!(
"SaBRe owned-child cleanup timed out after {result:?}"
));
}
}
result
}
fn run_inner(&mut self) -> Result<(ExitStatus, PathEvidence), Error> {
ptrace::attach(self.root).context("failed to attach SaBRe supervisor worker")?;
match waitpid(self.root, Some(WaitPidFlag::__WALL))? {
WaitStatus::Stopped(pid, Signal::SIGSTOP) if pid == self.root => {}
status => {
return Err(anyhow!(
"unexpected SaBRe supervisor attach stop: {status:?}"
));
}
}
tracing::trace!(
target: "hermit::sabre::fallback",
tid = self.root.as_raw(),
"received supervisor attach stop",
);
self.bootstrap = Some(super::sabre_bootstrap::Bootstrap::new(
self.root,
self.bootstrap_launch
.take()
.ok_or_else(|| anyhow!("bootstrap launch consumed twice"))?,
)?);
self.set_options(self.root)
.context("failed to set options on the initial SaBRe tracee")?;
ptrace::syscall(self.root, None)
.context("failed to resume the initial SaBRe tracee with PTRACE_SYSCALL")?;
let mut root_status = None;
while !self.tracees.is_empty() {
let status = match waitpid(Pid::from_raw(-1), Some(WaitPidFlag::__WALL)) {
Ok(status) => status,
Err(nix::errno::Errno::EINTR) => continue,
Err(error) => return Err(error.into()),
};
tracing::trace!(
target: "hermit::sabre::fallback",
?status,
"received ptrace wait status",
);
if let Some((pid, exit_status)) = final_physical_exit(&status) {
if let WaitStatus::Signaled(_, signal, _) = status
&& let Some(diagnostic) = self
.signal_diagnostics
.get(&pid)
.filter(|diagnostic| diagnostic.signal == signal)
{
let instruction_bytes = diagnostic.instruction_bytes.as_ref().map(|bytes| {
bytes
.iter()
.map(|byte| format!("{byte:02x}"))
.collect::<Vec<_>>()
.join(" ")
});
tracing::error!(
target: "hermit::sabre::fallback",
tid = pid.as_raw(),
?signal,
si_code = diagnostic.si_code,
si_errno = diagnostic.si_errno,
rip = format!("{:#x}", diagnostic.registers.rip),
rsp = format!("{:#x}", diagnostic.registers.rsp),
fault_address = diagnostic
.fault_address
.map(|address| format!("{address:#x}")),
mapping = diagnostic
.mapping
.as_ref()
.map(|mapping| mapping.line.as_str()),
file_offset = diagnostic
.mapping
.as_ref()
.map(|mapping| format!("{:#x}", mapping.file_offset)),
relative_offset = diagnostic
.mapping
.as_ref()
.map(|mapping| format!("{:#x}", mapping.relative_offset)),
instruction_bytes,
rax = format!("{:#x}", diagnostic.registers.rax),
rbx = format!("{:#x}", diagnostic.registers.rbx),
rcx = format!("{:#x}", diagnostic.registers.rcx),
rdx = format!("{:#x}", diagnostic.registers.rdx),
rsi = format!("{:#x}", diagnostic.registers.rsi),
rdi = format!("{:#x}", diagnostic.registers.rdi),
rbp = format!("{:#x}", diagnostic.registers.rbp),
r8 = format!("{:#x}", diagnostic.registers.r8),
r9 = format!("{:#x}", diagnostic.registers.r9),
r10 = format!("{:#x}", diagnostic.registers.r10),
r11 = format!("{:#x}", diagnostic.registers.r11),
r12 = format!("{:#x}", diagnostic.registers.r12),
r13 = format!("{:#x}", diagnostic.registers.r13),
r14 = format!("{:#x}", diagnostic.registers.r14),
r15 = format!("{:#x}", diagnostic.registers.r15),
orig_rax = format!("{:#x}", diagnostic.registers.orig_rax),
eflags = format!("{:#x}", diagnostic.registers.eflags),
cs = format!("{:#x}", diagnostic.registers.cs),
ss = format!("{:#x}", diagnostic.registers.ss),
fs_base = format!("{:#x}", diagnostic.registers.fs_base),
gs_base = format!("{:#x}", diagnostic.registers.gs_base),
"SaBRe tracee terminated by a fatal signal",
);
}
self.remove_tracee(pid);
self.physical_exit_observer
.complete_physical_process_exit(pid.as_raw());
if pid == self.root {
root_status = Some(exit_status);
}
continue;
}
match status {
WaitStatus::PtraceSyscall(pid) => self.handle_syscall_stop(pid)?,
WaitStatus::PtraceEvent(pid, _, event) => self.handle_ptrace_event(pid, event)?,
WaitStatus::Stopped(pid, signal) => {
self.bootstrap
.as_mut()
.expect("attached bootstrap")
.signal(pid, signal)?;
if signal == Signal::SIGSTOP && pid != self.root {
self.states.entry(pid).or_default();
self.tracees.insert(pid);
self.set_options(pid)?;
self.resume(pid, None)?;
} else {
if !matches!(signal, Signal::SIGSTOP | Signal::SIGCHLD) {
let registers = ptrace::getregs(pid).ok();
let rip = registers.as_ref().map_or(0, |registers| registers.rip);
let captures_fault_context = matches!(
signal,
Signal::SIGSEGV
| Signal::SIGILL
| Signal::SIGBUS
| Signal::SIGFPE
| Signal::SIGABRT
);
if captures_fault_context {
let siginfo = ptrace::getsiginfo(pid).ok();
let si_code = siginfo.as_ref().map(|info| info.si_code);
let existing =
self.signal_diagnostics.get(&pid).map(|diagnostic| {
(
diagnostic.signal,
diagnostic.si_code,
is_sabre_sigill_marker(
diagnostic.instruction_bytes.as_ref(),
),
)
});
if should_replace_signal_diagnostic(existing, signal, si_code) {
if let Some(registers) = registers {
let fault_address = siginfo
.as_ref()
.filter(|info| info.si_code > 0)
.map(|info| unsafe { info.si_addr() as usize });
let mapping = fs::read_to_string(format!(
"/proc/{}/maps",
pid.as_raw()
))
.ok()
.and_then(|maps| mapping_diagnostic(&maps, rip as usize));
let instruction_bytes =
read_diagnostic_bytes(pid, rip as usize).ok();
self.signal_diagnostics.insert(
pid,
SignalDiagnostic {
signal,
si_code,
si_errno: siginfo
.as_ref()
.map(|info| info.si_errno),
fault_address,
mapping,
instruction_bytes,
registers,
},
);
} else {
self.signal_diagnostics.remove(&pid);
}
}
}
tracing::debug!(
target: "hermit::sabre::fallback",
tid = pid.as_raw(),
?signal,
rip = format!("{rip:#x}"),
"forwarding signal to tracee",
);
}
self.resume(pid, Some(signal))?;
}
}
WaitStatus::Exited(..) | WaitStatus::Signaled(..) => unreachable!(),
WaitStatus::Continued(_) | WaitStatus::StillAlive => {}
}
}
let status = root_status.ok_or_else(|| anyhow!("SaBRe root tracee disappeared"))?;
let mut trusted_shared_objects = self
.trusted_shared_objects
.iter()
.map(|path| path.display().to_string())
.collect::<Vec<_>>();
trusted_shared_objects.sort();
Ok((
status,
PathEvidence {
schema: PathEvidence::SCHEMA,
guest_rpc_observed: self.readiness.load(Ordering::Acquire),
ptrace_fallback_sites: self.patched_sites.len(),
trusted_shared_object_sites: self.trusted_shared_object_sites.len(),
trusted_shared_objects,
},
))
}
fn set_options(&self, pid: Pid) -> Result<(), Error> {
ptrace::setoptions(
pid,
ptrace::Options::PTRACE_O_EXITKILL
| ptrace::Options::PTRACE_O_TRACESYSGOOD
| ptrace::Options::PTRACE_O_TRACECLONE
| ptrace::Options::PTRACE_O_TRACEFORK
| ptrace::Options::PTRACE_O_TRACEVFORK
| ptrace::Options::PTRACE_O_TRACEEXEC
| ptrace::Options::PTRACE_O_TRACEEXIT,
)?;
Ok(())
}
fn handle_syscall_stop(&mut self, pid: Pid) -> Result<(), Error> {
let syscall_info = get_syscall_info(pid)?;
tracing::trace!(
target: "hermit::sabre::fallback",
tid = pid.as_raw(),
op = syscall_info.op,
"decoded ptrace syscall stop",
);
match syscall_info.op {
libc::PTRACE_SYSCALL_INFO_ENTRY => {
self.stats
.increment_slow_path(SabreSlowPath::PtraceSyscallEntry);
let mut regs = ptrace::getregs(pid)?;
let state = self.states.entry(pid).or_default();
if state.pending_bootstrap.is_some() || state.pending_patch.is_some() {
return Err(anyhow!("overlapping SaBRe syscall-entry state for {pid}"));
}
if let Some(response) = self
.bootstrap
.as_mut()
.expect("attached bootstrap")
.request(pid, ®s)?
{
regs.orig_rax = u64::MAX;
ptrace::setregs(pid, regs)?;
self.states.entry(pid).or_default().pending_bootstrap = Some(response);
return self.resume(pid, None);
}
let site = regs
.rip
.checked_sub(SYSCALL_INSN.len() as u64)
.ok_or_else(|| anyhow!("invalid syscall RIP {:#x} in tracee {pid}", regs.rip))?
as usize;
let bytes = read_two_bytes(pid, site)?;
let fallback_ready = self.fallback_ready()?;
if pid != self.root {
tracing::trace!(
target: "hermit::sabre::fallback",
tid = pid.as_raw(),
nr = regs.orig_rax,
site = format!("{site:#x}"),
raw = bytes == SYSCALL_INSN,
"child syscall entry",
);
}
if bytes == SYSCALL_INSN && fallback_ready {
let mapping = self.classify_mapping(pid, site)?;
if let Some(path) = mapping.trusted_shared_object {
self.trusted_shared_object_sites.insert((pid, site));
self.trusted_shared_objects.insert(path);
}
if !mapping.trusted {
write_two_bytes(pid, site, SABRE_SYSCALL_MARKER)?;
self.stats.record_patch(
site as u64,
SABRE_SYSCALL_MARKER.len() as u8,
SabrePatchRoute::PtraceInstalledMarker,
);
self.stats
.increment_slow_path(SabreSlowPath::PtraceRawSyscallRedirect);
let syscall = regs.orig_rax;
regs.orig_rax = u64::MAX;
ptrace::setregs(pid, regs)?;
self.states.entry(pid).or_default().pending_patch =
Some(PendingPatch { site, syscall });
self.patched_sites.insert((pid, site));
tracing::debug!(
target: "hermit::sabre::fallback",
tid = pid.as_raw(),
address = site,
"redirecting raw syscall instruction through the SaBRe handler",
);
}
}
}
libc::PTRACE_SYSCALL_INFO_EXIT => {
self.stats
.increment_slow_path(SabreSlowPath::PtraceSyscallExit);
let exit_regs = ptrace::getregs(pid)?;
if let Some(response) = self.states.entry(pid).or_default().pending_bootstrap.take()
{
if exit_regs.orig_rax != u64::MAX
|| self.states.entry(pid).or_default().pending_patch.is_some()
{
return Err(anyhow!("mismatched bootstrap syscall exit for {pid}"));
}
let mut regs = exit_regs;
regs.rax = response as u64;
ptrace::setregs(pid, regs)?;
return self.resume(pid, None);
}
if mutates_address_space(exit_regs.orig_rax) {
self.mapping_cache
.evict(pid, &mutation_eviction(&exit_regs));
}
if let Some(pending) = self.states.entry(pid).or_default().pending_patch.take() {
let mut regs = exit_regs;
regs.rax = pending.syscall;
regs.orig_rax = pending.syscall;
regs.rip = pending.site as u64;
ptrace::setregs(pid, regs)?;
}
}
_ => {}
}
ptrace::syscall(pid, None)?;
Ok(())
}
fn handle_ptrace_event(&mut self, pid: Pid, event: libc::c_int) -> Result<(), Error> {
if !self.tracees.contains(&pid) {
return Err(anyhow!("ptrace event is not in the owned SaBRe lineage"));
}
if matches!(
event,
libc::PTRACE_EVENT_CLONE | libc::PTRACE_EVENT_FORK | libc::PTRACE_EVENT_VFORK
) {
let child = Pid::from_raw(ptrace::getevent(pid)? as i32);
self.tracees.insert(child);
self.states.entry(child).or_default();
let shares_mm = mm_sharing(pid, child);
self.mapping_cache.admit_child(pid, child, shares_mm);
} else if event == libc::PTRACE_EVENT_EXEC {
self.mapping_cache.replace_address_space(pid);
self.states.insert(pid, TraceeState::default());
self.signal_diagnostics.remove(&pid);
}
self.bootstrap
.as_mut()
.expect("attached bootstrap")
.event(pid, event)?;
self.resume(pid, None)
}
fn resume(&self, pid: Pid, signal: Option<Signal>) -> Result<(), Error> {
ptrace::syscall(pid, signal)?;
Ok(())
}
fn fallback_ready(&mut self) -> Result<bool, Error> {
let ready = self.readiness.load(Ordering::Acquire);
if ready && !self.ready_observed {
self.ready_observed = true;
tracing::debug!(
target: "hermit::sabre::fallback",
"SaBRe fallback readiness observed",
);
}
Ok(ready)
}
fn classify_mapping(
&mut self,
pid: Pid,
address: usize,
) -> Result<MappingClassification, Error> {
let page = address & !(PAGE_SIZE - 1);
if let Some(classification) = self.mapping_cache.get(pid, page) {
return Ok(classification.clone());
}
let maps = fs::read_to_string(format!("/proc/{}/maps", pid.as_raw()))?;
let classification = mapping_entry(&maps, address).map_or(
MappingClassification {
trusted: false,
trusted_shared_object: None,
},
|entry| classify_mapping(&entry, self.sabre_id.as_ref(), self.plugin_id.as_ref()),
);
self.mapping_cache.insert(pid, page, classification.clone());
Ok(classification)
}
fn remove_tracee(&mut self, pid: Pid) {
if let Some(bootstrap) = self.bootstrap.as_mut() {
bootstrap.forget(pid);
}
self.tracees.remove(&pid);
self.states.remove(&pid);
self.mapping_cache.forget(pid);
self.signal_diagnostics.remove(&pid);
}
}
fn get_syscall_info(pid: Pid) -> Result<libc::ptrace_syscall_info, Error> {
let mut info = std::mem::MaybeUninit::<libc::ptrace_syscall_info>::zeroed();
let size = std::mem::size_of::<libc::ptrace_syscall_info>();
let written = unsafe {
libc::ptrace(
libc::PTRACE_GET_SYSCALL_INFO,
pid.as_raw(),
size as *mut libc::c_void,
info.as_mut_ptr().cast::<libc::c_void>(),
)
};
if written < 0 {
return Err(std::io::Error::last_os_error().into());
}
if written == 0 {
return Err(anyhow!(
"PTRACE_GET_SYSCALL_INFO returned no data for {pid}"
));
}
Ok(unsafe { info.assume_init() })
}
fn mapping_line(maps: &str, address: usize) -> Option<&str> {
maps.lines().find_map(|line| {
let mut fields = line.split_whitespace();
let range = fields.next()?;
let mut limits = range.split('-');
let start = usize::from_str_radix(limits.next()?, 16).ok()?;
let end = usize::from_str_radix(limits.next()?, 16).ok()?;
(start <= address && address < end).then_some(line)
})
}
#[derive(Clone, Debug, Eq, PartialEq)]
struct FileId {
inode: u64,
path: PathBuf,
}
struct MappingEntry<'a> {
inode: Option<u64>,
path: &'a str,
}
impl MappingEntry<'_> {
fn file_id(&self) -> Option<FileId> {
match self.inode {
Some(inode) if inode != 0 => {
let path = fs::canonicalize(Path::new(self.path)).ok()?;
Some(FileId { inode, path })
}
_ => None,
}
}
}
fn launched_file_id(path: &Path) -> Option<FileId> {
use std::os::unix::fs::MetadataExt;
let metadata = fs::metadata(path).ok()?;
let canonical = fs::canonicalize(path).ok()?;
Some(FileId {
inode: metadata.ino(),
path: canonical,
})
}
fn mapping_entry(maps: &str, address: usize) -> Option<MappingEntry<'_>> {
let mut fields = mapping_line(maps, address)?.split_whitespace();
fields.next()?; fields.next()?; fields.next()?; fields.next()?; let inode = fields.next()?;
Some(MappingEntry {
inode: inode.parse::<u64>().ok(),
path: fields.next().unwrap_or(""),
})
}
fn mapping_diagnostic(maps: &str, address: usize) -> Option<MappingDiagnostic> {
let line = mapping_line(maps, address)?;
let mut fields = line.split_whitespace();
let mut limits = fields.next()?.split('-');
let start = usize::from_str_radix(limits.next()?, 16).ok()?;
fields.next()?;
let mapping_offset = usize::from_str_radix(fields.next()?, 16).ok()?;
let relative_offset = address.checked_sub(start)?;
Some(MappingDiagnostic {
line: line.to_owned(),
relative_offset,
file_offset: mapping_offset.checked_add(relative_offset)?,
})
}
const ADDRESS_SPACE_MUTATORS: [u64; 6] = [
libc::SYS_mmap as u64,
libc::SYS_munmap as u64,
libc::SYS_mremap as u64,
libc::SYS_mprotect as u64,
libc::SYS_brk as u64,
libc::SYS_shmat as u64,
];
fn mutates_address_space(nr: u64) -> bool {
ADDRESS_SPACE_MUTATORS.contains(&nr)
}
const PAGE_SIZE: usize = 4096;
const MIN_HUGE_PAGE_SIZE: usize = 2 * 1024 * 1024;
#[derive(Clone, Debug, Eq, PartialEq)]
enum Eviction {
Ranges(Vec<Range<usize>>),
WholeSpace,
}
fn page_range(addr: u64, len: u64) -> Option<Range<usize>> {
let start = usize::try_from(addr).ok()?;
let end = start
.checked_add(usize::try_from(len).ok()?)?
.checked_next_multiple_of(PAGE_SIZE)?;
Some(start & !(PAGE_SIZE - 1)..end)
}
fn contains_huge_page_boundary(range: &Range<usize>) -> bool {
range
.start
.checked_next_multiple_of(MIN_HUGE_PAGE_SIZE)
.is_none_or(|boundary| boundary < range.end)
}
fn mutation_eviction(regs: &libc::user_regs_struct) -> Eviction {
if (-4095..0).contains(&(regs.rax as i64)) {
return Eviction::WholeSpace;
}
let ranges = match regs.orig_rax as libc::c_long {
libc::SYS_mmap => {
let hugetlb_flag = (regs.r10 & libc::MAP_HUGETLB as u64) != 0;
let on_huge_page_boundary = regs.rax.is_multiple_of(MIN_HUGE_PAGE_SIZE as u64);
if hugetlb_flag || on_huge_page_boundary {
None
} else {
page_range(regs.rax, regs.rsi).map(|mapped| vec![mapped])
}
}
libc::SYS_munmap => page_range(regs.rdi, regs.rsi).map(|unmapped| vec![unmapped]),
libc::SYS_mprotect | libc::SYS_pkey_mprotect => {
let grows = (libc::PROT_GROWSDOWN | libc::PROT_GROWSUP) as u64;
if (regs.rdx & grows) != 0 {
None
} else {
page_range(regs.rdi, regs.rsi).map(|protected| vec![protected])
}
}
libc::SYS_mremap if regs.rsi != 0 => {
let new = page_range(regs.rax, regs.rdx);
let requested = page_range(regs.r8, regs.rdx);
let fixed = (regs.r10 & libc::MREMAP_FIXED as u64) != 0;
let mut ranges = vec![page_range(regs.rdi, regs.rsi), new.clone()];
if fixed && requested != new {
ranges.push(requested);
}
ranges
.into_iter()
.collect::<Option<Vec<_>>>()
.filter(|ranges| !ranges.iter().any(contains_huge_page_boundary))
}
_ => None,
};
ranges.map_or(Eviction::WholeSpace, Eviction::Ranges)
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
enum MmSharing {
Shared,
Private,
}
const KCMP_VM: libc::c_int = 1;
fn mm_sharing(parent: Pid, child: Pid) -> MmSharing {
let result = unsafe {
libc::syscall(
libc::SYS_kcmp,
parent.as_raw() as libc::c_long,
child.as_raw() as libc::c_long,
KCMP_VM as libc::c_long,
0 as libc::c_long,
0 as libc::c_long,
)
};
interpret_kcmp_vm(result)
}
fn interpret_kcmp_vm(result: libc::c_long) -> MmSharing {
match result {
0 => MmSharing::Shared,
r if r > 0 => MmSharing::Private,
_ => MmSharing::Shared,
}
}
const CAUSAL_KERNEL_MAPPINGS: [&str; 2] = ["[vdso]", "[vsyscall]"];
fn classify_mapping(
entry: &MappingEntry<'_>,
sabre: Option<&FileId>,
plugin: Option<&FileId>,
) -> MappingClassification {
let path = entry.path;
if path.starts_with('[') {
if CAUSAL_KERNEL_MAPPINGS.contains(&path) {
return MappingClassification {
trusted: true,
trusted_shared_object: Some(PathBuf::from(path)),
};
}
return MappingClassification {
trusted: false,
trusted_shared_object: None,
};
}
let path = Path::new(path.strip_suffix(" (deleted)").unwrap_or(path));
let mapped = entry.file_id();
let infrastructure = matches!((mapped.as_ref(), sabre), (Some(seen), Some(known)) if seen == known)
|| matches!((mapped.as_ref(), plugin), (Some(seen), Some(known)) if seen == known);
if infrastructure {
return MappingClassification {
trusted: true,
trusted_shared_object: None,
};
}
let trusted_shared_object = path
.file_name()
.is_some_and(|name| name.to_string_lossy().contains(".so"))
.then(|| path.to_path_buf());
MappingClassification {
trusted: trusted_shared_object.is_some(),
trusted_shared_object,
}
}
fn read_two_bytes(pid: Pid, address: usize) -> Result<[u8; 2], Error> {
let word_size = std::mem::size_of::<libc::c_long>();
let aligned = address & !(word_size - 1);
let offset = address - aligned;
let first = ptrace::read(pid, aligned as ptrace::AddressType)?.to_ne_bytes();
if offset + 1 < word_size {
Ok([first[offset], first[offset + 1]])
} else {
let second = ptrace::read(pid, (aligned + word_size) as ptrace::AddressType)?.to_ne_bytes();
Ok([first[offset], second[0]])
}
}
fn read_diagnostic_bytes(pid: Pid, address: usize) -> Result<[u8; 16], Error> {
let mut bytes = [0; 16];
let word_size = std::mem::size_of::<libc::c_long>();
for (index, chunk) in bytes.chunks_mut(word_size).enumerate() {
let word = ptrace::read(pid, (address + index * word_size) as ptrace::AddressType)?;
chunk.copy_from_slice(&word.to_ne_bytes()[..chunk.len()]);
}
Ok(bytes)
}
fn write_two_bytes(pid: Pid, address: usize, bytes: [u8; 2]) -> Result<(), Error> {
let word_size = std::mem::size_of::<libc::c_long>();
let aligned = address & !(word_size - 1);
let offset = address - aligned;
let mut first = ptrace::read(pid, aligned as ptrace::AddressType)?.to_ne_bytes();
first[offset] = bytes[0];
if offset + 1 < word_size {
first[offset + 1] = bytes[1];
ptrace::write(
pid,
aligned as ptrace::AddressType,
libc::c_long::from_ne_bytes(first),
)?;
} else {
ptrace::write(
pid,
aligned as ptrace::AddressType,
libc::c_long::from_ne_bytes(first),
)?;
let second_address = aligned + word_size;
let mut second = ptrace::read(pid, second_address as ptrace::AddressType)?.to_ne_bytes();
second[0] = bytes[1];
ptrace::write(
pid,
second_address as ptrace::AddressType,
libc::c_long::from_ne_bytes(second),
)?;
}
Ok(())
}
#[allow(clippy::too_many_arguments)]
pub async fn run(
mut command: std::process::Command,
sabre: PathBuf,
plugin: PathBuf,
readiness: Arc<AtomicBool>,
physical_exit_observer: Arc<detcore::GlobalState>,
bootstrap_launch: super::sabre_bootstrap::Launch,
stats: SabreStats,
capture_output: bool,
) -> Result<Output, Error> {
if capture_output {
command.stdout(Stdio::piped()).stderr(Stdio::piped());
}
let child = spawn_tracee(command)?;
let root = Pid::from_raw(child.id() as i32);
match waitpid(root, Some(WaitPidFlag::__WALL))? {
WaitStatus::Stopped(pid, Signal::SIGTRAP) if pid == root => {}
status => return Err(anyhow!("unexpected initial SaBRe ptrace stop: {status:?}")),
}
ptrace::detach(root, Some(Signal::SIGSTOP))
.context("failed to hand SaBRe tracee to supervisor worker")?;
tokio::task::spawn_blocking(move || {
run_blocking(
child,
sabre,
plugin,
readiness,
physical_exit_observer,
bootstrap_launch,
stats,
)
})
.await
.context("SaBRe ptrace supervisor task panicked")?
}
fn spawn_tracee(mut command: std::process::Command) -> Result<std::process::Child, Error> {
unsafe {
command.pre_exec(|| {
let current = libc::personality(0xffff_ffff);
if current == -1 {
return Err(std::io::Error::last_os_error());
}
let deterministic = current as libc::c_ulong | libc::ADDR_NO_RANDOMIZE as libc::c_ulong;
if libc::personality(deterministic) == -1 {
return Err(std::io::Error::last_os_error());
}
ptrace::traceme().map_err(std::io::Error::from)
});
}
command
.spawn()
.context("failed to spawn ptraced SaBRe guest")
}
fn run_blocking(
mut child: std::process::Child,
sabre: PathBuf,
plugin: PathBuf,
readiness: Arc<AtomicBool>,
physical_exit_observer: Arc<detcore::GlobalState>,
bootstrap_launch: super::sabre_bootstrap::Launch,
stats: SabreStats,
) -> Result<Output, Error> {
let root = Pid::from_raw(child.id() as i32);
let stdout = child.stdout.take();
let stderr = child.stderr.take();
drop(child);
let stdout_thread = std::thread::spawn(move || read_pipe(stdout));
let stderr_thread = std::thread::spawn(move || read_pipe(stderr));
let supervised = Supervisor::new(
root,
sabre,
plugin,
readiness,
physical_exit_observer,
bootstrap_launch,
stats,
)
.run();
let stdout = stdout_thread
.join()
.map_err(|_| anyhow!("SaBRe stdout reader panicked"))??;
let stderr = stderr_thread
.join()
.map_err(|_| anyhow!("SaBRe stderr reader panicked"))??;
let (status, path_evidence) = supervised?;
Ok(Output {
status,
stdout,
stderr,
path_evidence,
})
}
fn read_pipe<R: Read>(pipe: Option<R>) -> Result<Vec<u8>, std::io::Error> {
let mut bytes = Vec::new();
if let Some(mut pipe) = pipe {
pipe.read_to_end(&mut bytes)?;
}
Ok(bytes)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn acknowledges_only_final_physical_exit_statuses() {
let child = Pid::from_raw(17);
let parent = Pid::from_raw(11);
assert_eq!(
final_physical_exit(&WaitStatus::Exited(child, 0)).map(|(pid, _)| pid),
Some(child)
);
assert_eq!(
final_physical_exit(&WaitStatus::Signaled(child, Signal::SIGKILL, false))
.map(|(pid, _)| pid),
Some(child)
);
assert!(
final_physical_exit(&WaitStatus::PtraceEvent(
child,
Signal::SIGTRAP,
libc::PTRACE_EVENT_EXIT,
))
.is_none()
);
assert!(final_physical_exit(&WaitStatus::Stopped(parent, Signal::SIGCHLD)).is_none());
}
#[test]
fn preserves_hardware_sigill_across_userspace_reraise() {
const KERNEL_FAULT: i32 = 2;
const USER_RERAISE: i32 = -6;
let hardware = Some((Signal::SIGILL, Some(KERNEL_FAULT), false));
assert!(!should_replace_signal_diagnostic(
hardware,
Signal::SIGILL,
Some(USER_RERAISE),
));
assert!(!should_replace_signal_diagnostic(
hardware,
Signal::SIGILL,
None,
));
assert!(should_replace_signal_diagnostic(
hardware,
Signal::SIGILL,
Some(KERNEL_FAULT),
));
assert!(should_replace_signal_diagnostic(
hardware,
Signal::SIGSEGV,
Some(KERNEL_FAULT),
));
assert!(should_replace_signal_diagnostic(
Some((Signal::SIGILL, Some(KERNEL_FAULT), true)),
Signal::SIGILL,
Some(USER_RERAISE),
));
assert!(should_replace_signal_diagnostic(
Some((Signal::SIGILL, Some(USER_RERAISE), false)),
Signal::SIGILL,
Some(USER_RERAISE),
));
}
#[test]
fn recognizes_sabre_sigill_markers() {
for marker in [[0x0f, 0xff], [0x0f, 0x0b], [0x0f, 0x0c]] {
let mut bytes = [0; 16];
bytes[..2].copy_from_slice(&marker);
assert!(is_sabre_sigill_marker(Some(&bytes)));
}
let mut unknown = [0; 16];
unknown[..2].copy_from_slice(&[0x62, 0xf1]);
assert!(!is_sabre_sigill_marker(Some(&unknown)));
assert!(!is_sabre_sigill_marker(None));
}
#[test]
fn finds_mapping_path() {
let maps = concat!(
"1000-2000 r-xp 00002000 00:00 0 /tmp/sabre\n",
"3000-4000 rwxp 00000000 00:00 0 \n",
);
assert_eq!(
mapping_entry(maps, 0x1234).map(|entry| entry.path),
Some("/tmp/sabre")
);
assert_eq!(
mapping_entry(maps, 0x3456).map(|entry| entry.path),
Some("")
);
assert!(mapping_entry(maps, 0x2500).is_none());
let diagnostic = mapping_diagnostic(maps, 0x1234).unwrap();
assert_eq!(diagnostic.relative_offset, 0x234);
assert_eq!(diagnostic.file_offset, 0x2234);
assert_eq!(
diagnostic.line,
"1000-2000 r-xp 00002000 00:00 0 /tmp/sabre"
);
}
#[test]
fn classifies_runtime_mapping_attribution() {
let dir = tempfile::tempdir().expect("tempdir");
let (sabre_path, sabre) = real_file(dir.path(), "sabre");
let (plugin_path, plugin) = real_file(dir.path(), "libdetcore_sabre.so");
for (inode, path) in [
(sabre.inode, sabre_path.to_str().expect("utf8")),
(plugin.inode, plugin_path.to_str().expect("utf8")),
] {
let maps = maps_row("1000-2000", "00:2d", inode, path);
let entry = mapping_entry(&maps, 0x1234).unwrap();
assert_eq!(
classify_mapping(&entry, Some(&sabre), Some(&plugin)),
MappingClassification {
trusted: true,
trusted_shared_object: None
},
"{path} is a launched object and must be exempt"
);
}
let maps = maps_row(
"1000-2000",
"fd:01",
7777,
"/different/root/libdetcore_sabre.so (deleted)",
);
let entry = mapping_entry(&maps, 0x1234).unwrap();
assert_eq!(
classify_mapping(&entry, Some(&sabre), Some(&plugin)),
MappingClassification {
trusted: true,
trusted_shared_object: Some(PathBuf::from("/different/root/libdetcore_sabre.so")),
},
"a basename impostor must be counted as a trusted-native site, not exempted"
);
let maps = maps_row("1000-2000", "fd:01", 55, "/usr/lib/libc.so.6");
let entry = mapping_entry(&maps, 0x1234).unwrap();
assert_eq!(
classify_mapping(&entry, Some(&sabre), Some(&plugin)),
MappingClassification {
trusted: true,
trusted_shared_object: Some(PathBuf::from("/usr/lib/libc.so.6")),
}
);
for (inode, path) in [(66, "/usr/bin/echo"), (0, "")] {
let maps = maps_row("1000-2000", "fd:01", inode, path);
let entry = mapping_entry(&maps, 0x1234).unwrap();
assert_eq!(
classify_mapping(&entry, Some(&sabre), Some(&plugin)),
MappingClassification {
trusted: false,
trusted_shared_object: None
}
);
}
}
fn maps_row(range: &str, dev: &str, inode: u64, path: &str) -> String {
format!("{range} r-xp 00000000 {dev} {inode} {path}\n")
}
fn classify(maps: &str, address: usize, sabre: Option<&FileId>) -> MappingClassification {
let entry = mapping_entry(maps, address).expect("mapping row must parse");
classify_mapping(&entry, sabre, None)
}
fn real_file(dir: &std::path::Path, name: &str) -> (PathBuf, FileId) {
let path = dir.join(name);
fs::write(&path, b"object").expect("write fixture");
let id = launched_file_id(&path).expect("fixture must have an identity");
(path, id)
}
#[test]
fn vdso_raw_syscall_is_counted_not_silently_trusted() {
let maps = maps_row("7ffff7fc9000-7ffff7fcb000", "00:00", 0, "[vdso]");
let classification = classify(&maps, 0x7ffff7fc9010, None);
assert!(
classification.trusted,
"[vdso] is not writable, so it must not be redirected"
);
assert_eq!(
classification.trusted_shared_object,
Some(PathBuf::from("[vdso]")),
"a [vdso] raw syscall did not traverse the SaBRe handler and must be counted"
);
}
#[test]
fn non_infrastructure_bracket_mapping_is_refused() {
for name in ["[stack]", "[heap]", "[anon:jit]"] {
let maps = maps_row("7ffffffde000-7ffffffff000", "00:00", 0, name);
let classification = classify(&maps, 0x7ffffffde010, None);
assert!(
!classification.trusted,
"{name} must be redirected through the SaBRe handler, not exempted"
);
}
}
#[test]
fn a_real_mapping_and_stat_agree_on_the_same_object() {
use std::io::Write;
let mut file = tempfile::NamedTempFile::new().expect("tempfile");
file.write_all(&[0u8; 4096]).expect("write");
file.flush().expect("flush");
let path = file.path().to_path_buf();
let fd = fs::File::open(&path).expect("open");
let mapping = unsafe {
libc::mmap(
std::ptr::null_mut(),
4096,
libc::PROT_READ,
libc::MAP_PRIVATE,
std::os::unix::io::AsRawFd::as_raw_fd(&fd),
0,
)
};
assert_ne!(mapping, libc::MAP_FAILED, "mmap of the fixture failed");
let address = mapping as usize;
let maps = fs::read_to_string("/proc/self/maps").expect("read own maps");
let entry = mapping_entry(&maps, address).expect("our own mapping must have a row");
let from_maps = entry
.file_id()
.expect("a file-backed mapping has an identity");
let from_stat = launched_file_id(&path).expect("the fixture must stat");
unsafe { libc::munmap(mapping, 4096) };
assert_eq!(
from_maps, from_stat,
"the identity of one object must not depend on which side of the \
kernel it is observed from; a mismatch here is what patched the \
SaBRe loader's own text and SIGSEGV'd the guest"
);
}
#[test]
fn launched_sabre_object_is_recognised_despite_a_disagreeing_dev_column() {
let dir = tempfile::tempdir().expect("tempdir");
let (path, id) = real_file(dir.path(), "sabre");
let maps = maps_row(
"400000-401000",
"00:2d",
id.inode,
path.to_str().expect("utf8 path"),
);
let classification = classify(&maps, 0x400010, Some(&id));
assert!(
classification.trusted,
"the launched loader must stay infrastructure even when the maps dev \
column disagrees with stat() (btrfs subvolume anon dev)"
);
assert_eq!(classification.trusted_shared_object, None);
}
#[test]
fn same_basename_different_object_is_refused() {
let dir = tempfile::tempdir().expect("tempdir");
let (_, known) = real_file(dir.path(), "sabre");
let attacker = dir.path().join("attacker");
fs::create_dir(&attacker).expect("mkdir");
let (impostor_path, impostor) = real_file(&attacker, "sabre");
assert_ne!(
known.inode, impostor.inode,
"fixtures must be distinct files"
);
let maps = maps_row(
"500000-501000",
"fd:01",
impostor.inode,
impostor_path.to_str().expect("utf8 path"),
);
let classification = classify(&maps, 0x500010, Some(&known));
assert!(
!classification.trusted,
"basename `sabre` must not exempt a different object"
);
assert_eq!(classification.trusted_shared_object, None);
}
#[test]
fn unknown_launched_identity_grants_no_exemption() {
let dir = tempfile::tempdir().expect("tempdir");
let (path, id) = real_file(dir.path(), "sabre");
let maps = maps_row(
"400000-401000",
"fd:01",
id.inode,
path.to_str().expect("utf8 path"),
);
let classification = classify(&maps, 0x400010, None);
assert!(
!classification.trusted,
"an unresolved launched identity must not exempt anything"
);
}
#[test]
fn other_shared_object_remains_counted() {
let maps = maps_row(
"7f0000000000-7f0000001000",
"fd:01",
77,
"/usr/lib64/libc.so.6",
);
let dir = tempfile::tempdir().expect("tempdir");
let (_, known) = real_file(dir.path(), "sabre");
let classification = classify(&maps, 0x7f0000000010, Some(&known));
assert!(classification.trusted);
assert_eq!(
classification.trusted_shared_object,
Some(PathBuf::from("/usr/lib64/libc.so.6"))
);
}
fn trusted() -> MappingClassification {
MappingClassification {
trusted: true,
trusted_shared_object: None,
}
}
#[test]
fn sibling_thread_cannot_serve_a_stale_classification_after_a_shared_mm_mutation() {
const PAGE: usize = 0x1000;
let a = Pid::from_raw(100);
let b = Pid::from_raw(101);
let mut cache = MappingCache::new(a);
cache.share_address_space(a, b);
cache.insert(b, PAGE, trusted());
assert!(
cache.get(b, PAGE).is_some(),
"precondition: B has a cached verdict, so the eviction below is not vacuous"
);
cache.invalidate_address_space(a);
assert!(
cache.get(b, PAGE).is_none(),
"a sibling sharing the mm must lose its cached verdict when ANY thread \
in that mm mutates the address space"
);
a_sibling_thread_sees_a_range_eviction();
}
#[test]
fn unrelated_address_space_is_not_over_evicted() {
const PAGE: usize = 0x1000;
let a = Pid::from_raw(100);
let b = Pid::from_raw(101);
let unrelated = Pid::from_raw(200);
let mut cache = MappingCache::new(a);
cache.share_address_space(a, b);
cache.new_address_space(unrelated);
cache.insert(b, PAGE, trusted());
cache.insert(unrelated, PAGE, trusted());
cache.invalidate_address_space(a);
assert!(cache.get(b, PAGE).is_none(), "the mutated mm is evicted");
assert!(
cache.get(unrelated, PAGE).is_some(),
"a different address space maps a different page at the same address \
and must keep its verdict"
);
a_failed_mutator_evicts_the_whole_space();
a_range_eviction_leaves_an_unrelated_address_space_alone();
}
fn with_child<F: FnOnce(Pid, Pid)>(share_mm: bool, check: F) {
const STACK: usize = 1 << 20;
let mut stack = vec![0u8; STACK];
let child = if share_mm {
let top = unsafe { stack.as_mut_ptr().add(STACK) } as *mut libc::c_void;
extern "C" fn sleeper(_: *mut libc::c_void) -> libc::c_int {
unsafe { libc::pause() };
0
}
unsafe {
libc::clone(
sleeper,
top,
libc::CLONE_VM | libc::SIGCHLD,
std::ptr::null_mut(),
)
}
} else {
match unsafe { libc::fork() } {
0 => unsafe {
libc::pause();
libc::_exit(0)
},
pid => pid,
}
};
assert!(child > 0, "failed to create the {share_mm} child");
let parent = Pid::from_raw(unsafe { libc::getpid() });
let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
check(parent, Pid::from_raw(child))
}));
unsafe {
libc::kill(child, libc::SIGKILL);
libc::waitpid(child, std::ptr::null_mut(), libc::__WALL);
}
if let Err(payload) = result {
std::panic::resume_unwind(payload);
}
}
#[test]
fn the_kernel_distinguishes_a_shared_mm_from_a_private_one() {
with_child(true, |parent, child| {
assert_eq!(
mm_sharing(parent, child),
MmSharing::Shared,
"a CLONE_VM child shares the caller's address space"
);
});
with_child(false, |parent, child| {
assert_eq!(
mm_sharing(parent, child),
MmSharing::Private,
"a forked child gets its own address space"
);
});
}
#[test]
fn a_clone_vm_child_cannot_serve_a_stale_classification() {
const PAGE: usize = 0x5000;
with_child(true, |parent, child| {
let mut cache = MappingCache::new(parent);
cache.admit_child(parent, child, mm_sharing(parent, child));
cache.insert(child, PAGE, trusted());
assert!(
cache.get(child, PAGE).is_some(),
"precondition: the child has a cached verdict, so the eviction is not vacuous"
);
cache.invalidate_address_space(parent);
assert!(
cache.get(child, PAGE).is_none(),
"the child shares the mm the parent just mutated, so its cached \
verdict must be dropped"
);
});
a_clone_vm_child_sees_a_range_eviction();
}
#[test]
fn a_private_mm_child_keeps_its_own_address_space() {
const PAGE: usize = 0x6000;
with_child(false, |parent, child| {
let mut cache = MappingCache::new(parent);
cache.admit_child(parent, child, mm_sharing(parent, child));
cache.insert(child, PAGE, trusted());
cache.invalidate_address_space(parent);
assert!(
cache.get(child, PAGE).is_some(),
"a private mm cannot be reached by the parent's mutation"
);
});
}
#[test]
fn clone3_sharing_is_read_from_the_child_not_from_a_clobberable_buffer() {
#[repr(C)]
#[derive(Default)]
struct CloneArgs {
flags: u64,
pidfd: u64,
child_tid: u64,
parent_tid: u64,
exit_signal: u64,
stack: u64,
stack_size: u64,
tls: u64,
}
const STACK: usize = 1 << 20;
let mut stack = vec![0u8; STACK];
let mut args = CloneArgs {
flags: libc::CLONE_VM as u64,
exit_signal: libc::SIGCHLD as u64,
stack: stack.as_mut_ptr() as u64,
stack_size: STACK as u64,
..Default::default()
};
let child = unsafe {
libc::syscall(
libc::SYS_clone3,
&mut args as *mut CloneArgs,
std::mem::size_of::<CloneArgs>(),
)
};
if child == 0 {
unsafe {
libc::pause();
libc::_exit(0)
};
}
assert!(
child > 0,
"clone3 failed: {}",
std::io::Error::last_os_error()
);
args.flags = 0;
assert_eq!(
args.flags & libc::CLONE_VM as u64,
0,
"the request now reads private"
);
let parent = Pid::from_raw(unsafe { libc::getpid() });
let child = Pid::from_raw(child as i32);
let observed = mm_sharing(parent, child);
unsafe {
libc::kill(child.as_raw(), libc::SIGKILL);
libc::waitpid(child.as_raw(), std::ptr::null_mut(), libc::__WALL);
}
assert_eq!(
observed,
MmSharing::Shared,
"the child shares the mm; reading the clobbered request would have said private"
);
}
#[test]
fn an_unanswerable_kcmp_fails_safe_to_shared() {
assert_eq!(interpret_kcmp_vm(0), MmSharing::Shared, "0 means same mm");
for ordered in [1, 2, 3] {
assert_eq!(
interpret_kcmp_vm(ordered),
MmSharing::Private,
"a nonzero ordering value means the address spaces differ"
);
}
assert_eq!(
interpret_kcmp_vm(-1),
MmSharing::Shared,
"an error must fail safe to sharing, never to private"
);
}
#[test]
fn mm_sharing_of_an_unavailable_task_fails_safe_to_shared() {
let parent = Pid::from_raw(unsafe { libc::getpid() });
assert_eq!(
mm_sharing(parent, Pid::from_raw(-1)),
MmSharing::Shared,
"an unanswerable comparison must over-evict, not under-evict"
);
}
#[test]
fn forked_child_gets_its_own_address_space() {
const PAGE: usize = 0x2000;
let parent = Pid::from_raw(100);
let child = Pid::from_raw(101);
let mut cache = MappingCache::new(parent);
cache.new_address_space(child);
cache.insert(child, PAGE, trusted());
cache.invalidate_address_space(parent);
assert!(
cache.get(child, PAGE).is_some(),
"fork installs a separate mm; the parent's mutation cannot reach it"
);
a_mutation_before_admission_drops_every_cached_page();
}
#[test]
fn exec_discards_the_previous_address_space() {
const PAGE: usize = 0x3000;
let pid = Pid::from_raw(100);
let mut cache = MappingCache::new(pid);
cache.insert(pid, PAGE, trusted());
cache.replace_address_space(pid);
assert!(
cache.get(pid, PAGE).is_none(),
"exec installs a new mm; the old classification describes a dead address space"
);
a_range_eviction_drops_pages_in_range_and_keeps_the_rest();
a_zero_length_mprotect_evicts_nothing();
mremap_evicts_both_its_old_and_its_new_range();
}
#[test]
fn cached_pages_outlive_one_thread_but_not_the_whole_group() {
const PAGE: usize = 0x4000;
let a = Pid::from_raw(100);
let b = Pid::from_raw(101);
let mut cache = MappingCache::new(a);
cache.share_address_space(a, b);
cache.insert(a, PAGE, trusted());
cache.forget(b);
assert!(
cache.get(a, PAGE).is_some(),
"a sibling exiting does not invalidate the surviving thread's view"
);
cache.forget(a);
assert!(
cache.entries.is_empty(),
"the last user of an address space releases its cached pages"
);
}
#[test]
fn address_space_mutators_invalidate_and_others_do_not() {
for nr in [
libc::SYS_mmap,
libc::SYS_munmap,
libc::SYS_mremap,
libc::SYS_mprotect,
libc::SYS_brk,
libc::SYS_shmat,
] {
assert!(
mutates_address_space(nr as u64),
"syscall {nr} changes the address space and must invalidate the cache"
);
}
for nr in [libc::SYS_read, libc::SYS_write, libc::SYS_getpid] {
assert!(
!mutates_address_space(nr as u64),
"syscall {nr} cannot change a mapping and must not invalidate the cache"
);
}
mmap_and_mprotect_ranges();
calls_without_a_provable_range_evict_the_whole_space();
possible_hugetlb_mappings_evict_the_whole_space();
growing_mprotect_evicts_the_whole_space();
an_overflowing_range_evicts_the_whole_space();
}
fn exit_regs(nr: libc::c_long, args: [u64; 4], result: i64) -> libc::user_regs_struct {
let mut regs: libc::user_regs_struct = unsafe { std::mem::zeroed() };
regs.orig_rax = nr as u64;
regs.rdi = args[0];
regs.rsi = args[1];
regs.rdx = args[2];
regs.r10 = args[3];
regs.rax = result as u64;
regs
}
fn ranges(ranges: &[Range<usize>]) -> Eviction {
Eviction::Ranges(ranges.to_vec())
}
fn one_range(start: usize, end: usize) -> Eviction {
let range = start..end;
Eviction::Ranges(vec![range])
}
fn a_range_eviction_drops_pages_in_range_and_keeps_the_rest() {
let pid = Pid::from_raw(100);
let mut cache = MappingCache::new(pid);
for page in [0xf000, 0x10000, 0x11000, 0x12000] {
cache.insert(pid, page, trusted());
}
let munmap = exit_regs(libc::SYS_munmap, [0x10000, 0x1800, 0, 0], 0);
let eviction = mutation_eviction(&munmap);
assert_eq!(eviction, one_range(0x10000, 0x12000));
cache.evict(pid, &eviction);
assert!(cache.get(pid, 0x10000).is_none(), "first page in range");
assert!(
cache.get(pid, 0x11000).is_none(),
"the partial last page is unmapped too"
);
assert!(
cache.get(pid, 0xf000).is_some(),
"the page before the range"
);
assert!(
cache.get(pid, 0x12000).is_some(),
"the page after the range"
);
}
fn mmap_and_mprotect_ranges() {
let fixed = (libc::MAP_FIXED | libc::MAP_PRIVATE | libc::MAP_ANONYMOUS) as u64;
let prot = (libc::PROT_READ | libc::PROT_EXEC) as u64;
assert_eq!(
mutation_eviction(&exit_regs(
libc::SYS_mmap,
[0x30000, 0x2000, prot, fixed],
0x30000
)),
one_range(0x30000, 0x32000),
"MAP_FIXED over existing pages"
);
assert_eq!(
mutation_eviction(&exit_regs(
libc::SYS_mmap,
[0, 0x2001, prot, libc::MAP_PRIVATE as u64],
0x7f00_0000_1000
)),
one_range(0x7f00_0000_1000, 0x7f00_0000_4000),
"a kernel-chosen address, length rounded up"
);
for nr in [libc::SYS_mprotect, libc::SYS_pkey_mprotect] {
assert_eq!(
mutation_eviction(&exit_regs(nr, [0x40000, 0x1000, prot, 0], 0)),
one_range(0x40000, 0x41000),
"syscall {nr}"
);
}
}
fn a_zero_length_mprotect_evicts_nothing() {
let pid = Pid::from_raw(100);
let mut cache = MappingCache::new(pid);
cache.insert(pid, 0x40000, trusted());
let prot = libc::PROT_READ as u64;
let eviction = mutation_eviction(&exit_regs(libc::SYS_mprotect, [0x40000, 0, prot, 0], 0));
assert_eq!(eviction, one_range(0x40000, 0x40000));
cache.evict(pid, &eviction);
assert!(cache.get(pid, 0x40000).is_some());
}
fn mremap_evicts_both_its_old_and_its_new_range() {
let maymove = libc::MREMAP_MAYMOVE as u64;
let fixed = (libc::MREMAP_MAYMOVE | libc::MREMAP_FIXED) as u64;
let dontunmap = (libc::MREMAP_MAYMOVE | libc::MREMAP_DONTUNMAP) as u64;
for (flags, new_len) in [(maymove, 0x3000), (fixed, 0x3000), (dontunmap, 0x2000)] {
let mut regs = exit_regs(libc::SYS_mremap, [0x20000, 0x2000, new_len, flags], 0x40000);
regs.r8 = 0x40000;
let eviction = mutation_eviction(®s);
let new_end = 0x40000 + new_len as usize;
assert_eq!(
eviction,
ranges(&[0x20000..0x22000, 0x40000..new_end]),
"flags {flags:#x}"
);
let pid = Pid::from_raw(100);
let mut cache = MappingCache::new(pid);
for page in [
0x1f000, 0x20000, 0x21000, 0x22000, 0x40000, 0x41000, new_end,
] {
cache.insert(pid, page, trusted());
}
cache.evict(pid, &eviction);
for page in [0x20000, 0x21000, 0x40000, 0x41000] {
assert!(cache.get(pid, page).is_none(), "{page:#x} was remapped");
}
for page in [0x1f000, 0x22000, new_end] {
assert!(cache.get(pid, page).is_some(), "{page:#x} was not");
}
}
let mut elsewhere = exit_regs(libc::SYS_mremap, [0x20000, 0x2000, 0x3000, fixed], 0x40000);
elsewhere.r8 = 0x60000;
let eviction = mutation_eviction(&elsewhere);
assert_eq!(
eviction,
ranges(&[0x20000..0x22000, 0x40000..0x43000, 0x60000..0x63000]),
"MREMAP_FIXED unmapped r8 even though the move went elsewhere"
);
let pid = Pid::from_raw(100);
let mut cache = MappingCache::new(pid);
for page in [0x5f000, 0x60000, 0x62000, 0x63000] {
cache.insert(pid, page, trusted());
}
cache.evict(pid, &eviction);
for page in [0x60000, 0x62000] {
assert!(cache.get(pid, page).is_none(), "{page:#x} was unmapped");
}
for page in [0x5f000, 0x63000] {
assert!(cache.get(pid, page).is_some(), "{page:#x} was not");
}
assert_eq!(
mutation_eviction(&exit_regs(
libc::SYS_mremap,
[0x20000, 0x2000, 0x4000, 0],
0x20000
)),
ranges(&[0x20000..0x22000, 0x20000..0x24000]),
"growth in place"
);
}
fn a_failed_mutator_evicts_the_whole_space() {
let prot = libc::PROT_READ as u64;
let fixed = libc::MAP_FIXED as u64;
let calls = [
(libc::SYS_mmap, [0x10000, 0x1000, prot, fixed]),
(libc::SYS_munmap, [0x10000, 0x1000, 0, 0]),
(libc::SYS_mprotect, [0x10000, 0x1000, prot, 0]),
(libc::SYS_mremap, [0x10000, 0x1000, 0x1000, 0]),
(libc::SYS_brk, [0x10000, 0, 0, 0]),
(libc::SYS_shmat, [1, 0x10000, 0, 0]),
];
let errors = [-4095, -(libc::ENOMEM as i64), -(libc::EINVAL as i64), -1];
for (nr, args) in calls {
for error in errors {
let eviction = mutation_eviction(&exit_regs(nr, args, error));
assert_eq!(eviction, Eviction::WholeSpace, "syscall {nr} -> {error}");
let a = Pid::from_raw(100);
let b = Pid::from_raw(101);
let unrelated = Pid::from_raw(200);
let mut cache = MappingCache::new(a);
cache.share_address_space(a, b);
cache.new_address_space(unrelated);
cache.insert(b, 0x10000, trusted());
cache.insert(b, 0x900000, trusted());
cache.insert(unrelated, 0x10000, trusted());
cache.evict(a, &eviction);
assert!(cache.get(b, 0x10000).is_none(), "the named page");
assert!(cache.get(b, 0x900000).is_none(), "a page far outside it");
assert!(
cache.get(unrelated, 0x10000).is_some(),
"another address space"
);
}
}
}
fn calls_without_a_provable_range_evict_the_whole_space() {
for (nr, args, result) in [
(libc::SYS_brk, [0x60_0000, 0, 0, 0], 0x60_0000),
(libc::SYS_shmat, [3, 0, 0, 0], 0x7f00_0000_1000),
(libc::SYS_getpid, [0, 0, 0, 0], 100),
] {
assert_eq!(
mutation_eviction(&exit_regs(nr, args, result)),
Eviction::WholeSpace,
"syscall {nr}"
);
}
assert_eq!(
mutation_eviction(&exit_regs(
libc::SYS_mremap,
[0x20000, 0, 0x2000, libc::MREMAP_MAYMOVE as u64],
0x40000
)),
Eviction::WholeSpace,
"an old length of 0 duplicates a shared mapping"
);
}
fn possible_hugetlb_mappings_evict_the_whole_space() {
const HUGE: u64 = MIN_HUGE_PAGE_SIZE as u64;
let prot = libc::PROT_READ as u64;
let private = libc::MAP_PRIVATE as u64;
let huge_flag = private | libc::MAP_ANONYMOUS as u64 | libc::MAP_HUGETLB as u64;
let mmap = |flags: u64, result: u64| {
mutation_eviction(&exit_regs(
libc::SYS_mmap,
[0, 0x1000, prot, flags],
result as i64,
))
};
assert_eq!(
mmap(huge_flag, 3 * HUGE + 0x1000),
Eviction::WholeSpace,
"MAP_HUGETLB"
);
assert_eq!(
mmap(private, 3 * HUGE),
Eviction::WholeSpace,
"a hugetlbfs file needs no flag but always lands on a huge-page boundary"
);
assert_eq!(
mmap(private, 1 << 30),
Eviction::WholeSpace,
"a 1 GiB boundary"
);
assert_eq!(
mmap(private, 3 * HUGE + 0x1000),
one_range(0x60_1000, 0x60_2000),
"one page past a boundary cannot be hugetlb"
);
let mremap = |old: u64, old_len: u64, new_len: u64, result: u64| {
mutation_eviction(&exit_regs(
libc::SYS_mremap,
[old, old_len, new_len, libc::MREMAP_MAYMOVE as u64],
result as i64,
))
};
assert_eq!(
mremap(HUGE, 0x1000, 0x1000, 0x7f00_0000_1000),
Eviction::WholeSpace,
"an old range starting on a boundary"
);
assert_eq!(
mremap(HUGE - 0x1000, 0x2000, 0x1000, 0x7f00_0000_1000),
Eviction::WholeSpace,
"an old range containing a boundary"
);
assert_eq!(
mremap(0x1000, 0x1000, 0x2000, 5 * HUGE - 0x1000),
Eviction::WholeSpace,
"a new range containing a boundary"
);
assert_eq!(
mremap(HUGE - 0x2000, 0x2000, 0x1000, 0x7f00_0000_1000),
ranges(&[0x1f_e000..0x20_0000, 0x7f00_0000_1000..0x7f00_0000_2000]),
"an old range ending exactly at a boundary does not contain it"
);
}
fn growing_mprotect_evicts_the_whole_space() {
for grows in [libc::PROT_GROWSDOWN, libc::PROT_GROWSUP] {
let prot = (libc::PROT_READ | grows) as u64;
assert_eq!(
mutation_eviction(&exit_regs(
libc::SYS_mprotect,
[0x40000, 0x1000, prot, 0],
0
)),
Eviction::WholeSpace,
"prot {prot:#x}"
);
}
}
fn an_overflowing_range_evicts_the_whole_space() {
assert_eq!(
mutation_eviction(&exit_regs(
libc::SYS_munmap,
[u64::MAX - 0xfff, 0x2000, 0, 0],
0
)),
Eviction::WholeSpace
);
assert_eq!(
mutation_eviction(&exit_regs(
libc::SYS_mprotect,
[u64::MAX - 0xfff, 0x800, 0, 0],
0
)),
Eviction::WholeSpace,
"rounding the end up to a page overflows"
);
}
fn a_sibling_thread_sees_a_range_eviction() {
let a = Pid::from_raw(100);
let b = Pid::from_raw(101);
let mut cache = MappingCache::new(a);
cache.share_address_space(a, b);
cache.insert(b, 0x10000, trusted());
cache.insert(b, 0x20000, trusted());
let prot = libc::PROT_READ as u64;
let mprotect = exit_regs(libc::SYS_mprotect, [0x10000, 0x1000, prot, 0], 0);
cache.evict(a, &mutation_eviction(&mprotect));
assert!(
cache.get(b, 0x10000).is_none(),
"in range, cached by the sibling"
);
assert!(cache.get(b, 0x20000).is_some(), "out of range");
}
fn a_clone_vm_child_sees_a_range_eviction() {
with_child(true, |parent, child| {
let mut cache = MappingCache::new(parent);
cache.admit_child(parent, child, mm_sharing(parent, child));
let munmap = |addr| exit_regs(libc::SYS_munmap, [addr, 0x1000, 0, 0], 0);
cache.insert(child, 0x10000, trusted());
cache.insert(child, 0x20000, trusted());
cache.evict(parent, &mutation_eviction(&munmap(0x10000)));
assert!(cache.get(child, 0x10000).is_none(), "parent's mutation");
assert!(cache.get(child, 0x20000).is_some(), "out of range");
cache.insert(parent, 0x30000, trusted());
cache.evict(child, &mutation_eviction(&munmap(0x30000)));
assert!(cache.get(parent, 0x30000).is_none(), "child's mutation");
assert!(cache.get(parent, 0x20000).is_some(), "out of range");
});
}
fn a_range_eviction_leaves_an_unrelated_address_space_alone() {
let a = Pid::from_raw(100);
let unrelated = Pid::from_raw(200);
let mut cache = MappingCache::new(a);
cache.new_address_space(unrelated);
cache.insert(a, 0x10000, trusted());
cache.insert(unrelated, 0x10000, trusted());
let munmap = exit_regs(libc::SYS_munmap, [0x10000, 0x1000, 0, 0], 0);
cache.evict(a, &mutation_eviction(&munmap));
assert!(cache.get(a, 0x10000).is_none());
assert!(cache.get(unrelated, 0x10000).is_some());
}
fn a_mutation_before_admission_drops_every_cached_page() {
let parent = Pid::from_raw(100);
let early = Pid::from_raw(101);
let mut cache = MappingCache::new(parent);
cache.insert(parent, 0x10000, trusted());
cache.insert(parent, 0x20000, trusted());
let munmap = exit_regs(libc::SYS_munmap, [0x10000, 0x1000, 0, 0], 0);
cache.evict(early, &mutation_eviction(&munmap));
assert!(
cache.entries.is_empty(),
"an unplaced tracee may share the parent's mm, so nothing may survive"
);
cache.admit_child(parent, early, MmSharing::Shared);
cache.insert(parent, 0x10000, trusted());
cache.insert(parent, 0x20000, trusted());
cache.evict(early, &mutation_eviction(&munmap));
assert!(cache.get(parent, 0x10000).is_none(), "placed: in range");
assert!(cache.get(parent, 0x20000).is_some(), "placed: out of range");
}
}