use std::fs;
use std::fs::File;
use std::io::Write;
use std::os::fd::AsRawFd;
use std::os::unix::fs::FileExt;
use std::os::unix::fs::OpenOptionsExt;
use std::path::Path;
use std::path::PathBuf;
use std::sync::Arc;
use std::sync::atomic::AtomicBool;
use std::sync::atomic::Ordering;
use std::sync::mpsc::RecvTimeoutError;
use std::time::Duration;
use std::time::Instant;
use std::time::SystemTime;
use ::procfs::process::CoredumpFlags;
use ::procfs::process::MMPermissions;
use ::procfs::process::MMapPath;
use ::procfs::process::MemoryMap;
use ::procfs::process::Process;
use ::procfs::process::VmFlags;
use reverie::FatalSignalExit;
use ruzstd::encoding::CompressionLevel;
use crate::config::FatalCoreCapture;
const CHUNK: usize = 1 << 20;
const PAGE: u64 = 4096;
static ZEROS: [u8; CHUNK] = [0; CHUNK];
#[derive(Debug, PartialEq, Eq)]
pub(crate) enum Outcome {
Written { path: PathBuf },
NotDumping,
NoRoom { budget: u64 },
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(crate) enum Tier {
Full,
Stack,
NotesOnly,
}
pub(crate) fn capture_and_log(cfg: &FatalCoreCapture, exit: &FatalSignalExit) {
let report = match capture(cfg, exit) {
Ok(Outcome::Written { path }) => format!(
"kept as {}",
path.file_name().unwrap_or_default().to_string_lossy()
),
Ok(Outcome::NotDumping) => return,
Ok(Outcome::NoRoom { .. }) => {
"not kept: no core fits within its size and time limits".to_string()
}
Err(error) => format!("not kept: {error:#}"),
};
let line = format!(
"hermit: fatal core for thread {} ({}): {report}\n",
exit.tid, exit.signal
);
crate::nonwaiting_write::write_without_waiting(libc::STDERR_FILENO, line.as_bytes());
}
pub(crate) fn capture(cfg: &FatalCoreCapture, exit: &FatalSignalExit) -> anyhow::Result<Outcome> {
if !exit.dumping {
return Ok(Outcome::NotDumping);
}
let end = capture_end(Instant::now(), cfg.time_limit, exit.deadline) + HELPER_SLACK;
let (cfg, exit) = (cfg.clone(), *exit);
run_within(end, move |abandoned| {
let mem = File::open(format!("/proc/{}/mem", exit.tid.as_raw())).ok();
capture_abandonable(&cfg, &exit, mem.as_ref(), &FULL_TIER_TIMED_OUT, abandoned)
})
}
const HELPER_SLACK: Duration = Duration::from_millis(500);
fn run_within(
end: Instant,
work: impl FnOnce(&AtomicBool) -> anyhow::Result<Outcome> + Send + 'static,
) -> anyhow::Result<Outcome> {
let abandoned = Arc::new(AtomicBool::new(false));
let (send, receive) = std::sync::mpsc::sync_channel(1);
let flag = Arc::clone(&abandoned);
std::thread::Builder::new()
.name("fatal-core".to_string())
.spawn(move || {
let _ = send.send(work(&flag));
})
.map_err(|error| anyhow::anyhow!("cannot start the capture thread: {error}"))?;
match receive.recv_timeout(end.saturating_duration_since(Instant::now())) {
Ok(result) => result,
Err(RecvTimeoutError::Timeout) => {
abandoned.store(true, Ordering::SeqCst);
match receive.try_recv() {
Ok(result) => result,
Err(_) => anyhow::bail!("gave up: a read or write blocked past the time limit"),
}
}
Err(RecvTimeoutError::Disconnected) => anyhow::bail!("the capture thread panicked"),
}
}
static FULL_TIER_TIMED_OUT: AtomicBool = AtomicBool::new(false);
#[cfg(test)]
fn capture_from(
cfg: &FatalCoreCapture,
exit: &FatalSignalExit,
mem: Option<&File>,
full_timed_out: &AtomicBool,
) -> anyhow::Result<Outcome> {
capture_abandonable(cfg, exit, mem, full_timed_out, &AtomicBool::new(false))
}
fn capture_abandonable(
cfg: &FatalCoreCapture,
exit: &FatalSignalExit,
mem: Option<&File>,
full_timed_out: &AtomicBool,
abandoned: &AtomicBool,
) -> anyhow::Result<Outcome> {
let start = Instant::now();
let end = capture_end(start, cfg.time_limit, exit.deadline);
let tid = exit.tid.as_raw();
let process = Process::new(tid)?;
let snapshot = Snapshot::read(&process, exit)?;
fs::create_dir_all(&cfg.dir)?;
let _lock = DirLock::acquire(&cfg.dir, end)?;
remove_orphaned_temps(&cfg.dir, cfg.time_limit * 2)?;
let used = regular_file_bytes(&cfg.dir)?;
let budget = cfg
.max_core_bytes
.min(cfg.max_total_bytes.saturating_sub(used));
let stem = format!(
"{}core.{}.{tid}.{}",
cfg.file_prefix,
snapshot.tgid,
exit.signal.as_str()
);
let name = free_name(&cfg.dir, &stem)?;
let path = cfg.dir.join(&name);
let temp = cfg.dir.join(format!(".{name}.tmp"));
let span = end.saturating_duration_since(start);
let tiers: &[(Tier, u32)] = match mem {
Some(_) if !full_timed_out.load(Ordering::Relaxed) => {
&[(Tier::Full, 7), (Tier::Stack, 9), (Tier::NotesOnly, 10)]
}
Some(_) => &[(Tier::Stack, 9), (Tier::NotesOnly, 10)],
None => &[(Tier::NotesOnly, 10)],
};
for &(tier, tenths) in tiers {
let deadline = start + span * tenths / 10;
let segments = snapshot.segments(tier);
match write_core(&temp, budget, deadline, &snapshot, &segments, mem) {
Ok(_) => {
if abandoned.load(Ordering::SeqCst) {
let _ = fs::remove_file(&temp);
anyhow::bail!("the tracer gave up on this capture");
}
fs::rename(&temp, &path)?;
return Ok(Outcome::Written { path });
}
Err(error) => {
let _ = fs::remove_file(&temp);
match error {
WriteError::Deadline if tier == Tier::Full => {
full_timed_out.store(true, Ordering::Relaxed);
}
WriteError::OverBudget | WriteError::Deadline => {}
WriteError::Io(error) => return Err(error.into()),
}
}
}
}
Ok(Outcome::NoRoom { budget })
}
fn capture_end(start: Instant, time_limit: Duration, deadline: Option<Instant>) -> Instant {
let limit = start + time_limit;
deadline.map_or(limit, |deadline| deadline.min(limit))
}
struct DirLock {
_held: File,
}
impl DirLock {
fn acquire(dir: &Path, deadline: Instant) -> anyhow::Result<Self> {
let file = File::open(dir)?;
loop {
if unsafe { libc::flock(file.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) } == 0 {
return Ok(Self { _held: file });
}
let error = std::io::Error::last_os_error();
if error.raw_os_error() != Some(libc::EWOULDBLOCK) {
return Err(error.into());
}
if Instant::now() >= deadline {
anyhow::bail!("timed out waiting for the lock on {}", dir.display());
}
std::thread::sleep(Duration::from_millis(5));
}
}
}
fn free_name(dir: &Path, stem: &str) -> std::io::Result<String> {
let mut name = format!("{stem}.zst");
for n in 1u32.. {
if !dir.join(&name).try_exists()? {
break;
}
name = format!("{stem}.{n}.zst");
}
Ok(name)
}
fn remove_orphaned_temps(dir: &Path, min_age: Duration) -> std::io::Result<()> {
let now = SystemTime::now();
drop(crate::util::RawFd::open(
dir,
libc::O_RDONLY | libc::O_DIRECTORY,
)?);
let _ = crate::util::find_in_directory(dir, |name| {
let text = name.to_string_lossy();
if !(text.starts_with(".hermit-") && text.contains("core.") && text.ends_with(".zst.tmp")) {
return None::<()>;
}
let path = dir.join(name);
let Ok(metadata) = fs::symlink_metadata(&path) else {
return None;
};
let old = metadata
.modified()
.ok()
.and_then(|modified| now.duration_since(modified).ok())
.is_some_and(|age| age >= min_age);
if metadata.is_file() && old {
let _ = fs::remove_file(&path);
}
None
});
Ok(())
}
fn regular_file_bytes(dir: &Path) -> std::io::Result<u64> {
let mut total = 0u64;
let failed =
crate::util::find_in_directory(dir, |name| match fs::symlink_metadata(dir.join(name)) {
Ok(metadata) => {
if metadata.file_type().is_file() {
total = total.saturating_add(metadata.len());
}
None
}
Err(error) => Some(error),
})?;
match failed {
Some(error) => Err(error),
None => Ok(total),
}
}
struct Snapshot {
tgid: i32,
stack_pointer: u64,
notes: Vec<u8>,
maps: Vec<MemoryMap>,
rules: Vec<Dump>,
}
impl Snapshot {
fn read(process: &Process, exit: &FatalSignalExit) -> anyhow::Result<Self> {
let status = process.status()?;
let stat = process.stat()?;
let (maps, flags_known) = match process.smaps() {
Ok(maps) => (maps.0, true),
Err(_) => (process.maps()?.0, false),
};
let filter = coredump_filter(exit.tid.as_raw());
let rules = maps
.iter()
.map(|map| {
let flags = map.extension.vm_flags;
dump_rule(
map,
(flags_known && !flags.is_empty()).then_some(flags),
filter,
)
})
.collect();
let cmdline = process.cmdline().unwrap_or_default().join(" ");
let auxv = fs::read(format!("/proc/{}/auxv", exit.tid.as_raw())).ok();
let identity = Identity {
tid: exit.tid.as_raw(),
tgid: status.tgid,
ppid: stat.ppid,
pgrp: stat.pgrp,
sid: stat.session,
uid: status.ruid,
gid: status.rgid,
sigpend: status.sigpnd | status.shdpnd,
sighold: status.sigblk,
state: stat.state,
nice: stat.nice,
flags: stat.flags,
comm: stat.comm,
cmdline,
};
let mut notes = Vec::new();
note(
&mut notes,
NT_PRSTATUS,
&arch::prstatus(exit.signal as i32, &identity, &exit.regs),
);
note(&mut notes, NT_PRPSINFO, &prpsinfo(&identity));
if let Some(auxv) = auxv {
note(&mut notes, NT_AUXV, &auxv);
}
note(&mut notes, NT_FILE, &nt_file(&maps));
Ok(Self {
tgid: status.tgid,
stack_pointer: arch::stack_pointer(&exit.regs),
notes,
maps,
rules,
})
}
fn segments(&self, tier: Tier) -> Vec<Segment> {
self.maps
.iter()
.zip(&self.rules)
.filter(|(map, _)| dumpable(map))
.map(|(map, rule)| {
let (start, end) = map.address;
let len = end - start;
let all = rule.whole
&& match tier {
Tier::Full => true,
Tier::Stack => {
(start..end).contains(&self.stack_pointer)
|| matches!(map.pathname, MMapPath::Stack)
}
Tier::NotesOnly => false,
};
let elf_header = tier != Tier::NotesOnly && rule.elf_header;
let filesz = if all {
len
} else if elf_header {
len.min(PAGE)
} else {
0
};
Segment {
vaddr: start,
memsz: len,
filesz,
flags: segment_flags(map.perms),
}
})
.collect()
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
struct Dump {
whole: bool,
elf_header: bool,
}
fn dump_rule(map: &MemoryMap, flags: Option<VmFlags>, filter: CoredumpFlags) -> Dump {
const NOTHING: Dump = Dump {
whole: false,
elf_header: false,
};
let Some(flags) = flags else {
return NOTHING;
};
if flags.intersects(VmFlags::DD | VmFlags::IO | VmFlags::PF) {
return NOTHING;
}
let file = matches!(map.pathname, MMapPath::Path(_));
let shared = flags.contains(VmFlags::SH);
let whole = if flags.contains(VmFlags::HT) {
filter.contains(if shared {
CoredumpFlags::SHARED_HUGEPAGES
} else {
CoredumpFlags::PROVATE_HUGEPAGES
})
} else if shared {
filter.contains(if file {
CoredumpFlags::FILEBACKED_SHARED_MAPPINGS
} else {
CoredumpFlags::ANONYMOUS_SHARED_MAPPINGS
})
} else if !file || map.perms.contains(MMPermissions::WRITE) {
filter.contains(CoredumpFlags::ANONYMOUS_PRIVATE_MAPPINGS)
|| (file && filter.contains(CoredumpFlags::FILEBACKED_PRIVATE_MAPPINGS))
} else {
filter.contains(CoredumpFlags::FILEBACKED_PRIVATE_MAPPINGS)
};
Dump {
whole,
elf_header: file && map.offset == 0 && filter.contains(CoredumpFlags::ELF_HEADERS),
}
}
fn coredump_filter(tid: i32) -> CoredumpFlags {
fs::read_to_string(format!("/proc/{tid}/coredump_filter"))
.ok()
.and_then(|text| u32::from_str_radix(text.trim(), 16).ok())
.map_or(
CoredumpFlags::from_bits_truncate(0x33),
CoredumpFlags::from_bits_truncate,
)
}
fn dumpable(map: &MemoryMap) -> bool {
map.perms.contains(MMPermissions::READ)
&& !matches!(map.pathname, MMapPath::Vvar | MMapPath::Vsyscall)
&& !matches!(&map.pathname, MMapPath::Other(name) if name.starts_with("vvar"))
}
fn segment_flags(perms: MMPermissions) -> u32 {
let mut flags = 0;
if perms.contains(MMPermissions::EXECUTE) {
flags |= PF_X;
}
if perms.contains(MMPermissions::WRITE) {
flags |= PF_W;
}
if perms.contains(MMPermissions::READ) {
flags |= PF_R;
}
flags
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
struct Segment {
vaddr: u64,
memsz: u64,
filesz: u64,
flags: u32,
}
const PT_LOAD: u32 = 1;
const PT_NOTE: u32 = 4;
const PF_X: u32 = 1;
const PF_W: u32 = 2;
const PF_R: u32 = 4;
const NT_PRSTATUS: u32 = 1;
const NT_PRPSINFO: u32 = 3;
const NT_AUXV: u32 = 6;
const NT_FILE: u32 = 0x4649_4c45;
const EHDR_SIZE: u64 = 64;
const PHDR_SIZE: u64 = 56;
struct Identity {
tid: i32,
tgid: i32,
ppid: i32,
pgrp: i32,
sid: i32,
uid: u32,
gid: u32,
sigpend: u64,
sighold: u64,
state: char,
nice: i64,
flags: u32,
comm: String,
cmdline: String,
}
fn note(out: &mut Vec<u8>, kind: u32, desc: &[u8]) {
const NAME: &[u8] = b"CORE\0";
out.extend_from_slice(&(NAME.len() as u32).to_le_bytes());
out.extend_from_slice(&(desc.len() as u32).to_le_bytes());
out.extend_from_slice(&kind.to_le_bytes());
out.extend_from_slice(NAME);
pad4(out);
out.extend_from_slice(desc);
pad4(out);
}
fn pad4(out: &mut Vec<u8>) {
out.resize(out.len().next_multiple_of(4), 0);
}
fn put(out: &mut [u8], offset: usize, bytes: &[u8]) {
out[offset..offset + bytes.len()].copy_from_slice(bytes);
}
fn prpsinfo(id: &Identity) -> Vec<u8> {
let mut out = vec![0u8; 136];
let state = id.state as u8;
out[0] = b"RSDTZW".iter().position(|&c| c == state).unwrap_or(0) as u8;
out[1] = state;
out[2] = u8::from(state == b'Z');
out[3] = id.nice as i8 as u8;
put(&mut out, 8, &u64::from(id.flags).to_le_bytes());
put(&mut out, 16, &id.uid.to_le_bytes());
put(&mut out, 20, &id.gid.to_le_bytes());
put(&mut out, 24, &id.tgid.to_le_bytes());
put(&mut out, 28, &id.ppid.to_le_bytes());
put(&mut out, 32, &id.pgrp.to_le_bytes());
put(&mut out, 36, &id.sid.to_le_bytes());
let comm = id.comm.as_bytes();
put(&mut out, 40, &comm[..comm.len().min(15)]);
let args = id.cmdline.as_bytes();
put(&mut out, 56, &args[..args.len().min(79)]);
out
}
fn nt_file(maps: &[MemoryMap]) -> Vec<u8> {
let files: Vec<(&MemoryMap, &Path)> = maps
.iter()
.filter_map(|map| match &map.pathname {
MMapPath::Path(path) => Some((map, path.as_path())),
_ => None,
})
.collect();
let mut out = Vec::new();
out.extend_from_slice(&(files.len() as u64).to_le_bytes());
out.extend_from_slice(&PAGE.to_le_bytes());
for (map, _) in &files {
out.extend_from_slice(&map.address.0.to_le_bytes());
out.extend_from_slice(&map.address.1.to_le_bytes());
out.extend_from_slice(&(map.offset / PAGE).to_le_bytes());
}
for (_, path) in &files {
use std::os::unix::ffi::OsStrExt;
out.extend_from_slice(path.as_os_str().as_bytes());
out.push(0);
}
out
}
#[cfg(target_arch = "x86_64")]
mod arch {
use super::Identity;
use super::put;
pub(super) const MACHINE: u16 = 62;
pub(super) fn stack_pointer(regs: &libc::user_regs_struct) -> u64 {
regs.rsp
}
pub(super) fn prstatus(signal: i32, id: &Identity, regs: &libc::user_regs_struct) -> Vec<u8> {
const REGS: usize = std::mem::size_of::<libc::user_regs_struct>();
const _: () = assert!(REGS == 216);
let mut out = vec![0u8; 336];
put(&mut out, 0, &signal.to_le_bytes()); put(&mut out, 12, &(signal as i16).to_le_bytes()); put(&mut out, 16, &id.sigpend.to_le_bytes());
put(&mut out, 24, &id.sighold.to_le_bytes());
put(&mut out, 32, &id.tid.to_le_bytes());
put(&mut out, 36, &id.ppid.to_le_bytes());
put(&mut out, 40, &id.pgrp.to_le_bytes());
put(&mut out, 44, &id.sid.to_le_bytes());
let bytes = unsafe { std::slice::from_raw_parts(regs as *const _ as *const u8, REGS) };
put(&mut out, 112, bytes);
out
}
}
#[cfg(not(target_arch = "x86_64"))]
mod arch {
use super::Identity;
pub(super) const MACHINE: u16 = 0;
pub(super) fn stack_pointer(regs: &libc::user_regs_struct) -> u64 {
regs.sp
}
pub(super) fn prstatus(_: i32, _: &Identity, _: &libc::user_regs_struct) -> Vec<u8> {
Vec::new()
}
}
#[derive(Debug)]
enum WriteError {
OverBudget,
Deadline,
Io(std::io::Error),
}
impl From<std::io::Error> for WriteError {
fn from(error: std::io::Error) -> Self {
Self::Io(error)
}
}
struct Compressed {
file: File,
buf: Vec<u8>,
written: u64,
budget: u64,
deadline: Instant,
}
impl Compressed {
fn push(&mut self, mut data: &[u8]) -> Result<(), WriteError> {
while !data.is_empty() {
let take = data.len().min(CHUNK - self.buf.len());
self.buf.extend_from_slice(&data[..take]);
data = &data[take..];
if self.buf.len() == CHUNK {
self.flush()?;
}
}
Ok(())
}
fn push_zeros(&mut self, mut len: u64) -> Result<(), WriteError> {
while len > 0 {
let take = len.min(CHUNK as u64) as usize;
self.push(&ZEROS[..take])?;
len -= take as u64;
}
Ok(())
}
fn flush(&mut self) -> Result<(), WriteError> {
if self.buf.is_empty() {
return Ok(());
}
if Instant::now() >= self.deadline {
return Err(WriteError::Deadline);
}
let frame = ruzstd::encoding::compress_to_vec(&self.buf[..], CompressionLevel::Fastest);
self.buf.clear();
if self.written + frame.len() as u64 > self.budget {
return Err(WriteError::OverBudget);
}
self.file.write_all(&frame)?;
self.written += frame.len() as u64;
Ok(())
}
}
fn headers(notes_len: u64, segments: &[Segment]) -> (Vec<u8>, u64) {
let phnum = 1 + segments.len() as u64;
let notes_offset = EHDR_SIZE + PHDR_SIZE * phnum;
let data_offset = (notes_offset + notes_len).next_multiple_of(PAGE);
let mut out = Vec::with_capacity(notes_offset as usize);
out.extend_from_slice(b"\x7fELF\x02\x01\x01");
out.resize(16, 0);
out.extend_from_slice(&4u16.to_le_bytes()); out.extend_from_slice(&arch::MACHINE.to_le_bytes());
out.extend_from_slice(&1u32.to_le_bytes()); out.extend_from_slice(&0u64.to_le_bytes()); out.extend_from_slice(&EHDR_SIZE.to_le_bytes()); out.extend_from_slice(&0u64.to_le_bytes()); out.extend_from_slice(&0u32.to_le_bytes()); out.extend_from_slice(&(EHDR_SIZE as u16).to_le_bytes());
out.extend_from_slice(&(PHDR_SIZE as u16).to_le_bytes());
out.extend_from_slice(&(phnum as u16).to_le_bytes());
out.extend_from_slice(&[0; 6]); let mut phdr =
|kind: u32, flags: u32, offset: u64, vaddr: u64, filesz: u64, memsz: u64, align: u64| {
out.extend_from_slice(&kind.to_le_bytes());
out.extend_from_slice(&flags.to_le_bytes());
out.extend_from_slice(&offset.to_le_bytes());
out.extend_from_slice(&vaddr.to_le_bytes());
out.extend_from_slice(&0u64.to_le_bytes()); out.extend_from_slice(&filesz.to_le_bytes());
out.extend_from_slice(&memsz.to_le_bytes());
out.extend_from_slice(&align.to_le_bytes());
};
phdr(PT_NOTE, 0, notes_offset, 0, notes_len, 0, 4);
let mut offset = data_offset;
for segment in segments {
phdr(
PT_LOAD,
segment.flags,
offset,
segment.vaddr,
segment.filesz,
segment.memsz,
PAGE,
);
offset += segment.filesz.next_multiple_of(PAGE);
}
(out, data_offset)
}
fn write_core(
temp: &Path,
budget: u64,
deadline: Instant,
snapshot: &Snapshot,
segments: &[Segment],
mem: Option<&File>,
) -> Result<u64, WriteError> {
let _ = fs::remove_file(temp);
let file = fs::OpenOptions::new()
.write(true)
.create_new(true)
.mode(0o600)
.open(temp)?;
let mut out = Compressed {
file,
buf: Vec::with_capacity(CHUNK),
written: 0,
budget,
deadline,
};
let (headers, data_offset) = headers(snapshot.notes.len() as u64, segments);
out.push(&headers)?;
out.push(&snapshot.notes)?;
out.push_zeros(data_offset - headers.len() as u64 - snapshot.notes.len() as u64)?;
let mut page = vec![0u8; CHUNK];
for segment in segments {
match mem {
Some(mem) => copy_memory(&mut out, mem, segment.vaddr, segment.filesz, &mut page)?,
None => out.push_zeros(segment.filesz)?,
}
out.push_zeros(segment.filesz.next_multiple_of(PAGE) - segment.filesz)?;
}
out.flush()?;
Ok(out.written)
}
fn copy_memory(
out: &mut Compressed,
mem: &File,
mut addr: u64,
len: u64,
buf: &mut [u8],
) -> Result<(), WriteError> {
let end = addr + len;
while addr < end {
let want = (end - addr).min(buf.len() as u64) as usize;
match read_fully(mem, &mut buf[..want], addr) {
Ok(()) => out.push(&buf[..want])?,
Err(_) => {
for page in buf[..want].chunks_mut(PAGE as usize) {
let page_addr = addr;
addr += page.len() as u64;
if read_fully(mem, page, page_addr).is_err() {
page.fill(0);
}
}
out.push(&buf[..want])?;
continue;
}
}
addr += want as u64;
}
Ok(())
}
fn read_fully(mem: &File, buf: &mut [u8], addr: u64) -> std::io::Result<()> {
let mut done = 0;
while done < buf.len() {
match mem.read_at(&mut buf[done..], addr + done as u64)? {
0 => return Err(std::io::ErrorKind::UnexpectedEof.into()),
n => done += n,
}
}
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
#[allow(clippy::disallowed_methods)]
fn entries(dir: &std::path::Path) -> usize {
fs::read_dir(dir).unwrap().count()
}
#[test]
fn headers_lay_out_notes_then_page_aligned_segments() {
let segments = [
Segment {
vaddr: 0x1000,
memsz: 0x3000,
filesz: 0x3000,
flags: PF_R | PF_W,
},
Segment {
vaddr: 0x10000,
memsz: 0x2000,
filesz: 0,
flags: PF_R,
},
Segment {
vaddr: 0x20000,
memsz: 0x5000,
filesz: 0x1000,
flags: PF_R | PF_X,
},
];
let (bytes, data_offset) = headers(100, &segments);
assert_eq!(bytes.len() as u64, EHDR_SIZE + 4 * PHDR_SIZE);
assert_eq!(data_offset, PAGE);
assert_eq!(&bytes[..4], b"\x7fELF");
let u16_at = |at: usize| u16::from_le_bytes(bytes[at..at + 2].try_into().unwrap());
assert_eq!(u16_at(16), 4, "e_type is ET_CORE");
assert_eq!(u16_at(56), 4, "e_phnum");
let u64_at = |at: usize| u64::from_le_bytes(bytes[at..at + 8].try_into().unwrap());
let phdr = |i: u64| (EHDR_SIZE + i * PHDR_SIZE) as usize;
assert_eq!(u64_at(phdr(0) + 8), EHDR_SIZE + 4 * PHDR_SIZE);
assert_eq!(u64_at(phdr(0) + 32), 100);
assert_eq!(u64_at(phdr(1) + 8), PAGE);
assert_eq!(u64_at(phdr(2) + 8), PAGE + 0x3000);
assert_eq!(u64_at(phdr(3) + 8), PAGE + 0x3000);
assert_eq!(u64_at(phdr(3) + 32), 0x1000);
assert_eq!(u64_at(phdr(3) + 40), 0x5000);
}
#[test]
fn notes_are_four_byte_aligned() {
let mut notes = Vec::new();
note(&mut notes, NT_AUXV, &[1, 2, 3]);
assert_eq!(notes.len(), 12 + 8 + 4);
assert_eq!(¬es[12..17], b"CORE\0");
assert_eq!(¬es[20..23], &[1, 2, 3]);
}
fn compressed(dir: &Path, budget: u64, deadline: Instant) -> Compressed {
Compressed {
file: File::create(dir.join("out")).unwrap(),
buf: Vec::new(),
written: 0,
budget,
deadline,
}
}
#[test]
fn frames_decode_back_to_the_input() {
let dir = tempfile::tempdir().unwrap();
let mut out = compressed(
dir.path(),
u64::MAX,
Instant::now() + Duration::from_secs(60),
);
let input: Vec<u8> = (0..(CHUNK * 2 + 77)).map(|i| (i % 251) as u8).collect();
out.push(&input).unwrap();
out.flush().unwrap();
let stored = fs::read(dir.path().join("out")).unwrap();
assert_eq!(stored.len() as u64, out.written);
let mut decoded = Vec::with_capacity(input.len());
ruzstd::decoding::FrameDecoder::new()
.decode_all_to_vec(&stored, &mut decoded)
.unwrap();
assert_eq!(decoded, input);
}
#[test]
fn a_frame_past_the_budget_is_refused_and_not_written() {
let dir = tempfile::tempdir().unwrap();
let mut out = compressed(dir.path(), 64, Instant::now() + Duration::from_secs(60));
let noise: Vec<u8> = (0..4096u32)
.map(|i| (i.wrapping_mul(2654435761) >> 13) as u8)
.collect();
out.push(&noise).unwrap();
assert!(matches!(out.flush(), Err(WriteError::OverBudget)));
assert_eq!(fs::metadata(dir.path().join("out")).unwrap().len(), 0);
}
#[test]
fn a_frame_after_the_deadline_is_refused() {
let dir = tempfile::tempdir().unwrap();
let mut out = compressed(dir.path(), u64::MAX, Instant::now());
out.push(b"late").unwrap();
assert!(matches!(out.flush(), Err(WriteError::Deadline)));
}
#[test]
fn the_capture_ends_at_the_earlier_of_its_time_limit_and_the_cleanup_deadline() {
let start = Instant::now();
let limit = Duration::from_secs(10);
assert_eq!(capture_end(start, limit, None), start + limit);
let soon = start + Duration::from_secs(2);
assert_eq!(capture_end(start, limit, Some(soon)), soon);
let late = start + Duration::from_secs(60);
assert_eq!(capture_end(start, limit, Some(late)), start + limit);
}
fn config(dir: &Path) -> FatalCoreCapture {
FatalCoreCapture {
dir: dir.to_path_buf(),
file_prefix: "test-".to_string(),
max_core_bytes: 64 << 20,
max_total_bytes: 64 << 20,
time_limit: Duration::from_secs(10),
}
}
fn own_exit(dumping: bool, deadline: Option<Instant>) -> FatalSignalExit {
let tid = reverie::Pid::from_raw(unsafe { libc::gettid() });
let regs: libc::user_regs_struct = unsafe { std::mem::zeroed() };
let status = reverie::ExitStatus::Signaled(reverie::Signal::SIGSEGV, false);
FatalSignalExit::new(tid, status, dumping, deadline, regs).unwrap()
}
fn stored_memory(path: &Path) -> u64 {
use std::io::Read;
let stored = fs::read(path).unwrap();
let mut rest = stored.as_slice();
let mut core = Vec::new();
while !rest.is_empty() {
ruzstd::decoding::StreamingDecoder::new(&mut rest)
.unwrap()
.read_to_end(&mut core)
.unwrap();
}
let u16_at = |at: usize| u16::from_le_bytes(core[at..at + 2].try_into().unwrap());
let u32_at = |at: usize| u32::from_le_bytes(core[at..at + 4].try_into().unwrap());
let u64_at = |at: usize| u64::from_le_bytes(core[at..at + 8].try_into().unwrap());
assert_eq!(u16_at(16), 4, "not an ET_CORE file");
(0..u64::from(u16_at(56)))
.map(|i| (EHDR_SIZE + i * PHDR_SIZE) as usize)
.filter(|&phdr| u32_at(phdr) == PT_LOAD)
.map(|phdr| u64_at(phdr + 32))
.sum()
}
fn map(pathname: MMapPath, perms: MMPermissions, offset: u64) -> MemoryMap {
MemoryMap {
address: (0x1000, 0x3000),
perms,
offset,
dev: (0, 0),
inode: u64::from(matches!(pathname, MMapPath::Path(_))),
pathname,
extension: Default::default(),
}
}
#[test]
fn the_dump_rule_keeps_what_the_kernel_would() {
use MMPermissions as P;
let default = CoredumpFlags::from_bits_truncate(0x33);
let rd = VmFlags::RD | VmFlags::MR;
let rw = rd | VmFlags::WR | VmFlags::MW;
let lib = || MMapPath::Path("/lib/libc.so.6".into());
let whole = |m: &MemoryMap, flags, filter| dump_rule(m, Some(flags), filter).whole;
let heap = map(MMapPath::Heap, P::READ | P::WRITE | P::PRIVATE, 0);
assert!(whole(&heap, rw, default), "anonymous private memory");
assert!(!whole(&heap, rw | VmFlags::DD, default), "MADV_DONTDUMP");
assert!(!whole(&heap, rw | VmFlags::IO, default), "an I/O mapping");
assert!(
!whole(&heap, rw | VmFlags::PF, default),
"a raw-PFN mapping"
);
assert!(!whole(&heap, rw, CoredumpFlags::empty()), "filter 0");
assert_eq!(
dump_rule(&heap, None, default),
dump_rule(&heap, Some(rw | VmFlags::DD), default)
);
let shared_file = map(lib(), P::READ | P::WRITE | P::SHARED, 0);
assert!(
!whole(&shared_file, rw | VmFlags::SH, default),
"shared file, default"
);
assert!(whole(
&shared_file,
rw | VmFlags::SH,
default | CoredumpFlags::FILEBACKED_SHARED_MAPPINGS
));
let data = map(lib(), P::READ | P::WRITE | P::PRIVATE, 0x2000);
assert!(whole(&data, rw, default), "a written private file mapping");
let text = map(lib(), P::READ | P::EXECUTE | P::PRIVATE, 0);
let rule = dump_rule(&text, Some(rd | VmFlags::EX), default);
assert!(!rule.whole && rule.elf_header, "{rule:?}");
let no_headers = default - CoredumpFlags::ELF_HEADERS;
assert!(!dump_rule(&text, Some(rd | VmFlags::EX), no_headers).elf_header);
assert!(!dump_rule(&text, Some(rd | VmFlags::DD), default).elf_header);
}
#[test]
fn the_lock_wait_gives_up_at_the_deadline() {
let dir = tempfile::tempdir().unwrap();
let held = DirLock::acquire(dir.path(), Instant::now()).unwrap();
let start = Instant::now();
let error = DirLock::acquire(dir.path(), start + Duration::from_millis(50))
.err()
.expect("a second writer took a held lock");
assert!(error.to_string().contains("timed out"), "{error:#}");
assert!(start.elapsed() >= Duration::from_millis(50));
drop(held);
DirLock::acquire(dir.path(), Instant::now()).unwrap();
}
#[test]
fn a_repeated_process_id_gets_a_new_name_instead_of_being_skipped() {
let dir = tempfile::tempdir().unwrap();
let cfg = config(dir.path());
let exit = own_exit(true, None);
let fresh = AtomicBool::new(false);
let mut paths = Vec::new();
for _ in 0..2 {
let Outcome::Written { path } = capture_from(&cfg, &exit, None, &fresh).unwrap() else {
panic!("the second core of a repeated process ID was not kept");
};
paths.push(path);
}
assert_ne!(paths[0], paths[1]);
assert!(paths.iter().all(|path| path.exists()));
let name = paths[1].file_name().unwrap().to_string_lossy().into_owned();
assert!(name.ends_with(".SIGSEGV.1.zst"), "{name}");
}
#[test]
fn after_a_full_core_runs_out_of_time_later_cores_start_smaller() {
let dir = tempfile::tempdir().unwrap();
let cfg = config(dir.path());
let mem = File::open("/proc/self/mem").unwrap();
let flag = AtomicBool::new(false);
let late = own_exit(true, Some(Instant::now()));
capture_from(&cfg, &late, Some(&mem), &flag).unwrap();
assert!(
flag.load(Ordering::Relaxed),
"a timed-out Full core was not remembered"
);
let full = AtomicBool::new(false);
let Outcome::Written { path: big } =
capture_from(&cfg, &own_exit(true, None), Some(&mem), &full).unwrap()
else {
panic!("no Full core");
};
let Outcome::Written { path: small } =
capture_from(&cfg, &own_exit(true, None), Some(&mem), &flag).unwrap()
else {
panic!("no core after the timeout");
};
assert!(
stored_memory(&small) < stored_memory(&big),
"the core after a timeout was not smaller"
);
}
#[test]
fn a_thread_that_did_not_dump_gets_no_core() {
let dir = tempfile::tempdir().unwrap();
let cores = dir.path().join("cores");
let outcome = capture(&config(&cores), &own_exit(false, None)).unwrap();
assert_eq!(outcome, Outcome::NotDumping);
assert!(
!cores.exists(),
"a non-dumping thread created the directory"
);
}
#[test]
fn unreadable_memory_still_keeps_the_notes() {
let dir = tempfile::tempdir().unwrap();
let Outcome::Written { path } = capture_from(
&config(dir.path()),
&own_exit(true, None),
None,
&AtomicBool::new(false),
)
.unwrap() else {
panic!("no core was kept without memory");
};
assert_eq!(
stored_memory(&path),
0,
"memory was stored without a mem file"
);
}
#[test]
fn readable_memory_is_stored() {
let dir = tempfile::tempdir().unwrap();
let Outcome::Written { path } =
capture(&config(dir.path()), &own_exit(true, None)).unwrap()
else {
panic!("no core was kept");
};
assert!(stored_memory(&path) > 0, "the core stores no memory");
}
#[test]
fn a_passed_cleanup_deadline_keeps_nothing() {
let dir = tempfile::tempdir().unwrap();
let mem = File::open("/proc/self/mem").unwrap();
let flag = AtomicBool::new(false);
let late = own_exit(true, Some(Instant::now()));
let outcome = capture_from(&config(dir.path()), &late, Some(&mem), &flag).unwrap();
assert!(matches!(outcome, Outcome::NoRoom { .. }), "{outcome:?}");
assert_eq!(entries(dir.path()), 0);
}
#[test]
fn a_dead_writers_temp_file_stops_counting_against_the_total() {
let dir = tempfile::tempdir().unwrap();
let orphan = dir
.path()
.join(".hermit-1-0000000000000000-core.3.3.SIGSEGV.zst.tmp");
let unrelated = dir.path().join(".someone-else.tmp");
let not_a_core = dir.path().join(".hermit-notes.tmp");
let young = dir
.path()
.join(".hermit-2-0000000000000000-core.4.4.SIGSEGV.zst.tmp");
fs::write(&orphan, vec![0u8; 1 << 20]).unwrap();
for path in [&unrelated, ¬_a_core, &young] {
fs::write(path, b"x").unwrap();
}
let hour_ago = SystemTime::now() - Duration::from_secs(3600);
for path in [&orphan, &unrelated, ¬_a_core] {
File::options()
.write(true)
.open(path)
.unwrap()
.set_modified(hour_ago)
.unwrap();
}
let cfg = FatalCoreCapture {
max_total_bytes: 1 << 20,
..config(dir.path())
};
let outcome = capture_from(&cfg, &own_exit(true, None), None, &AtomicBool::new(false));
assert!(
matches!(outcome, Ok(Outcome::Written { .. })),
"the orphan still took the whole total: {outcome:?}"
);
assert!(!orphan.exists(), "the orphaned temp file was kept");
assert!(
unrelated.exists(),
"a file that is not hermit's was removed"
);
assert!(
not_a_core.exists(),
"a hermit file that is not a core temp was removed"
);
assert!(young.exists(), "a temp written just now was removed");
}
#[test]
fn the_tracer_stops_waiting_at_the_end_and_the_helper_then_keeps_nothing() {
let (tell, told) = std::sync::mpsc::channel();
let start = Instant::now();
let outcome = run_within(start + Duration::from_millis(100), move |abandoned| {
std::thread::sleep(Duration::from_millis(600));
tell.send(abandoned.load(Ordering::SeqCst)).unwrap();
Ok(Outcome::NotDumping)
});
let waited = start.elapsed();
let error = outcome.expect_err("a blocked capture was reported as done");
assert!(error.to_string().contains("gave up"), "{error:#}");
assert!(
waited < Duration::from_millis(500),
"the tracer waited {waited:?}"
);
assert!(
told.recv_timeout(Duration::from_secs(5)).unwrap(),
"the helper was not told it was abandoned"
);
}
#[test]
fn a_capture_that_finishes_in_time_is_returned_and_a_panic_is_an_error() {
let done = run_within(Instant::now() + Duration::from_secs(5), |_| {
Ok(Outcome::NotDumping)
});
assert_eq!(done.unwrap(), Outcome::NotDumping);
let panicked = run_within(Instant::now() + Duration::from_secs(5), |_| panic!("boom"));
assert!(panicked.unwrap_err().to_string().contains("panicked"));
}
#[test]
fn an_abandoned_capture_deletes_its_core_instead_of_keeping_it() {
let dir = tempfile::tempdir().unwrap();
let abandoned = AtomicBool::new(true);
let outcome = capture_abandonable(
&config(dir.path()),
&own_exit(true, None),
None,
&AtomicBool::new(false),
&abandoned,
);
assert!(outcome.is_err(), "{outcome:?}");
assert_eq!(entries(dir.path()), 0, "an abandoned capture left a file");
}
#[test]
fn regular_file_bytes_counts_only_regular_files() {
let dir = tempfile::tempdir().unwrap();
fs::write(dir.path().join("a"), [0u8; 10]).unwrap();
fs::write(dir.path().join("b"), [0u8; 5]).unwrap();
fs::create_dir(dir.path().join("sub")).unwrap();
fs::write(dir.path().join("sub/c"), [0u8; 100]).unwrap();
assert_eq!(regular_file_bytes(dir.path()).unwrap(), 15);
}
}