use std::{
path::{Path, PathBuf},
sync::{
atomic::{AtomicBool, Ordering},
Arc, Mutex,
},
thread,
time::Duration,
};
pub const DEFAULT_MAX_PROCESSES: u64 = 128;
pub const DEFAULT_MAX_OPEN_FILES: u64 = 1024;
pub const DEFAULT_MAX_FILE_SIZE_BYTES: u64 = 4 * 1024 * 1024 * 1024;
pub const DEFAULT_MEMORY_BYTES: u64 = 2 * 1024 * 1024 * 1024;
pub const DEFAULT_CGROUP_MEMORY_BYTES: u64 = 4 * 1024 * 1024 * 1024;
pub const DEFAULT_CGROUP_PIDS_MAX: u64 = 256;
pub const DEFAULT_CGROUP_IO_BYTES_PER_SEC: u64 = 100 * 1024 * 1024;
pub const DEFAULT_WORKDIR_MAX_BYTES: u64 = 10 * 1024 * 1024 * 1024;
pub(crate) const WORKDIR_CHECK_INTERVAL: Duration = Duration::from_secs(10);
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct HostResourceLimits {
pub max_processes: u64,
pub max_open_files: u64,
pub max_file_size_bytes: u64,
pub cpu_seconds: Option<u64>,
pub memory_bytes: u64,
pub cgroup_memory_bytes: u64,
pub cgroup_pids_max: u64,
pub cgroup_cpu_percent: Option<u32>,
pub cgroup_io_bytes_per_sec: u64,
pub workdir_max_bytes: u64,
}
impl Default for HostResourceLimits {
fn default() -> Self {
Self {
max_processes: DEFAULT_MAX_PROCESSES,
max_open_files: DEFAULT_MAX_OPEN_FILES,
max_file_size_bytes: DEFAULT_MAX_FILE_SIZE_BYTES,
cpu_seconds: None,
memory_bytes: DEFAULT_MEMORY_BYTES,
cgroup_memory_bytes: DEFAULT_CGROUP_MEMORY_BYTES,
cgroup_pids_max: DEFAULT_CGROUP_PIDS_MAX,
cgroup_cpu_percent: None,
cgroup_io_bytes_per_sec: DEFAULT_CGROUP_IO_BYTES_PER_SEC,
workdir_max_bytes: DEFAULT_WORKDIR_MAX_BYTES,
}
}
}
impl HostResourceLimits {
#[must_use]
pub fn resolve(declared: Option<&murmur_artifact::ResourceCapabilities>) -> Self {
let defaults = Self::default();
let Some(declared) = declared else {
return defaults;
};
Self {
max_processes: declared.max_processes.unwrap_or(defaults.max_processes),
max_open_files: declared.max_open_files.unwrap_or(defaults.max_open_files),
max_file_size_bytes: declared
.max_file_size_bytes
.unwrap_or(defaults.max_file_size_bytes),
cpu_seconds: declared.cpu_seconds,
memory_bytes: declared.memory_bytes.unwrap_or(defaults.memory_bytes),
cgroup_memory_bytes: declared
.cgroup_memory_bytes
.unwrap_or(defaults.cgroup_memory_bytes),
cgroup_pids_max: declared.cgroup_pids_max.unwrap_or(defaults.cgroup_pids_max),
cgroup_cpu_percent: declared.cgroup_cpu_percent,
cgroup_io_bytes_per_sec: declared
.cgroup_io_bytes_per_sec
.unwrap_or(defaults.cgroup_io_bytes_per_sec),
workdir_max_bytes: declared
.workdir_max_bytes
.unwrap_or(defaults.workdir_max_bytes),
}
}
}
#[must_use]
pub(crate) fn resolve(
declared: Option<&murmur_artifact::ResourceCapabilities>,
) -> HostResourceLimits {
HostResourceLimits::resolve(declared)
}
#[cfg_attr(not(unix), allow(dead_code))]
pub(crate) fn limit_from_signal(signal: i32) -> Option<&'static str> {
match signal {
libc::SIGXCPU => Some("cpu_seconds"),
libc::SIGXFSZ => Some("max_file_size_bytes"),
_ => None,
}
}
pub(crate) fn resource_limit_line(limit: &str) -> String {
let explanation = match limit {
"cgroup_pids_max" => "the capsule reached its process limit while this command ran, so \
starting a new process failed (EAGAIN). Retry with fewer parallel processes"
.to_string(),
"cgroup_memory_bytes" => "the capsule reached its memory limit while this command ran, \
and the kernel killed a process in it to free memory. Retry using less memory at \
once (smaller inputs, fewer parallel jobs)"
.to_string(),
"cpu_seconds" => "this command used up its CPU-time limit and was stopped. Retry with \
less work per command"
.to_string(),
"max_file_size_bytes" => "this command tried to write a file larger than the capsule's \
file-size limit and was stopped. Retry writing smaller files"
.to_string(),
other => format!(
"this command was stopped by the capsule's capabilities.resources.{other} limit. \
Retry with less work"
),
};
format!(
"resource_limit: {limit} — {explanation}, or ask the operator to raise \
capabilities.resources.{limit}."
)
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum NprocBound {
CgroupPidsMax,
Headroom { baseline: u64 },
}
#[cfg(unix)]
pub(crate) fn apply_hard_rlimits(
limits: &HostResourceLimits,
nproc: NprocBound,
) -> std::io::Result<()> {
apply_nproc_bound(limits, nproc)?;
apply_per_process_rlimits(limits)
}
#[cfg(unix)]
#[allow(clippy::unnecessary_cast)]
pub(crate) fn apply_declared_rlimits(
declared: &murmur_artifact::ResourceCapabilities,
nproc: NprocBound,
) -> std::io::Result<()> {
if let (Some(max_processes), NprocBound::Headroom { baseline }) =
(declared.max_processes, nproc)
{
set_hard_rlimit(
libc::RLIMIT_NPROC as u32,
baseline.saturating_add(max_processes),
)?;
}
if let Some(max_open_files) = declared.max_open_files {
set_hard_rlimit(libc::RLIMIT_NOFILE as u32, max_open_files)?;
}
if let Some(max_file_size_bytes) = declared.max_file_size_bytes {
set_hard_rlimit(libc::RLIMIT_FSIZE as u32, max_file_size_bytes)?;
}
if let Some(cpu_seconds) = declared.cpu_seconds {
set_cpu_rlimit(cpu_seconds)?;
}
if let Some(memory_bytes) = declared.memory_bytes {
#[cfg(target_os = "linux")]
set_hard_rlimit(libc::RLIMIT_AS as u32, memory_bytes)?;
#[cfg(not(target_os = "linux"))]
let _ = set_hard_rlimit(libc::RLIMIT_DATA as u32, memory_bytes);
}
Ok(())
}
pub(crate) fn declares_cgroup_limit(declared: &murmur_artifact::ResourceCapabilities) -> bool {
declared.cgroup_memory_bytes.is_some()
|| declared.cgroup_pids_max.is_some()
|| declared.cgroup_cpu_percent.is_some()
|| declared.cgroup_io_bytes_per_sec.is_some()
}
pub(crate) fn workdir_guard_max_bytes(policy: &crate::types::CapabilityPolicy) -> Option<u64> {
if policy.containment_floor == murmur_artifact::ContainmentClass::Unconfined {
policy.declared_resources.workdir_max_bytes
} else {
Some(policy.resources.workdir_max_bytes)
}
}
pub(crate) fn declared_resource_fields(
declared: &murmur_artifact::ResourceCapabilities,
) -> Vec<&'static str> {
[
("max_processes", declared.max_processes.is_some()),
("max_open_files", declared.max_open_files.is_some()),
(
"max_file_size_bytes",
declared.max_file_size_bytes.is_some(),
),
("cpu_seconds", declared.cpu_seconds.is_some()),
("memory_bytes", declared.memory_bytes.is_some()),
(
"cgroup_memory_bytes",
declared.cgroup_memory_bytes.is_some(),
),
("cgroup_pids_max", declared.cgroup_pids_max.is_some()),
("cgroup_cpu_percent", declared.cgroup_cpu_percent.is_some()),
(
"cgroup_io_bytes_per_sec",
declared.cgroup_io_bytes_per_sec.is_some(),
),
("workdir_max_bytes", declared.workdir_max_bytes.is_some()),
]
.into_iter()
.filter_map(|(name, set)| set.then_some(name))
.collect()
}
#[cfg(unix)]
#[allow(clippy::unnecessary_cast)]
pub(crate) fn apply_nproc_bound(
limits: &HostResourceLimits,
nproc: NprocBound,
) -> std::io::Result<()> {
if let NprocBound::Headroom { baseline } = nproc {
set_hard_rlimit(
libc::RLIMIT_NPROC as u32,
baseline.saturating_add(limits.max_processes),
)?;
}
Ok(())
}
#[cfg(unix)]
#[allow(clippy::unnecessary_cast)]
pub(crate) fn apply_per_process_rlimits(limits: &HostResourceLimits) -> std::io::Result<()> {
set_hard_rlimit(libc::RLIMIT_NOFILE as u32, limits.max_open_files)?;
set_hard_rlimit(libc::RLIMIT_FSIZE as u32, limits.max_file_size_bytes)?;
if let Some(cpu_seconds) = limits.cpu_seconds {
set_cpu_rlimit(cpu_seconds)?;
}
set_hard_rlimit(libc::RLIMIT_CORE as u32, 0)?;
#[cfg(target_os = "linux")]
set_hard_rlimit(libc::RLIMIT_AS as u32, limits.memory_bytes)?;
#[cfg(not(target_os = "linux"))]
let _ = set_hard_rlimit(libc::RLIMIT_DATA as u32, limits.memory_bytes);
Ok(())
}
#[cfg(unix)]
fn set_hard_rlimit(resource: u32, value: u64) -> std::io::Result<()> {
let current = read_rlimit(resource)?;
let requested = libc::rlim_t::try_from(value).unwrap_or(libc::rlim_t::MAX);
let target = if current.rlim_max == libc::RLIM_INFINITY {
requested
} else {
requested.min(current.rlim_max)
};
write_rlimit(
resource,
&libc::rlimit {
rlim_cur: target,
rlim_max: target,
},
)
}
#[cfg(unix)]
#[allow(clippy::unnecessary_cast)]
fn set_cpu_rlimit(cpu_seconds: u64) -> std::io::Result<()> {
let resource = libc::RLIMIT_CPU as u32;
let current = read_rlimit(resource)?;
write_rlimit(resource, &cpu_rlimit_slots(cpu_seconds, current.rlim_max))
}
#[cfg(unix)]
#[allow(clippy::unnecessary_min_or_max)]
fn cpu_rlimit_slots(cpu_seconds: u64, inherited_hard: libc::rlim_t) -> libc::rlimit {
let requested = libc::rlim_t::try_from(cpu_seconds)
.unwrap_or(libc::RLIM_INFINITY)
.min(libc::RLIM_INFINITY);
let mut hard = requested.saturating_add(1).min(libc::RLIM_INFINITY);
if inherited_hard != libc::RLIM_INFINITY {
hard = hard.min(inherited_hard);
}
let soft = if hard == libc::RLIM_INFINITY || hard <= 1 {
hard
} else {
hard - 1
};
libc::rlimit {
rlim_cur: soft,
rlim_max: hard,
}
}
#[cfg(unix)]
#[allow(unsafe_code)]
pub(crate) fn read_rlimit(resource: u32) -> std::io::Result<libc::rlimit> {
let mut current = libc::rlimit {
rlim_cur: 0,
rlim_max: 0,
};
if unsafe { libc::getrlimit(resource as _, &mut current) } != 0 {
return Err(std::io::Error::last_os_error());
}
Ok(current)
}
#[cfg(unix)]
#[allow(unsafe_code)]
fn write_rlimit(resource: u32, limit: &libc::rlimit) -> std::io::Result<()> {
if unsafe { libc::setrlimit(resource as _, limit) } != 0 {
return Err(std::io::Error::last_os_error());
}
Ok(())
}
#[cfg_attr(not(unix), allow(dead_code))]
#[allow(clippy::needless_return)]
pub(crate) fn uid_task_count() -> Option<u64> {
#[cfg(target_os = "linux")]
{
let uid = {
#[allow(unsafe_code)]
unsafe {
libc::geteuid()
}
};
return uid_task_count_in(Path::new("/proc"), uid);
}
#[cfg(target_os = "macos")]
{
const PROC_UID_ONLY: u32 = 4;
extern "C" {
fn proc_listpids(
r#type: u32,
typeinfo: u32,
buffer: *mut libc::c_void,
buffersize: libc::c_int,
) -> libc::c_int;
}
#[allow(unsafe_code)]
let bytes = unsafe {
let uid = libc::geteuid();
proc_listpids(PROC_UID_ONLY, uid, std::ptr::null_mut(), 0)
};
if bytes <= 0 {
return None;
}
return Some(bytes as u64 / std::mem::size_of::<u32>() as u64);
}
#[cfg(not(any(target_os = "linux", target_os = "macos")))]
None
}
#[cfg(target_os = "linux")]
pub(crate) fn uid_task_count_in(proc_root: &Path, uid: u32) -> Option<u64> {
use std::os::unix::fs::MetadataExt;
let entries = std::fs::read_dir(proc_root).ok()?;
let count = entries
.flatten()
.filter(|entry| {
entry
.file_name()
.to_str()
.is_some_and(|name| name.bytes().all(|b| b.is_ascii_digit()))
&& entry.metadata().is_ok_and(|metadata| metadata.uid() == uid)
})
.map(|entry| {
std::fs::read_dir(entry.path().join("task"))
.map(|threads| threads.flatten().count() as u64)
.unwrap_or(0)
})
.sum();
Some(count)
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) struct WorkdirBreach {
pub(crate) max_bytes: u64,
pub(crate) observed_bytes: u64,
}
#[derive(Debug)]
pub(crate) struct WorkdirGuard {
max_bytes: u64,
breach: Mutex<Option<WorkdirBreach>>,
stop: AtomicBool,
}
impl WorkdirGuard {
pub(crate) fn spawn(
workdir: &crate::workdir_fs::WorkdirRoot,
temp_dir: Option<&crate::workdir_fs::WorkdirRoot>,
home_dir: Option<&crate::workdir_fs::WorkdirRoot>,
records_root: Option<&crate::workdir_fs::WorkdirRoot>,
max_bytes: u64,
) -> Arc<Self> {
let guard = Arc::new(Self {
max_bytes,
breach: Mutex::new(None),
stop: AtomicBool::new(false),
});
let weak = Arc::downgrade(&guard);
let workdir = workdir.clone();
let temp_dir = temp_dir.cloned();
let home_dir = home_dir.cloned();
let records_root = records_root.cloned();
thread::spawn(move || loop {
thread::sleep(WORKDIR_CHECK_INTERVAL);
let Some(guard) = weak.upgrade() else {
break;
};
if guard.stop.load(Ordering::Relaxed) || guard.breach().is_some() {
break;
}
if guard.check(
&workdir,
temp_dir.as_ref(),
home_dir.as_ref(),
records_root.as_ref(),
) {
break;
}
});
guard
}
fn check(
&self,
workdir: &crate::workdir_fs::WorkdirRoot,
temp_dir: Option<&crate::workdir_fs::WorkdirRoot>,
home_dir: Option<&crate::workdir_fs::WorkdirRoot>,
records_root: Option<&crate::workdir_fs::WorkdirRoot>,
) -> bool {
let measured: Vec<&Path> = std::iter::once(workdir)
.chain(temp_dir)
.chain(home_dir)
.map(crate::workdir_fs::WorkdirRoot::path)
.collect();
let observed = measured.iter().fold(0u64, |sum, dir| {
sum.saturating_add(directory_size_bytes(dir))
});
if observed > self.max_bytes {
self.record_breach(records_root, observed, &measured);
return true;
}
false
}
pub(crate) fn breach(&self) -> Option<WorkdirBreach> {
self.breach.lock().ok().and_then(|breach| *breach)
}
fn record_breach(
&self,
records_root: Option<&crate::workdir_fs::WorkdirRoot>,
observed_bytes: u64,
measured: &[&Path],
) {
let breach = WorkdirBreach {
max_bytes: self.max_bytes,
observed_bytes,
};
if let Ok(mut slot) = self.breach.lock() {
if slot.is_none() {
*slot = Some(breach);
}
}
let measured = measured
.iter()
.map(|dir| dir.display().to_string())
.collect::<Vec<_>>()
.join(", ");
let message = format!("{}; measured: {measured}", workdir_breach_message(breach));
crate::runtime_err!("[capsule-runtime] {message}");
if let Some(records_root) = records_root {
crate::agent::append_bootstrap_log(records_root, &format!("[resources] {message}"));
}
}
}
impl Drop for WorkdirGuard {
fn drop(&mut self) {
self.stop.store(true, Ordering::Relaxed);
}
}
pub(crate) fn workdir_breach_message(breach: WorkdirBreach) -> String {
format!(
"work folder, $HOME and $TMPDIR together grew to {} bytes, past the {} byte ceiling \
(capabilities.resources.workdir_max_bytes)",
breach.observed_bytes, breach.max_bytes
)
}
pub(crate) fn directory_size_bytes(root: &Path) -> u64 {
let mut total: u64 = 0;
let mut stack: Vec<PathBuf> = vec![root.to_path_buf()];
while let Some(dir) = stack.pop() {
let Ok(entries) = std::fs::read_dir(&dir) else {
continue;
};
for entry in entries.flatten() {
let Ok(metadata) = entry.path().symlink_metadata() else {
continue;
};
if metadata.is_dir() {
stack.push(entry.path());
} else {
total = total.saturating_add(metadata.len());
}
}
}
total
}
#[cfg(all(test, target_os = "linux"))]
pub(crate) mod test_support {
use std::fs::{File, OpenOptions};
use std::sync::{Arc, Barrier};
pub(crate) fn host_threads_lock() -> File {
let path = std::env::temp_dir().join("murmur-capsule-runtime-host-threads.lock");
let file = OpenOptions::new()
.create(true)
.truncate(false)
.write(true)
.open(&path)
.unwrap_or_else(|error| panic!("opening {}: {error}", path.display()));
file.lock()
.unwrap_or_else(|error| panic!("locking {}: {error}", path.display()));
file
}
pub(crate) struct HeldThreads {
release: Arc<Barrier>,
handles: Vec<std::thread::JoinHandle<()>>,
}
impl HeldThreads {
pub(crate) fn start(count: usize) -> Self {
let started = Arc::new(Barrier::new(count + 1));
let release = Arc::new(Barrier::new(count + 1));
let handles = (0..count)
.map(|_| {
let started = Arc::clone(&started);
let release = Arc::clone(&release);
std::thread::Builder::new()
.stack_size(64 * 1024)
.spawn(move || {
started.wait();
release.wait();
})
.expect("a holder thread must start")
})
.collect();
started.wait();
Self { release, handles }
}
}
impl Drop for HeldThreads {
fn drop(&mut self) {
self.release.wait();
for handle in self.handles.drain(..) {
let _ = handle.join();
}
}
}
#[allow(unsafe_code)]
pub(crate) fn own_rlimit_nproc() -> (libc::rlim_t, libc::rlim_t) {
let mut current = libc::rlimit {
rlim_cur: 0,
rlim_max: 0,
};
assert_eq!(
unsafe { libc::getrlimit(libc::RLIMIT_NPROC, &mut current) },
0
);
(current.rlim_cur, current.rlim_max)
}
pub(crate) fn rendered_rlimit(value: libc::rlim_t) -> String {
if value == libc::RLIM_INFINITY {
"unlimited".to_string()
} else {
value.to_string()
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn an_unconfined_session_applies_only_the_declared_limits() {
use murmur_artifact::{ContainmentClass, ResourceCapabilities};
let empty = ResourceCapabilities::default();
assert!(declared_resource_fields(&empty).is_empty());
assert!(!declares_cgroup_limit(&empty));
let declared = ResourceCapabilities {
max_open_files: Some(256),
cgroup_pids_max: Some(64),
workdir_max_bytes: Some(1 << 20),
..Default::default()
};
assert_eq!(
declared_resource_fields(&declared),
vec!["max_open_files", "cgroup_pids_max", "workdir_max_bytes"]
);
assert!(declares_cgroup_limit(&declared));
let policy = |containment_floor, declared_resources| crate::types::CapabilityPolicy {
containment_floor,
declared_resources,
..Default::default()
};
assert_eq!(
workdir_guard_max_bytes(&policy(ContainmentClass::Unconfined, empty)),
None
);
assert_eq!(
workdir_guard_max_bytes(&policy(ContainmentClass::Unconfined, declared)),
Some(1 << 20)
);
assert_eq!(
workdir_guard_max_bytes(&policy(ContainmentClass::Advisory, empty)),
Some(DEFAULT_WORKDIR_MAX_BYTES)
);
}
#[test]
fn resolve_uses_defaults_when_manifest_is_silent() {
assert_eq!(
HostResourceLimits::resolve(None),
HostResourceLimits::default()
);
}
#[test]
fn the_cpu_fields_are_unset_unless_declared() {
let silent = HostResourceLimits::resolve(None);
assert_eq!(silent.cpu_seconds, None);
assert_eq!(silent.cgroup_cpu_percent, None);
let declared = murmur_artifact::ResourceCapabilities {
cpu_seconds: Some(7),
cgroup_cpu_percent: Some(50),
..murmur_artifact::ResourceCapabilities::default()
};
let limits = HostResourceLimits::resolve(Some(&declared));
assert_eq!(limits.cpu_seconds, Some(7));
assert_eq!(limits.cgroup_cpu_percent, Some(50));
assert_eq!(limits.memory_bytes, DEFAULT_MEMORY_BYTES);
}
#[test]
fn resolve_fills_each_omitted_field_independently() {
let declared = murmur_artifact::ResourceCapabilities {
max_open_files: Some(16),
cgroup_pids_max: Some(32),
..murmur_artifact::ResourceCapabilities::default()
};
let limits = HostResourceLimits::resolve(Some(&declared));
assert_eq!(limits.max_open_files, 16);
assert_eq!(limits.cgroup_pids_max, 32);
assert_eq!(limits.max_processes, DEFAULT_MAX_PROCESSES);
assert_eq!(limits.max_file_size_bytes, DEFAULT_MAX_FILE_SIZE_BYTES);
assert_eq!(limits.cpu_seconds, None);
assert_eq!(limits.memory_bytes, DEFAULT_MEMORY_BYTES);
assert_eq!(limits.cgroup_memory_bytes, DEFAULT_CGROUP_MEMORY_BYTES);
assert_eq!(limits.cgroup_cpu_percent, None);
assert_eq!(
limits.cgroup_io_bytes_per_sec,
DEFAULT_CGROUP_IO_BYTES_PER_SEC
);
assert_eq!(limits.workdir_max_bytes, DEFAULT_WORKDIR_MAX_BYTES);
}
#[test]
fn free_resolve_matches_the_inherent_one() {
let declared = murmur_artifact::ResourceCapabilities {
cpu_seconds: Some(5),
..murmur_artifact::ResourceCapabilities::default()
};
assert_eq!(
resolve(Some(&declared)),
HostResourceLimits::resolve(Some(&declared))
);
}
#[test]
fn only_the_two_unambiguous_signals_name_a_limit() {
assert_eq!(limit_from_signal(libc::SIGXCPU), Some("cpu_seconds"));
assert_eq!(
limit_from_signal(libc::SIGXFSZ),
Some("max_file_size_bytes")
);
for ambiguous in [libc::SIGKILL, libc::SIGSEGV, libc::SIGABRT, libc::SIGTERM] {
assert_eq!(
limit_from_signal(ambiguous),
None,
"signal {ambiguous} has more than one possible cause and must stay unattributed"
);
}
}
#[test]
fn every_attributable_limit_has_its_own_line() {
use crate::cgroup::CgroupEventCounters;
let cpu = limit_from_signal(libc::SIGXCPU).unwrap();
let fsize = limit_from_signal(libc::SIGXFSZ).unwrap();
let memory = CgroupEventCounters {
oom_kill: 1,
pids_max: 0,
}
.attribution_since(CgroupEventCounters::default())
.unwrap();
let pids = CgroupEventCounters {
oom_kill: 0,
pids_max: 1,
}
.attribution_since(CgroupEventCounters::default())
.unwrap();
let expected = [
(
pids,
"resource_limit: cgroup_pids_max — the capsule reached its process limit while \
this command ran, so starting a new process failed (EAGAIN). Retry with fewer \
parallel processes, or ask the operator to raise \
capabilities.resources.cgroup_pids_max.",
),
(
memory,
"resource_limit: cgroup_memory_bytes — the capsule reached its memory limit while \
this command ran, and the kernel killed a process in it to free memory. Retry \
using less memory at once (smaller inputs, fewer parallel jobs), or ask the \
operator to raise capabilities.resources.cgroup_memory_bytes.",
),
(
cpu,
"resource_limit: cpu_seconds — this command used up its CPU-time limit and was \
stopped. Retry with less work per command, or ask the operator to raise \
capabilities.resources.cpu_seconds.",
),
(
fsize,
"resource_limit: max_file_size_bytes — this command tried to write a file larger \
than the capsule's file-size limit and was stopped. Retry writing smaller files, \
or ask the operator to raise capabilities.resources.max_file_size_bytes.",
),
];
for (name, text) in expected {
let line = resource_limit_line(name);
assert_eq!(line, text);
assert!(line.starts_with(&format!("resource_limit: {name} — ")));
assert!(line.contains(&format!("capabilities.resources.{name}")));
assert!(line.contains("ask the operator to raise"));
assert!(!line.contains('\n'), "{line:?} must be a single line");
}
for cgroup in [pids, memory] {
let line = resource_limit_line(cgroup);
assert!(line.contains("the capsule"), "{line}");
assert!(line.contains("while this command ran"), "{line}");
}
for signal in [cpu, fsize] {
assert!(resource_limit_line(signal).contains("this command"));
}
assert_eq!(
resource_limit_line("some_future_limit"),
"resource_limit: some_future_limit — this command was stopped by the capsule's \
capabilities.resources.some_future_limit limit. Retry with less work, or ask the \
operator to raise capabilities.resources.some_future_limit."
);
}
#[test]
fn directory_size_sums_nested_files() {
let temp = tempfile::tempdir().unwrap();
std::fs::write(temp.path().join("a"), vec![0u8; 100]).unwrap();
std::fs::create_dir(temp.path().join("nested")).unwrap();
std::fs::write(temp.path().join("nested").join("b"), vec![0u8; 250]).unwrap();
assert_eq!(directory_size_bytes(temp.path()), 350);
}
#[test]
fn directory_size_of_a_missing_path_is_zero() {
assert_eq!(
directory_size_bytes(Path::new("/nonexistent-murmur-workdir-probe")),
0
);
}
#[test]
fn guard_reports_no_breach_for_a_workdir_under_the_ceiling() {
let temp = tempfile::tempdir().unwrap();
std::fs::write(temp.path().join("small"), vec![0u8; 64]).unwrap();
let guard = WorkdirGuard::spawn(
&crate::workdir_fs::WorkdirRoot::open(temp.path()).unwrap(),
None,
None,
None,
DEFAULT_WORKDIR_MAX_BYTES,
);
assert_eq!(guard.breach(), None);
}
fn workdir_guard_breaches_on_bytes_in(dir: usize) {
const MIB: u64 = 1024 * 1024;
let dirs = [(); 3].map(|()| tempfile::tempdir().unwrap());
std::fs::write(dirs[0].path().join("small"), vec![0u8; 100]).unwrap();
std::fs::write(dirs[dir].path().join("big"), vec![0u8; 2 * MIB as usize]).unwrap();
let [workdir, temp, home] =
[0, 1, 2].map(|i| crate::workdir_fs::WorkdirRoot::open(dirs[i].path()).unwrap());
let guard = WorkdirGuard::spawn(&workdir, Some(&temp), Some(&home), None, MIB);
if dir != 0 {
assert!(
!guard.check(&workdir, None, None, None),
"the work folder alone is under 1 MiB"
);
assert_eq!(guard.breach(), None);
}
assert!(guard.check(&workdir, Some(&temp), Some(&home), None));
let breach = guard.breach().expect("the bytes breach the ceiling");
assert_eq!(breach.max_bytes, MIB);
assert_eq!(breach.observed_bytes, 2 * MIB + 100);
let message = workdir_breach_message(breach);
assert!(
message.contains("work folder, $HOME and $TMPDIR"),
"{message}"
);
let error = crate::errors::RuntimeError::WorkdirSizeExceeded {
max_bytes: breach.max_bytes,
observed_bytes: breach.observed_bytes,
}
.to_string();
assert!(error.contains("work folder, $HOME and $TMPDIR"), "{error}");
}
#[test]
fn workdir_guard_counts_the_work_folder() {
workdir_guard_breaches_on_bytes_in(0);
}
#[test]
fn workdir_guard_counts_the_temp_folder() {
workdir_guard_breaches_on_bytes_in(1);
}
#[test]
fn workdir_guard_counts_the_home_folder() {
workdir_guard_breaches_on_bytes_in(2);
}
#[test]
fn workdir_guard_logs_what_it_measured() {
let dirs = [(); 4].map(|()| tempfile::tempdir().unwrap());
std::fs::write(dirs[0].path().join("big"), vec![0u8; 4096]).unwrap();
let [workdir, temp, home, records] =
[0, 1, 2, 3].map(|i| crate::workdir_fs::WorkdirRoot::open(dirs[i].path()).unwrap());
let guard = WorkdirGuard::spawn(&workdir, Some(&temp), Some(&home), None, 1024);
assert!(guard.check(&workdir, Some(&temp), Some(&home), Some(&records)));
let log = std::fs::read_to_string(dirs[3].path().join("logs/bootstrap.log")).unwrap();
for dir in &dirs[..3] {
assert!(log.contains(&dir.path().display().to_string()), "{log}");
}
assert!(log.contains("measured: "), "{log}");
}
#[cfg(any(target_os = "linux", target_os = "macos"))]
#[test]
fn uid_task_count_reports_a_plausible_baseline() {
let count = uid_task_count().expect("this platform can be asked for its task count");
assert!(
count > 0,
"the test process itself is owned by this uid, so the count cannot be zero"
);
}
#[cfg(target_os = "linux")]
#[test]
fn uid_task_count_sums_threads_not_processes() {
let temp = tempfile::tempdir().unwrap();
let root = temp.path();
for (pid, threads) in [("100", 1), ("200", 4), ("300", 2), ("self", 5)] {
let task = root.join(pid).join("task");
std::fs::create_dir_all(&task).unwrap();
for thread in 0..threads {
std::fs::write(task.join(thread.to_string()), b"").unwrap();
}
}
std::fs::create_dir(root.join("400")).unwrap();
let own_uid = {
use std::os::unix::fs::MetadataExt;
std::fs::metadata(root.join("100")).unwrap().uid()
};
assert_eq!(uid_task_count_in(root, own_uid), Some(7));
assert_eq!(uid_task_count_in(root, own_uid.wrapping_add(1)), Some(0));
assert_eq!(uid_task_count_in(&root.join("missing"), own_uid), None);
}
#[cfg(unix)]
#[test]
#[allow(unsafe_code)]
#[allow(clippy::unnecessary_cast)]
fn set_hard_rlimit_writes_both_the_soft_and_hard_slot() {
set_hard_rlimit(libc::RLIMIT_CORE as u32, 0).unwrap();
let mut current = libc::rlimit {
rlim_cur: 1,
rlim_max: 1,
};
assert_eq!(
unsafe { libc::getrlimit(libc::RLIMIT_CORE as _, &mut current) },
0
);
assert_eq!(current.rlim_cur, 0);
assert_eq!(
current.rlim_max, 0,
"the hard slot must be set, not only the soft one"
);
}
#[cfg(unix)]
#[test]
fn cpu_rlimit_slots_warn_one_second_before_the_hard_ceiling() {
let inf = libc::RLIM_INFINITY;
let cases: [(u64, libc::rlim_t, (libc::rlim_t, libc::rlim_t)); 5] = [
(5, inf, (5, 6)),
(3600, 100, (99, 100)),
(5, 6, (5, 6)),
(5, 1, (1, 1)),
(u64::MAX, inf, (inf, inf)),
];
for (cpu_seconds, inherited_hard, expected) in cases {
let slots = cpu_rlimit_slots(cpu_seconds, inherited_hard);
assert_eq!(
(slots.rlim_cur, slots.rlim_max),
expected,
"cpu_seconds {cpu_seconds}, inherited hard {inherited_hard}"
);
}
}
}