use core::fmt;
use std::ffi::OsStr;
use std::fs;
use std::io;
use std::path::{Path, PathBuf};
use shep_core::config::CpuCores;
use crate::runner::{RunnerError, SpawnSpec};
const MOUNT: &str = "/sys/fs/cgroup";
const LEAF: &str = "shepherd";
const CPU_PERIOD_US: u64 = CpuCores::PERIOD_US;
const CONTROLLERS: &str = "+memory +cpu +pids";
#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
pub(crate) struct Limits {
pub memory_max: Option<u64>,
pub cpu_quota: Option<u64>,
pub pids_max: Option<u32>,
}
impl From<&SpawnSpec> for Limits {
fn from(spec: &SpawnSpec) -> Self {
Self {
memory_max: spec.memory_max,
cpu_quota: spec.cpu_quota,
pids_max: spec.pids_max,
}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) enum Cgroups {
Available { base: PathBuf },
Unavailable(Reason),
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) enum Reason {
NotV2,
NotDelegated,
Busy,
Setup(io::ErrorKind),
}
impl fmt::Display for Reason {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::NotV2 => f.write_str("this host is not on cgroup v2 alone"),
Self::NotDelegated => f.write_str("the shepherd's cgroup is not delegated to it"),
Self::Busy => f.write_str("the shepherd's cgroup holds another process"),
Self::Setup(kind) => write!(f, "setting up the shepherd's cgroup failed: {kind}"),
}
}
}
impl fmt::Display for Cgroups {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::Available { base } => {
write!(f, "cgroup v2 available under {}", base.display())
}
Self::Unavailable(reason) => write!(f, "cgroups unavailable: {reason}"),
}
}
}
pub(crate) fn detect() -> Cgroups {
detect_at(
Path::new("/proc/self/cgroup"),
Path::new(MOUNT),
std::process::id(),
)
}
fn detect_at(proc_cgroup: &Path, mount: &Path, pid: u32) -> Cgroups {
match try_detect(proc_cgroup, mount, pid) {
Ok(found) => found,
Err(reason) => Cgroups::Unavailable(reason),
}
}
fn try_detect(proc_cgroup: &Path, mount: &Path, pid: u32) -> Result<Cgroups, Reason> {
let text = fs::read_to_string(proc_cgroup).map_err(|err| Reason::Setup(err.kind()))?;
let own = unified_path(&text).ok_or(Reason::NotV2)?;
if !mount.join("cgroup.controllers").is_file() {
return Err(Reason::NotV2);
}
let own_dir = mount.join(own.trim_start_matches('/'));
let base = if own_dir.file_name().is_some_and(|name| name == LEAF) {
own_dir.parent().unwrap_or(mount).to_path_buf()
} else {
if holds_another_process(&own_dir, pid) {
return Err(Reason::Busy);
}
move_into_leaf(&own_dir.join(LEAF), pid)?;
own_dir
};
fs::write(base.join("cgroup.subtree_control"), CONTROLLERS).map_err(|err| {
match err.kind() {
io::ErrorKind::ResourceBusy => Reason::Busy,
_ => classify(&err),
}
})?;
Ok(Cgroups::Available { base })
}
fn holds_another_process(dir: &Path, pid: u32) -> bool {
fs::read_to_string(dir.join("cgroup.procs")).is_ok_and(|procs| {
procs
.split_whitespace()
.any(|listed| listed.parse::<u32>().is_ok_and(|listed| listed != pid))
})
}
fn move_into_leaf(leaf: &Path, pid: u32) -> Result<(), Reason> {
let created = match fs::create_dir(leaf) {
Ok(()) => true,
Err(err) if err.kind() == io::ErrorKind::AlreadyExists => false,
Err(err) => return Err(classify(&err)),
};
fs::write(leaf.join("cgroup.procs"), pid.to_string()).map_err(|err| {
if created {
let _ = fs::remove_dir(leaf);
}
classify(&err)
})
}
fn unified_path(text: &str) -> Option<&str> {
let mut lines = text.lines().filter(|line| !line.is_empty());
let path = lines.next()?.strip_prefix("0::")?;
lines.next().is_none().then_some(path)
}
fn classify(err: &io::Error) -> Reason {
match err.kind() {
io::ErrorKind::PermissionDenied | io::ErrorKind::ReadOnlyFilesystem => Reason::NotDelegated,
kind => Reason::Setup(kind),
}
}
pub(crate) fn sheep_dir(base: &Path, id: u32, instance: u32) -> PathBuf {
base.join(format!("sheep-{id}-{instance}"))
}
fn is_sheep_dir(name: &OsStr) -> bool {
name.to_str()
.and_then(|name| name.strip_prefix("sheep-"))
.and_then(|rest| rest.split_once('-'))
.is_some_and(|(id, instance)| id.parse::<u32>().is_ok() && instance.parse::<u32>().is_ok())
}
pub(crate) fn join_on_exec(
command: &mut std::process::Command,
dir: &Path,
limits: Limits,
) -> io::Result<()> {
let created = match fs::create_dir(dir) {
Ok(()) => true,
Err(err) if err.kind() == io::ErrorKind::AlreadyExists => false,
Err(err) => return Err(err),
};
let procs = if is_occupied(dir) {
Err(io::Error::from(io::ErrorKind::ResourceBusy))
} else {
fs::write(dir.join("memory.oom.group"), "1")
.and_then(|()| {
write_unless_fresh(created, limits.memory_max, |l| write_memory_max(dir, l))
})
.and_then(|()| write_unless_fresh(created, limits.cpu_quota, |l| write_cpu_max(dir, l)))
.and_then(|()| write_unless_fresh(created, limits.pids_max, |l| write_pids_max(dir, l)))
.and_then(|()| {
fs::OpenOptions::new()
.write(true)
.open(dir.join("cgroup.procs"))
})
}
.inspect_err(|_| {
if created {
let _ = fs::remove_dir(dir);
}
})?;
crate::sys::join_cgroup_on_exec(command, procs);
Ok(())
}
fn write_unless_fresh<T>(
created: bool,
limit: Option<T>,
write: impl FnOnce(Option<T>) -> io::Result<()>,
) -> io::Result<()> {
if limit.is_none() && created {
Ok(())
} else {
write(limit)
}
}
pub(crate) fn write_memory_max(dir: &Path, limit: Option<u64>) -> io::Result<()> {
let (max, swap) = match limit {
Some(bytes) => (bytes.to_string(), "0"),
None => ("max".to_string(), "max"),
};
fs::write(dir.join("memory.max"), max)?;
let _ = fs::write(dir.join("memory.swap.max"), swap);
Ok(())
}
pub(crate) fn cpu_quota(cores: CpuCores) -> u64 {
cores.quota_us()
}
pub(crate) fn write_cpu_max(dir: &Path, quota: Option<u64>) -> io::Result<()> {
let quota = quota.map_or_else(|| "max".to_string(), |micros| micros.to_string());
fs::write(dir.join("cpu.max"), format!("{quota} {CPU_PERIOD_US}"))
}
pub(crate) fn write_pids_max(dir: &Path, limit: Option<u32>) -> io::Result<()> {
let limit = limit.map_or_else(|| "max".to_string(), |tasks| tasks.to_string());
fs::write(dir.join("pids.max"), limit)
}
pub(crate) fn oom_kills(dir: &Path) -> u64 {
fs::read_to_string(dir.join("memory.events.local")).map_or(0, |text| parse_oom_kill(&text))
}
fn parse_oom_kill(text: &str) -> u64 {
text.lines()
.filter_map(|line| line.strip_prefix("oom_kill "))
.find_map(|value| value.trim().parse().ok())
.unwrap_or(0)
}
fn is_occupied(dir: &Path) -> bool {
let mut first = [0u8; 1];
fs::File::open(dir.join("cgroup.procs"))
.and_then(|mut procs| io::Read::read(&mut procs, &mut first))
.is_ok_and(|read| read == 1 && first[0].is_ascii_digit())
}
pub(crate) fn spawn_failed(name: &str, dir: &Path, error: &io::Error) -> RunnerError {
let _ = remove_if_empty(dir);
RunnerError::SpawnFailed(format!(
"{name} could not start in cgroup {}: {error}",
dir.display()
))
}
pub(crate) fn remove_if_empty(dir: &Path) -> io::Result<()> {
if is_occupied(dir) {
return Ok(());
}
match fs::remove_dir(dir) {
Err(err) if err.kind() != io::ErrorKind::NotFound => Err(err),
_ => Ok(()),
}
}
pub(crate) fn release(dir: Option<&Path>) {
if let Some(dir) = dir {
let _ = remove_if_empty(dir);
}
}
pub(crate) fn remove_stale(base: &Path) {
let Ok(entries) = fs::read_dir(base) else {
return;
};
for entry in entries.flatten() {
if !is_sheep_dir(&entry.file_name()) {
continue;
}
if let Err(error) = remove_if_empty(&entry.path()) {
tracing::warn!(%error, cgroup = %entry.path().display(), "stale sheep cgroup not removed");
}
}
}
#[cfg(test)]
mod cpu_tests;
#[cfg(test)]
mod task_tests;
#[cfg(test)]
mod tests;