use super::*;
use cortexkit_test_support::ScratchDir;
use std::{fs, path::Path};
struct KernelCgroup {
root: PathBuf,
placement: subc_cgroup::Placement,
modules: PathBuf,
}
impl KernelCgroup {
fn new(test_name: &str) -> Option<Self> {
let required = std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_some()
|| std::env::var_os("SUBC_TEST_CGROUP_REQUIRED").is_some();
let base = std::env::var_os("SUBC_TEST_CGROUP_ROOT")
.map(PathBuf::from)
.or_else(|| {
std::env::var_os("SUBC_REQUIRE_CGROUP_TEST")
.map(|_| PathBuf::from("/sys/fs/cgroup"))
});
let Some(base) = base else {
assert!(
!required,
"required cgroup test needs SUBC_TEST_CGROUP_ROOT"
);
eprintln!("SKIP {test_name}: set SUBC_TEST_CGROUP_ROOT to an owned delegated cgroup");
return None;
};
let root = base.join(format!(
"subc-reap-test-{test_name}-{}-{}",
std::process::id(),
unix_ms_now()
));
if let Err(error) = fs::create_dir(&root) {
assert!(!required, "required cgroup test cannot execute: {error}");
eprintln!("SKIP {test_name}: no writable delegated cgroup: {error}");
return None;
}
let placement = subc_cgroup::prepare_at(&root).unwrap().unwrap();
let modules = root.join("subc-modules");
let fixture = Self {
root,
placement,
modules,
};
if !fixture.root.join("cgroup.kill").exists()
|| !fixture.root.join("cgroup.freeze").exists()
{
assert!(
!required,
"required cgroup.kill/cgroup.freeze interface unavailable"
);
eprintln!("SKIP {test_name}: cgroup.kill/cgroup.freeze unavailable");
return None;
}
Some(fixture)
}
fn module_cgroups(&self) -> Vec<PathBuf> {
fs::read_dir(&self.modules)
.map(|entries| {
entries
.filter_map(Result::ok)
.filter(|entry| entry.file_type().is_ok_and(|kind| kind.is_dir()))
.map(|entry| entry.path())
.collect()
})
.unwrap_or_default()
}
}
impl Drop for KernelCgroup {
fn drop(&mut self) {
for module in self.module_cgroups() {
let _ = fs::write(module.join("cgroup.kill"), "1");
let deadline = std::time::Instant::now() + Duration::from_secs(1);
while cgroup_is_populated(&module) && std::time::Instant::now() < deadline {
std::thread::sleep(Duration::from_millis(1));
}
fn remove_empty_tree(path: &Path) {
if let Ok(entries) = fs::read_dir(path) {
for entry in entries.flatten() {
if entry.file_type().is_ok_and(|kind| kind.is_dir()) {
remove_empty_tree(&entry.path());
}
}
}
let _ = fs::remove_dir(path);
}
remove_empty_tree(&module);
}
let _ = fs::remove_dir(&self.modules);
let _ = fs::remove_dir(&self.root);
}
}
fn spawn_module(
placement: subc_cgroup::Placement,
dir: &Path,
args: Vec<String>,
) -> SupervisedChild {
let supervisor = Supervisor::new_for_test(
Arc::new(Registry::default()),
RestartPolicy::new(3, Duration::ZERO),
)
.with_cgroup_placement(Some(placement));
let spec = ModuleSpec {
carry_containment: false,
module_id: "reaped-module".to_string(),
program: PathBuf::from("/bin/sh"),
args,
env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
.into_iter()
.map(|key| (key.to_string(), dir.display().to_string()))
.collect(),
reserved: false,
reserved_prefixes: Vec::new(),
protocol: ModuleProtocol::None,
overlap: Default::default(),
};
spawn_and_mark_running(
&spec,
&supervisor.runtime_config(),
&Arc::new(Mutex::new(SupervisorSnapshot::starting())),
)
.expect("spawn module that exits without waiting for its child")
}
async fn reap_leftover(test_name: &str, ignore_term: bool) {
let Some(kernel) = KernelCgroup::new(test_name) else {
return;
};
let dir = ScratchDir::new(test_name);
let marker = dir.join("terminated");
let pid_file = dir.join("leftover.pid");
let script = dir.join("leftover.sh");
let body = if ignore_term {
"trap '' TERM\necho $$ > \"$2\"\nexec sleep 600\n"
} else {
"trap 'printf terminated > \"$1\"; exit 0' TERM\necho $$ > \"$2\"\nwhile :; do :; done\n"
};
fs::write(&script, body).unwrap();
let mut child = spawn_module(
kernel.placement.clone(),
&dir,
vec![
"-c".into(),
"/bin/sh \"$1\" \"$2\" \"$3\" & while [ ! -s \"$3\" ]; do sleep 0.01; done".into(),
"fixture".into(),
script.display().to_string(),
marker.display().to_string(),
pid_file.display().to_string(),
],
);
assert_eq!(
kernel.module_cgroups().len(),
1,
"the spawn must create exactly one module cgroup"
);
let started = Instant::now();
timeout(Duration::from_secs(10), async {
child.wait().await?;
child.finish_exit(false).await
})
.await
.expect("leftover cleanup must finish")
.expect("reap module");
let elapsed = started.elapsed();
let pid: i32 = fs::read_to_string(&pid_file)
.unwrap()
.trim()
.parse()
.unwrap();
let running = fs::read_to_string(format!("/proc/{pid}/stat"))
.ok()
.and_then(|stat| {
stat.rsplit_once(") ")
.map(|(_, rest)| !rest.starts_with('Z'))
})
.unwrap_or(false);
assert!(!running, "leftover {pid} must not outlive cleanup");
assert_eq!(
kernel.module_cgroups(),
Vec::<PathBuf>::new(),
"module cgroup must be removed"
);
if ignore_term {
assert!(
elapsed >= Duration::from_millis(1900),
"SIGTERM grace was skipped: {elapsed:?}"
);
assert!(
elapsed < Duration::from_secs(5),
"SIGKILL escalation was late: {elapsed:?}"
);
} else {
assert!(
marker.exists(),
"leftover must receive SIGTERM before cgroup.kill"
);
assert!(
elapsed < Duration::from_secs(1),
"a cooperative child should not use the full grace: {elapsed:?}"
);
}
eprintln!("EXECUTED {test_name}: leftover {pid} stopped in {elapsed:?}");
}
#[tokio::test]
async fn linux_reaped_cgroup_sigterm_then_exit() {
reap_leftover("linux_reaped_cgroup_sigterm_then_exit", false).await;
}
#[tokio::test]
async fn linux_reaped_cgroup_ignored_sigterm_escalates_after_two_seconds() {
reap_leftover("linux_reaped_cgroup_ignored_sigterm", true).await;
}
#[tokio::test]
async fn linux_reaped_cgroup_empty_has_no_grace_wait() {
let Some(kernel) = KernelCgroup::new("linux_reaped_cgroup_empty") else {
return;
};
let dir = ScratchDir::new("linux-reaped-cgroup-empty");
let mut child = spawn_module(
kernel.placement.clone(),
&dir,
vec!["-c".into(), "exit 0".into()],
);
assert_eq!(
kernel.module_cgroups().len(),
1,
"the spawn must create exactly one module cgroup"
);
let started = Instant::now();
child.wait().await.unwrap();
child.finish_exit(false).await.unwrap();
assert!(started.elapsed() < Duration::from_secs(1));
assert_eq!(
kernel.module_cgroups(),
Vec::<PathBuf>::new(),
"empty module cgroup must be removed"
);
eprintln!("EXECUTED linux_reaped_cgroup_empty: no grace wait");
}
#[path = "carried_cgroup_tests.rs"]
mod carried;
async fn scratch_leftover(test_name: &str, ignore_term: bool, freezer: &str) -> (bool, Duration) {
let dir = ScratchDir::new(test_name);
fs::write(dir.join("cgroup.procs"), "").unwrap();
let placement = subc_cgroup::prepare_at(&dir).unwrap().unwrap();
let module = placement.module_path("reaped-module").unwrap();
fs::write(module.join("cgroup.kill"), "0").unwrap();
let frozen = if freezer == "confirmed" { 1 } else { 0 };
fs::write(
module.join("cgroup.events"),
format!("populated 1\nfrozen {frozen}\n"),
)
.unwrap();
match freezer {
"unavailable" => {}
"io-error" => fs::create_dir(module.join("cgroup.freeze")).unwrap(),
_ => fs::write(module.join("cgroup.freeze"), "0").unwrap(),
}
let marker = dir.join("terminated");
let ready = dir.join("ready");
let body = if ignore_term {
"trap '' TERM; echo ready > \"$2\"; exec sleep 600"
} else {
"trap 'printf terminated > \"$1\"; exit 0' TERM; echo ready > \"$2\"; while :; do :; done"
};
let mut child = Command::new("/bin/sh")
.args(["-c", body, "fixture"])
.arg(&marker)
.arg(&ready)
.kill_on_drop(true)
.spawn()
.unwrap();
let pid = child.id().unwrap();
timeout(Duration::from_secs(5), async {
while !ready.exists() {
sleep(Duration::from_millis(1)).await;
}
})
.await
.unwrap();
fs::write(module.join("cgroup.procs"), format!("{pid}\n")).unwrap();
let kernel = async {
loop {
tokio::select! {
status = child.wait() => { status.unwrap(); break; }
_ = sleep(Duration::from_millis(1)) => {
if fs::read(module.join("cgroup.kill")).unwrap() == b"1" {
let pid = rustix::process::Pid::from_raw(pid as i32).unwrap();
rustix::process::kill_process(pid, rustix::process::Signal::KILL).unwrap();
child.wait().await.unwrap();
break;
}
}
}
}
fs::write(module.join("cgroup.events"), "populated 0\nfrozen 0\n").unwrap();
for entry in fs::read_dir(&module).unwrap() {
let path = entry.unwrap().path();
if path.is_dir() {
fs::remove_dir(path).unwrap();
} else {
fs::remove_file(path).unwrap();
}
}
};
let started = Instant::now();
timeout(Duration::from_secs(5), async {
tokio::join!(cleanup_reaped_cgroup(&placement, "reaped-module"), kernel);
})
.await
.expect("scratch cleanup must finish");
assert!(!module.exists(), "scratch module directory must be removed");
(marker.exists(), started.elapsed())
}
#[tokio::test]
async fn scratch_reaped_cgroup_sigterm_then_exit() {
let (marker, elapsed) = scratch_leftover("scratch-reap-term", false, "confirmed").await;
assert!(marker, "leftover must receive SIGTERM before cgroup.kill");
assert!(elapsed < Duration::from_secs(1));
}
#[tokio::test]
async fn scratch_reaped_cgroup_ignored_sigterm_escalates_after_two_seconds() {
let (_, elapsed) = scratch_leftover("scratch-reap-ignore", true, "confirmed").await;
assert!(
elapsed >= Duration::from_millis(1900),
"grace was skipped: {elapsed:?}"
);
assert!(elapsed < Duration::from_secs(5));
}
#[tokio::test]
async fn scratch_reaped_cgroup_freeze_failures_never_send_sigterm() {
for freezer in ["unavailable", "timeout", "io-error"] {
let (marker, elapsed) =
scratch_leftover(&format!("scratch-reap-{freezer}"), false, freezer).await;
assert!(!marker, "unfrozen PIDs must not be signalled: {freezer}");
assert!(
elapsed < Duration::from_secs(1),
"fallback must skip grace: {elapsed:?}"
);
}
}
#[tokio::test(start_paused = true)]
async fn scratch_reaped_cgroup_empty_skips_freeze_and_kill() {
let dir = ScratchDir::new("scratch-reap-empty");
fs::write(dir.join("cgroup.procs"), "").unwrap();
let placement = subc_cgroup::prepare_at(&dir).unwrap().unwrap();
let module = placement.module_path("reaped-module").unwrap();
fs::write(module.join("cgroup.events"), "populated 0\nfrozen 0\n").unwrap();
fs::write(module.join("cgroup.freeze"), "unchanged").unwrap();
fs::write(module.join("cgroup.kill"), "unchanged").unwrap();
let started = Instant::now();
cleanup_reaped_cgroup(&placement, "reaped-module").await;
assert_eq!(started.elapsed(), Duration::ZERO);
assert_eq!(
fs::read(module.join("cgroup.freeze")).unwrap(),
b"unchanged"
);
assert_eq!(fs::read(module.join("cgroup.kill")).unwrap(), b"unchanged");
}
#[tokio::test]
async fn linux_cgroup_cpu_survives_child_reap() {
let Some(kernel) = KernelCgroup::new("cpu-reap") else {
return;
};
let dir = ScratchDir::new("cgroup-cpu-reap");
let script = r#"
(while [ ! -f "$1/reap" ]; do sleep 0.01; done; echo exit) |
SUBC_RESOURCE_COMMAND_FIXTURE=1 "$2" --exact resource_totals::tests::resource_command_fixture --nocapture > "$1/child.out" &
command=$!
echo "$command" > "$1/child.pid"
wait "$command"
touch "$1/reaped"
while [ ! -f "$1/finish" ]; do sleep 0.01; done
"#;
let mut child = spawn_module(
kernel.placement.clone(),
&dir,
vec![
"-c".into(),
script.into(),
"fixture".into(),
dir.display().to_string(),
std::env::current_exe().unwrap().display().to_string(),
],
);
let deadline = tokio::time::Instant::now() + Duration::from_secs(10);
while !fs::read_to_string(dir.join("child.out")).is_ok_and(|s| s.contains("RESOURCE_READY")) {
assert!(
tokio::time::Instant::now() < deadline,
"CPU child never became ready"
);
tokio::time::sleep(Duration::from_millis(10)).await;
}
let pid: u32 = fs::read_to_string(dir.join("child.pid"))
.unwrap()
.trim()
.parse()
.unwrap();
let usage = subc_os::resource_usage(pid).unwrap();
let child_cpu = (usage.cpu_user + usage.cpu_system).as_millis() as u64;
assert!(child_cpu >= 100);
let group = kernel.module_cgroups().pop().unwrap();
let before = subc_cgroup::read_cpu_ms(&group).unwrap();
assert!(before >= child_cpu);
fs::write(dir.join("reap"), "").unwrap();
while !dir.join("reaped").exists() {
assert!(
tokio::time::Instant::now() < deadline,
"CPU child was not reaped"
);
tokio::time::sleep(Duration::from_millis(10)).await;
}
let after = subc_cgroup::read_cpu_ms(&group).unwrap();
fs::write(dir.join("finish"), "").unwrap();
child.wait().await.unwrap();
assert!(
after >= before,
"cgroup CPU dropped across reap: {before} -> {after}"
);
assert!(after >= child_cpu);
eprintln!("EXECUTED linux_cgroup_cpu_survives_child_reap: {before} -> {after}");
}