use std::ffi::OsString;
use std::path::{Path, PathBuf};
use std::process::{Child, Command, Output, Stdio};
use std::time::{Duration, Instant};
const PHASE_TIMEOUT_ENV: &str = "AVER_CERT_PHASE_TIMEOUT_SECS";
const DEFAULT_PHASE_TIMEOUT: Duration = Duration::from_secs(15 * 60);
const MAX_PHASE_TIMEOUT_SECS: u64 = 24 * 60 * 60;
pub(crate) const MEMORY_LIMIT_ENV: &str = "AVER_CERT_MEMORY_LIMIT_MB";
const DEFAULT_MEMORY_LIMIT_MB: u64 = 16 * 1024;
pub(crate) const BUILD_JOBS_ENV: &str = "AVER_CERT_BUILD_JOBS";
const DEFAULT_BUILD_JOBS: u64 = 1;
const MAX_MEMORY_LIMIT_MB: u64 = 1024 * 1024;
const CLEANUP_LIMIT: Duration = Duration::from_secs(5);
const WAIT_POLL: Duration = Duration::from_millis(50);
#[derive(Debug)]
pub(crate) enum LeanStepError {
Timeout { phase: String, limit: Duration },
Failed(String),
}
impl std::fmt::Display for LeanStepError {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Self::Timeout { phase, limit } => write!(
formatter,
"the {phase} step exceeded its {}-second time limit and was stopped; \
the certificate is not accepted (set {PHASE_TIMEOUT_ENV} to allow more time)",
limit.as_secs()
),
Self::Failed(reason) => formatter.write_str(reason),
}
}
}
pub(crate) struct LeanRunner {
elan: PathBuf,
elan_home: PathBuf,
toolchain: String,
system_environment: Vec<(&'static str, OsString)>,
phase_timeout: Duration,
memory_limit_mb: u64,
build_jobs: u64,
}
impl LeanRunner {
pub(crate) fn new(toolchain: &str) -> Result<Self, String> {
Self::with_environment(toolchain, |name| std::env::var_os(name))
}
fn with_environment(
toolchain: &str,
environment: impl Fn(&str) -> Option<OsString>,
) -> Result<Self, String> {
let toolchain = toolchain.trim();
if toolchain.is_empty() || toolchain.chars().any(char::is_whitespace) {
return Err("embedded Lean toolchain name is empty or malformed".to_string());
}
let phase_timeout = phase_timeout(&environment)?;
let memory_limit_mb = memory_limit_mb(&environment)?;
let build_jobs = build_jobs(&environment)?;
#[cfg(windows)]
let user_home =
nonempty(environment("USERPROFILE")).or_else(|| nonempty(environment("HOME")));
#[cfg(not(windows))]
let user_home = nonempty(environment("HOME"));
let elan_home = nonempty(environment("ELAN_HOME"))
.map(PathBuf::from)
.or_else(|| user_home.map(|home| PathBuf::from(home).join(".elan")))
.ok_or_else(|| {
"cannot resolve the pinned Lean toolchain: set ELAN_HOME or HOME".to_string()
})?;
let elan_home = std::fs::canonicalize(&elan_home).map_err(|error| {
format!(
"cannot resolve trusted Elan home {}: {error}",
elan_home.display()
)
})?;
let elan_name = format!("elan{}", std::env::consts::EXE_SUFFIX);
let elan = elan_home.join("bin").join(elan_name);
let elan = std::fs::canonicalize(&elan).map_err(|error| {
format!(
"cannot resolve trusted Elan executable {}: {error}; install Elan there or set ELAN_HOME",
elan.display()
)
})?;
if !elan.is_file() {
return Err(format!(
"trusted Elan executable is not a regular file: {}",
elan.display()
));
}
let system_environment = Vec::new();
#[cfg(windows)]
let mut system_environment = system_environment;
#[cfg(windows)]
{
copy_if_present(&mut system_environment, "SYSTEMROOT", &environment);
copy_if_present(&mut system_environment, "WINDIR", &environment);
}
Ok(Self {
elan,
elan_home,
toolchain: toolchain.to_string(),
system_environment,
phase_timeout,
memory_limit_mb,
build_jobs,
})
}
pub(crate) fn memory_limit_mb(&self) -> u64 {
self.memory_limit_mb
}
pub(crate) fn run_lake(
&self,
work_dir: &Path,
phase: &str,
arguments: &[&str],
) -> Result<Output, LeanStepError> {
let work_dir = std::fs::canonicalize(work_dir).map_err(|error| {
LeanStepError::Failed(format!(
"cannot resolve checker-owned Lean work dir {}: {error}",
work_dir.display()
))
})?;
let temp_dir = work_dir.join(format!(
".aver-cert-tmp-{}-{}",
std::process::id(),
unique_nanos()
));
let mut builder = std::fs::DirBuilder::new();
#[cfg(unix)]
{
use std::os::unix::fs::DirBuilderExt;
builder.mode(0o700);
}
builder.create(&temp_dir).map_err(|error| {
LeanStepError::Failed(format!(
"cannot create checker-owned Lean temp dir {}: {error}",
temp_dir.display()
))
})?;
let output = run_with_timeout(
self.lake_command(&work_dir, &temp_dir, arguments),
self.phase_timeout,
phase,
);
let _ = std::fs::remove_dir_all(&temp_dir);
output
}
fn lake_command(&self, work_dir: &Path, temp_dir: &Path, arguments: &[&str]) -> Command {
let mut command = Command::new(&self.elan);
command.env_clear();
command
.env("ELAN_HOME", &self.elan_home)
.env("LAKE_ARTIFACT_CACHE", "0")
.env("LAKE_RESTORE_ARTIFACTS", "0")
.env("LAKE_NO_CACHE", "1")
.env("LAKE_CACHE_DIR", "");
for (name, value) in &self.system_environment {
command.env(name, value);
}
command
.env("TMPDIR", temp_dir)
.env("TMP", temp_dir)
.env("TEMP", temp_dir)
.current_dir(work_dir)
.arg("run")
.arg("--install")
.arg(&self.toolchain)
.arg("lake")
.args(arguments.iter().take(DIRECT_LEAN_PREFIX.len()));
if arguments.starts_with(&DIRECT_LEAN_PREFIX) {
command.arg(format!("--memory={}", self.memory_limit_mb));
}
command.args(arguments.iter().skip(DIRECT_LEAN_PREFIX.len()));
if arguments.starts_with(&BUILD_PREFIX) {
command.env("LEAN_NUM_THREADS", self.build_jobs.to_string());
}
command
}
}
const DIRECT_LEAN_PREFIX: [&str; 2] = ["env", "lean"];
const BUILD_PREFIX: [&str; 1] = ["build"];
fn build_jobs(environment: &impl Fn(&str) -> Option<OsString>) -> Result<u64, String> {
let Some(value) = nonempty(environment(BUILD_JOBS_ENV)) else {
return Ok(DEFAULT_BUILD_JOBS);
};
let text = value.to_string_lossy();
text.trim()
.parse::<u64>()
.ok()
.filter(|&jobs| jobs > 0)
.ok_or_else(|| {
format!(
"{BUILD_JOBS_ENV} must be a positive whole number of workers, got `{}`",
text.trim()
)
})
}
fn memory_limit_mb(environment: &impl Fn(&str) -> Option<OsString>) -> Result<u64, String> {
let Some(value) = nonempty(environment(MEMORY_LIMIT_ENV)) else {
return Ok(DEFAULT_MEMORY_LIMIT_MB);
};
let text = value.to_string_lossy();
let limit = text
.trim()
.parse::<u64>()
.ok()
.filter(|&limit| limit > 0)
.ok_or_else(|| {
format!(
"{MEMORY_LIMIT_ENV} must be a positive whole number of MiB, got `{}`",
text.trim()
)
})?;
if limit > MAX_MEMORY_LIMIT_MB {
return Err(format!(
"{MEMORY_LIMIT_ENV} must be at most {MAX_MEMORY_LIMIT_MB} MiB (1 TiB), got `{}`",
text.trim()
));
}
Ok(limit)
}
fn phase_timeout(environment: &impl Fn(&str) -> Option<OsString>) -> Result<Duration, String> {
let Some(value) = nonempty(environment(PHASE_TIMEOUT_ENV)) else {
return Ok(DEFAULT_PHASE_TIMEOUT);
};
let text = value.to_string_lossy();
let seconds = text
.trim()
.parse::<u64>()
.ok()
.filter(|&seconds| seconds > 0)
.ok_or_else(|| {
format!(
"{PHASE_TIMEOUT_ENV} must be a positive whole number of seconds, got `{}`",
text.trim()
)
})?;
if seconds > MAX_PHASE_TIMEOUT_SECS {
return Err(format!(
"{PHASE_TIMEOUT_ENV} must be at most {MAX_PHASE_TIMEOUT_SECS} seconds (one day), got `{}`",
text.trim()
));
}
Ok(Duration::from_secs(seconds))
}
fn run_with_timeout(
mut command: Command,
limit: Duration,
phase: &str,
) -> Result<Output, LeanStepError> {
command
.stdin(Stdio::null())
.stdout(Stdio::piped())
.stderr(Stdio::piped());
#[cfg(unix)]
{
use std::os::unix::process::CommandExt;
command.process_group(0);
}
let mut child = command.spawn().map_err(|error| {
LeanStepError::Failed(format!(
"could not start the trusted Lean toolchain for the {phase} step: {error}"
))
})?;
let stdout = drain(child.stdout.take());
let stderr = drain(child.stderr.take());
let started = Instant::now();
let mut status = None;
let finished = loop {
if status.is_none() {
status = match child.try_wait() {
Ok(exited) => exited,
Err(error) => {
kill_process_tree(&mut child);
drop(stdout);
drop(stderr);
return Err(LeanStepError::Failed(format!(
"could not observe the {phase} step: {error}"
)));
}
};
}
match status {
Some(status) if stdout.is_finished() && stderr.is_finished() => break Some(status),
_ if started.elapsed() >= limit => break None,
_ => std::thread::sleep(WAIT_POLL),
}
};
let Some(status) = finished else {
kill_process_tree(&mut child);
drop(stdout);
drop(stderr);
return Err(LeanStepError::Timeout {
phase: phase.to_string(),
limit,
});
};
Ok(Output {
status,
stdout: join_drain(stdout),
stderr: join_drain(stderr),
})
}
fn kill_process_tree(child: &mut Child) {
#[cfg(unix)]
let killer = Command::new("kill")
.args(["-KILL", "--", &format!("-{}", child.id())])
.stdin(Stdio::null())
.stdout(Stdio::null())
.stderr(Stdio::null())
.spawn();
#[cfg(windows)]
let killer = Command::new("taskkill")
.args(["/T", "/F", "/PID", &child.id().to_string()])
.stdin(Stdio::null())
.stdout(Stdio::null())
.stderr(Stdio::null())
.spawn();
if let Ok(mut killer) = killer
&& reap_with_deadline(&mut killer, CLEANUP_LIMIT).is_none()
{
let _ = killer.kill();
let _ = killer.try_wait();
}
let _ = child.kill();
let _ = reap_with_deadline(child, CLEANUP_LIMIT);
}
fn reap_with_deadline(child: &mut Child, limit: Duration) -> Option<std::process::ExitStatus> {
let started = Instant::now();
loop {
match child.try_wait() {
Ok(Some(status)) => return Some(status),
Ok(None) if started.elapsed() < limit => std::thread::sleep(WAIT_POLL),
Ok(None) | Err(_) => return None,
}
}
}
fn drain(pipe: Option<impl std::io::Read + Send + 'static>) -> std::thread::JoinHandle<Vec<u8>> {
std::thread::spawn(move || {
let mut buffer = Vec::new();
if let Some(mut pipe) = pipe {
let _ = pipe.read_to_end(&mut buffer);
}
buffer
})
}
fn join_drain(reader: std::thread::JoinHandle<Vec<u8>>) -> Vec<u8> {
reader.join().unwrap_or_default()
}
fn unique_nanos() -> u128 {
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|duration| duration.as_nanos())
.unwrap_or(0)
}
#[cfg(windows)]
fn copy_if_present(
destination: &mut Vec<(&'static str, OsString)>,
name: &'static str,
environment: &impl Fn(&str) -> Option<OsString>,
) {
if let Some(value) = nonempty(environment(name)) {
destination.push((name, value));
}
}
fn nonempty(value: Option<OsString>) -> Option<OsString> {
value.filter(|value| !value.is_empty())
}
#[cfg(test)]
mod tests {
use super::*;
use crate::wall;
use std::collections::BTreeMap;
use std::path::PathBuf;
use std::process::Command;
fn command_environment(command: &Command) -> BTreeMap<String, OsString> {
command
.get_envs()
.filter_map(|(name, value)| {
value.map(|value| (name.to_string_lossy().into_owned(), value.to_os_string()))
})
.collect()
}
#[test]
fn command_whitelists_only_toolchain_bootstrap_and_checker_temp() {
let runner = LeanRunner {
elan: PathBuf::from("/trusted/elan/bin/elan"),
elan_home: PathBuf::from("/trusted/elan"),
toolchain: wall::LEAN_TOOLCHAIN.trim().to_string(),
system_environment: Vec::new(),
phase_timeout: DEFAULT_PHASE_TIMEOUT,
memory_limit_mb: DEFAULT_MEMORY_LIMIT_MB,
build_jobs: DEFAULT_BUILD_JOBS,
};
let command = runner.lake_command(
Path::new("/checker/build"),
Path::new("/checker/temp"),
&["env", "lean", "Probe.lean"],
);
assert_eq!(command.get_program(), "/trusted/elan/bin/elan");
assert_eq!(
command
.get_args()
.map(|argument| argument.to_string_lossy().into_owned())
.collect::<Vec<_>>(),
[
"run",
"--install",
wall::LEAN_TOOLCHAIN.trim(),
"lake",
"env",
"lean",
"--memory=16384",
"Probe.lean"
]
);
for untouched in [
&["build"] as &[&str],
&["env", "leanchecker", "--fresh", "ArtifactCertificate"],
] {
let command = runner.lake_command(
Path::new("/checker/build"),
Path::new("/checker/temp"),
untouched,
);
let arguments = command
.get_args()
.map(|argument| argument.to_string_lossy().into_owned())
.collect::<Vec<_>>();
assert!(
!arguments
.iter()
.any(|argument| argument.contains("--memory")),
"{untouched:?} must not receive a lean flag: {arguments:?}"
);
assert_eq!(&arguments[4..], untouched, "arguments reordered");
}
let build = runner.lake_command(
Path::new("/checker/build"),
Path::new("/checker/temp"),
&["build"],
);
assert_eq!(
command_environment(&build).get("LEAN_NUM_THREADS"),
Some(&OsString::from("1")),
"build step did not carry the worker count"
);
assert!(
!command_environment(&command).contains_key("LEAN_NUM_THREADS"),
"worker count leaked onto a direct elaboration step"
);
let environment = command_environment(&command);
for forbidden in [
"LEAN_PATH",
"LEAN_SRC_PATH",
"LEAN_SYSROOT",
"ELAN_TOOLCHAIN",
"PATH",
"HOME",
"LAKE_HOME",
"LAKE_OVERRIDE_LEAN",
] {
assert!(
!environment.contains_key(forbidden),
"forwarded {forbidden}"
);
}
assert_eq!(environment.get("ELAN_HOME").unwrap(), "/trusted/elan");
assert_eq!(environment.get("LAKE_ARTIFACT_CACHE").unwrap(), "0");
assert_eq!(environment.get("LAKE_RESTORE_ARTIFACTS").unwrap(), "0");
assert_eq!(environment.get("LAKE_NO_CACHE").unwrap(), "1");
assert_eq!(environment.get("LAKE_CACHE_DIR").unwrap(), "");
for temp in ["TMPDIR", "TMP", "TEMP"] {
assert_eq!(environment.get(temp).unwrap(), "/checker/temp");
}
}
struct TestTree(PathBuf);
impl TestTree {
fn new() -> Self {
static NEXT: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
let path = std::env::temp_dir().join(format!(
"aver-cert-lean-env-{}-{}-{}",
std::process::id(),
NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed),
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|duration| duration.as_nanos())
.unwrap_or(0)
));
std::fs::create_dir(&path).unwrap();
Self(path)
}
}
impl Drop for TestTree {
fn drop(&mut self) {
let _ = std::fs::remove_dir_all(&self.0);
}
}
fn report(output: &std::process::Output) -> String {
format!(
"{}{}",
String::from_utf8_lossy(&output.stdout),
String::from_utf8_lossy(&output.stderr)
)
}
#[test]
fn hostile_lean_path_olean_is_not_visible() {
let toolchain = wall::LEAN_TOOLCHAIN.trim();
let available = Command::new("elan")
.args(["run", "--install", toolchain, "lake", "--version"])
.output();
if !matches!(available, Ok(output) if output.status.success()) {
assert!(
std::env::var_os("AVER_CERT_REQUIRE_PINNED_LEAN").is_none(),
"pinned Elan toolchain is required for this security gate"
);
eprintln!("skipping hostile LEAN_PATH test: pinned Elan toolchain unavailable");
return;
}
let tree = TestTree::new();
let ambient = tree.0.join("ambient");
let consumer = tree.0.join("consumer");
std::fs::create_dir(&ambient).unwrap();
std::fs::create_dir(&consumer).unwrap();
for directory in [&ambient, &consumer] {
std::fs::write(directory.join("lean-toolchain"), wall::LEAN_TOOLCHAIN).unwrap();
std::fs::write(
directory.join("lakefile.lean"),
"import Lake\nopen Lake DSL\npackage «envprobe» where\n",
)
.unwrap();
}
std::fs::write(
ambient.join("CertPrelude.lean"),
"def ambientMarker : Nat := 666\n",
)
.unwrap();
let runner = LeanRunner::new(wall::LEAN_TOOLCHAIN).unwrap();
let built = runner
.run_lake(
&ambient,
"shadow olean build",
&["env", "lean", "-o", "CertPrelude.olean", "CertPrelude.lean"],
)
.unwrap();
assert!(
built.status.success(),
"shadow olean build failed:\n{}",
report(&built)
);
std::fs::remove_file(ambient.join("CertPrelude.lean")).unwrap();
assert!(ambient.join("CertPrelude.olean").is_file());
std::fs::write(
consumer.join("Probe.lean"),
"import CertPrelude\n\ndef observed : Nat := ambientMarker\n",
)
.unwrap();
let baseline = Command::new("elan")
.current_dir(&consumer)
.env("LEAN_PATH", &ambient)
.args(["run", toolchain, "lake", "env", "lean", "Probe.lean"])
.output()
.unwrap();
assert!(
baseline.status.success(),
"Lean 4.32 baseline did not retain ambient LEAN_PATH:\n{}",
report(&baseline)
);
let hostile = |name: &str| match name {
"LEAN_PATH" | "LEAN_SRC_PATH" => Some(ambient.as_os_str().to_os_string()),
"ELAN_TOOLCHAIN" => Some(OsString::from("leanprover/lean4:v4.31.0")),
_ => std::env::var_os(name),
};
let hardened = LeanRunner::with_environment(wall::LEAN_TOOLCHAIN, hostile)
.unwrap()
.run_lake(&consumer, "hostile probe", &["env", "lean", "Probe.lean"])
.unwrap();
let hardened_report = report(&hardened);
assert!(
!hardened.status.success(),
"hostile ambient olean crossed the cleared environment:\n{hardened_report}"
);
assert!(
hardened_report.contains("CertPrelude")
&& (hardened_report.contains("unknown module")
|| hardened_report.contains("not found")),
"hardened failure was not the missing hostile module:\n{hardened_report}"
);
}
fn runner_with_setting(knob: &'static str, value: Option<&str>) -> Result<LeanRunner, String> {
let tree = TestTree::new();
let elan_home = tree.0.join("elan");
std::fs::create_dir_all(elan_home.join("bin")).unwrap();
std::fs::write(
elan_home
.join("bin")
.join(format!("elan{}", std::env::consts::EXE_SUFFIX)),
b"placeholder",
)
.unwrap();
let elan_home = elan_home.into_os_string();
let setting = value.map(OsString::from);
LeanRunner::with_environment(wall::LEAN_TOOLCHAIN, move |name| {
if name == "ELAN_HOME" {
Some(elan_home.clone())
} else if name == knob {
setting.clone()
} else {
None
}
})
}
fn runner_with_timeout_setting(value: Option<&str>) -> Result<LeanRunner, String> {
runner_with_setting(PHASE_TIMEOUT_ENV, value)
}
#[test]
fn phase_timeout_defaults_generously_and_validates_the_override() {
let default = runner_with_timeout_setting(None).unwrap();
assert_eq!(default.phase_timeout, DEFAULT_PHASE_TIMEOUT);
assert_eq!(default.phase_timeout, Duration::from_secs(900));
let overridden = runner_with_timeout_setting(Some("120")).unwrap();
assert_eq!(overridden.phase_timeout, Duration::from_secs(120));
let at_cap = runner_with_timeout_setting(Some(&MAX_PHASE_TIMEOUT_SECS.to_string()))
.expect("the cap itself is a valid override");
assert_eq!(
at_cap.phase_timeout,
Duration::from_secs(MAX_PHASE_TIMEOUT_SECS)
);
for invalid in ["0", "-5", "soon", "1.5"] {
let Err(error) = runner_with_timeout_setting(Some(invalid)) else {
panic!("malformed timeout override `{invalid}` must fail closed");
};
assert!(error.contains(PHASE_TIMEOUT_ENV), "{error}");
}
let over_cap = (MAX_PHASE_TIMEOUT_SECS + 1).to_string();
for oversized in [over_cap.as_str(), &u64::MAX.to_string()] {
let Err(error) = runner_with_timeout_setting(Some(oversized)) else {
panic!("oversized timeout override `{oversized}` must fail closed");
};
assert!(error.contains(PHASE_TIMEOUT_ENV), "{error}");
assert!(error.contains("at most"), "{error}");
}
}
#[test]
fn memory_limit_defaults_and_validates_the_override() {
let default = runner_with_setting(MEMORY_LIMIT_ENV, None).unwrap();
assert_eq!(default.memory_limit_mb(), DEFAULT_MEMORY_LIMIT_MB);
assert_eq!(default.memory_limit_mb(), 16384);
let overridden = runner_with_setting(MEMORY_LIMIT_ENV, Some("256")).unwrap();
assert_eq!(overridden.memory_limit_mb(), 256);
let command = overridden.lake_command(
Path::new("/checker/build"),
Path::new("/checker/temp"),
&["env", "lean", "Probe.lean"],
);
assert!(
command
.get_args()
.any(|argument| argument.to_string_lossy() == "--memory=256"),
"override did not reach the lean command line"
);
for invalid in ["0", "-5", "lots", "1.5"] {
let Err(error) = runner_with_setting(MEMORY_LIMIT_ENV, Some(invalid)) else {
panic!("malformed memory override `{invalid}` must fail closed");
};
assert!(error.contains(MEMORY_LIMIT_ENV), "{error}");
}
let over_cap = (MAX_MEMORY_LIMIT_MB + 1).to_string();
let Err(error) = runner_with_setting(MEMORY_LIMIT_ENV, Some(&over_cap)) else {
panic!("oversized memory override must fail closed");
};
assert!(error.contains("at most"), "{error}");
}
#[cfg(unix)]
#[test]
fn expired_phase_kills_the_whole_process_group() {
let tree = TestTree::new();
let pid_file = tree.0.join("grandchild.pid");
let mut command = Command::new("sh");
command.arg("-c").arg(format!(
"tail -f /dev/null & echo $! > '{}'; wait",
pid_file.display()
));
let error = run_with_timeout(command, Duration::from_secs(1), "test probe")
.expect_err("a hung step must fail closed")
.to_string();
assert!(error.contains("test probe"), "{error}");
assert!(error.contains("time limit"), "{error}");
assert!(error.contains(PHASE_TIMEOUT_ENV), "{error}");
let grandchild =
std::fs::read_to_string(&pid_file).expect("grandchild pid recorded before the timeout");
let grandchild = grandchild.trim().to_string();
assert!(!grandchild.is_empty(), "grandchild pid file was empty");
let mut alive = true;
for _ in 0..100 {
let probe = Command::new("ps")
.args(["-p", &grandchild, "-o", "pid="])
.output()
.expect("ps runs");
if !probe.status.success() || String::from_utf8_lossy(&probe.stdout).trim().is_empty() {
alive = false;
break;
}
std::thread::sleep(Duration::from_millis(50));
}
assert!(
!alive,
"grandchild process {grandchild} survived the group kill"
);
}
}