#![cfg(target_os = "linux")]
use std::fs::{self, File};
use std::io::Write;
use std::path::{Path, PathBuf};
use std::process::{Command, Stdio};
use std::time::{Duration, Instant};
const REQUIRED_ENV: &[&str] = &[
"CELLOS_FIRECRACKER_BINARY",
"CELLOS_FIRECRACKER_KERNEL_IMAGE",
"CELLOS_FIRECRACKER_ROOTFS_IMAGE",
"CELLOS_FIRECRACKER_SOCKET_DIR",
];
const GRACEFUL_SHUTDOWN_TIMEOUT_SECS: u64 = 5;
const TEARDOWN_SLACK_SECS: u64 = 2;
const NO_VSOCK_TIMEOUT_SECS: u64 = 2;
const SUPERVISOR_DEADLINE: Duration = Duration::from_secs(30);
fn skip(reason: &str) {
eprintln!("firecracker_e2e: skipping — {reason}");
}
fn supervisor_exe() -> Option<PathBuf> {
if let Some(p) = std::env::var_os("CELLOS_SUPERVISOR_BIN") {
let path = PathBuf::from(p);
if path.is_file() {
return Some(path);
}
}
let crate_dir = Path::new(env!("CARGO_MANIFEST_DIR"));
let workspace = crate_dir.parent()?.parent()?;
for profile in ["release", "debug"] {
let candidate = workspace
.join("target")
.join(profile)
.join("cellos-supervisor");
if candidate.is_file() {
return Some(candidate);
}
}
None
}
fn handle_rootfs_alias() {
let long = std::env::var_os("CELLOS_FIRECRACKER_ROOTFS_IMAGE");
let short = std::env::var_os("CELLOS_FIRECRACKER_ROOTFS");
match (long, short) {
(Some(_), _) => {}
(None, Some(s)) => std::env::set_var("CELLOS_FIRECRACKER_ROOTFS_IMAGE", s),
_ => {}
}
}
fn collect_jsonl(dir: &Path) -> Vec<PathBuf> {
let mut out = Vec::new();
let mut walker = vec![dir.to_path_buf()];
while let Some(d) = walker.pop() {
let entries = match fs::read_dir(&d) {
Ok(it) => it,
Err(_) => continue,
};
for entry in entries.flatten() {
let path = entry.path();
if path.is_dir() {
walker.push(path);
} else if path.extension().and_then(|s| s.to_str()) == Some("jsonl") {
out.push(path);
}
}
}
out
}
fn read_events(paths: &[PathBuf]) -> Vec<(String, String, serde_json::Value)> {
let mut events = Vec::new();
for p in paths {
let raw = match fs::read_to_string(p) {
Ok(s) => s,
Err(_) => continue,
};
for line in raw.lines() {
let line = line.trim();
if line.is_empty() {
continue;
}
let v: serde_json::Value = match serde_json::from_str(line) {
Ok(v) => v,
Err(_) => continue,
};
let ty = v
.get("type")
.and_then(|x| x.as_str())
.unwrap_or("")
.to_string();
let time = v
.get("time")
.and_then(|x| x.as_str())
.unwrap_or("")
.to_string();
let data = v.get("data").cloned().unwrap_or(serde_json::Value::Null);
events.push((ty, time, data));
}
}
events
}
fn parse_rfc3339_to_unix_nanos(ts: &str) -> Option<i128> {
let (date_part, time_part_with_tz) = ts.split_once('T')?;
let mut dp = date_part.split('-');
let y: i32 = dp.next()?.parse().ok()?;
let mo: u32 = dp.next()?.parse().ok()?;
let d: u32 = dp.next()?.parse().ok()?;
let (hms_frac, tz) = if let Some(idx) = time_part_with_tz.rfind(['+', '-', 'Z']) {
(&time_part_with_tz[..idx], &time_part_with_tz[idx..])
} else {
(time_part_with_tz, "")
};
let mut tp = hms_frac.split(':');
let h: u32 = tp.next()?.parse().ok()?;
let mi: u32 = tp.next()?.parse().ok()?;
let s_with_frac = tp.next()?;
let (sec_str, frac_str) = match s_with_frac.split_once('.') {
Some((s, f)) => (s, f),
None => (s_with_frac, ""),
};
let s: u32 = sec_str.parse().ok()?;
let mut nanos: u32 = 0;
let mut chars = frac_str.chars();
for i in 0..9 {
let digit = chars.next().unwrap_or('0');
let v = digit.to_digit(10)?;
nanos += v * 10u32.pow(8 - i);
}
let yy = y - if mo <= 2 { 1 } else { 0 };
let era = if yy >= 0 { yy } else { yy - 399 } / 400;
let yoe = (yy - era * 400) as i64;
let doy = (153 * (mo as i64 + if mo > 2 { -3 } else { 9 }) + 2) / 5 + d as i64 - 1;
let doe = yoe * 365 + yoe / 4 - yoe / 100 + doy;
let days_since_epoch = era as i64 * 146097 + doe - 719468;
let secs: i128 =
days_since_epoch as i128 * 86_400 + h as i128 * 3_600 + mi as i128 * 60 + s as i128;
let mut total_nanos: i128 = secs * 1_000_000_000 + nanos as i128;
if !tz.is_empty() && tz != "Z" {
let sign: i128 = if tz.starts_with('+') { -1 } else { 1 };
let body = &tz[1..];
let mut bp = body.split(':');
let oh: i128 = bp.next()?.parse().ok()?;
let om: i128 = bp.next().unwrap_or("0").parse().ok()?;
total_nanos += sign * (oh * 3_600 + om * 60) * 1_000_000_000;
}
Some(total_nanos)
}
#[test]
fn fc22_sigkill_fallback_after_sigterm_ignoring_workload() {
if !Path::new("/dev/kvm").exists() {
skip("/dev/kvm not present (no KVM on this host)");
return;
}
handle_rootfs_alias();
let missing: Vec<&str> = REQUIRED_ENV
.iter()
.copied()
.filter(|k| std::env::var_os(k).is_none())
.collect();
if !missing.is_empty() {
skip(&format!("missing env: {}", missing.join(", ")));
return;
}
for key in [
"CELLOS_FIRECRACKER_BINARY",
"CELLOS_FIRECRACKER_KERNEL_IMAGE",
"CELLOS_FIRECRACKER_ROOTFS_IMAGE",
] {
let path = std::env::var(key).expect("checked above");
if !Path::new(&path).exists() {
skip(&format!("{key}={path} does not exist on disk"));
return;
}
}
let sock_dir = std::env::var("CELLOS_FIRECRACKER_SOCKET_DIR").expect("checked");
if !Path::new(&sock_dir).is_dir() && fs::create_dir_all(&sock_dir).is_err() {
skip(&format!("socket dir {sock_dir} not creatable"));
return;
}
let exe = match supervisor_exe() {
Some(e) => e,
None => {
skip(
"supervisor binary not found — set CELLOS_SUPERVISOR_BIN or build cellos-supervisor",
);
return;
}
};
let tmp = tempfile::tempdir().expect("tempdir");
let spec_path = tmp.path().join("cell.json");
let spec_json = r#"{
"apiVersion": "cellos.io/v1",
"kind": "ExecutionCell",
"spec": {
"id": "fc-22-sigkill",
"authority": { "secretRefs": [], "egressRules": [] },
"lifetime": { "ttlSeconds": 1 },
"run": {
"argv": ["/bin/sh", "-c", "trap '' TERM INT; sleep 9999"],
"limits": { "memoryMaxBytes": 67108864 }
}
}
}"#;
File::create(&spec_path)
.and_then(|mut f| f.write_all(spec_json.as_bytes()))
.expect("write cell spec");
let export_dir = tmp.path().join("events");
fs::create_dir_all(&export_dir).expect("mkdir export dir");
let mut cmd = Command::new(&exe);
cmd.env("CELL_OS_USE_NOOP_SINK", "1")
.env("CELLOS_CELL_BACKEND", "firecracker")
.env("CELLOS_EXPORT_DIR", &export_dir)
.env("RUST_BACKTRACE", "1")
.arg(&spec_path)
.stdout(Stdio::piped())
.stderr(Stdio::piped());
for (k, v) in std::env::vars_os() {
if k.to_string_lossy().starts_with("CELLOS_FIRECRACKER_") {
cmd.env(&k, &v);
}
}
cmd.env("CELLOS_FIRECRACKER_ALLOW_NO_VSOCK", "1");
cmd.env(
"CELLOS_FIRECRACKER_NO_VSOCK_TIMEOUT_SECS",
NO_VSOCK_TIMEOUT_SECS.to_string(),
);
eprintln!(
"fc22_sigkill_fallback: spawning supervisor {}",
exe.display()
);
let supervisor_started = Instant::now();
let mut child = cmd.spawn().expect("spawn supervisor");
let deadline = supervisor_started + SUPERVISOR_DEADLINE;
let status = loop {
match child.try_wait().expect("try_wait") {
Some(status) => break status,
None if Instant::now() >= deadline => {
let _ = child.kill();
let _ = child.wait();
panic!(
"supervisor did not exit within {:?} — SIGKILL fallback regression \
(or a deeper hang in `destroy()`); FC-22 budget is \
GRACEFUL_SHUTDOWN_TIMEOUT ({GRACEFUL_SHUTDOWN_TIMEOUT_SECS}s) + \
{TEARDOWN_SLACK_SECS}s slack from teardown start",
SUPERVISOR_DEADLINE
);
}
None => std::thread::sleep(Duration::from_millis(100)),
}
};
let mut stderr_buf = String::new();
let mut stdout_buf = String::new();
if let Some(mut s) = child.stderr.take() {
use std::io::Read;
let _ = s.read_to_string(&mut stderr_buf);
}
if let Some(mut s) = child.stdout.take() {
use std::io::Read;
let _ = s.read_to_string(&mut stdout_buf);
}
eprintln!(
"fc22_sigkill_fallback: supervisor exit status = {status:?} \
(non-zero is expected for the failed-then-forced path)"
);
let sigkill_marker = "VM did not exit gracefully";
assert!(
stderr_buf.contains(sigkill_marker),
"expected `{sigkill_marker}` in supervisor stderr — SIGKILL fallback did not fire \
(FC-22 regression). \n--- stderr ---\n{stderr_buf}\n--- stdout ---\n{stdout_buf}"
);
let jsonl_paths = collect_jsonl(&export_dir);
assert!(
!jsonl_paths.is_empty(),
"no JSONL files under {} — the supervisor did not export any events. \
\n--- stderr ---\n{stderr_buf}",
export_dir.display()
);
let events = read_events(&jsonl_paths);
let destroyed = events
.iter()
.find(|(ty, _, _)| ty == "dev.cellos.events.cell.lifecycle.v1.destroyed");
let (_, destroyed_time, destroyed_data) = destroyed.unwrap_or_else(|| {
panic!(
"no `cell.lifecycle.v1.destroyed` event found in JSONL export. \
Got types: {:?}\n--- stderr ---\n{}",
events.iter().map(|(t, _, _)| t).collect::<Vec<_>>(),
stderr_buf
)
});
let outcome = destroyed_data
.get("outcome")
.and_then(|v| v.as_str())
.unwrap_or("");
assert_eq!(
outcome, "failed",
"destroyed.outcome must be `failed` (workload trapped TERM/INT and was force-killed); \
got {outcome:?}. data = {destroyed_data}"
);
let terminal_state = destroyed_data
.get("terminalState")
.and_then(|v| v.as_str())
.unwrap_or("");
assert_eq!(
terminal_state, "forced",
"destroyed.terminalState must be `forced` (in-VM bridge did not deliver an \
authenticated exit code); got {terminal_state:?}. data = {destroyed_data}"
);
let teardown_start_ns = events
.iter()
.filter(|(ty, _, _)| ty != "dev.cellos.events.cell.lifecycle.v1.destroyed")
.filter_map(|(_, t, _)| parse_rfc3339_to_unix_nanos(t))
.max();
let teardown_end_ns = parse_rfc3339_to_unix_nanos(destroyed_time);
match (teardown_start_ns, teardown_end_ns) {
(Some(start_ns), Some(end_ns)) => {
let teardown_ns = end_ns - start_ns;
let budget_ns: i128 =
(GRACEFUL_SHUTDOWN_TIMEOUT_SECS + TEARDOWN_SLACK_SECS) as i128 * 1_000_000_000;
assert!(
teardown_ns >= 0 && teardown_ns <= budget_ns,
"teardown took {} ms — must be ≤ {} ms (= GRACEFUL_SHUTDOWN_TIMEOUT \
{GRACEFUL_SHUTDOWN_TIMEOUT_SECS}s + slack {TEARDOWN_SLACK_SECS}s). \
destroyed time = {destroyed_time:?}",
teardown_ns / 1_000_000,
budget_ns / 1_000_000
);
}
_ => {
eprintln!(
"fc22_sigkill_fallback: could not derive teardown window from JSONL \
timestamps; relying on the supervisor-deadline ceiling and the SIGKILL \
warning + destroyed-event shape assertions"
);
}
}
drop(tmp);
}