use super::*;
#[tokio::test(start_paused = true)]
async fn shutdown_kills_all_and_stops_the_engine() {
let dir = tempfile::tempdir().unwrap();
let mut app = AppConfig::minimal("web", "./srv");
app.instances = 2;
let (handle, _runner, _rx) = started_apps(
&dir,
vec![app],
vec![ProcScript::never_exits(), ProcScript::never_exits()],
64,
)
.await;
handle.shutdown().await; assert!(handle.list_checked().await.is_err());
}
#[tokio::test(start_paused = true)]
async fn a_call_after_shutdown_reports_engine_stopped() {
let dir = tempfile::tempdir().unwrap();
let app = AppConfig::minimal("web", "./srv");
let (handle, _runner, _rx) =
started_apps(&dir, vec![app], vec![ProcScript::never_exits()], 64).await;
handle.shutdown().await;
assert!(
matches!(
handle.list_checked().await,
Err(SupervisorError::EngineStopped)
),
"a listing after shutdown must report the actor gone"
);
assert!(
matches!(
handle.stop(ProcessSelector::All).await,
Err(SupervisorError::EngineStopped)
),
"a command whose reply is already a `Result` must report the actor \
gone the same way"
);
}
#[tokio::test(start_paused = true)]
async fn a_second_shutdown_and_a_forget_after_it_never_panic() {
let dir = tempfile::tempdir().unwrap();
let app = AppConfig::minimal("web", "./srv");
let (handle, _runner, _rx) =
started_apps(&dir, vec![app], vec![ProcScript::never_exits()], 64).await;
handle.shutdown().await;
handle.shutdown().await;
handle.forget_smits(ConnId::next()).await;
}
async fn drain_kinds(
rx: &mut tokio::sync::broadcast::Receiver<SharedEvent>,
) -> Vec<(u32, ProcessEventKind)> {
let mut out = Vec::new();
while let Ok(ev) = rx.try_recv().map(|event| event.to_event()) {
if let BusEvent::Process { event, info, .. } = ev {
out.push((info.id, event));
}
}
out
}
#[tokio::test(start_paused = true)]
async fn shutdown_ignores_a_pending_restart_timer() {
let dir = tempfile::tempdir().unwrap();
let mut crash = AppConfig::minimal("crash", "./boom");
crash.exp_backoff_restart_delay = Some("500".parse().unwrap());
let web = AppConfig::minimal("web", "./srv");
let (handle, _runner, mut rx) = started_apps(
&dir,
vec![crash, web],
vec![
ProcScript::const_exit(1), ProcScript::ignores_signals(), ProcScript::never_exits(), ],
1024,
)
.await;
await_event(&mut rx, 0, ProcessEventKind::Exit).await;
handle.shutdown().await;
let seen = drain_kinds(&mut rx).await;
let ghost = seen
.iter()
.any(|(id, k)| *id == 0 && *k == ProcessEventKind::Restart);
assert!(
!ghost,
"GHOST RESPAWN during shutdown: events after shutdown = {seen:?}"
);
}
#[tokio::test(start_paused = true)]
async fn shutdown_ignores_a_pending_readiness_wait() {
let dir = tempfile::tempdir().unwrap();
let mut app = AppConfig::minimal("gated", "./g");
app.wait_ready = true; app.listen_timeout = UpDuration::from_millis(1_000);
let (handle, _runner, mut rx) =
started_apps(&dir, vec![app], vec![ProcScript::ignores_signals()], 1024).await;
assert_eq!(
handle.list().await[0].status,
ProcStatus::Starting,
"the readiness wait has to be armed for its result to be droppable"
);
handle.shutdown().await;
let seen = drain_kinds(&mut rx).await;
assert!(
seen.contains(&(0, ProcessEventKind::Stop)),
"the kill ladder must have outlasted the readiness deadline: events = {seen:?}"
);
assert!(
!seen.contains(&(0, ProcessEventKind::Online)),
"a readiness wait resolved during shutdown marked a dying sheep online: \
events = {seen:?}"
);
}
#[tokio::test(start_paused = true)]
async fn late_start_racing_shutdown_never_orphans() {
let dir = tempfile::tempdir().unwrap();
let web = AppConfig::minimal("web", "./srv");
let (handle, _runner, mut rx) = started_apps(
&dir,
vec![web],
vec![
ProcScript::ignores_signals(), ProcScript::never_exits(), ],
1024,
)
.await;
let h2 = handle.clone();
let late = tokio::spawn(async move {
let app = AppConfig::minimal("late", "./l");
h2.start(vec![normalize(app).unwrap()]).await
});
handle.shutdown().await;
let outcome = late.await.unwrap();
let seen = drain_kinds(&mut rx).await;
match outcome {
Err(SupervisorError::EngineStopped) => {} Ok(infos) => {
let late_id = infos[0].id;
assert!(
seen.iter().any(|(id, k)| *id == late_id
&& matches!(k, ProcessEventKind::Stop | ProcessEventKind::Exit)),
"late Start raced ahead of shutdown but was never killed: events = {seen:?}"
);
}
Err(other) => panic!("unexpected error from a late Start during shutdown: {other:?}"),
}
}
#[tokio::test(start_paused = true)]
async fn stale_restart_timer_never_short_circuits_a_newer_backoff() {
let dir = tempfile::tempdir().unwrap();
let mut app = AppConfig::minimal("crash", "./boom");
app.exp_backoff_restart_delay = Some("2000".parse().unwrap());
app.min_uptime = "5000".parse().unwrap(); let (handle, _runner, mut rx) = started_apps(
&dir,
vec![app],
vec![
ProcScript::const_exit(1), ProcScript::stable_then_exit(1500, 1), ProcScript::never_exits(), ProcScript::never_exits(),
],
1024,
)
.await;
await_event(&mut rx, 0, ProcessEventKind::Exit).await;
let out = handle.restart(ProcessSelector::All).await.unwrap();
assert_eq!(
out[0].status,
ProcStatus::Online,
"manual restart respawned"
);
tokio::time::advance(Duration::from_millis(2500)).await;
for _ in 0..20 {
tokio::task::yield_now().await;
}
let info = handle.list().await.remove(0);
assert_eq!(
(info.status, info.restarts),
(ProcStatus::WaitingRestart, 1),
"at t=2500 the sheep should still be waiting on its 3500ms backoff; \
got {info:?} -- the stale 2000ms timer fired early"
);
}
#[tokio::test(start_paused = true)]
async fn overlapping_stop_and_restart_agree_on_one_outcome() {
let dir = tempfile::tempdir().unwrap();
let app = AppConfig::minimal("svc", "./svc");
let (handle, _runner, _rx) = started_apps(
&dir,
vec![app],
vec![
ProcScript::ignores_signals(), ProcScript::never_exits(),
],
1024,
)
.await;
let h2 = handle.clone();
let stopper = tokio::spawn(async move { h2.stop(ProcessSelector::All).await });
for _ in 0..10 {
tokio::task::yield_now().await;
}
let restarted = handle.restart(ProcessSelector::All).await.unwrap();
let stopped = stopper.await.unwrap().unwrap();
assert_eq!(
stopped[0].status,
ProcStatus::Stopped,
"stop() reported a non-stopped sheep"
);
assert_eq!(
restarted[0].status,
ProcStatus::Stopped,
"restart() lost the race to the earlier stop() but got a different \
answer than the stop() caller -- the two callers disagree about \
what happened to the same sheep"
);
}
#[tokio::test(start_paused = true)]
async fn actor_never_blocks_behind_a_busy_kill_ladder() {
let dir = tempfile::tempdir().unwrap();
let a = AppConfig::minimal("a", "./a");
let mut b = AppConfig::minimal("b", "./b");
b.autorestart = false;
let (handle, _runner, mut rx) = started_apps(
&dir,
vec![a, b],
vec![
ProcScript::ignores_signals(), ProcScript::stable_then_exit(800, 0), ],
1024,
)
.await;
let t0 = tokio::time::Instant::now();
let mut tasks = Vec::new();
for _ in 0..8 {
let h = handle.clone();
tasks.push(tokio::spawn(async move {
h.stop(ProcessSelector::Name("a".to_string())).await
}));
}
await_event(&mut rx, 1, ProcessEventKind::Stop).await;
let seen_at = t0.elapsed();
for t in tasks {
let _ = t.await;
}
assert!(
seen_at < Duration::from_millis(1000),
"sheep b's own exit was only processed at {seen_at:?} -- the actor \
was parked inside ctl.send() for sheep a's kill ladder"
);
}
#[tokio::test(start_paused = true)]
async fn mailbox_flood_during_a_kill_never_deadlocks() {
let dir = tempfile::tempdir().unwrap();
let a = AppConfig::minimal("a", "./a");
let (handle, _runner, mut rx) =
started_apps(&dir, vec![a], vec![ProcScript::ignores_signals()], 4096).await;
for _ in 0..8 {
let h = handle.clone();
tokio::spawn(async move {
let _ = h.stop(ProcessSelector::All).await;
});
}
for _ in 0..40 {
tokio::task::yield_now().await;
}
for _ in 0..400 {
let h = handle.clone();
tokio::spawn(async move {
let _ = h.list_checked().await;
});
}
for _ in 0..200 {
tokio::task::yield_now().await;
}
let r = tokio::time::timeout(
Duration::from_secs(600),
await_event(&mut rx, 0, ProcessEventKind::Stop),
)
.await;
assert!(
r.is_ok(),
"DEADLOCK: actor parked in ctl.send() while the sheep task is \
parked in actor_tx.send() -- the daemon never recovers"
);
}
#[tokio::test(start_paused = true)]
async fn delete_racing_shutdown_still_deregisters_the_sheep() {
let dir = tempfile::tempdir().unwrap();
let target = AppConfig::minimal("svc", "./svc");
let mut decoy = AppConfig::minimal("decoy", "./decoy");
decoy.kill_timeout = "600000".parse().unwrap(); let (handle, _runner, _rx) = started_apps(
&dir,
vec![target, decoy],
vec![
ProcScript::ignores_signals(), ProcScript::ignores_signals(), ],
1024,
)
.await;
let id = handle
.list()
.await
.iter()
.find(|info| info.name == "svc")
.expect("target sheep registered")
.id;
let h2 = handle.clone();
let shutter = tokio::spawn(async move { h2.shutdown().await });
for _ in 0..10 {
tokio::task::yield_now().await; }
let deleted = handle.delete(ProcessSelector::Id(id)).await.unwrap();
assert_eq!(deleted, vec![id], "the caller was told this id was deleted");
assert!(
handle.list().await.iter().all(|info| info.id != id),
"a Delete that raced a Shutdown must still deregister the sheep, \
not just tell its caller it did"
);
drop(shutter);
}
#[tokio::test(start_paused = true)]
async fn delete_racing_restart_still_deregisters_the_sheep() {
let dir = tempfile::tempdir().unwrap();
let app = AppConfig::minimal("svc", "./svc");
let (handle, _runner, _rx) = started_apps(
&dir,
vec![app],
vec![
ProcScript::ignores_signals(), ProcScript::never_exits(),
],
1024,
)
.await;
let id = handle.list().await.remove(0).id;
let h2 = handle.clone();
let restarter = tokio::spawn(async move { h2.restart(ProcessSelector::All).await });
for _ in 0..10 {
tokio::task::yield_now().await; }
let deleted = handle.delete(ProcessSelector::Id(id)).await.unwrap();
let restarted = restarter.await.unwrap().unwrap();
assert_eq!(deleted, vec![id], "the caller was told this id was deleted");
assert_eq!(restarted[0].id, id);
assert_eq!(
restarted[0].status,
ProcStatus::Stopped,
"restart() must not report a respawned process once a racing \
Delete has claimed this id -- got {restarted:?}"
);
assert!(
handle.list().await.iter().all(|info| info.id != id),
"a Delete that raced a Restart must still deregister the sheep, \
not respawn a brand-new live process while telling its caller \
the sheep was deleted"
);
}
#[tokio::test(start_paused = true)]
async fn an_operators_stop_beats_an_automatic_restart_mid_ladder() {
let dir = tempfile::tempdir().unwrap();
let app = AppConfig::minimal("svc", "./svc");
let (handle, _runner, _rx) = started_apps(
&dir,
vec![app],
vec![
ProcScript::ignores_signals(), ProcScript::never_exits(), ],
1024,
)
.await;
let running = handle.list().await.remove(0);
let pid = running.pid.expect("an online sheep has a pid");
handle.extra_restart(running.id, pid, None, None).await;
let stopped = handle.stop(ProcessSelector::All).await.unwrap();
assert_eq!(stopped.len(), 1);
assert_eq!(stopped[0].id, running.id);
assert_eq!(
(stopped[0].status, stopped[0].restarts),
(ProcStatus::Stopped, 0),
"an operator's stop was silently converted into the automatic \
restart it raced -- got {stopped:?}"
);
let listed = handle.list().await;
assert_eq!(
(listed[0].status, listed[0].pid),
(ProcStatus::Stopped, None),
"the sheep an operator stopped is running again -- got {listed:?}"
);
}
#[tokio::test(start_paused = true)]
async fn an_operators_delete_beats_an_automatic_restart_mid_ladder() {
let dir = tempfile::tempdir().unwrap();
let app = AppConfig::minimal("svc", "./svc");
let (handle, _runner, _rx) = started_apps(
&dir,
vec![app],
vec![ProcScript::ignores_signals(), ProcScript::never_exits()],
1024,
)
.await;
let running = handle.list().await.remove(0);
let pid = running.pid.expect("an online sheep has a pid");
handle.extra_restart(running.id, pid, None, None).await;
let deleted = handle
.delete(ProcessSelector::Id(running.id))
.await
.unwrap();
assert_eq!(deleted, vec![running.id]);
assert!(
handle.list().await.is_empty(),
"a delete that raced an automatic restart must still deregister \
the sheep, not leave one behind for the restart to bring back"
);
}
#[tokio::test(start_paused = true)]
async fn an_operators_stop_still_shows_its_signal_as_the_last_exit() {
let dir = tempfile::tempdir().unwrap();
let app = AppConfig::minimal("svc", "./svc");
let (handle, _runner, _rx) =
started_apps(&dir, vec![app], vec![ProcScript::never_exits()], 64).await;
let stopped = handle.stop(ProcessSelector::All).await.unwrap();
assert_eq!(stopped.len(), 1);
assert_eq!(stopped[0].status, ProcStatus::Stopped);
assert_eq!(
stopped[0].last_exit,
Some(ExitInfo {
code: None,
signal: Some(15),
}),
"an operator's own stop must still show up as a last exit: {stopped:?}"
);
}
#[tokio::test(start_paused = true)]
async fn a_respawn_that_fails_to_spawn_clears_the_previous_exit_code() {
let dir = tempfile::tempdir().unwrap();
let app = AppConfig::minimal("svc", "./svc");
let (handle, _runner, _rx) =
started_apps(&dir, vec![app], vec![ProcScript::const_exit(1)], 64).await;
tokio::time::sleep(Duration::from_secs(30)).await;
let listing = handle.list().await;
assert_eq!(listing.len(), 1);
assert_eq!(
listing[0].status,
ProcStatus::Errored,
"a respawn that cannot spawn is still terminal: {listing:?}"
);
assert_eq!(
listing[0].last_exit, None,
"nothing exited on the failed respawn, so the earlier code must \
not still be showing: {listing:?}"
);
}