use super::*;
#[tokio::test(start_paused = true)]
async fn wait_ready_app_stays_starting_until_the_channel_signals() {
let (events, mut rx) = crate::bus::test_bus(64);
let runner = ScriptedRunner::new(vec![ProcScript::never_exits()]);
let dir = tempfile::tempdir().unwrap();
let handle = spawn_supervisor(runner, test_paths(&dir), events);
let mut app = AppConfig::minimal("web", "./srv");
app.wait_ready = true;
handle.start(vec![normalize(app).unwrap()]).await.unwrap();
assert_eq!(handle.list().await[0].status, ProcStatus::Starting);
assert_no_event_within(
&mut rx,
0,
ProcessEventKind::Online,
Duration::from_millis(500),
)
.await;
assert_eq!(handle.list().await[0].status, ProcStatus::Starting);
handle.tx.send(Msg::Ready { id: 0 }).await.unwrap();
tokio::time::timeout(
Duration::from_secs(1),
await_event(&mut rx, 0, ProcessEventKind::Online),
)
.await
.expect("Online once the channel signals");
assert_eq!(handle.list().await[0].status, ProcStatus::Online);
}
#[tokio::test]
async fn readiness_probe_app_stays_starting_until_the_probe_passes() {
let reserved = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
let addr = reserved.local_addr().unwrap();
drop(reserved);
let (events, mut rx) = crate::bus::test_bus(64);
let runner = ScriptedRunner::new(vec![ProcScript::never_exits()]);
let dir = tempfile::tempdir().unwrap();
let handle = spawn_supervisor(runner, test_paths(&dir), events);
let mut app = AppConfig::minimal("web", "./srv");
app.readiness_probe = Some(ProbeConfig {
interval: UpDuration::from_millis(50),
timeout: UpDuration::from_millis(200),
..probe_config(ProbeKind::Tcp, &addr.to_string())
});
handle.start(vec![normalize(app).unwrap()]).await.unwrap();
assert_eq!(handle.list().await[0].status, ProcStatus::Starting);
assert_no_event_within(
&mut rx,
0,
ProcessEventKind::Online,
Duration::from_millis(220),
)
.await;
assert_eq!(handle.list().await[0].status, ProcStatus::Starting);
let listener = tokio::net::TcpListener::bind(addr).await.unwrap();
let _accept = tokio::spawn(async move { while listener.accept().await.is_ok() {} });
tokio::time::timeout(
Duration::from_secs(2),
await_event(&mut rx, 0, ProcessEventKind::Online),
)
.await
.expect("Online once the probe starts passing");
assert_eq!(handle.list().await[0].status, ProcStatus::Online);
}
#[cfg(unix)]
#[tokio::test]
async fn an_exec_readiness_probe_sees_the_assembled_env_not_the_apps_own() {
let (events, mut rx) = crate::bus::test_bus(64);
let runner = ScriptedRunner::new(vec![ProcScript::never_exits()]);
let dir = tempfile::tempdir().unwrap();
let ready_file = dir.path().join("ready-0");
let handle = spawn_supervisor(runner, test_paths(&dir), events);
let mut app = AppConfig::minimal("web", "./srv");
app.readiness_probe = Some(ProbeConfig {
interval: UpDuration::from_millis(50),
timeout: UpDuration::from_millis(500),
..probe_config(
ProbeKind::Exec,
&format!(r#"test -f "{}/ready-$SHEP_INSTANCE""#, dir.path().display()),
)
});
app.listen_timeout = UpDuration::from_millis(60_000);
handle.start(vec![normalize(app).unwrap()]).await.unwrap();
assert_eq!(handle.list().await[0].status, ProcStatus::Starting);
assert_no_event_within(
&mut rx,
0,
ProcessEventKind::Online,
Duration::from_millis(220),
)
.await;
assert_eq!(handle.list().await[0].status, ProcStatus::Starting);
std::fs::write(&ready_file, b"").unwrap();
tokio::time::timeout(
Duration::from_secs(5),
await_event(&mut rx, 0, ProcessEventKind::Online),
)
.await
.expect("Online once the exec probe can resolve $SHEP_INSTANCE");
assert_eq!(handle.list().await[0].status, ProcStatus::Online);
}
#[tokio::test(start_paused = true)]
async fn a_gated_apps_online_carries_the_same_manually_flag_an_ungated_one_does() {
let (events, mut rx) = crate::bus::test_bus(64);
let runner = ScriptedRunner::new(vec![
ProcScript::never_exits(), ProcScript::never_exits(), ProcScript::never_exits(), ]);
let dir = tempfile::tempdir().unwrap();
let handle = spawn_supervisor(runner, test_paths(&dir), events);
let mut gated = AppConfig::minimal("gated", "./srv");
gated.wait_ready = true;
let ungated = AppConfig::minimal("ungated", "./srv");
handle
.start(vec![normalize(gated).unwrap(), normalize(ungated).unwrap()])
.await
.unwrap();
let ungated_manually = tokio::time::timeout(
Duration::from_secs(1),
await_event(&mut rx, 1, ProcessEventKind::Online),
)
.await
.expect("an ungated app is Online at spawn");
assert!(ungated_manually, "sanity: a Start is a manual event");
handle.tx.send(Msg::Ready { id: 0 }).await.unwrap();
let gated_manually = tokio::time::timeout(
Duration::from_secs(1),
await_event(&mut rx, 0, ProcessEventKind::Online),
)
.await
.expect("Online once the channel signals");
assert_eq!(
gated_manually, ungated_manually,
"the same `shep start` must report the same flag, gated or not"
);
let restarted = handle.restart(ProcessSelector::Id(0)).await.unwrap();
assert_eq!(restarted[0].status, ProcStatus::Starting);
handle.tx.send(Msg::Ready { id: 0 }).await.unwrap();
let restarted_manually = tokio::time::timeout(
Duration::from_secs(1),
await_event(&mut rx, 0, ProcessEventKind::Online),
)
.await
.expect("Online once the respawned sheep signals");
assert!(
restarted_manually,
"a manual Restart's Online must stay manual through the readiness gate"
);
}
#[tokio::test(start_paused = true)]
async fn an_operators_restart_is_reported_as_a_user_action() {
let (events, mut rx) = crate::bus::test_bus(64);
let runner = ScriptedRunner::new(vec![ProcScript::never_exits(); 2]);
let dir = tempfile::tempdir().unwrap();
let handle = spawn_supervisor(runner, test_paths(&dir), events);
handle
.start(vec![normalize(AppConfig::minimal("web", "./srv")).unwrap()])
.await
.unwrap();
let restarted = handle.restart(ProcessSelector::All).await.unwrap();
assert_eq!(restarted[0].restarts, 1, "the restart really respawned");
let manually = tokio::time::timeout(
Duration::from_secs(1),
await_event(&mut rx, 0, ProcessEventKind::Restart),
)
.await
.expect("the respawn `restart` performed");
assert!(
manually,
"a person typed `shep restart`; the bus must say a user action caused it"
);
}
#[tokio::test(start_paused = true)]
async fn a_gated_app_whose_deadline_elapses_goes_online_anyway() {
let (events, mut rx) = crate::bus::test_bus(64);
let runner = ScriptedRunner::new(vec![ProcScript::never_exits()]);
let dir = tempfile::tempdir().unwrap();
let handle = spawn_supervisor(runner, test_paths(&dir), events);
let mut app = AppConfig::minimal("web", "./srv");
app.wait_ready = true; handle.start(vec![normalize(app).unwrap()]).await.unwrap();
assert_eq!(handle.list().await[0].status, ProcStatus::Starting);
tokio::time::timeout(
Duration::from_secs(4), await_event(&mut rx, 0, ProcessEventKind::Online),
)
.await
.expect("Online once the readiness deadline elapses");
assert_eq!(handle.list().await[0].status, ProcStatus::Online);
}
#[tokio::test(start_paused = true)]
async fn a_gated_app_that_exits_while_starting_never_reaches_online_from_the_old_wait() {
let (events, mut rx) = crate::bus::test_bus(64);
let runner = ScriptedRunner::new(vec![
ProcScript::stable_then_exit(500, 1), ProcScript::never_exits(), ]);
let dir = tempfile::tempdir().unwrap();
let handle = spawn_supervisor(runner, test_paths(&dir), events);
let mut app = AppConfig::minimal("web", "./srv");
app.wait_ready = true; handle.start(vec![normalize(app).unwrap()]).await.unwrap();
assert_eq!(handle.list().await[0].status, ProcStatus::Starting);
tokio::time::timeout(
Duration::from_secs(1),
await_event(&mut rx, 0, ProcessEventKind::Restart),
)
.await
.expect("the automatic respawn after the unstable exit");
assert_eq!(handle.list().await[0].status, ProcStatus::Starting);
assert_eq!(handle.list().await[0].restarts, 1);
assert_no_event_within(
&mut rx,
0,
ProcessEventKind::Online,
Duration::from_millis(2_700),
)
.await;
assert_eq!(
handle.list().await[0].status,
ProcStatus::Starting,
"the OLD wait's stale TimedOut must not have marked the respawned process online"
);
tokio::time::timeout(
Duration::from_secs(2),
await_event(&mut rx, 0, ProcessEventKind::Online),
)
.await
.expect("the new process's own readiness deadline");
assert_eq!(handle.list().await[0].status, ProcStatus::Online);
}
#[tokio::test(start_paused = true)]
async fn a_gated_app_stopped_while_starting_ignores_the_old_wait() {
let (events, mut rx) = crate::bus::test_bus(64);
let runner = ScriptedRunner::new(vec![ProcScript::stable_then_exit(500, 1)]);
let dir = tempfile::tempdir().unwrap();
let handle = spawn_supervisor(runner, test_paths(&dir), events);
let mut app = AppConfig::minimal("web", "./srv");
app.wait_ready = true; app.autorestart = false; handle.start(vec![normalize(app).unwrap()]).await.unwrap();
assert_eq!(handle.list().await[0].status, ProcStatus::Starting);
tokio::time::timeout(
Duration::from_secs(1),
await_event(&mut rx, 0, ProcessEventKind::Stop),
)
.await
.expect("the natural exit at 500ms");
assert_eq!(handle.list().await[0].status, ProcStatus::Stopped);
assert_no_event_within(&mut rx, 0, ProcessEventKind::Online, Duration::from_secs(3)).await;
assert_eq!(handle.list().await[0].status, ProcStatus::Stopped);
}
#[tokio::test(start_paused = true)]
async fn a_gated_app_restarted_while_starting_ignores_the_old_wait() {
let (events, mut rx) = crate::bus::test_bus(64);
let runner = ScriptedRunner::new(vec![ProcScript::never_exits(), ProcScript::never_exits()]);
let dir = tempfile::tempdir().unwrap();
let handle = spawn_supervisor(runner, test_paths(&dir), events);
let mut app = AppConfig::minimal("web", "./srv");
app.wait_ready = true; handle.start(vec![normalize(app).unwrap()]).await.unwrap();
assert_eq!(handle.list().await[0].status, ProcStatus::Starting);
tokio::time::sleep(Duration::from_millis(500)).await;
handle
.restart(ProcessSelector::Name("web".to_string()))
.await
.unwrap();
assert_eq!(
handle.list().await[0].status,
ProcStatus::Starting,
"the respawned process is gated too, so it must still be Starting"
);
assert_no_event_within(
&mut rx,
0,
ProcessEventKind::Online,
Duration::from_millis(2_700),
)
.await;
assert_eq!(
handle.list().await[0].status,
ProcStatus::Starting,
"the old wait's stale TimedOut must not have marked the new process online"
);
tokio::time::timeout(
Duration::from_secs(2),
await_event(&mut rx, 0, ProcessEventKind::Online),
)
.await
.expect("the new process's own readiness deadline");
assert_eq!(handle.list().await[0].status, ProcStatus::Online);
}