use super::*;
use crate::testing_http::{HttpReply, loopback_http};
#[tokio::test(start_paused = true)]
async fn wait_ready_app_stays_starting_until_the_channel_signals() {
let (events, mut rx) = crate::bus::test_bus(64);
let runner = ScriptedRunner::new(vec![ProcScript::never_exits()]);
let dir = tempfile::tempdir().unwrap();
let handle = spawn_supervisor(runner, test_paths(&dir), events);
let mut app = AppConfig::minimal("web", "./srv");
app.wait_ready = true;
handle.start(vec![normalize(app).unwrap()]).await.unwrap();
assert_eq!(handle.list().await[0].status, ProcStatus::Starting);
assert_no_event_within(
&mut rx,
0,
ProcessEventKind::Online,
Duration::from_millis(500),
)
.await;
assert_eq!(handle.list().await[0].status, ProcStatus::Starting);
handle.tx.send(Msg::Ready { id: 0 }).await.unwrap();
tokio::time::timeout(
Duration::from_secs(1),
await_event(&mut rx, 0, ProcessEventKind::Online),
)
.await
.expect("Online once the channel signals");
assert_eq!(handle.list().await[0].status, ProcStatus::Online);
}
#[tokio::test]
async fn readiness_probe_app_stays_starting_until_the_probe_passes() {
let mut gate = loopback_http(Vec::new()).await;
let (events, mut rx) = crate::bus::test_bus(64);
let runner = ScriptedRunner::new(vec![ProcScript::never_exits()]);
let dir = tempfile::tempdir().unwrap();
let handle = spawn_supervisor(runner, test_paths(&dir), events);
let mut app = AppConfig::minimal("web", "./srv");
app.readiness_probe = Some(ProbeConfig {
interval: UpDuration::from_millis(50),
timeout: UpDuration::from_millis(2_000),
..probe_config(
ProbeKind::Http,
&format!("http://127.0.0.1:{}/healthz", gate.addr.port()),
)
});
app.listen_timeout = UpDuration::from_millis(60_000);
handle.start(vec![normalize(app).unwrap()]).await.unwrap();
assert_eq!(handle.list().await[0].status, ProcStatus::Starting);
gate.next_request().await;
gate.queue_reply(HttpReply::Status(500));
assert_eq!(handle.list().await[0].status, ProcStatus::Starting);
gate.next_request().await;
gate.queue_reply(HttpReply::Status(200));
tokio::time::timeout(
Duration::from_secs(5),
await_event(&mut rx, 0, ProcessEventKind::Online),
)
.await
.expect("Online once the queued reply passes the probe");
assert_eq!(handle.list().await[0].status, ProcStatus::Online);
}
#[cfg(unix)]
#[tokio::test]
async fn an_exec_readiness_probe_sees_the_assembled_env_not_the_apps_own() {
let (events, mut rx) = crate::bus::test_bus(64);
let runner = ScriptedRunner::new(vec![ProcScript::never_exits()]);
let dir = tempfile::tempdir().unwrap();
let ready_file = dir.path().join("ready-0");
let attempt_file = dir.path().join("probe-attempt-0");
let handle = spawn_supervisor(runner, test_paths(&dir), events);
let mut app = AppConfig::minimal("web", "./srv");
app.readiness_probe = Some(ProbeConfig {
interval: UpDuration::from_millis(50),
timeout: UpDuration::from_millis(2_000),
..probe_config(
ProbeKind::Exec,
&format!(
r#": > "{dir}/probe-attempt-$SHEP_INSTANCE"; test -f "{dir}/ready-$SHEP_INSTANCE""#,
dir = dir.path().display(),
),
)
});
app.listen_timeout = UpDuration::from_millis(60_000);
handle.start(vec![normalize(app).unwrap()]).await.unwrap();
assert_eq!(handle.list().await[0].status, ProcStatus::Starting);
await_file(&attempt_file, Duration::from_secs(5)).await;
assert_eq!(handle.list().await[0].status, ProcStatus::Starting);
std::fs::write(&ready_file, b"").unwrap();
tokio::time::timeout(
Duration::from_secs(5),
await_event(&mut rx, 0, ProcessEventKind::Online),
)
.await
.expect("Online once the exec probe can resolve $SHEP_INSTANCE");
assert_eq!(handle.list().await[0].status, ProcStatus::Online);
}
#[cfg(unix)]
#[tokio::test]
async fn a_reload_of_a_probed_app_reaches_online_at_the_default_probe_interval() {
let dir = tempfile::tempdir().unwrap();
let ready_file = dir.path().join("ready-0");
let attempts_file = dir.path().join("probe-attempts-0");
let mut app = AppConfig::minimal("web", "./srv");
app.readiness_probe = Some(ProbeConfig {
timeout: UpDuration::from_millis(2_000),
..probe_config(
ProbeKind::Exec,
&format!(
r#"echo >> "{dir}/probe-attempts-$SHEP_INSTANCE"; test -f "{dir}/ready-$SHEP_INSTANCE""#,
dir = dir.path().display(),
),
)
});
std::fs::write(&ready_file, b"").unwrap();
let (handle, _runner, mut rx) = started(
&dir,
app,
vec![ProcScript::never_exits(), ProcScript::never_exits()],
)
.await;
tokio::time::timeout(
Duration::from_secs(10),
await_event(&mut rx, 0, ProcessEventKind::Online),
)
.await
.expect("the start's probe passes the marker written for it");
assert_eq!(probe_attempts(&attempts_file), 1, "one probe for the start");
std::fs::remove_file(&ready_file).unwrap();
handle
.reload(ProcessSelector::Name("web".to_string()))
.await
.expect("the reload is accepted");
await_probe_attempts(&attempts_file, 2, Duration::from_secs(10)).await;
std::fs::write(&ready_file, b"").unwrap();
tokio::time::timeout(
Duration::from_secs(10),
await_event(&mut rx, 1, ProcessEventKind::Reloaded),
)
.await
.expect("a replacement that failed its first probe and passed a later one is serving");
let after = handle.list().await;
assert_eq!(after.len(), 1, "the drainee's registration goes with it");
assert_eq!(after[0].id, 1);
assert_eq!(
after[0].status,
ProcStatus::Online,
"a probe that passed inside listen_timeout must put the replacement online"
);
}
fn probe_attempts(path: &std::path::Path) -> usize {
match std::fs::read_to_string(path) {
Ok(text) => text.lines().count(),
Err(_) => 0,
}
}
async fn await_probe_attempts(path: &std::path::Path, want: usize, bound: Duration) {
let reached = tokio::time::timeout(bound, async {
loop {
if probe_attempts(path) >= want {
break;
}
tokio::time::sleep(Duration::from_millis(10)).await;
}
})
.await;
assert!(
reached.is_ok(),
"{} never reached {want} probe attempts",
path.display(),
);
}
async fn await_file(path: &std::path::Path, bound: Duration) {
let found = tokio::time::timeout(bound, async {
loop {
match path.try_exists() {
Ok(true) => break,
Ok(false) => {}
Err(err) => panic!("cannot poll {}: {err}", path.display()),
}
tokio::time::sleep(Duration::from_millis(20)).await;
}
})
.await;
assert!(
found.is_ok(),
"{} never appeared: the exec probe never ran to completion with \
$SHEP_INSTANCE expanded to 0",
path.display()
);
}
#[tokio::test(start_paused = true)]
async fn a_gated_apps_online_carries_the_same_manually_flag_an_ungated_one_does() {
let (events, mut rx) = crate::bus::test_bus(64);
let runner = ScriptedRunner::new(vec![
ProcScript::never_exits(), ProcScript::never_exits(), ProcScript::never_exits(), ]);
let dir = tempfile::tempdir().unwrap();
let handle = spawn_supervisor(runner, test_paths(&dir), events);
let mut gated = AppConfig::minimal("gated", "./srv");
gated.wait_ready = true;
let ungated = AppConfig::minimal("ungated", "./srv");
handle
.start(vec![normalize(gated).unwrap(), normalize(ungated).unwrap()])
.await
.unwrap();
let ungated_manually = tokio::time::timeout(
Duration::from_secs(1),
await_event(&mut rx, 1, ProcessEventKind::Online),
)
.await
.expect("an ungated app is Online at spawn");
assert!(ungated_manually, "sanity: a Start is a manual event");
handle.tx.send(Msg::Ready { id: 0 }).await.unwrap();
let gated_manually = tokio::time::timeout(
Duration::from_secs(1),
await_event(&mut rx, 0, ProcessEventKind::Online),
)
.await
.expect("Online once the channel signals");
assert_eq!(
gated_manually, ungated_manually,
"the same `shep start` must report the same flag, gated or not"
);
let restarted = handle.restart(ProcessSelector::Id(0)).await.unwrap();
assert_eq!(restarted[0].status, ProcStatus::Starting);
handle.tx.send(Msg::Ready { id: 0 }).await.unwrap();
let restarted_manually = tokio::time::timeout(
Duration::from_secs(1),
await_event(&mut rx, 0, ProcessEventKind::Online),
)
.await
.expect("Online once the respawned sheep signals");
assert!(
restarted_manually,
"a manual Restart's Online must stay manual through the readiness gate"
);
}
#[tokio::test(start_paused = true)]
async fn an_operators_restart_is_reported_as_a_user_action() {
let (events, mut rx) = crate::bus::test_bus(64);
let runner = ScriptedRunner::new(vec![ProcScript::never_exits(); 2]);
let dir = tempfile::tempdir().unwrap();
let handle = spawn_supervisor(runner, test_paths(&dir), events);
handle
.start(vec![normalize(AppConfig::minimal("web", "./srv")).unwrap()])
.await
.unwrap();
let restarted = handle.restart(ProcessSelector::All).await.unwrap();
assert_eq!(restarted[0].restarts, 1, "the restart really respawned");
let manually = tokio::time::timeout(
Duration::from_secs(1),
await_event(&mut rx, 0, ProcessEventKind::Restart),
)
.await
.expect("the respawn `restart` performed");
assert!(
manually,
"a person typed `shep restart`; the bus must say a user action caused it"
);
}
#[tokio::test(start_paused = true)]
async fn a_gated_app_whose_deadline_elapses_goes_online_anyway() {
let (events, mut rx) = crate::bus::test_bus(64);
let runner = ScriptedRunner::new(vec![ProcScript::never_exits()]);
let dir = tempfile::tempdir().unwrap();
let handle = spawn_supervisor(runner, test_paths(&dir), events);
let mut app = AppConfig::minimal("web", "./srv");
app.wait_ready = true; handle.start(vec![normalize(app).unwrap()]).await.unwrap();
assert_eq!(handle.list().await[0].status, ProcStatus::Starting);
tokio::time::timeout(
Duration::from_secs(4), await_event(&mut rx, 0, ProcessEventKind::Online),
)
.await
.expect("Online once the readiness deadline elapses");
assert_eq!(handle.list().await[0].status, ProcStatus::Online);
}
#[tokio::test(start_paused = true)]
async fn a_gated_app_that_exits_while_starting_never_reaches_online_from_the_old_wait() {
let (events, mut rx) = crate::bus::test_bus(64);
let runner = ScriptedRunner::new(vec![
ProcScript::stable_then_exit(500, 1), ProcScript::never_exits(), ]);
let dir = tempfile::tempdir().unwrap();
let handle = spawn_supervisor(runner, test_paths(&dir), events);
let mut app = AppConfig::minimal("web", "./srv");
app.wait_ready = true; handle.start(vec![normalize(app).unwrap()]).await.unwrap();
assert_eq!(handle.list().await[0].status, ProcStatus::Starting);
tokio::time::timeout(
Duration::from_secs(1),
await_event(&mut rx, 0, ProcessEventKind::Restart),
)
.await
.expect("the automatic respawn after the unstable exit");
assert_eq!(handle.list().await[0].status, ProcStatus::Starting);
assert_eq!(handle.list().await[0].restarts, 1);
assert_no_event_within(
&mut rx,
0,
ProcessEventKind::Online,
Duration::from_millis(2_700),
)
.await;
assert_eq!(
handle.list().await[0].status,
ProcStatus::Starting,
"the OLD wait's stale TimedOut must not have marked the respawned process online"
);
tokio::time::timeout(
Duration::from_secs(2),
await_event(&mut rx, 0, ProcessEventKind::Online),
)
.await
.expect("the new process's own readiness deadline");
assert_eq!(handle.list().await[0].status, ProcStatus::Online);
}
#[tokio::test(start_paused = true)]
async fn a_gated_app_stopped_while_starting_ignores_the_old_wait() {
let (events, mut rx) = crate::bus::test_bus(64);
let runner = ScriptedRunner::new(vec![ProcScript::stable_then_exit(500, 1)]);
let dir = tempfile::tempdir().unwrap();
let handle = spawn_supervisor(runner, test_paths(&dir), events);
let mut app = AppConfig::minimal("web", "./srv");
app.wait_ready = true; app.autorestart = false; handle.start(vec![normalize(app).unwrap()]).await.unwrap();
assert_eq!(handle.list().await[0].status, ProcStatus::Starting);
tokio::time::timeout(
Duration::from_secs(1),
await_event(&mut rx, 0, ProcessEventKind::Stop),
)
.await
.expect("the natural exit at 500ms");
assert_eq!(handle.list().await[0].status, ProcStatus::Stopped);
assert_no_event_within(&mut rx, 0, ProcessEventKind::Online, Duration::from_secs(3)).await;
assert_eq!(handle.list().await[0].status, ProcStatus::Stopped);
}
#[tokio::test(start_paused = true)]
async fn a_gated_app_restarted_while_starting_ignores_the_old_wait() {
let (events, mut rx) = crate::bus::test_bus(64);
let runner = ScriptedRunner::new(vec![ProcScript::never_exits(), ProcScript::never_exits()]);
let dir = tempfile::tempdir().unwrap();
let handle = spawn_supervisor(runner, test_paths(&dir), events);
let mut app = AppConfig::minimal("web", "./srv");
app.wait_ready = true; handle.start(vec![normalize(app).unwrap()]).await.unwrap();
assert_eq!(handle.list().await[0].status, ProcStatus::Starting);
tokio::time::sleep(Duration::from_millis(500)).await;
handle
.restart(ProcessSelector::Name("web".to_string()))
.await
.unwrap();
assert_eq!(
handle.list().await[0].status,
ProcStatus::Starting,
"the respawned process is gated too, so it must still be Starting"
);
assert_no_event_within(
&mut rx,
0,
ProcessEventKind::Online,
Duration::from_millis(2_700),
)
.await;
assert_eq!(
handle.list().await[0].status,
ProcStatus::Starting,
"the old wait's stale TimedOut must not have marked the new process online"
);
tokio::time::timeout(
Duration::from_secs(2),
await_event(&mut rx, 0, ProcessEventKind::Online),
)
.await
.expect("the new process's own readiness deadline");
assert_eq!(handle.list().await[0].status, ProcStatus::Online);
}