use super::*;
#[tokio::test(start_paused = true)]
async fn a_completed_swap_reports_itself_on_the_bus() {
let dir = tempfile::tempdir().unwrap();
let (handle, _runner, mut rx) = started(
&dir,
AppConfig::minimal("web", "./srv"),
vec![ProcScript::never_exits(); 3],
)
.await;
handle
.reload(ProcessSelector::Name("web".to_string()))
.await
.expect("the reload is accepted");
let seen = events_through(&mut rx, 1, ProcessEventKind::Reloaded).await;
assert!(
at(&seen, 0, ProcessEventKind::Reload) < at(&seen, 1, ProcessEventKind::Start),
"the instance being replaced is named before its replacement starts: {seen:?}"
);
assert_eq!(
seen[at(&seen, 0, ProcessEventKind::Reload)],
Seen {
id: 0,
kind: ProcessEventKind::Reload,
status: ProcStatus::Stopping,
manually: true,
}
);
assert!(
at(&seen, 0, ProcessEventKind::Delete) < at(&seen, 1, ProcessEventKind::Reloaded),
"a swap is not over until the instance it replaced is gone: {seen:?}"
);
assert_eq!(
seen[at(&seen, 1, ProcessEventKind::Reloaded)],
Seen {
id: 1,
kind: ProcessEventKind::Reloaded,
status: ProcStatus::Online,
manually: true,
}
);
}
#[tokio::test(start_paused = true)]
async fn a_swap_is_not_announced_for_a_replacement_that_is_no_longer_serving() {
let dir = tempfile::tempdir().unwrap();
let mut app = AppConfig::minimal("web", "./srv");
app.autorestart = false; let (handle, _runner, mut rx) = started(
&dir,
app,
vec![
ProcScript::ignores_signals(),
ProcScript::stable_then_exit(5_000, 1),
],
)
.await;
handle
.reload(ProcessSelector::Name("web".to_string()))
.await
.expect("the reload is accepted");
let seen = events_through(&mut rx, 1, ProcessEventKind::ReloadAbandoned).await;
assert_eq!(
seen[at(&seen, 1, ProcessEventKind::ReloadAbandoned)],
Seen {
id: 1,
kind: ProcessEventKind::ReloadAbandoned,
status: ProcStatus::Stopped,
manually: true,
},
"a replacement that died inside the drain window is what the \
abandonment names, carrying the status it actually reached"
);
assert!(
!seen.iter().any(|e| e.kind == ProcessEventKind::Reloaded),
"no swap succeeded, so nothing may say one did: {seen:?}"
);
}
#[tokio::test(start_paused = true)]
async fn a_swap_whose_drainee_never_reports_its_exit_gives_up_on_its_own() {
let dir = tempfile::tempdir().unwrap();
let (handle, runner, mut rx) = started(
&dir,
AppConfig::minimal("web", "./srv"),
vec![
ProcScript::never_reports_its_exit(),
ProcScript::never_exits(),
ProcScript::never_exits(),
],
)
.await;
handle
.reload(ProcessSelector::Name("web".to_string()))
.await
.expect("the reload is accepted");
let seen = events_through(&mut rx, 1, ProcessEventKind::ReloadAbandoned).await;
assert_eq!(
seen[at(&seen, 1, ProcessEventKind::ReloadAbandoned)],
Seen {
id: 1,
kind: ProcessEventKind::ReloadAbandoned,
status: ProcStatus::Online,
manually: true,
},
"the replacement took the slot over and is what is left holding it"
);
assert_eq!(
runner.kill_counts(),
vec![1, 0],
"the drain did reach `SIGKILL`; what never came back was the exit"
);
let again = handle
.reload(ProcessSelector::Name("web".to_string()))
.await
.map(|infos| infos.iter().map(|info| info.id).collect::<Vec<_>>());
assert_eq!(
again,
Ok(vec![0, 1]),
"a wedged instance must not refuse the app's next reload"
);
}
#[tokio::test(start_paused = true)]
async fn a_deadline_from_a_finished_swap_never_ends_the_one_that_followed_it() {
let dir = tempfile::tempdir().unwrap();
let mut app = AppConfig::minimal("web", "./srv");
app.instances = 2;
let (handle, _runner, mut rx) = started(
&dir,
app,
vec![
ProcScript::ignores_signals(),
ProcScript::ignores_signals(),
ProcScript::never_exits(),
ProcScript::never_exits(),
],
)
.await;
handle
.reload(ProcessSelector::Name("web".to_string()))
.await
.expect("the reload is accepted");
expect_event(&mut rx, 2, ProcessEventKind::Reloaded).await;
expect_event(&mut rx, 3, ProcessEventKind::Reloaded).await;
let after = handle.list().await;
assert_eq!(
after.iter().map(|info| info.id).collect::<Vec<_>>(),
vec![2, 3],
"both instances were replaced: {after:?}"
);
assert!(after.iter().all(|info| info.status == ProcStatus::Online));
}
#[tokio::test(start_paused = true)]
async fn a_deadline_before_the_commit_puts_the_instance_being_replaced_back() {
let dir = tempfile::tempdir().unwrap();
let (mut actor, _mailbox) = actor_with_one_online_sheep(&dir, vec![ProcScript::never_exits()]);
let (ctl_tx, _ctl_rx) = mpsc::channel(SHEEP_CTL_CAPACITY);
actor.sheep.get_mut(&0).expect("the fixture's sheep").ctl = Some(ctl_tx);
let new_id = actor
.spawn_replacement(0, ReloadMode::Overlap)
.expect("the fixture's one script covers this spawn");
actor.reloads.insert(
"web".to_string(),
ReloadJob {
queue: VecDeque::new(),
mode: ReloadMode::Overlap,
deadline: 0,
swap: ReloadSwap {
old_id: 0,
new_id: Some(new_id),
phase: ReloadPhase::AwaitReady,
},
},
);
actor.handle_reload_deadline("web", actor.reloads["web"].deadline);
assert!(
actor.reloads.is_empty(),
"the job is gone, so the app is reloadable again"
);
let drainee = &actor.sheep[&0];
assert_eq!(
drainee.entry.status,
ProcStatus::Online,
"nothing was ever killed, so the instance being replaced goes back to serving"
);
assert_eq!(drainee.entry.reload, ReloadState::None);
assert_eq!(
actor.sheep[&new_id].manual.map(|pending| pending.kind),
Some(ManualKind::Delete),
"the replacement that never proved itself is taken back down"
);
}
#[tokio::test(start_paused = true)]
async fn an_abandoned_reload_says_so_on_the_bus() {
let dir = tempfile::tempdir().unwrap();
let mut app = AppConfig::minimal("web", "./srv");
app.wait_ready = true; let (handle, _runner, mut rx) = started(&dir, app, vec![ProcScript::never_exits(); 2]).await;
handle.tx.send(Msg::Ready { id: 0 }).await.unwrap();
expect_event(&mut rx, 0, ProcessEventKind::Online).await;
handle
.reload(ProcessSelector::Name("web".to_string()))
.await
.expect("the reload is accepted");
let seen = events_through(&mut rx, 0, ProcessEventKind::ReloadAbandoned).await;
assert_eq!(
seen[at(&seen, 0, ProcessEventKind::ReloadAbandoned)],
Seen {
id: 0,
kind: ProcessEventKind::ReloadAbandoned,
status: ProcStatus::Online,
manually: true,
},
"the abandoned reload's own instance is still the one serving"
);
}
#[tokio::test(start_paused = true)]
async fn a_reload_whose_replacement_cannot_spawn_says_so_on_the_bus() {
let dir = tempfile::tempdir().unwrap();
let (handle, _runner, mut rx) = started(
&dir,
AppConfig::minimal("web", "./srv"),
vec![ProcScript::never_exits()],
)
.await;
handle
.reload(ProcessSelector::Name("web".to_string()))
.await
.expect("the reload is accepted before anything is spawned");
let seen = events_through(&mut rx, 0, ProcessEventKind::ReloadAbandoned).await;
assert_eq!(
seen[at(&seen, 0, ProcessEventKind::ReloadAbandoned)],
Seen {
id: 0,
kind: ProcessEventKind::ReloadAbandoned,
status: ProcStatus::Online,
manually: true,
},
"a failed spawn leaves the instance it was replacing serving"
);
let after = handle.list().await;
assert_eq!(after.len(), 1, "no replacement was registered: {after:?}");
assert_eq!(after[0].id, 0);
}
#[tokio::test(start_paused = true)]
async fn a_live_sheep_keeps_holding_its_log_control_sender() {
let runner = ScriptedRunner::new(vec![ProcScript::never_exits()]);
let (proc, io) = runner.spawn(&log_ctl_spec()).unwrap();
assert!(runner.log_ctl_live(0), "sanity: the fake starts it live");
let (events, _rx) = crate::bus::test_bus(64);
let (_ctl_tx, ctl_rx) = mpsc::channel(8);
let (_signal_tx, signal_rx) = mpsc::channel(8);
let (actor_tx, _actor_rx) = mpsc::channel(8);
let app = normalize(AppConfig::minimal("svc", "./svc")).unwrap();
tokio::spawn(run_sheep(
7, proc, io, app, ctl_rx, signal_rx, events, actor_tx,
));
for _ in 0..100 {
tokio::task::yield_now().await;
}
assert!(
runner.log_ctl_live(0),
"run_sheep dropped ProcIo::log_ctl while its sheep was still \
running: against the real runner that closes the read ends of \
the child's stdout and stderr"
);
}
#[tokio::test(start_paused = true)]
async fn a_pump_is_reaped_when_its_sheep_ends_even_with_a_lamb_on_the_pipe() {
let (events, mut rx) = crate::bus::test_bus(64);
let runner = Arc::new(ScriptedRunner::new(vec![
ProcScript::stable_then_exit(1_000, 0).with_a_lamb_holding_the_pipe(),
]));
let dir = tempfile::tempdir().unwrap();
let handle = spawn_supervisor(SharedRunner(Arc::clone(&runner)), test_paths(&dir), events);
let mut app = AppConfig::minimal("web", "./srv");
app.autorestart = false;
handle.start(vec![normalize(app).unwrap()]).await.unwrap();
assert!(
runner.log_ctl_live(0),
"sanity: a running sheep has a live pump"
);
await_event(&mut rx, 0, ProcessEventKind::Stop).await;
assert_eq!(
handle.list().await.len(),
1,
"sanity: the sheep is still registered, so its slot still holds a \
clone of the control sender"
);
let reaped = tokio::time::timeout(Duration::from_secs(5), async {
while runner.log_ctl_live(0) {
tokio::time::sleep(Duration::from_millis(10)).await;
}
})
.await;
assert!(
reaped.is_ok(),
"the pump outlived its sheep task, holding both log files and \
both pipe read ends open"
);
}
#[tokio::test(start_paused = true)]
async fn a_writer_task_is_reaped_when_its_sheep_exits() {
let dir = tempfile::tempdir().unwrap();
let mut app = AppConfig::minimal("web", "./srv");
app.autorestart = false;
app.channel = true;
let (handle, runner, mut rx) =
started(&dir, app, vec![ProcScript::stable_then_exit(1_000, 0)]).await;
let mut io = runner.io_handles(0);
assert_eq!(
io.to_child_rx.try_recv(),
Err(mpsc::error::TryRecvError::Empty),
"sanity: a running sheep's channel is open and quiet, so the \
`None` below is a close and not a channel that never opened"
);
await_event(&mut rx, 0, ProcessEventKind::Stop).await;
assert_eq!(
handle.list().await.len(),
1,
"sanity: the sheep is still registered, so nothing but the \
clearing can have let go of the slot's clone"
);
let reaped = tokio::time::timeout(Duration::from_secs(5), io.to_child_rx.recv()).await;
assert_eq!(
reaped.ok(),
Some(None),
"the writer task outlived its sheep, parked on `recv()` and \
holding the daemon's end of the shepherd channel"
);
}
#[tokio::test(start_paused = true)]
async fn a_failed_spawn_names_the_sheep_and_the_path_it_tried() {
let dir = tempfile::tempdir().unwrap();
let (mut actor, _mailbox) = actor_with_one_online_sheep(&dir, Vec::new());
let (reply, answer) = oneshot::channel();
actor.handle_command(Command::Start {
apps: vec![normalize(AppConfig::minimal("api", "./api")).unwrap()],
policy: BatchPolicy::AllOrNothing,
gate: BTreeSet::new(),
reply,
});
let err = answer
.await
.expect("the actor answers every Start")
.expect_err("an empty script pool cannot spawn");
assert_eq!(
err.to_string(),
"spawn failed: api: process spawn failed: script exhausted; tried `./api`"
);
}
#[tokio::test(start_paused = true)]
async fn config_drift_names_an_edited_sheeps_fields_and_changes_nothing() {
let dir = tempfile::tempdir().unwrap();
let (actor, _mailbox) = actor_with_one_online_sheep(&dir, vec![ProcScript::never_exits()]);
let mut edited = AppConfig::minimal("web", "./srv");
edited.cwd = Some("/srv/new".to_string());
edited
.env
.insert("DATABASE_URL".to_string(), "postgres://hunter2".to_string());
let unknown = AppConfig::minimal("api", "./api");
let drift = actor.config_drift(&[normalize(edited).unwrap(), normalize(unknown).unwrap()]);
assert_eq!(
drift,
vec![SheepDrift::new(
"web",
vec!["cwd".to_string(), "env".to_string()]
)]
);
assert!(
!format!("{drift:?}").contains("hunter2"),
"a value must never travel with the field name that changed: {drift:?}"
);
assert_eq!(
actor.sheep[&0].entry.spec.config().cwd,
None,
"asking which fields differ must not apply them"
);
}
#[tokio::test(start_paused = true)]
async fn every_spawn_leaves_the_daemons_end_of_the_channel_on_the_slot() {
let dir = tempfile::tempdir().unwrap();
let (mut actor, _mailbox) = actor_with_one_online_sheep(
&dir,
vec![
ProcScript::never_exits(),
ProcScript::never_exits(),
ProcScript::never_exits(),
],
);
let mut app = AppConfig::minimal("api", "./api");
app.autorestart = false;
let (reply, _answer) = oneshot::channel();
actor.handle_command(Command::Start {
apps: vec![normalize(app).unwrap()],
policy: BatchPolicy::AllOrNothing,
gate: BTreeSet::new(),
reply,
});
assert!(
actor.sheep[&1].to_child.is_some(),
"a fresh spawn's slot holds the daemon's end of its shepherd channel"
);
actor.handle_exited(
1,
ExitOutcome {
code: Some(0),
signal: None,
},
);
assert!(
actor.sheep[&1].to_child.is_none(),
"the clone goes with the process it was cloned for"
);
actor.respawn(1, false);
assert!(
actor.sheep[&1].to_child.is_some(),
"a respawn hands the slot the new process's channel, not the dead \
one's"
);
actor.advance_reload("web", VecDeque::from([0]));
let new_id = actor.reloads["web"]
.swap
.new_id
.expect("an overlapping reload spawns its replacement at once");
assert!(
actor.sheep[&new_id].to_child.is_some(),
"a reload's replacement holds the daemon's end of its own channel"
);
}