use super::*;
fn probed_app(name: &str) -> AppConfig {
let mut app = AppConfig::minimal(name, "./srv");
app.readiness_probe = Some(probe_config(ProbeKind::Tcp, "127.0.0.1:9"));
app
}
fn park_ready_failed(actor: &mut Actor<ScriptedRunner>) {
let slot = actor.sheep.get_mut(&0).expect("the fixture's sheep");
slot.entry.status = ProcStatus::Starting;
slot.ready_failed = true;
}
#[test]
fn reload_mode_serialises_exactly_the_apps_whose_probe_can_lie() {
let cases = [
(false, false, false, ReloadMode::Overlap),
(false, true, false, ReloadMode::Overlap),
(true, false, false, ReloadMode::Serial),
(true, true, false, ReloadMode::Overlap),
(true, false, true, ReloadMode::Overlap),
];
for (probe, wait_ready, reuse_port, want) in cases {
let mut app = if probe {
probed_app("web")
} else {
AppConfig::minimal("web", "./srv")
};
app.wait_ready = wait_ready;
app.reuse_port = reuse_port;
let source = ReadinessSource::of(&app).expect("the target parses");
assert_eq!(
ReloadMode::of(&app, &source),
want,
"probe={probe} wait_ready={wait_ready} reuse_port={reuse_port}"
);
}
}
#[tokio::test(start_paused = true)]
async fn a_probed_reload_drains_before_it_spawns_anything() {
let dir = tempfile::tempdir().unwrap();
let (mut actor, _mailbox) = actor_with_one_online_sheep_of(&dir, probed_app("web"), vec![]);
let mut ctl_rx = attach_ctl(&mut actor);
actor.advance_reload("web", VecDeque::from([0]));
let job = &actor.reloads["web"];
assert_eq!(job.mode, ReloadMode::Serial);
assert_eq!(job.swap.phase, ReloadPhase::DrainFirst);
assert_eq!(
job.swap.new_id, None,
"a serial reload has no replacement until the drain is over"
);
assert_eq!(
actor.sheep.len(),
1,
"nothing was spawned into the slot the drain is emptying"
);
let drainee = &actor.sheep[&0];
assert_eq!(drainee.entry.status, ProcStatus::Stopping);
assert_eq!(
drainee.entry.reload,
ReloadState::Drainee { new_id: None },
"the marker is what keeps this exit off the ordinary respawn path"
);
assert!(
drainee.manual.is_some(),
"the drain owns the drainee's next exit"
);
assert!(
ctl_rx.try_recv().is_ok(),
"the drain asked the instance to go"
);
}
#[tokio::test(start_paused = true)]
async fn reuse_port_buys_back_the_overlap_a_probe_would_otherwise_cost() {
let dir = tempfile::tempdir().unwrap();
let mut app = probed_app("web");
app.reuse_port = true;
let (mut actor, _mailbox) =
actor_with_one_online_sheep_of(&dir, app, vec![ProcScript::never_exits()]);
actor.advance_reload("web", VecDeque::from([0]));
let job = &actor.reloads["web"];
assert_eq!(job.mode, ReloadMode::Overlap);
assert_eq!(job.swap.phase, ReloadPhase::AwaitReady);
let new_id = job
.swap
.new_id
.expect("an overlapping reload spawns at once");
assert_eq!(
actor.sheep[&new_id].entry.instance, actor.sheep[&0].entry.instance,
"the replacement takes the same instance slot"
);
assert_eq!(
actor.sheep[&0].entry.status,
ProcStatus::Stopping,
"the instance being replaced is still there, marked and serving"
);
}
#[tokio::test(start_paused = true)]
async fn a_serial_reload_spawns_into_the_slot_the_drain_emptied() {
let dir = tempfile::tempdir().unwrap();
let (mut actor, _mailbox) =
actor_with_one_online_sheep_of(&dir, probed_app("web"), vec![ProcScript::never_exits()]);
let _ctl_rx = attach_ctl(&mut actor);
let instance = actor.sheep[&0].entry.instance;
let restarts = actor.sheep[&0].entry.restarts;
actor.advance_reload("web", VecDeque::from([0]));
actor.handle_exited(
0,
ExitOutcome {
code: Some(0),
signal: None,
},
);
assert!(
!actor.sheep.contains_key(&0),
"the drained instance is deregistered, not left as a second row"
);
let new_id = actor.reloads["web"]
.swap
.new_id
.expect("the reap is where a serial reload spawns");
let replacement = &actor.sheep[&new_id].entry;
assert_eq!(replacement.instance, instance);
assert_eq!(replacement.restarts, restarts);
assert_eq!(replacement.status, ProcStatus::Starting);
assert_eq!(replacement.reload, ReloadState::Replacement);
assert_eq!(
actor.reloads["web"].swap.phase,
ReloadPhase::DrainOld,
"with the drainee gone there is nothing left to abandon back to"
);
}
#[tokio::test(start_paused = true)]
async fn a_reload_can_still_replace_the_instance_a_failed_reload_parked() {
let dir = tempfile::tempdir().unwrap();
let (mut actor, _mailbox) = actor_with_one_online_sheep_of(&dir, probed_app("web"), vec![]);
let _ctl_rx = attach_ctl(&mut actor);
park_ready_failed(&mut actor);
let (reply, _rx) = oneshot::channel();
actor.handle_reload(&ProcessSelector::Name("web".to_string()), reply);
assert!(
actor.reloads.contains_key("web"),
"a parked instance is still replaceable, or nothing can roll it back"
);
assert_eq!(actor.reloads["web"].swap.old_id, 0);
}
#[tokio::test(start_paused = true)]
async fn an_ordinarily_starting_sheep_is_still_skipped_by_a_reload() {
let dir = tempfile::tempdir().unwrap();
let (mut actor, _mailbox) = actor_with_one_online_sheep_of(&dir, probed_app("web"), vec![]);
let _ctl_rx = attach_ctl(&mut actor);
let slot = actor.sheep.get_mut(&0).expect("the fixture's sheep");
slot.entry.status = ProcStatus::Starting;
let (reply, _rx) = oneshot::channel();
actor.handle_reload(&ProcessSelector::Name("web".to_string()), reply);
assert!(
actor.reloads.is_empty(),
"a sheep still waiting on its own readiness is left to finish"
);
assert_eq!(actor.sheep[&0].entry.status, ProcStatus::Starting);
}
#[tokio::test(start_paused = true)]
async fn undoing_a_swap_never_promotes_a_parked_drainee_to_online() {
let dir = tempfile::tempdir().unwrap();
let mut app = probed_app("web");
app.reuse_port = true;
let (mut actor, _mailbox) = actor_with_one_online_sheep_of(&dir, app, vec![]);
park_ready_failed(&mut actor);
actor
.spawn_replacement(0, ReloadMode::Overlap)
.expect_err("a runner with no scripts left cannot spawn");
let slot = actor.sheep.get(&0).expect("the drainee stays registered");
assert_eq!(
slot.entry.status,
ProcStatus::Starting,
"an instance that never proved itself is not restored to online"
);
assert_eq!(slot.entry.reload, ReloadState::None);
assert!(
slot.ready_failed,
"and it stays replaceable, or the next attempt cannot reach it"
);
}
#[tokio::test(start_paused = true)]
async fn abandoning_a_swap_never_promotes_a_parked_drainee_to_online() {
let dir = tempfile::tempdir().unwrap();
let mut app = probed_app("web");
app.reuse_port = true;
let (mut actor, _mailbox) =
actor_with_one_online_sheep_of(&dir, app, vec![ProcScript::never_exits()]);
let _ctl_rx = attach_ctl(&mut actor);
park_ready_failed(&mut actor);
let (reply, _rx) = oneshot::channel();
actor.handle_reload(&ProcessSelector::Name("web".to_string()), reply);
assert!(
actor.reloads.contains_key("web"),
"the parked instance is the one being replaced"
);
actor.abort_reload("web", "the replacement was not ready inside listen_timeout");
assert_eq!(
actor.sheep[&0].entry.status,
ProcStatus::Starting,
"an instance that never proved itself is not restored to online"
);
assert!(actor.sheep[&0].ready_failed);
}
#[test]
fn only_an_overlapping_probe_is_re_asked_after_the_drain() {
let dir = tempfile::tempdir().unwrap();
let mut probed = probed_app("web");
probed.reuse_port = true;
let mut channel = AppConfig::minimal("web", "./srv");
channel.wait_ready = true;
for (app, mode, want) in [
(probed.clone(), ReloadMode::Overlap, true),
(probed, ReloadMode::Serial, false),
(channel, ReloadMode::Overlap, false),
(
AppConfig::minimal("web", "./srv"),
ReloadMode::Overlap,
false,
),
] {
let (actor, _mailbox) = actor_with_one_online_sheep_of(&dir, app, vec![]);
assert_eq!(
actor.post_drain_probe(0, mode).is_some(),
want,
"mode={mode:?}"
);
}
}
fn actor_awaiting_a_verdict(
dir: &tempfile::TempDir,
) -> (Actor<ScriptedRunner>, mpsc::Receiver<Msg>) {
let mut app = probed_app("web");
app.reuse_port = true;
let (mut actor, mailbox) = actor_with_one_online_sheep_of(dir, app, vec![]);
actor
.sheep
.get_mut(&0)
.expect("the fixture's sheep")
.entry
.reload = ReloadState::Replacement;
actor.reloads.insert(
"web".to_string(),
ReloadJob {
queue: VecDeque::new(),
mode: ReloadMode::Overlap,
deadline: 0,
swap: ReloadSwap {
old_id: 99,
new_id: Some(0),
phase: ReloadPhase::Verify,
},
},
);
(actor, mailbox)
}
#[tokio::test(start_paused = true)]
async fn a_replacement_that_fails_the_second_probe_is_demoted_and_abandoned() {
let dir = tempfile::tempdir().unwrap();
let (mut actor, _mailbox) = actor_awaiting_a_verdict(&dir);
let mut rx = actor.events.subscribe();
actor.handle_reload_verified("web", 0, Readiness::TimedOut);
assert!(actor.reloads.is_empty(), "the reload is over either way");
assert!(
actor.sheep.contains_key(&0),
"the only instance the app has left is not killed as well"
);
assert_eq!(
actor.sheep[&0].entry.status,
ProcStatus::Starting,
"an instance that never answered alone is not online"
);
assert_eq!(
drained_process_kinds(&mut rx),
vec![ProcessEventKind::ReloadAbandoned],
"an abandonment, and never the `Reloaded` that claims a success"
);
}
#[tokio::test(start_paused = true)]
async fn a_replacement_that_answers_alone_finishes_the_reload() {
let dir = tempfile::tempdir().unwrap();
let (mut actor, _mailbox) = actor_awaiting_a_verdict(&dir);
let mut rx = actor.events.subscribe();
actor.handle_reload_verified("web", 0, Readiness::Ready);
assert!(actor.reloads.is_empty());
assert_eq!(actor.sheep[&0].entry.status, ProcStatus::Online);
assert_eq!(actor.sheep[&0].entry.reload, ReloadState::None);
assert_eq!(
drained_process_kinds(&mut rx),
vec![ProcessEventKind::Reloaded]
);
}
#[tokio::test(start_paused = true)]
async fn a_re_armed_watchdog_retires_the_one_it_replaced() {
let dir = tempfile::tempdir().unwrap();
let (mut actor, _mailbox) = actor_awaiting_a_verdict(&dir);
actor.arm_reload_deadline("web", 0);
let first = actor.reloads["web"].deadline;
actor.arm_reload_deadline("web", 0);
let second = actor.reloads["web"].deadline;
assert_ne!(first, second, "each arming takes a stamp of its own");
actor.handle_reload_deadline("web", first);
assert!(
actor.reloads.contains_key("web"),
"the retired watchdog must not end a swap the live one is still watching"
);
actor.handle_reload_deadline("web", second);
assert!(actor.reloads.is_empty(), "the live one still ends it");
}
#[tokio::test(start_paused = true)]
async fn a_delete_during_a_serial_drain_spawns_no_replacement() {
let dir = tempfile::tempdir().unwrap();
let (mut actor, _mailbox) = actor_with_one_online_sheep_of(&dir, probed_app("web"), vec![]);
let _ctl_rx = attach_ctl(&mut actor);
actor.advance_reload("web", VecDeque::from([0]));
assert_eq!(actor.reloads["web"].swap.phase, ReloadPhase::DrainFirst);
actor.sheep.get_mut(&0).expect("the drainee").pending_delete = true;
actor.handle_exited(
0,
ExitOutcome {
code: Some(0),
signal: None,
},
);
assert!(actor.sheep.is_empty(), "the delete emptied the flock");
assert!(actor.reloads.is_empty(), "and the reload ended with it");
}
#[tokio::test(start_paused = true)]
async fn a_parked_instances_verdict_does_not_outlive_its_process() {
let dir = tempfile::tempdir().unwrap();
let mut app = probed_app("web");
app.autorestart = false;
let (mut actor, _mailbox) = actor_with_one_online_sheep_of(&dir, app, vec![]);
let _ctl_rx = attach_ctl(&mut actor);
park_ready_failed(&mut actor);
actor.handle_exited(
0,
ExitOutcome {
code: Some(0),
signal: None,
},
);
let slot = actor.sheep.get(&0).expect("a clean stop keeps the row");
assert_ne!(slot.entry.status, ProcStatus::Online);
assert!(!slot.ready_failed);
assert!(
!reload_eligible(slot),
"a row with no process behind it is not something a reload may replace"
);
}