use super::*;
use shep_core::config::graph::plan;
use shep_core::config::{AppConfig, normalize_all};
use shep_core::protocol::DogSource;
use shep_core::values::UpDuration;
use crate::fake::ProcScript;
use crate::testing::{harness, harness_failing_to_spawn, harness_refusing};
const SHORT_BOUND: Duration = Duration::from_millis(500);
const DEFAULT_KILL_TIMEOUT: Duration = Duration::from_millis(1600);
fn drain(rx: &mut broadcast::Receiver<SharedEvent>) -> Vec<String> {
let mut order = Vec::new();
while let Ok(event) = rx.try_recv() {
let BusEvent::Process {
event: kind, info, ..
} = &*event
else {
continue;
};
order.push(format!("{kind:?} {}", info.name));
}
order
}
fn online_event(info: ProcessInfo) -> BusEvent {
BusEvent::Process {
event: ProcessEventKind::Online,
info,
manually: true,
at_ms: 0,
}
}
fn db_then_api() -> Vec<ResolvedApp> {
let mut db = AppConfig::minimal("db", "./sleep");
db.listen_timeout = UpDuration::from_millis(50);
let mut api = AppConfig::minimal("api", "./sleep");
api.depends_on = vec!["db".to_string()];
normalize_all(vec![db, api]).expect("two apps, one edge, no cycle")
}
fn db_then_worker() -> Vec<ResolvedApp> {
let db = AppConfig::minimal("db", "./sleep");
let mut worker = AppConfig::minimal("worker", "./sleep");
worker.depends_on = vec!["db".to_string()];
normalize_all(vec![db, worker]).expect("two apps, one edge, no cycle")
}
#[test]
fn a_dog_named_after_a_sheep_is_not_a_node_of_its_own() {
let apps =
normalize_all(vec![AppConfig::minimal("metrics", "./sleep")]).expect("one app, no edges");
let nodes = nodes_for_with_dogs(&apps, &["metrics".to_string()], &[]);
assert_eq!(nodes.len(), 1, "one name is one node: {nodes:?}");
assert_eq!(nodes[0].kind, NodeKind::Sheep);
}
#[tokio::test(start_paused = true)]
async fn a_later_stage_does_not_start_until_the_earlier_one_is_online() {
let h = harness(vec![ProcScript::never_exits(), ProcScript::never_exits()]);
let apps = db_then_api();
let plan = plan(&nodes_for(&apps, &[]));
assert_eq!(plan.stages, vec![vec!["db"], vec!["api"]]);
let mut rx = h.ctx.events.subscribe();
start_in_stages(
&plan,
&apps,
&h.ctx.supervisor,
&h.ctx.events,
BatchPolicy::PerApp,
)
.await
.expect("`PerApp` never refuses a stage");
assert_eq!(
drain(&mut rx),
vec!["Start db", "Online db", "Start api", "Online api"],
);
}
#[tokio::test(start_paused = true)]
async fn one_stage_starts_its_members_in_the_order_the_caller_handed_them() {
let h = harness(vec![ProcScript::never_exits(), ProcScript::never_exits()]);
let apps = normalize_all(vec![
AppConfig::minimal("zulu", "./sleep"),
AppConfig::minimal("alpha", "./sleep"),
])
.expect("two apps, no edges");
let plan = plan(&nodes_for(&apps, &[]));
assert_eq!(plan.stages, vec![vec!["alpha", "zulu"]], "one sorted stage");
let mut rx = h.ctx.events.subscribe();
start_in_stages(
&plan,
&apps,
&h.ctx.supervisor,
&h.ctx.events,
BatchPolicy::AllOrNothing,
)
.await
.expect("both scripts spawn");
let starts: Vec<String> = drain(&mut rx)
.into_iter()
.filter(|line| line.starts_with("Start "))
.collect();
assert_eq!(starts, vec!["Start zulu", "Start alpha"]);
}
#[tokio::test(start_paused = true)]
async fn a_member_that_exits_does_not_hold_its_stage() {
let h = harness(vec![ProcScript::const_exit(1), ProcScript::never_exits()]);
let mut db = AppConfig::minimal("db", "./does-not-exist");
db.listen_timeout = UpDuration::from_millis(30_000);
db.autorestart = false;
let mut api = AppConfig::minimal("api", "./sleep");
api.depends_on = vec!["db".to_string()];
let apps = normalize_all(vec![db, api]).expect("two apps, one edge, no cycle");
let plan = plan(&nodes_for(&apps, &[]));
let started = tokio::time::timeout(
Duration::from_secs(5),
start_in_stages(
&plan,
&apps,
&h.ctx.supervisor,
&h.ctx.events,
BatchPolicy::PerApp,
),
)
.await
.expect("a dead dependency must not hold the stage for its deadline")
.expect("`PerApp` never refuses a stage");
assert!(started.iter().any(|info| info.name == "api"));
}
#[tokio::test]
async fn a_dog_lands_in_the_final_stage_and_boot_first_moves_it() {
let apps = normalize_all(vec![AppConfig::minimal("web", "./sleep")]).expect("one app");
let dogs = ["metrics".to_string()];
let plain = plan(&nodes_for(&apps, &dogs));
assert_eq!(plain.stages, vec![vec!["web"], vec!["metrics"]]);
let promoted = plan_for(&apps, &dogs, &["metrics".to_string()]);
assert_eq!(promoted.stages, vec![vec!["metrics"], vec!["web"]]);
}
#[tokio::test(start_paused = true)]
async fn a_refused_stage_names_what_earlier_stages_already_started() {
let h = harness_refusing(vec![ProcScript::never_exits()], &["api"]);
let apps = db_then_api();
let plan = plan(&nodes_for(&apps, &[]));
let err = start_in_stages(
&plan,
&apps,
&h.ctx.supervisor,
&h.ctx.events,
BatchPolicy::AllOrNothing,
)
.await
.expect_err("a refused preflight ends an `AllOrNothing` batch");
let SupervisorError::CannotStart(message) = &err else {
panic!("a refused preflight is `CannotStart`, got {err:?}");
};
assert!(
message.contains("db"),
"the refusal must name the stage that is running: {message}"
);
let flock = h
.ctx
.supervisor
.list_checked()
.await
.expect("the actor is up");
assert_eq!(
flock
.iter()
.map(|info| info.name.as_str())
.collect::<Vec<_>>(),
vec!["db"],
"the earlier stage is deliberately left running"
);
}
#[tokio::test(start_paused = true)]
async fn a_stage_that_fails_part_way_names_what_it_left_running() {
let h = harness_failing_to_spawn(
vec![ProcScript::never_exits(), ProcScript::never_exits()],
&["zulu"],
);
let apps = normalize_all(vec![
AppConfig::minimal("alpha", "./sleep"),
AppConfig::minimal("zulu", "./sleep"),
])
.expect("two apps, no edges");
let plan = plan(&nodes_for(&apps, &[]));
assert_eq!(plan.stages, vec![vec!["alpha", "zulu"]], "one stage");
let err = start_in_stages(
&plan,
&apps,
&h.ctx.supervisor,
&h.ctx.events,
BatchPolicy::AllOrNothing,
)
.await
.expect_err("a failed spawn ends an `AllOrNothing` batch");
let SupervisorError::SpawnFailed(message) = &err else {
panic!("a failed spawn is `SpawnFailed`, got {err:?}");
};
assert!(
message.contains("alpha"),
"the member that came up first must be named: {message}"
);
}
#[tokio::test(start_paused = true)]
async fn one_stage_stops_its_members_at_the_same_time() {
let h = harness(vec![
ProcScript::ignores_signals(),
ProcScript::ignores_signals(),
]);
let apps = normalize_all(vec![
AppConfig::minimal("alpha", "./sleep"),
AppConfig::minimal("zulu", "./sleep"),
])
.expect("two apps, no edges");
let plan = plan(&nodes_for(&apps, &[]));
assert_eq!(plan.stages, vec![vec!["alpha", "zulu"]], "one stage");
h.ctx
.supervisor
.start(apps)
.await
.expect("two scripted apps start");
let began = tokio::time::Instant::now();
stop_in_reverse(&plan, &h.ctx.supervisor).await;
let spent = began.elapsed();
assert!(
spent < DEFAULT_KILL_TIMEOUT * 2,
"one stage's ladders must overlap; spent {spent:?} on two of them"
);
}
#[tokio::test(start_paused = true)]
async fn stopping_walks_the_stages_backwards() {
let h = harness(vec![
ProcScript::never_exits(),
ProcScript::ignores_signals(),
ProcScript::ignores_signals(),
]);
let db = AppConfig::minimal("db", "./sleep");
let mut api = AppConfig::minimal("api", "./sleep");
api.depends_on = vec!["db".to_string()];
let mut worker = AppConfig::minimal("worker", "./sleep");
worker.depends_on = vec!["db".to_string()];
let apps = normalize_all(vec![db, api, worker]).expect("three apps, two edges, no cycle");
let plan = plan(&nodes_for(&apps, &[]));
assert_eq!(plan.stages, vec![vec!["db"], vec!["api", "worker"]]);
start_in_stages(
&plan,
&apps,
&h.ctx.supervisor,
&h.ctx.events,
BatchPolicy::PerApp,
)
.await
.expect("`PerApp` never refuses a stage");
let mut rx = h.ctx.events.subscribe();
stop_in_reverse(&plan, &h.ctx.supervisor).await;
let stops = drain(&mut rx);
let (last_stage, first_stage) = stops.split_at(2);
assert_eq!(
last_stage.iter().collect::<BTreeSet<_>>(),
["Stop api".to_string(), "Stop worker".to_string()]
.iter()
.collect::<BTreeSet<_>>(),
"the later stage stops first, in no order of its own: {stops:?}"
);
assert_eq!(first_stage, ["Stop db".to_string()], "{stops:?}");
}
#[tokio::test(start_paused = true)]
async fn a_shutdown_stops_dependents_before_their_dependencies() {
let h = harness(vec![
ProcScript::never_exits(),
ProcScript::ignores_signals(),
]);
let apps = db_then_worker();
h.ctx.registry.record(&apps);
let plan = plan(&nodes_for(&apps, &[]));
start_in_stages(
&plan,
&apps,
&h.ctx.supervisor,
&h.ctx.events,
BatchPolicy::PerApp,
)
.await
.expect("`PerApp` never refuses a stage");
let mut rx = h.ctx.events.subscribe();
stop_edges_in_reverse(
h.ctx.registry.depends_on_by_name(),
&h.ctx.dog_names,
&h.ctx.supervisor,
)
.await;
assert_eq!(drain(&mut rx), vec!["Stop worker", "Stop db"]);
}
#[tokio::test(start_paused = true)]
async fn a_dog_stops_after_every_sheep() {
let h = harness(vec![ProcScript::never_exits(), ProcScript::never_exits()]);
let apps = normalize_all(vec![AppConfig::minimal("web", "./sleep")]).expect("one app");
h.ctx.registry.record(&apps);
h.ctx
.supervisor
.start(apps)
.await
.expect("a scripted app that never exits starts");
let dog = normalize_all(vec![AppConfig::minimal("metrics", "./sleep")])
.expect("one dog app")
.remove(0);
h.ctx.registry.record(std::slice::from_ref(&dog));
h.ctx
.supervisor
.start_dog(dog, DogSource::BuiltIn)
.await
.expect("a scripted dog that never exits starts");
let mut rx = h.ctx.events.subscribe();
stop_edges_in_reverse(
h.ctx.registry.depends_on_by_name(),
&["metrics".to_string()],
&h.ctx.supervisor,
)
.await;
assert_eq!(
drain(&mut rx),
vec!["Stop web"],
"the staged walk must leave the dog running"
);
h.ctx.supervisor.shutdown().await;
assert_eq!(
drain(&mut rx),
vec!["Stop metrics"],
"the backstop is what stops a dog"
);
}
#[tokio::test(start_paused = true)]
async fn a_stage_whose_members_have_already_settled_waits_for_no_event_at_all() {
let h = harness(vec![ProcScript::never_exits()]);
let apps = normalize_all(vec![AppConfig::minimal("db", "./sleep")]).expect("one app");
h.ctx
.supervisor
.start(apps)
.await
.expect("a scripted app that never exits starts");
let rx = h.ctx.events.subscribe();
let waiting = ["db".to_string()].into_iter().collect();
let unsettled = await_stage(rx, waiting, SHORT_BOUND, &h.ctx.supervisor).await;
assert!(unsettled.is_empty(), "db is online, so nothing is waiting");
}
#[tokio::test(start_paused = true)]
async fn one_instance_going_online_does_not_settle_a_multi_instance_dependency() {
let h = harness(vec![
ProcScript::never_exits(),
ProcScript::never_exits(),
ProcScript::never_exits(),
]);
let mut web = AppConfig::minimal("web", "./sleep");
web.instances = 3;
web.listen_timeout = UpDuration::from_millis(30_000);
let apps = normalize_all(vec![web]).expect("one app");
let gate = ["web".to_string()].into_iter().collect();
let rx = h.ctx.events.subscribe();
let started = h
.ctx
.supervisor
.start_staged(apps, gate, BatchPolicy::PerApp)
.await
.expect("a scripted app that never exits starts");
assert_eq!(started.len(), 3);
assert!(
h.ctx
.supervisor
.list_checked()
.await
.expect("the actor is up")
.iter()
.all(|info| info.status == ProcStatus::Starting),
"a gated app parks every instance at Starting"
);
let mut first = started[0].clone();
first.status = ProcStatus::Online;
h.ctx
.events
.send(online_event(first).into())
.expect("the harness holds a receiver open");
let waiting = ["web".to_string()].into_iter().collect();
let unsettled = tokio::time::timeout(
Duration::from_secs(5),
await_stage(rx, waiting, SHORT_BOUND, &h.ctx.supervisor),
)
.await
.expect("the wait is bounded, so it ends either way");
assert!(
unsettled.contains("web"),
"two of web's three instances are still starting"
);
}