use core::time::Duration;
use std::collections::{BTreeMap, BTreeSet};
use shep_core::config::ResolvedApp;
use shep_core::config::graph::{BootNode, BootPlan, NodeKind};
use shep_core::protocol::{BusEvent, ProcessEventKind, ProcessInfo};
use shep_core::selector::ProcessSelector;
use shep_core::status::ProcStatus;
use tokio::sync::broadcast::{self, error::RecvError};
use crate::bus::{Bus, SharedEvent};
use crate::supervisor::{BatchPolicy, SupervisorError, SupervisorHandle};
pub(crate) const STAGE_SLACK: Duration = Duration::from_secs(5);
#[cfg_attr(not(test), expect(dead_code))]
#[must_use]
pub(crate) fn nodes_for(apps: &[ResolvedApp], dogs: &[String]) -> Vec<BootNode> {
nodes_for_with_dogs(apps, dogs, &[])
}
#[must_use]
pub(crate) fn plan_for(apps: &[ResolvedApp], dogs: &[String], boot_first: &[String]) -> BootPlan {
shep_core::config::graph::plan(&nodes_for_with_dogs(apps, dogs, boot_first))
}
#[must_use]
pub(crate) fn plan_for_names(edges: &BTreeMap<String, Vec<String>>) -> BootPlan {
let nodes: Vec<BootNode> = edges
.iter()
.map(|(name, depends_on)| BootNode {
name: name.clone(),
depends_on: depends_on.clone(),
kind: NodeKind::Sheep,
})
.collect();
shep_core::config::graph::plan(&nodes)
}
#[must_use]
pub(crate) fn nodes_for_with_dogs(
apps: &[ResolvedApp],
dogs: &[String],
boot_first: &[String],
) -> Vec<BootNode> {
let promoted: BTreeSet<&str> = boot_first.iter().map(String::as_str).collect();
let taken: BTreeSet<&str> = apps.iter().map(|app| app.config().name.as_str()).collect();
apps.iter()
.map(|app| BootNode {
name: app.config().name.clone(),
depends_on: app.config().depends_on.clone(),
kind: NodeKind::Sheep,
})
.chain(
dogs.iter()
.filter(|name| !taken.contains(name.as_str()))
.map(|name| BootNode {
name: name.clone(),
depends_on: Vec::new(),
kind: NodeKind::Dog {
boot_first: promoted.contains(name.as_str()),
},
}),
)
.collect()
}
pub(crate) async fn start_in_stages(
plan: &BootPlan,
apps: &[ResolvedApp],
supervisor: &SupervisorHandle,
events: &Bus,
policy: BatchPolicy,
) -> Result<Vec<ProcessInfo>, SupervisorError> {
let mut depended_on: BTreeSet<&str> = BTreeSet::new();
for app in apps {
depended_on.extend(app.config().depends_on.iter().map(String::as_str));
}
let mut started = Vec::new();
for (index, stage) in plan.stages.iter().enumerate() {
let names: BTreeSet<&str> = stage.iter().map(String::as_str).collect();
let members: Vec<ResolvedApp> = apps
.iter()
.filter(|app| names.contains(app.config().name.as_str()))
.cloned()
.collect();
if members.is_empty() {
continue;
}
let gate: BTreeSet<String> = members
.iter()
.map(|app| app.config().name.clone())
.filter(|name| depended_on.contains(name.as_str()))
.collect();
let waiting = gate.clone();
let bound = members
.iter()
.map(|app| app.config().listen_timeout.as_duration())
.max()
.unwrap_or_default()
+ STAGE_SLACK;
let rx = events.subscribe();
tracing::info!(stage = index, members = ?stage, "boot stage starting");
match supervisor.start_staged(members, gate, policy).await {
Ok(infos) => started.extend(infos),
Err(err) if policy == BatchPolicy::AllOrNothing => {
let left = left_running(supervisor, &started, &names).await;
return Err(corrected_for_earlier_stages(err, &left));
}
Err(err) => tracing::warn!(stage = index, %err, "a boot stage did not start"),
}
if waiting.is_empty() {
continue;
}
let unsettled = await_stage(rx, waiting, bound, supervisor).await;
if !unsettled.is_empty() {
let names: Vec<&str> = unsettled.iter().map(String::as_str).collect();
tracing::warn!(
stage = index,
unsettled = ?names,
"a boot stage did not settle inside its bound; advancing anyway"
);
}
}
Ok(started)
}
async fn left_running(
supervisor: &SupervisorHandle,
started: &[ProcessInfo],
stage: &BTreeSet<&str>,
) -> BTreeSet<String> {
let mut names: BTreeSet<String> = started.iter().map(|info| info.name.clone()).collect();
let Ok(flock) = supervisor.list_checked().await else {
return names;
};
names.extend(
flock
.iter()
.filter(|info| stage.contains(info.name.as_str()))
.filter(|info| !matches!(info.status, ProcStatus::Stopped | ProcStatus::Errored))
.map(|info| info.name.clone()),
);
names
}
fn corrected_for_earlier_stages(
err: SupervisorError,
left_running: &BTreeSet<String>,
) -> SupervisorError {
if left_running.is_empty() {
return err;
}
let running: Vec<&str> = left_running.iter().map(String::as_str).collect();
let note = format!(
" (this start already has children running and leaves them alone: {})",
running.join(", ")
);
match err {
SupervisorError::CannotStart(message) => SupervisorError::CannotStart(message + ¬e),
SupervisorError::SpawnFailed(message) => SupervisorError::SpawnFailed(message + ¬e),
other => other,
}
}
pub(crate) async fn await_stage(
mut rx: broadcast::Receiver<SharedEvent>,
mut waiting: BTreeSet<String>,
bound: Duration,
supervisor: &SupervisorHandle,
) -> BTreeSet<String> {
let settle = async {
drop_settled(supervisor, &mut waiting).await;
while !waiting.is_empty() {
match rx.recv().await {
Ok(event) => {
let BusEvent::Process { info, .. } = &*event else {
continue;
};
if waiting.contains(&info.name) && is_settled(info.status) {
drop_settled(supervisor, &mut waiting).await;
}
}
Err(RecvError::Lagged(_)) => drop_settled(supervisor, &mut waiting).await,
Err(RecvError::Closed) => return,
}
}
};
let _ = tokio::time::timeout(bound, settle).await;
waiting
}
pub(crate) async fn await_reloads(
mut rx: broadcast::Receiver<SharedEvent>,
mut waiting: BTreeMap<String, usize>,
bound: Duration,
) -> BTreeSet<String> {
let settle = async {
while !waiting.is_empty() {
match rx.recv().await {
Ok(event) => {
let BusEvent::Process {
event: kind, info, ..
} = &*event
else {
continue;
};
match kind {
ProcessEventKind::Reloaded => {
if let Some(left) = waiting.get_mut(&info.name) {
*left = left.saturating_sub(1);
if *left == 0 {
waiting.remove(&info.name);
}
}
}
ProcessEventKind::ReloadAbandoned => {
waiting.remove(&info.name);
}
_ => {}
}
}
Err(RecvError::Lagged(_)) => continue,
Err(RecvError::Closed) => return,
}
}
};
let _ = tokio::time::timeout(bound, settle).await;
waiting.into_keys().collect()
}
fn is_settled(status: ProcStatus) -> bool {
!matches!(status, ProcStatus::Starting)
}
async fn drop_settled(supervisor: &SupervisorHandle, waiting: &mut BTreeSet<String>) {
let Ok(flock) = supervisor.list_checked().await else {
waiting.clear();
return;
};
waiting.retain(|name| {
flock
.iter()
.any(|info| &info.name == name && !is_settled(info.status))
});
}
pub(crate) async fn stop_in_reverse(plan: &BootPlan, supervisor: &SupervisorHandle) {
for stage in plan.stages.iter().rev() {
futures_util::future::join_all(stage.iter().map(|name| async move {
if let Err(err) = supervisor.stop(ProcessSelector::Name(name.clone())).await {
tracing::warn!(sheep = %name, %err, "a sheep did not stop in its stage");
}
}))
.await;
}
}
pub(crate) async fn stop_edges_in_reverse(
mut edges: BTreeMap<String, Vec<String>>,
dogs: &[String],
supervisor: &SupervisorHandle,
) {
for dog in dogs {
edges.remove(dog);
}
stop_in_reverse(&plan_for_names(&edges), supervisor).await;
}
#[cfg(test)]
mod tests {
use super::*;
use shep_core::config::graph::plan;
use shep_core::config::{AppConfig, normalize_all};
use shep_core::protocol::DogSource;
use shep_core::values::UpDuration;
use crate::fake::ProcScript;
use crate::testing::{harness, harness_failing_to_spawn, harness_refusing};
const SHORT_BOUND: Duration = Duration::from_millis(500);
const DEFAULT_KILL_TIMEOUT: Duration = Duration::from_millis(1600);
fn drain(rx: &mut broadcast::Receiver<SharedEvent>) -> Vec<String> {
let mut order = Vec::new();
while let Ok(event) = rx.try_recv() {
let BusEvent::Process {
event: kind, info, ..
} = &*event
else {
continue;
};
order.push(format!("{kind:?} {}", info.name));
}
order
}
fn online_event(info: ProcessInfo) -> BusEvent {
BusEvent::Process {
event: ProcessEventKind::Online,
info,
manually: true,
at_ms: 0,
}
}
fn db_then_api() -> Vec<ResolvedApp> {
let mut db = AppConfig::minimal("db", "./sleep");
db.listen_timeout = UpDuration::from_millis(50);
let mut api = AppConfig::minimal("api", "./sleep");
api.depends_on = vec!["db".to_string()];
normalize_all(vec![db, api]).expect("two apps, one edge, no cycle")
}
fn db_then_worker() -> Vec<ResolvedApp> {
let db = AppConfig::minimal("db", "./sleep");
let mut worker = AppConfig::minimal("worker", "./sleep");
worker.depends_on = vec!["db".to_string()];
normalize_all(vec![db, worker]).expect("two apps, one edge, no cycle")
}
#[test]
fn a_dog_named_after_a_sheep_is_not_a_node_of_its_own() {
let apps = normalize_all(vec![AppConfig::minimal("metrics", "./sleep")])
.expect("one app, no edges");
let nodes = nodes_for_with_dogs(&apps, &["metrics".to_string()], &[]);
assert_eq!(nodes.len(), 1, "one name is one node: {nodes:?}");
assert_eq!(nodes[0].kind, NodeKind::Sheep);
}
#[tokio::test(start_paused = true)]
async fn a_later_stage_does_not_start_until_the_earlier_one_is_online() {
let h = harness(vec![ProcScript::never_exits(), ProcScript::never_exits()]);
let apps = db_then_api();
let plan = plan(&nodes_for(&apps, &[]));
assert_eq!(plan.stages, vec![vec!["db"], vec!["api"]]);
let mut rx = h.ctx.events.subscribe();
start_in_stages(
&plan,
&apps,
&h.ctx.supervisor,
&h.ctx.events,
BatchPolicy::PerApp,
)
.await
.expect("`PerApp` never refuses a stage");
assert_eq!(
drain(&mut rx),
vec!["Start db", "Online db", "Start api", "Online api"],
);
}
#[tokio::test(start_paused = true)]
async fn one_stage_starts_its_members_in_the_order_the_caller_handed_them() {
let h = harness(vec![ProcScript::never_exits(), ProcScript::never_exits()]);
let apps = normalize_all(vec![
AppConfig::minimal("zulu", "./sleep"),
AppConfig::minimal("alpha", "./sleep"),
])
.expect("two apps, no edges");
let plan = plan(&nodes_for(&apps, &[]));
assert_eq!(plan.stages, vec![vec!["alpha", "zulu"]], "one sorted stage");
let mut rx = h.ctx.events.subscribe();
start_in_stages(
&plan,
&apps,
&h.ctx.supervisor,
&h.ctx.events,
BatchPolicy::AllOrNothing,
)
.await
.expect("both scripts spawn");
let starts: Vec<String> = drain(&mut rx)
.into_iter()
.filter(|line| line.starts_with("Start "))
.collect();
assert_eq!(starts, vec!["Start zulu", "Start alpha"]);
}
#[tokio::test(start_paused = true)]
async fn a_member_that_exits_does_not_hold_its_stage() {
let h = harness(vec![ProcScript::const_exit(1), ProcScript::never_exits()]);
let mut db = AppConfig::minimal("db", "./does-not-exist");
db.listen_timeout = UpDuration::from_millis(30_000);
db.autorestart = false;
let mut api = AppConfig::minimal("api", "./sleep");
api.depends_on = vec!["db".to_string()];
let apps = normalize_all(vec![db, api]).expect("two apps, one edge, no cycle");
let plan = plan(&nodes_for(&apps, &[]));
let started = tokio::time::timeout(
Duration::from_secs(5),
start_in_stages(
&plan,
&apps,
&h.ctx.supervisor,
&h.ctx.events,
BatchPolicy::PerApp,
),
)
.await
.expect("a dead dependency must not hold the stage for its deadline")
.expect("`PerApp` never refuses a stage");
assert!(started.iter().any(|info| info.name == "api"));
}
#[tokio::test]
async fn a_dog_lands_in_the_final_stage_and_boot_first_moves_it() {
let apps = normalize_all(vec![AppConfig::minimal("web", "./sleep")]).expect("one app");
let dogs = ["metrics".to_string()];
let plain = plan(&nodes_for(&apps, &dogs));
assert_eq!(plain.stages, vec![vec!["web"], vec!["metrics"]]);
let promoted = plan_for(&apps, &dogs, &["metrics".to_string()]);
assert_eq!(promoted.stages, vec![vec!["metrics"], vec!["web"]]);
}
#[tokio::test(start_paused = true)]
async fn a_refused_stage_names_what_earlier_stages_already_started() {
let h = harness_refusing(vec![ProcScript::never_exits()], &["api"]);
let apps = db_then_api();
let plan = plan(&nodes_for(&apps, &[]));
let err = start_in_stages(
&plan,
&apps,
&h.ctx.supervisor,
&h.ctx.events,
BatchPolicy::AllOrNothing,
)
.await
.expect_err("a refused preflight ends an `AllOrNothing` batch");
let SupervisorError::CannotStart(message) = &err else {
panic!("a refused preflight is `CannotStart`, got {err:?}");
};
assert!(
message.contains("db"),
"the refusal must name the stage that is running: {message}"
);
let flock = h
.ctx
.supervisor
.list_checked()
.await
.expect("the actor is up");
assert_eq!(
flock
.iter()
.map(|info| info.name.as_str())
.collect::<Vec<_>>(),
vec!["db"],
"the earlier stage is deliberately left running"
);
}
#[tokio::test(start_paused = true)]
async fn a_stage_that_fails_part_way_names_what_it_left_running() {
let h = harness_failing_to_spawn(
vec![ProcScript::never_exits(), ProcScript::never_exits()],
&["zulu"],
);
let apps = normalize_all(vec![
AppConfig::minimal("alpha", "./sleep"),
AppConfig::minimal("zulu", "./sleep"),
])
.expect("two apps, no edges");
let plan = plan(&nodes_for(&apps, &[]));
assert_eq!(plan.stages, vec![vec!["alpha", "zulu"]], "one stage");
let err = start_in_stages(
&plan,
&apps,
&h.ctx.supervisor,
&h.ctx.events,
BatchPolicy::AllOrNothing,
)
.await
.expect_err("a failed spawn ends an `AllOrNothing` batch");
let SupervisorError::SpawnFailed(message) = &err else {
panic!("a failed spawn is `SpawnFailed`, got {err:?}");
};
assert!(
message.contains("alpha"),
"the member that came up first must be named: {message}"
);
}
#[tokio::test(start_paused = true)]
async fn one_stage_stops_its_members_at_the_same_time() {
let h = harness(vec![
ProcScript::ignores_signals(),
ProcScript::ignores_signals(),
]);
let apps = normalize_all(vec![
AppConfig::minimal("alpha", "./sleep"),
AppConfig::minimal("zulu", "./sleep"),
])
.expect("two apps, no edges");
let plan = plan(&nodes_for(&apps, &[]));
assert_eq!(plan.stages, vec![vec!["alpha", "zulu"]], "one stage");
h.ctx
.supervisor
.start(apps)
.await
.expect("two scripted apps start");
let began = tokio::time::Instant::now();
stop_in_reverse(&plan, &h.ctx.supervisor).await;
let spent = began.elapsed();
assert!(
spent < DEFAULT_KILL_TIMEOUT * 2,
"one stage's ladders must overlap; spent {spent:?} on two of them"
);
}
#[tokio::test(start_paused = true)]
async fn stopping_walks_the_stages_backwards() {
let h = harness(vec![
ProcScript::never_exits(),
ProcScript::ignores_signals(),
ProcScript::ignores_signals(),
]);
let db = AppConfig::minimal("db", "./sleep");
let mut api = AppConfig::minimal("api", "./sleep");
api.depends_on = vec!["db".to_string()];
let mut worker = AppConfig::minimal("worker", "./sleep");
worker.depends_on = vec!["db".to_string()];
let apps = normalize_all(vec![db, api, worker]).expect("three apps, two edges, no cycle");
let plan = plan(&nodes_for(&apps, &[]));
assert_eq!(plan.stages, vec![vec!["db"], vec!["api", "worker"]]);
start_in_stages(
&plan,
&apps,
&h.ctx.supervisor,
&h.ctx.events,
BatchPolicy::PerApp,
)
.await
.expect("`PerApp` never refuses a stage");
let mut rx = h.ctx.events.subscribe();
stop_in_reverse(&plan, &h.ctx.supervisor).await;
let stops = drain(&mut rx);
let (last_stage, first_stage) = stops.split_at(2);
assert_eq!(
last_stage.iter().collect::<BTreeSet<_>>(),
["Stop api".to_string(), "Stop worker".to_string()]
.iter()
.collect::<BTreeSet<_>>(),
"the later stage stops first, in no order of its own: {stops:?}"
);
assert_eq!(first_stage, ["Stop db".to_string()], "{stops:?}");
}
#[tokio::test(start_paused = true)]
async fn a_shutdown_stops_dependents_before_their_dependencies() {
let h = harness(vec![
ProcScript::never_exits(),
ProcScript::ignores_signals(),
]);
let apps = db_then_worker();
h.ctx.registry.record(&apps);
let plan = plan(&nodes_for(&apps, &[]));
start_in_stages(
&plan,
&apps,
&h.ctx.supervisor,
&h.ctx.events,
BatchPolicy::PerApp,
)
.await
.expect("`PerApp` never refuses a stage");
let mut rx = h.ctx.events.subscribe();
stop_edges_in_reverse(
h.ctx.registry.depends_on_by_name(),
&h.ctx.dog_names,
&h.ctx.supervisor,
)
.await;
assert_eq!(drain(&mut rx), vec!["Stop worker", "Stop db"]);
}
#[tokio::test(start_paused = true)]
async fn a_dog_stops_after_every_sheep() {
let h = harness(vec![ProcScript::never_exits(), ProcScript::never_exits()]);
let apps = normalize_all(vec![AppConfig::minimal("web", "./sleep")]).expect("one app");
h.ctx.registry.record(&apps);
h.ctx
.supervisor
.start(apps)
.await
.expect("a scripted app that never exits starts");
let dog = normalize_all(vec![AppConfig::minimal("metrics", "./sleep")])
.expect("one dog app")
.remove(0);
h.ctx.registry.record(std::slice::from_ref(&dog));
h.ctx
.supervisor
.start_dog(dog, DogSource::BuiltIn)
.await
.expect("a scripted dog that never exits starts");
let mut rx = h.ctx.events.subscribe();
stop_edges_in_reverse(
h.ctx.registry.depends_on_by_name(),
&["metrics".to_string()],
&h.ctx.supervisor,
)
.await;
assert_eq!(
drain(&mut rx),
vec!["Stop web"],
"the staged walk must leave the dog running"
);
h.ctx.supervisor.shutdown().await;
assert_eq!(
drain(&mut rx),
vec!["Stop metrics"],
"the backstop is what stops a dog"
);
}
#[tokio::test(start_paused = true)]
async fn a_stage_whose_members_have_already_settled_waits_for_no_event_at_all() {
let h = harness(vec![ProcScript::never_exits()]);
let apps = normalize_all(vec![AppConfig::minimal("db", "./sleep")]).expect("one app");
h.ctx
.supervisor
.start(apps)
.await
.expect("a scripted app that never exits starts");
let rx = h.ctx.events.subscribe();
let waiting = ["db".to_string()].into_iter().collect();
let unsettled = await_stage(rx, waiting, SHORT_BOUND, &h.ctx.supervisor).await;
assert!(unsettled.is_empty(), "db is online, so nothing is waiting");
}
#[tokio::test(start_paused = true)]
async fn one_instance_going_online_does_not_settle_a_multi_instance_dependency() {
let h = harness(vec![
ProcScript::never_exits(),
ProcScript::never_exits(),
ProcScript::never_exits(),
]);
let mut web = AppConfig::minimal("web", "./sleep");
web.instances = 3;
web.listen_timeout = UpDuration::from_millis(30_000);
let apps = normalize_all(vec![web]).expect("one app");
let gate = ["web".to_string()].into_iter().collect();
let rx = h.ctx.events.subscribe();
let started = h
.ctx
.supervisor
.start_staged(apps, gate, BatchPolicy::PerApp)
.await
.expect("a scripted app that never exits starts");
assert_eq!(started.len(), 3);
assert!(
h.ctx
.supervisor
.list_checked()
.await
.expect("the actor is up")
.iter()
.all(|info| info.status == ProcStatus::Starting),
"a gated app parks every instance at Starting"
);
let mut first = started[0].clone();
first.status = ProcStatus::Online;
h.ctx
.events
.send(online_event(first).into())
.expect("the harness holds a receiver open");
let waiting = ["web".to_string()].into_iter().collect();
let unsettled = tokio::time::timeout(
Duration::from_secs(5),
await_stage(rx, waiting, SHORT_BOUND, &h.ctx.supervisor),
)
.await
.expect("the wait is bounded, so it ends either way");
assert!(
unsettled.contains("web"),
"two of web's three instances are still starting"
);
}
}