use core::time::Duration;
use std::collections::{BTreeMap, BTreeSet};
use std::ffi::OsStr;
use shep_core::config::ResolvedApp;
use shep_core::config::graph::{BootNode, BootPlan, NodeKind};
use shep_core::protocol::{BusEvent, ProcessEventKind, ProcessInfo};
use shep_core::selector::ProcessSelector;
use shep_core::status::ProcStatus;
use tokio::sync::broadcast::{self, error::RecvError};
use crate::bus::{Bus, SharedEvent};
use crate::supervisor::{BatchPolicy, SupervisorError, SupervisorHandle};
pub(crate) const STAGE_SLACK: Duration = Duration::from_secs(5);
#[cfg_attr(not(test), expect(dead_code))]
#[must_use]
pub(crate) fn nodes_for(apps: &[ResolvedApp], dogs: &[String]) -> Vec<BootNode> {
nodes_for_with_dogs(apps, dogs, &[])
}
#[must_use]
pub(crate) fn plan_for(apps: &[ResolvedApp], dogs: &[String], boot_first: &[String]) -> BootPlan {
shep_core::config::graph::plan(&nodes_for_with_dogs(apps, dogs, boot_first))
}
#[must_use]
pub(crate) fn plan_for_names(edges: &BTreeMap<String, Vec<String>>) -> BootPlan {
let nodes: Vec<BootNode> = edges
.iter()
.map(|(name, depends_on)| BootNode {
name: name.clone(),
depends_on: depends_on.clone(),
kind: NodeKind::Sheep,
})
.collect();
shep_core::config::graph::plan(&nodes)
}
#[must_use]
pub(crate) fn nodes_for_with_dogs(
apps: &[ResolvedApp],
dogs: &[String],
boot_first: &[String],
) -> Vec<BootNode> {
let promoted: BTreeSet<&str> = boot_first.iter().map(String::as_str).collect();
let taken: BTreeSet<&str> = apps.iter().map(|app| app.config().name.as_str()).collect();
apps.iter()
.map(|app| BootNode {
name: app.config().name.clone(),
depends_on: app.config().depends_on.clone(),
kind: NodeKind::Sheep,
})
.chain(
dogs.iter()
.filter(|name| !taken.contains(name.as_str()))
.map(|name| BootNode {
name: name.clone(),
depends_on: Vec::new(),
kind: NodeKind::Dog {
boot_first: promoted.contains(name.as_str()),
},
}),
)
.collect()
}
pub(crate) async fn start_in_stages(
plan: &BootPlan,
apps: &[ResolvedApp],
supervisor: &SupervisorHandle,
events: &Bus,
policy: BatchPolicy,
notify_socket: Option<&OsStr>,
) -> Result<Vec<ProcessInfo>, SupervisorError> {
let mut depended_on: BTreeSet<&str> = BTreeSet::new();
for app in apps {
depended_on.extend(app.config().depends_on.iter().map(String::as_str));
}
let mut started = Vec::new();
for (index, stage) in plan.stages.iter().enumerate() {
let names: BTreeSet<&str> = stage.iter().map(String::as_str).collect();
let members: Vec<ResolvedApp> = apps
.iter()
.filter(|app| names.contains(app.config().name.as_str()))
.cloned()
.collect();
if members.is_empty() {
continue;
}
let gate: BTreeSet<String> = members
.iter()
.map(|app| app.config().name.clone())
.filter(|name| depended_on.contains(name.as_str()))
.collect();
let waiting = gate.clone();
let bound = members
.iter()
.map(|app| app.config().listen_timeout.as_duration())
.max()
.unwrap_or_default()
+ STAGE_SLACK;
let rx = events.subscribe();
tracing::info!(stage = index, members = ?stage, "boot stage starting");
extend_start_deadline(notify_socket, bound);
match supervisor.start_staged(members, gate, policy).await {
Ok(infos) => started.extend(infos),
Err(err) if policy == BatchPolicy::AllOrNothing => {
let left = left_running(supervisor, &started, &names).await;
return Err(corrected_for_earlier_stages(err, &left));
}
Err(err) => tracing::warn!(stage = index, %err, "a boot stage did not start"),
}
if waiting.is_empty() {
continue;
}
extend_start_deadline(notify_socket, bound);
let unsettled = await_stage(rx, waiting, bound, supervisor).await;
if !unsettled.is_empty() {
let names: Vec<&str> = unsettled.iter().map(String::as_str).collect();
tracing::warn!(
stage = index,
unsettled = ?names,
"a boot stage did not settle inside its bound; advancing anyway"
);
}
}
Ok(started)
}
pub(crate) fn extend_start_deadline(notify_socket: Option<&OsStr>, bound: Duration) {
#[cfg(unix)]
if let Some(target) = notify_socket
&& let Err(err) = crate::notify::extend_timeout(target, bound)
{
tracing::warn!(
%err,
"the start deadline could not be extended through $NOTIFY_SOCKET; carrying on"
);
}
#[cfg(not(unix))]
let _ = (notify_socket, bound);
}
async fn left_running(
supervisor: &SupervisorHandle,
started: &[ProcessInfo],
stage: &BTreeSet<&str>,
) -> BTreeSet<String> {
let mut names: BTreeSet<String> = started.iter().map(|info| info.name.clone()).collect();
let Ok(flock) = supervisor.list_checked().await else {
return names;
};
names.extend(
flock
.iter()
.filter(|info| stage.contains(info.name.as_str()))
.filter(|info| !matches!(info.status, ProcStatus::Stopped | ProcStatus::Errored))
.map(|info| info.name.clone()),
);
names
}
fn corrected_for_earlier_stages(
err: SupervisorError,
left_running: &BTreeSet<String>,
) -> SupervisorError {
if left_running.is_empty() {
return err;
}
let running: Vec<&str> = left_running.iter().map(String::as_str).collect();
let note = format!(
" (this start already has children running and leaves them alone: {})",
running.join(", ")
);
match err {
SupervisorError::CannotStart(message) => SupervisorError::CannotStart(message + ¬e),
SupervisorError::SpawnFailed(message) => SupervisorError::SpawnFailed(message + ¬e),
other => other,
}
}
pub(crate) async fn await_stage(
mut rx: broadcast::Receiver<SharedEvent>,
mut waiting: BTreeSet<String>,
bound: Duration,
supervisor: &SupervisorHandle,
) -> BTreeSet<String> {
let settle = async {
drop_settled(supervisor, &mut waiting).await;
while !waiting.is_empty() {
match rx.recv().await {
Ok(event) => {
let BusEvent::Process { info, .. } = &*event else {
continue;
};
if waiting.contains(&info.name) && is_settled(info.status) {
drop_settled(supervisor, &mut waiting).await;
}
}
Err(RecvError::Lagged(_)) => drop_settled(supervisor, &mut waiting).await,
Err(RecvError::Closed) => return,
}
}
};
let _ = tokio::time::timeout(bound, settle).await;
waiting
}
pub(crate) async fn await_reloads(
mut rx: broadcast::Receiver<SharedEvent>,
mut waiting: BTreeMap<String, usize>,
bound: Duration,
) -> BTreeSet<String> {
let settle = async {
while !waiting.is_empty() {
match rx.recv().await {
Ok(event) => {
let BusEvent::Process {
event: kind, info, ..
} = &*event
else {
continue;
};
match kind {
ProcessEventKind::Reloaded => {
if let Some(left) = waiting.get_mut(&info.name) {
*left = left.saturating_sub(1);
if *left == 0 {
waiting.remove(&info.name);
}
}
}
ProcessEventKind::ReloadAbandoned => {
waiting.remove(&info.name);
}
_ => {}
}
}
Err(RecvError::Lagged(_)) => continue,
Err(RecvError::Closed) => return,
}
}
};
let _ = tokio::time::timeout(bound, settle).await;
waiting.into_keys().collect()
}
fn is_settled(status: ProcStatus) -> bool {
!matches!(status, ProcStatus::Starting)
}
async fn drop_settled(supervisor: &SupervisorHandle, waiting: &mut BTreeSet<String>) {
let Ok(flock) = supervisor.list_checked().await else {
waiting.clear();
return;
};
waiting.retain(|name| {
flock
.iter()
.any(|info| &info.name == name && !is_settled(info.status))
});
}
pub(crate) async fn stop_in_reverse(plan: &BootPlan, supervisor: &SupervisorHandle) {
for stage in plan.stages.iter().rev() {
futures_util::future::join_all(stage.iter().map(|name| async move {
if let Err(err) = supervisor.stop(ProcessSelector::Name(name.clone())).await {
tracing::warn!(sheep = %name, %err, "a sheep did not stop in its stage");
}
}))
.await;
}
}
pub(crate) async fn stop_edges_in_reverse(
mut edges: BTreeMap<String, Vec<String>>,
dogs: &[String],
supervisor: &SupervisorHandle,
) {
for dog in dogs {
edges.remove(dog);
}
stop_in_reverse(&plan_for_names(&edges), supervisor).await;
}
#[cfg(test)]
mod tests;