use core::time::Duration;
use std::collections::{BTreeMap, BTreeSet};
use shep_core::protocol::{ProcessInfo, Reply, Response, RpcError, SelectorSpec, SheepRefusal};
use shep_core::selector::ProcessSelector;
use shep_core::status::ProcStatus;
use crate::supervisor::SupervisorError;
use super::context::{Outcome, RpcContext};
use super::error::rpc_error;
use super::selector_verbs::selector_of;
pub(super) struct OrderedWalk {
pub(super) stages: Vec<Vec<String>>,
pub(super) depended_on: BTreeSet<String>,
}
pub(super) fn ordered_walk(
ctx: &RpcContext,
selector: &ProcessSelector,
flock: &[ProcessInfo],
) -> Option<OrderedWalk> {
let exact = selector.is_exact();
let matched: BTreeSet<String> = flock
.iter()
.filter(|info| exact || info.dog.is_none())
.filter(|info| exact || info.disabled != Some(true))
.filter(|info| selector.matches(&info.name, info.id, info.fold.as_deref(), info.instance))
.map(|info| info.name.clone())
.collect();
if matched.len() < 2 {
return None;
}
let edges = ctx.registry.depends_on_by_name();
let plan = crate::boot_order::plan_for_names(&edges);
let mut stages: Vec<Vec<String>> = plan
.stages
.iter()
.map(|stage| {
stage
.iter()
.filter(|name| matched.contains(name.as_str()))
.cloned()
.collect::<Vec<String>>()
})
.filter(|stage| !stage.is_empty())
.collect();
let placed: BTreeSet<&str> = stages.iter().flatten().map(String::as_str).collect();
let unplaced: Vec<String> = matched
.iter()
.filter(|name| !placed.contains(name.as_str()))
.cloned()
.collect();
if !unplaced.is_empty() {
stages.push(unplaced);
}
let depended_on = matched
.iter()
.flat_map(|name| matched_dependencies(&edges, name, &matched))
.collect();
Some(OrderedWalk {
stages,
depended_on,
})
}
fn matched_dependencies(
edges: &BTreeMap<String, Vec<String>>,
start: &str,
matched: &BTreeSet<String>,
) -> BTreeSet<String> {
let mut found = BTreeSet::new();
let mut seen: BTreeSet<&str> = [start].into_iter().collect();
let mut frontier: Vec<&str> = vec![start];
while let Some(name) = frontier.pop() {
for dependency in edges.get(name).into_iter().flatten() {
if !seen.insert(dependency.as_str()) {
continue;
}
if matched.contains(dependency.as_str()) {
found.insert(dependency.clone());
}
frontier.push(dependency.as_str());
}
}
found
}
fn settle_bound(ctx: &RpcContext, name: &str, reloading: bool) -> Duration {
let (readiness, drain) = ctx.registry.timeouts_of(name).unwrap_or_default();
let drain = if reloading { drain } else { Duration::ZERO };
readiness + drain
}
fn stage_bound(ctx: &RpcContext, stage: &[String]) -> Duration {
stage
.iter()
.map(|name| settle_bound(ctx, name, false))
.max()
.unwrap_or_default()
+ crate::boot_order::STAGE_SLACK
}
pub(super) fn reload_stage_bound(ctx: &RpcContext, waiting: &BTreeMap<String, usize>) -> Duration {
waiting
.iter()
.map(|(name, swaps)| {
settle_bound(ctx, name, true).saturating_mul(u32::try_from(*swaps).unwrap_or(u32::MAX))
})
.max()
.unwrap_or_default()
+ crate::boot_order::STAGE_SLACK
}
pub(super) async fn restart_request(id: u64, spec: SelectorSpec, ctx: &RpcContext) -> Outcome {
let result = match selector_of(spec) {
Err(err) => Err(err),
Ok(selector) => match walk_for(ctx, &selector).await {
None => ctx
.supervisor
.restart(selector)
.await
.map(|accepted| Response::Restarted {
accepted,
refused: Vec::new(),
})
.map_err(|err| rpc_error(&err)),
Some(walk) => restart_in_stages(ctx, &walk)
.await
.map(|(accepted, refused)| Response::Restarted { accepted, refused }),
},
};
Outcome::Reply(Reply { id, result })
}
pub(super) async fn reload_request(id: u64, spec: SelectorSpec, ctx: &RpcContext) -> Outcome {
let result = match selector_of(spec) {
Err(err) => Err(err),
Ok(selector) => match walk_for(ctx, &selector).await {
None => ctx
.supervisor
.reload(selector)
.await
.map(|accepted| Response::Reloading {
accepted,
refused: Vec::new(),
})
.map_err(|err| rpc_error(&err)),
Some(walk) => reload_in_stages(ctx, &walk)
.await
.map(|(accepted, refused)| Response::Reloading { accepted, refused }),
},
};
Outcome::Reply(Reply { id, result })
}
pub(super) async fn walk_for(ctx: &RpcContext, selector: &ProcessSelector) -> Option<OrderedWalk> {
let flock = ctx.supervisor.list_checked().await.ok()?;
ordered_walk(ctx, selector, &flock)
}
pub(super) async fn restart_in_stages(
ctx: &RpcContext,
walk: &OrderedWalk,
) -> Result<(Vec<ProcessInfo>, Vec<SheepRefusal>), RpcError> {
let mut restarted = Vec::new();
let mut refused: Vec<SheepRefusal> = Vec::new();
let mut refusal = None;
for stage in &walk.stages {
let rx = ctx.events.subscribe();
let outcomes = futures_util::future::join_all(stage.iter().map(|name| async move {
(
name,
ctx.supervisor
.restart(ProcessSelector::Name(name.clone()))
.await,
)
}))
.await;
for (name, outcome) in outcomes {
match outcome {
Ok(infos) => restarted.extend(infos),
Err(err) => {
tracing::warn!(
sheep = %name,
%err,
"a sheep did not restart in its stage; it may have left the flock since \
the walk was planned"
);
refused.push(SheepRefusal::new(name.clone(), err.to_string()));
refusal.get_or_insert(err);
}
}
}
let waiting: BTreeSet<String> = stage
.iter()
.filter(|name| walk.depended_on.contains(name.as_str()))
.cloned()
.collect();
if waiting.is_empty() {
continue;
}
let bound = stage_bound(ctx, stage);
let unsettled = crate::boot_order::await_stage(rx, waiting, bound, &ctx.supervisor).await;
warn_about_unsettled(&unsettled);
}
finished(restarted, refusal).map(|rows| (rows, refused))
}
async fn reload_in_stages(
ctx: &RpcContext,
walk: &OrderedWalk,
) -> Result<(Vec<ProcessInfo>, Vec<SheepRefusal>), RpcError> {
let mut accepted = Vec::new();
let mut refused: Vec<SheepRefusal> = Vec::new();
let mut refusal = None;
for stage in &walk.stages {
let rx = ctx.events.subscribe();
let outcomes = futures_util::future::join_all(stage.iter().map(|name| async move {
(
name,
ctx.supervisor
.reload(ProcessSelector::Name(name.clone()))
.await,
)
}))
.await;
let mut waiting: BTreeMap<String, usize> = BTreeMap::new();
for (name, outcome) in outcomes {
match outcome {
Ok(infos) => {
if walk.depended_on.contains(name.as_str()) {
let swaps = infos
.iter()
.filter(|info| info.status == ProcStatus::Online)
.count();
if swaps > 0 {
waiting.insert(name.clone(), swaps);
}
}
accepted.extend(infos);
}
Err(err) => {
if matches!(err, SupervisorError::ReloadInFlight(_))
&& walk.depended_on.contains(name.as_str())
{
waiting.insert(name.clone(), 1);
}
tracing::warn!(
sheep = %name,
%err,
"a sheep did not reload in its stage; it may have left the flock since \
the walk was planned"
);
refused.push(SheepRefusal::new(name.clone(), err.to_string()));
refusal.get_or_insert(err);
}
}
}
if waiting.is_empty() {
continue;
}
let bound = reload_stage_bound(ctx, &waiting);
let unsettled = crate::boot_order::await_reloads(rx, waiting, bound).await;
warn_about_unsettled(&unsettled);
}
finished(accepted, refusal).map(|rows| (rows, refused))
}
fn warn_about_unsettled(unsettled: &BTreeSet<String>) {
if unsettled.is_empty() {
return;
}
let names: Vec<&str> = unsettled.iter().map(String::as_str).collect();
tracing::warn!(
unsettled = ?names,
"a stage did not settle inside its bound; advancing anyway"
);
}
fn finished(
mut rows: Vec<ProcessInfo>,
refusal: Option<SupervisorError>,
) -> Result<Vec<ProcessInfo>, RpcError> {
match refusal {
Some(err) if rows.is_empty() => Err(rpc_error(&err)),
_ => {
shep_core::protocol::sort_flock(&mut rows);
Ok(rows)
}
}
}