use core::future::Future;
use core::time::Duration;
use std::collections::{BTreeMap, BTreeSet, HashMap};
use std::path::PathBuf;
use std::sync::{Arc, Mutex, MutexGuard, PoisonError};
use tokio::sync::watch;
use shep_core::config::graph::BootPlan;
use shep_core::config::{DeclaredApp, NormalizeError, ResolvedApp, normalize_all};
use shep_core::paths::ShepPaths;
use shep_core::protocol::{
Envelope, Lamb, PROTOCOL_VERSION, ProcessInfo, Reply, Request, Response, RpcError,
RpcErrorCode, SelectorSpec, SheepApplied, SheepRefusal,
};
use shep_core::selector::ProcessSelector;
use shep_core::signals::OperatorSignal;
use shep_core::status::ProcStatus;
use crate::bus::{Bus, TopicFilter};
use crate::dogs::DogSpec;
use crate::limits::stats::StatsState;
use crate::snapshot::{FlockRegistry, SnapshotError, write_atomic};
use crate::supervisor::{Applied, BatchPolicy, ConnId, SupervisorError, SupervisorHandle};
pub(crate) const DEFAULT_DEADLINE_MS: u64 = 5_000;
pub(crate) const MAX_DEADLINE_MS: u64 = 60_000;
#[derive(Debug, Clone)]
pub(crate) struct KnownDogs {
names: Arc<Mutex<BTreeSet<String>>>,
}
impl KnownDogs {
pub(crate) fn new(names: BTreeSet<String>) -> Self {
Self {
names: Arc::new(Mutex::new(names)),
}
}
pub(crate) fn contains(&self, name: &str) -> bool {
self.lock().contains(name)
}
pub(crate) fn insert(&self, name: &str) {
self.lock().insert(name.to_owned());
}
fn lock(&self) -> MutexGuard<'_, BTreeSet<String>> {
self.names.lock().unwrap_or_else(PoisonError::into_inner)
}
}
#[derive(Clone, Debug)]
pub struct RpcContext {
pub(crate) supervisor: SupervisorHandle,
pub(crate) events: Bus,
pub(crate) registry: FlockRegistry,
pub(crate) snapshot_path: PathBuf,
pub(crate) dogs_config: PathBuf,
pub(crate) known_dogs: KnownDogs,
pub(crate) dog_names: Vec<String>,
pub(crate) boot_first_dogs: Vec<String>,
pub(crate) paths: ShepPaths,
pub(crate) daemon_version: String,
pub(crate) dog_refusals: crate::dogs::DogRefusals,
pub(crate) peer_contacts: crate::dogs::PeerContacts,
pub(crate) pid: u32,
pub(crate) shutdown: Arc<watch::Sender<bool>>,
pub(crate) stats: Arc<StatsState>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct SavedRoll {
pub path: PathBuf,
pub apps: u32,
}
impl RpcContext {
pub fn shutdown(&self) {
let _ = self.shutdown.send(true);
}
pub fn announce_dog_config(&self, dogs: &[String]) {
crate::bus::publish_dog_config_changed(&self.events, dogs);
}
pub async fn save_roll_now(&self) -> Result<Option<SavedRoll>, SnapshotError> {
let Ok(infos) = self.supervisor.list_checked().await else {
return Ok(None);
};
let roll = self.registry.roll(&infos, crate::now_ms());
write_atomic(&self.snapshot_path, &roll)?;
Ok(Some(SavedRoll {
path: self.snapshot_path.clone(),
apps: u32::try_from(roll.apps.len()).unwrap_or(u32::MAX),
}))
}
pub async fn snapshot_now(&self) -> Result<(), SnapshotError> {
self.save_roll_now().await.map(|_| ())
}
}
#[derive(Debug)]
pub(crate) enum Outcome {
Reply(Reply),
Subscribe {
reply: Reply,
filter: TopicFilter,
},
Shutdown(Reply),
}
#[must_use]
pub(crate) fn budget(deadline_ms: Option<u64>) -> Duration {
Duration::from_millis(
deadline_ms
.unwrap_or(DEFAULT_DEADLINE_MS)
.clamp(1, MAX_DEADLINE_MS),
)
}
pub(crate) async fn dispatch(envelope: Envelope, conn: ConnId, ctx: &RpcContext) -> Outcome {
let id = envelope.id;
with_deadline(
id,
budget(envelope.deadline_ms),
run(id, conn, envelope.body, ctx),
)
.await
}
async fn with_deadline<F: Future<Output = Outcome> + Send>(
id: u64,
budget: Duration,
work: F,
) -> Outcome {
match tokio::time::timeout(budget, work).await {
Ok(outcome) => outcome,
Err(_) => Outcome::Reply(Reply {
id,
result: Err(RpcError {
code: RpcErrorCode::DeadlineExceeded,
message: format!(
"the request deadline of {} ms expired before the daemon finished",
budget.as_millis()
),
daemon_version: None,
}),
}),
}
}
async fn run(id: u64, conn: ConnId, request: Request, ctx: &RpcContext) -> Outcome {
let reply = |result| Outcome::Reply(Reply { id, result });
match request {
Request::Ping => reply(Ok(Response::Pong)),
Request::ListFlock => match ctx.supervisor.list_checked().await {
Ok(infos) => reply(Ok(Response::Flock(with_dog_contact(
&ctx.dog_refusals,
with_live_stats(&ctx.stats, infos).await,
)))),
Err(err) => reply(Err(rpc_error(&err))),
},
Request::Describe { selector } => match selector_of(selector) {
Err(err) => reply(Err(err)),
Ok(selector) => match ctx.supervisor.list_checked().await {
Err(err) => reply(Err(rpc_error(&err))),
Ok(infos) => {
let exact = selector.is_exact();
let hits: Vec<_> = infos
.into_iter()
.filter(|i| exact || i.dog.is_none())
.filter(|i| selector.matches(&i.name, i.id, i.fold.as_deref(), i.instance))
.collect();
if hits.is_empty() {
reply(Err(not_found()))
} else {
let hits = with_live_stats(&ctx.stats, hits).await;
let hits = with_lambs(&ctx.stats, hits).await;
reply(Ok(Response::Described(with_dog_contact(
&ctx.dog_refusals,
hits,
))))
}
}
},
},
Request::Start { apps } => match normalize_all(apps) {
Err(err) => reply(Err(RpcError {
code: RpcErrorCode::InvalidConfig,
message: err.to_string(),
daemon_version: None,
})),
Ok(resolved) => match staged_plan(ctx, &resolved) {
Err(err) => reply(Err(err)),
Ok(plan) => {
ctx.registry.record(&resolved);
match crate::boot_order::start_in_stages(
&plan,
&resolved,
&ctx.supervisor,
&ctx.events,
BatchPolicy::AllOrNothing,
)
.await
{
Ok(infos) => reply(Ok(Response::Started(infos))),
Err(err) => reply(Err(rpc_error(&err))),
}
}
},
},
Request::Add { apps } => match normalize_all(apps) {
Err(err) => reply(Err(RpcError {
code: RpcErrorCode::InvalidConfig,
message: err.to_string(),
daemon_version: None,
})),
Ok(resolved) => match staged_plan(ctx, &resolved) {
Err(err) => reply(Err(err)),
Ok(_) => {
ctx.registry.record(&resolved);
match ctx.supervisor.register_at_rest(resolved).await {
Ok(infos) => reply(Ok(Response::Added(infos))),
Err(err) => reply(Err(rpc_error(&err))),
}
}
},
},
Request::ConfigDrift { apps } => match normalize_all(apps) {
Err(err) => reply(Err(RpcError {
code: RpcErrorCode::InvalidConfig,
message: err.to_string(),
daemon_version: None,
})),
Ok(resolved) => match ctx.supervisor.config_drift(resolved).await {
Ok(drifted) => reply(Ok(Response::Drifted(drifted))),
Err(err) => reply(Err(rpc_error(&err))),
},
},
Request::Stop { selector } => {
selector_call(id, selector, |s| ctx.supervisor.stop(s), Response::Stopped).await
}
Request::Restart { selector } => restart_request(id, selector, ctx).await,
Request::Reload { selector } => reload_request(id, selector, ctx).await,
Request::Reopen { selector } => {
selector_call(
id,
selector,
|s| ctx.supervisor.reopen(s),
Response::Reopened,
)
.await
}
Request::Flush { selector } => {
selector_call(id, selector, |s| ctx.supervisor.flush(s), Response::Flushed).await
}
Request::Trigger {
selector,
action,
params,
} => trigger(id, selector, action, params, ctx).await,
Request::Signal { selector, signal } => signal_request(id, selector, signal, ctx).await,
Request::SendLine { selector, line } => {
if line.contains(['\n', '\r']) {
return reply(Err(RpcError {
code: RpcErrorCode::InvalidConfig,
message: "a line may not contain a newline or a carriage return; \
send one line per request"
.to_string(),
daemon_version: None,
}));
}
match selector_of(selector) {
Ok(selector) => match ctx.supervisor.send_line(selector, line).await {
Ok(rows) => reply(Ok(Response::SentLine(rows))),
Err(err) => reply(Err(rpc_error(&err))),
},
Err(err) => reply(Err(err)),
}
}
Request::Delete { selector } => match selector_of(selector) {
Err(err) => reply(Err(err)),
Ok(selector) => match ctx.supervisor.delete(selector).await {
Ok(ids) => reply(Ok(Response::Deleted(ids))),
Err(err) => reply(Err(rpc_error(&err))),
},
},
Request::Scale { name, count } => match ctx.supervisor.scale(&name, count).await {
Ok(scaled) => {
let achieved = scaled.achieved();
let requested = scaled.requested;
ctx.registry.record(&[scaled.app]);
match scaled.shortfall {
None => reply(Ok(Response::Scaled(scaled.instances))),
Some(message) => reply(Err(RpcError {
code: RpcErrorCode::SpawnFailed,
message: format!(
"scaled {name} to {achieved} of {requested} requested; \
the next instance would not spawn: {message}"
),
daemon_version: None,
})),
}
}
Err(err) => reply(Err(rpc_error(&err))),
},
Request::SetSmit { sheep, smit } => {
match ctx.supervisor.set_smit(conn, &sheep, smit).await {
Ok(infos) => reply(Ok(Response::SmitPainted(infos))),
Err(err) => reply(Err(rpc_error(&err))),
}
}
Request::SaveRoll => match ctx.save_roll_now().await {
Ok(Some(saved)) => reply(Ok(Response::RollSaved {
path: saved.path.to_string_lossy().into_owned(),
apps: saved.apps,
})),
Ok(None) => reply(Err(RpcError {
code: RpcErrorCode::Internal,
message: "the supervisor engine has stopped; no roll was written".to_string(),
daemon_version: None,
})),
Err(err) => reply(Err(RpcError {
code: RpcErrorCode::Internal,
message: err.to_string(),
daemon_version: None,
})),
},
Request::Muster => {
match crate::snapshot::muster(
&ctx.snapshot_path,
&ctx.registry,
&ctx.supervisor,
&ctx.events,
&ctx.dog_names,
&ctx.boot_first_dogs,
)
.await
{
Err(err) => reply(Err(RpcError {
code: RpcErrorCode::Internal,
message: err.to_string(),
daemon_version: None,
})),
Ok(names) => match ctx.supervisor.list_checked().await {
Err(err) => reply(Err(rpc_error(&err))),
Ok(infos) => reply(Ok(Response::Mustered(
infos
.into_iter()
.filter(|info| names.contains(&info.name))
.collect(),
))),
},
}
}
Request::DogConfig { name } => match crate::dogs::dog_section(&ctx.dogs_config, &name) {
Ok(toml) => reply(Ok(Response::DogSection { toml: toml.into() })),
Err(err) => reply(Err(RpcError {
code: RpcErrorCode::InvalidConfig,
message: err.to_string(),
daemon_version: None,
})),
},
Request::EnableDog { name, source } => {
let spec = DogSpec { name, source };
match crate::dogs::dog_app(&spec, &ctx.paths) {
Err(err) => reply(Err(RpcError {
code: RpcErrorCode::InvalidConfig,
message: err.to_string(),
daemon_version: None,
})),
Ok(app) => {
let script = app.config().script.clone();
match ctx.supervisor.start_dog(app, spec.source).await {
Ok(info) if info.dog.is_none() => reply(Err(RpcError {
code: RpcErrorCode::InvalidConfig,
message: format!(
"a sheep is already registered as `{}`; rename it or give the dog another name",
spec.name
),
daemon_version: None,
})),
Ok(info) => {
ctx.known_dogs.insert(&info.name);
crate::dogs::narrate(
&ctx.events,
&info,
&format!(
"shep has this dog enabled, running the binary at {script}"
),
)
.await;
reply(Ok(Response::DogStarted(info)))
}
Err(err) => reply(Err(rpc_error(&err))),
}
}
}
}
Request::DisableDog { name } => {
match ctx.supervisor.delete(ProcessSelector::Name(name)).await {
Ok(ids) => reply(Ok(Response::Deleted(ids))),
Err(err) => reply(Err(rpc_error(&err))),
}
}
Request::Subscribe { topics } => match TopicFilter::new(&topics) {
Ok(filter) => Outcome::Subscribe {
reply: Reply {
id,
result: Ok(Response::Subscribed),
},
filter,
},
Err(err) => reply(Err(RpcError {
code: RpcErrorCode::InvalidConfig,
message: err.to_string(),
daemon_version: None,
})),
},
Request::DogStaleness => {
let (stale, pending) = dog_staleness(ctx).await;
reply(Ok(Response::DogStaleness { stale, pending }))
}
Request::HandoverFitness => reply(Ok(Response::HandoverFitness {
refusal: handover_refusal(ctx).await,
})),
Request::KillDaemon => Outcome::Shutdown(Reply {
id,
result: Ok(Response::ShuttingDown),
}),
Request::ApplyConfig { apps, reset } => match duplicate_name(&apps) {
Some(name) => reply(Err(RpcError {
code: RpcErrorCode::InvalidConfig,
message: NormalizeError::DuplicateName(name).to_string(),
daemon_version: None,
})),
None => match ctx.supervisor.apply_config(apps, reset).await {
Ok(applied) => {
let recorded: Vec<ResolvedApp> =
applied.iter().filter_map(|a| a.app.clone()).collect();
ctx.registry.record(&recorded);
reply(Ok(Response::Applied(
applied.into_iter().map(SheepApplied::from).collect(),
)))
}
Err(err) => reply(Err(rpc_error(&err))),
},
},
Request::SheepConfig { name } => match ctx.supervisor.sheep_config(name.clone()).await {
Ok(Some(view)) => reply(Ok(Response::SheepConfig(Box::new(view)))),
Ok(None) => reply(Err(RpcError {
code: RpcErrorCode::NotFound,
message: format!("no sheep named {name}"),
daemon_version: None,
})),
Err(err) => reply(Err(rpc_error(&err))),
},
Request::SetSheepEnv { name, key, value } => {
match ctx
.supervisor
.set_sheep_env(name.clone(), key.clone(), value)
.await
{
Ok(Some(app)) => {
ctx.registry.record(&[app]);
reply(Ok(Response::SheepEnvSet { name, key }))
}
Ok(None) => reply(Err(RpcError {
code: RpcErrorCode::NotFound,
message: format!("no sheep named {name}"),
daemon_version: None,
})),
Err(err) => reply(Err(rpc_error(&err))),
}
}
Request::SetSheepField { name, key, value } => {
match ctx
.supervisor
.set_sheep_field(name.clone(), key.clone(), value)
.await
{
Ok(Some(set)) => {
ctx.registry.record(&[set.app]);
reply(Ok(Response::SheepFieldSet {
name,
key,
pending: set.pending,
}))
}
Ok(None) => reply(Err(RpcError {
code: RpcErrorCode::NotFound,
message: format!("no sheep named {name}"),
daemon_version: None,
})),
Err(err) => reply(Err(rpc_error(&err))),
}
}
Request::SetDogConfig { name, toml } => {
let running_dog = || async {
ctx.supervisor
.list_checked()
.await
.unwrap_or_default()
.iter()
.any(|info| info.name == name && info.dog.is_some())
};
if !ctx.known_dogs.contains(&name) && !running_dog().await {
return reply(Err(RpcError {
code: RpcErrorCode::NotFound,
message: format!(
"no dog named {name}; `shep adopt` or `shep enable` makes one known \
to this shepherd"
),
daemon_version: None,
}));
}
match crate::dogs::set_dog_section(&ctx.dogs_config, &name, toml.as_str()) {
Ok(()) => {
crate::bus::publish_dog_config_changed(
&ctx.events,
std::slice::from_ref(&name),
);
reply(Ok(Response::DogConfigSet { name }))
}
Err(err) => reply(Err(RpcError {
code: RpcErrorCode::InvalidConfig,
message: err.to_string(),
daemon_version: None,
})),
}
}
Request::Unrecognized => reply(Err(RpcError {
code: RpcErrorCode::Unsupported,
message: format!(
"this shepherd speaks protocol {PROTOCOL_VERSION} and does not \
implement the request the client sent"
),
daemon_version: None,
})),
_ => reply(Err(RpcError {
code: RpcErrorCode::Internal,
message: "this daemon does not implement that request".to_string(),
daemon_version: None,
})),
}
}
fn staged_plan(ctx: &RpcContext, apps: &[ResolvedApp]) -> Result<BootPlan, RpcError> {
let mut edges = ctx.registry.depends_on_by_name();
for dog in &ctx.dog_names {
edges.entry(dog.clone()).or_default();
}
for app in apps {
edges.insert(app.config().name.clone(), app.config().depends_on.clone());
}
let plan = crate::boot_order::plan_for_names(&edges);
let batch: BTreeSet<&str> = apps.iter().map(|app| app.config().name.as_str()).collect();
let cycle = plan
.knots
.iter()
.position(|knot| knot.iter().any(|name| batch.contains(name.as_str())))
.and_then(|knot| plan.cycles.get(knot));
if let Some(cycle) = cycle {
return Err(RpcError {
code: RpcErrorCode::InvalidConfig,
message: format!(
"dependency cycle: {}",
shep_core::config::graph::render_cycle(cycle)
),
daemon_version: None,
});
}
for unresolved in &plan.unresolved {
if batch.contains(unresolved.dependent.as_str()) {
tracing::warn!(
sheep = %unresolved.dependent,
missing = %unresolved.missing,
"a dependency names nothing this flock has; starting without it"
);
}
}
Ok(BootPlan {
stages: plan
.stages
.iter()
.map(|stage| {
stage
.iter()
.filter(|name| batch.contains(name.as_str()))
.cloned()
.collect::<Vec<String>>()
})
.filter(|stage| !stage.is_empty())
.collect(),
unresolved: plan.unresolved,
cycles: Vec::new(),
knots: Vec::new(),
})
}
fn duplicate_name(apps: &[DeclaredApp]) -> Option<String> {
let mut seen = BTreeSet::new();
apps.iter()
.find(|app| !seen.insert(app.config.name.as_str()))
.map(|app| app.config.name.clone())
}
impl From<Applied> for SheepApplied {
fn from(applied: Applied) -> Self {
Self::new(
applied.name,
applied.applied,
applied.pending,
applied.refused,
)
}
}
async fn dog_staleness(ctx: &RpcContext) -> (Vec<String>, Vec<String>) {
let stale = ctx.dog_refusals.stale();
let mut pending = ctx.dog_refusals.restarting();
if let Ok(infos) = ctx.supervisor.list_checked().await {
pending.extend(crate::dogs::silent_dogs(&infos, &ctx.dog_refusals));
}
pending.sort();
pending.dedup();
(stale, pending)
}
#[cfg(unix)]
async fn handover_refusal(ctx: &RpcContext) -> Option<String> {
match ctx.supervisor.handover_fitness().await {
Ok(crate::handover::Fitness::Carryable) => None,
Ok(crate::handover::Fitness::Refused(reason)) => Some(reason.to_string()),
Err(err) => Some(format!(
"this shepherd could not check whether its flock can be handed over ({err})"
)),
}
}
#[cfg(windows)]
#[expect(
clippy::unused_async,
reason = "one signature for both platforms; the unix arm awaits the supervisor"
)]
async fn handover_refusal(_ctx: &RpcContext) -> Option<String> {
Some(
"this shepherd runs on Windows, which has no `execve`, so its flock cannot be handed to \
a successor in place"
.to_string(),
)
}
async fn with_live_stats(stats: &Arc<StatsState>, mut infos: Vec<ProcessInfo>) -> Vec<ProcessInfo> {
let stats = Arc::clone(stats);
let Ok(sample) = tokio::task::spawn_blocking(move || stats.sample_now()).await else {
return infos;
};
for info in &mut infos {
if let Some(reading) = info.pid.and_then(|pid| sample.get(&pid)) {
info.cpu_percent = reading.cpu_percent;
info.memory_bytes = Some(reading.memory_bytes);
}
}
infos
}
fn with_dog_contact(
refusals: &crate::dogs::DogRefusals,
mut infos: Vec<ProcessInfo>,
) -> Vec<ProcessInfo> {
let stale = refusals.stale();
for info in &mut infos {
if info.dog.is_some() {
info.handshook = Some(refusals.has_handshook(&info.name));
info.dog_stale = Some(stale.contains(&info.name));
}
}
infos
}
async fn with_lambs(stats: &Arc<StatsState>, mut infos: Vec<ProcessInfo>) -> Vec<ProcessInfo> {
if infos.iter().all(|info| info.pid.is_none()) {
return infos;
}
let stats = Arc::clone(stats);
let pids: Vec<u32> = infos.iter().filter_map(|info| info.pid).collect();
let Ok(walked) = tokio::task::spawn_blocking(move || {
let index = stats.lamb_index();
pids.into_iter()
.map(|pid| (pid, stats.lambs_of(&index, pid)))
.collect::<HashMap<u32, Vec<Lamb>>>()
})
.await
else {
return infos;
};
for info in &mut infos {
if let Some(lambs) = info.pid.and_then(|pid| walked.get(&pid)) {
info.lambs = Some(lambs.clone());
}
}
infos
}
fn rpc_error(err: &SupervisorError) -> RpcError {
match err {
SupervisorError::NotFound => not_found(),
SupervisorError::SpawnFailed(msg) => RpcError {
code: RpcErrorCode::SpawnFailed,
message: msg.clone(),
daemon_version: None,
},
SupervisorError::CannotStart(msg) => RpcError {
code: RpcErrorCode::SpawnFailed,
message: msg.clone(),
daemon_version: None,
},
SupervisorError::ReopenFailed(_) | SupervisorError::FlushFailed(_) => RpcError {
code: RpcErrorCode::Internal,
message: err.to_string(),
daemon_version: None,
},
SupervisorError::ReloadInFlight(_) => RpcError {
code: RpcErrorCode::Internal,
message: err.to_string(),
daemon_version: None,
},
SupervisorError::InvalidScale(msg) => RpcError {
code: RpcErrorCode::InvalidConfig,
message: msg.clone(),
daemon_version: None,
},
SupervisorError::IsADog(msg) => RpcError {
code: RpcErrorCode::InvalidConfig,
message: msg.clone(),
daemon_version: None,
},
SupervisorError::InvalidEnv(msg) => RpcError {
code: RpcErrorCode::InvalidConfig,
message: msg.clone(),
daemon_version: None,
},
SupervisorError::InvalidField(msg) => RpcError {
code: RpcErrorCode::InvalidConfig,
message: msg.clone(),
daemon_version: None,
},
SupervisorError::Overrides(_) => RpcError {
code: RpcErrorCode::Internal,
message: err.to_string(),
daemon_version: None,
},
SupervisorError::EngineStopped => RpcError {
code: RpcErrorCode::Internal,
message: "the supervisor engine has stopped".to_string(),
daemon_version: None,
},
}
}
struct OrderedWalk {
stages: Vec<Vec<String>>,
depended_on: BTreeSet<String>,
}
fn ordered_walk(
ctx: &RpcContext,
selector: &ProcessSelector,
flock: &[ProcessInfo],
) -> Option<OrderedWalk> {
let exact = selector.is_exact();
let matched: BTreeSet<String> = flock
.iter()
.filter(|info| exact || info.dog.is_none())
.filter(|info| selector.matches(&info.name, info.id, info.fold.as_deref(), info.instance))
.map(|info| info.name.clone())
.collect();
if matched.len() < 2 {
return None;
}
let edges = ctx.registry.depends_on_by_name();
let plan = crate::boot_order::plan_for_names(&edges);
let mut stages: Vec<Vec<String>> = plan
.stages
.iter()
.map(|stage| {
stage
.iter()
.filter(|name| matched.contains(name.as_str()))
.cloned()
.collect::<Vec<String>>()
})
.filter(|stage| !stage.is_empty())
.collect();
let placed: BTreeSet<&str> = stages.iter().flatten().map(String::as_str).collect();
let unplaced: Vec<String> = matched
.iter()
.filter(|name| !placed.contains(name.as_str()))
.cloned()
.collect();
if !unplaced.is_empty() {
stages.push(unplaced);
}
let depended_on = matched
.iter()
.flat_map(|name| matched_dependencies(&edges, name, &matched))
.collect();
Some(OrderedWalk {
stages,
depended_on,
})
}
fn matched_dependencies(
edges: &BTreeMap<String, Vec<String>>,
start: &str,
matched: &BTreeSet<String>,
) -> BTreeSet<String> {
let mut found = BTreeSet::new();
let mut seen: BTreeSet<&str> = [start].into_iter().collect();
let mut frontier: Vec<&str> = vec![start];
while let Some(name) = frontier.pop() {
for dependency in edges.get(name).into_iter().flatten() {
if !seen.insert(dependency.as_str()) {
continue;
}
if matched.contains(dependency.as_str()) {
found.insert(dependency.clone());
}
frontier.push(dependency.as_str());
}
}
found
}
fn settle_bound(ctx: &RpcContext, name: &str, reloading: bool) -> Duration {
let (readiness, drain) = ctx.registry.timeouts_of(name).unwrap_or_default();
let drain = if reloading { drain } else { Duration::ZERO };
readiness + drain
}
fn stage_bound(ctx: &RpcContext, stage: &[String]) -> Duration {
stage
.iter()
.map(|name| settle_bound(ctx, name, false))
.max()
.unwrap_or_default()
+ crate::boot_order::STAGE_SLACK
}
fn reload_stage_bound(ctx: &RpcContext, waiting: &BTreeMap<String, usize>) -> Duration {
waiting
.iter()
.map(|(name, swaps)| {
settle_bound(ctx, name, true).saturating_mul(u32::try_from(*swaps).unwrap_or(u32::MAX))
})
.max()
.unwrap_or_default()
+ crate::boot_order::STAGE_SLACK
}
async fn restart_request(id: u64, spec: SelectorSpec, ctx: &RpcContext) -> Outcome {
let result = match selector_of(spec) {
Err(err) => Err(err),
Ok(selector) => match walk_for(ctx, &selector).await {
None => ctx
.supervisor
.restart(selector)
.await
.map(|accepted| Response::Restarted {
accepted,
refused: Vec::new(),
})
.map_err(|err| rpc_error(&err)),
Some(walk) => restart_in_stages(ctx, &walk)
.await
.map(|(accepted, refused)| Response::Restarted { accepted, refused }),
},
};
Outcome::Reply(Reply { id, result })
}
async fn reload_request(id: u64, spec: SelectorSpec, ctx: &RpcContext) -> Outcome {
let result = match selector_of(spec) {
Err(err) => Err(err),
Ok(selector) => match walk_for(ctx, &selector).await {
None => ctx
.supervisor
.reload(selector)
.await
.map(|accepted| Response::Reloading {
accepted,
refused: Vec::new(),
})
.map_err(|err| rpc_error(&err)),
Some(walk) => reload_in_stages(ctx, &walk)
.await
.map(|(accepted, refused)| Response::Reloading { accepted, refused }),
},
};
Outcome::Reply(Reply { id, result })
}
async fn walk_for(ctx: &RpcContext, selector: &ProcessSelector) -> Option<OrderedWalk> {
let flock = ctx.supervisor.list_checked().await.ok()?;
ordered_walk(ctx, selector, &flock)
}
async fn restart_in_stages(
ctx: &RpcContext,
walk: &OrderedWalk,
) -> Result<(Vec<ProcessInfo>, Vec<SheepRefusal>), RpcError> {
let mut restarted = Vec::new();
let mut refused: Vec<SheepRefusal> = Vec::new();
let mut refusal = None;
for stage in &walk.stages {
let rx = ctx.events.subscribe();
let outcomes = futures_util::future::join_all(stage.iter().map(|name| async move {
(
name,
ctx.supervisor
.restart(ProcessSelector::Name(name.clone()))
.await,
)
}))
.await;
for (name, outcome) in outcomes {
match outcome {
Ok(infos) => restarted.extend(infos),
Err(err) => {
tracing::warn!(
sheep = %name,
%err,
"a sheep did not restart in its stage; it may have left the flock since \
the walk was planned"
);
refused.push(SheepRefusal::new(name.clone(), err.to_string()));
refusal.get_or_insert(err);
}
}
}
let waiting: BTreeSet<String> = stage
.iter()
.filter(|name| walk.depended_on.contains(name.as_str()))
.cloned()
.collect();
if waiting.is_empty() {
continue;
}
let bound = stage_bound(ctx, stage);
let unsettled = crate::boot_order::await_stage(rx, waiting, bound, &ctx.supervisor).await;
warn_about_unsettled(&unsettled);
}
finished(restarted, refusal).map(|rows| (rows, refused))
}
async fn reload_in_stages(
ctx: &RpcContext,
walk: &OrderedWalk,
) -> Result<(Vec<ProcessInfo>, Vec<SheepRefusal>), RpcError> {
let mut accepted = Vec::new();
let mut refused: Vec<SheepRefusal> = Vec::new();
let mut refusal = None;
for stage in &walk.stages {
let rx = ctx.events.subscribe();
let outcomes = futures_util::future::join_all(stage.iter().map(|name| async move {
(
name,
ctx.supervisor
.reload(ProcessSelector::Name(name.clone()))
.await,
)
}))
.await;
let mut waiting: BTreeMap<String, usize> = BTreeMap::new();
for (name, outcome) in outcomes {
match outcome {
Ok(infos) => {
if walk.depended_on.contains(name.as_str()) {
let swaps = infos
.iter()
.filter(|info| info.status == ProcStatus::Online)
.count();
if swaps > 0 {
waiting.insert(name.clone(), swaps);
}
}
accepted.extend(infos);
}
Err(err) => {
if matches!(err, SupervisorError::ReloadInFlight(_))
&& walk.depended_on.contains(name.as_str())
{
waiting.insert(name.clone(), 1);
}
tracing::warn!(
sheep = %name,
%err,
"a sheep did not reload in its stage; it may have left the flock since \
the walk was planned"
);
refused.push(SheepRefusal::new(name.clone(), err.to_string()));
refusal.get_or_insert(err);
}
}
}
if waiting.is_empty() {
continue;
}
let bound = reload_stage_bound(ctx, &waiting);
let unsettled = crate::boot_order::await_reloads(rx, waiting, bound).await;
warn_about_unsettled(&unsettled);
}
finished(accepted, refusal).map(|rows| (rows, refused))
}
fn warn_about_unsettled(unsettled: &BTreeSet<String>) {
if unsettled.is_empty() {
return;
}
let names: Vec<&str> = unsettled.iter().map(String::as_str).collect();
tracing::warn!(
unsettled = ?names,
"a stage did not settle inside its bound; advancing anyway"
);
}
fn finished(
mut rows: Vec<ProcessInfo>,
refusal: Option<SupervisorError>,
) -> Result<Vec<ProcessInfo>, RpcError> {
match refusal {
Some(err) if rows.is_empty() => Err(rpc_error(&err)),
_ => {
shep_core::protocol::sort_flock(&mut rows);
Ok(rows)
}
}
}
fn not_found() -> RpcError {
RpcError {
code: RpcErrorCode::NotFound,
message: "selector matched no registered sheep".to_string(),
daemon_version: None,
}
}
fn selector_of(spec: SelectorSpec) -> Result<ProcessSelector, RpcError> {
ProcessSelector::try_from(spec).map_err(|err| RpcError {
code: RpcErrorCode::InvalidConfig,
message: err.to_string(),
daemon_version: None,
})
}
async fn selector_call<F, Fut>(
id: u64,
spec: SelectorSpec,
call: F,
ok: fn(Vec<ProcessInfo>) -> Response,
) -> Outcome
where
F: FnOnce(ProcessSelector) -> Fut + Send,
Fut: Future<Output = Result<Vec<ProcessInfo>, SupervisorError>> + Send,
{
let result = match selector_of(spec) {
Ok(selector) => call(selector).await.map(ok).map_err(|err| rpc_error(&err)),
Err(err) => Err(err),
};
Outcome::Reply(Reply { id, result })
}
async fn trigger(
id: u64,
spec: SelectorSpec,
action: String,
params: Option<String>,
ctx: &RpcContext,
) -> Outcome {
let result = match selector_of(spec) {
Err(err) => Err(err),
Ok(selector) => ctx
.supervisor
.trigger(selector, action, params)
.await
.map(Response::Triggered)
.map_err(|err| rpc_error(&err)),
};
Outcome::Reply(Reply { id, result })
}
async fn signal_request(id: u64, spec: SelectorSpec, signal: String, ctx: &RpcContext) -> Outcome {
let result = match OperatorSignal::parse(&signal) {
None => Err(RpcError {
code: RpcErrorCode::InvalidConfig,
message: format!(
"`{signal}` is not a signal shep will send; accepted: {}",
OperatorSignal::ACCEPTED.join(", ")
),
daemon_version: None,
}),
Some(sig) => match selector_of(spec) {
Err(err) => Err(err),
Ok(selector) => ctx
.supervisor
.signal(selector, sig)
.await
.map(Response::Signalled)
.map_err(|err| rpc_error(&err)),
},
};
Outcome::Reply(Reply { id, result })
}
#[cfg(test)]
mod tests {
use super::*;
use crate::fake::{FIRST_SCRIPTED_PID, ProcScript};
use crate::limits::MEMORY_POLL_INTERVAL;
use crate::testing::{
Harness, SCRIPTED_TREE_BYTES, harness, harness_identifying, harness_with_stats, identity,
};
use shep_core::config::{AppConfig, ApplyGroup, DeclaredApp, ResetDepth, apply_group};
use shep_core::protocol::{
ActionOutcome, ActionReply, DogSource, ProcessEventKind, Request, Response, RpcErrorCode,
SelectorSpec,
};
use shep_core::values::UpDuration;
use std::collections::BTreeSet;
use tokio::time::Instant;
async fn dispatch(envelope: Envelope, ctx: &RpcContext) -> Outcome {
super::dispatch(envelope, ConnId::next(), ctx).await
}
fn envelope(id: u64, body: Request) -> Envelope {
Envelope {
id,
deadline_ms: None,
body,
}
}
fn reply_of(outcome: Outcome) -> Reply {
match outcome {
Outcome::Reply(reply) | Outcome::Subscribe { reply, .. } | Outcome::Shutdown(reply) => {
reply
}
}
}
#[tokio::test(start_paused = true)]
async fn ping_answers_pong_on_the_same_envelope_id() {
let h = harness(vec![]);
let reply = reply_of(dispatch(envelope(9, Request::Ping), &h.ctx).await);
assert_eq!(reply.id, 9);
assert_eq!(reply.result.unwrap(), Response::Pong);
}
#[tokio::test(start_paused = true)]
async fn start_registers_the_config_and_lists_it() {
let h = harness(vec![ProcScript::never_exits()]);
let started = reply_of(
dispatch(
envelope(
1,
Request::Start {
apps: vec![AppConfig::minimal("web", "./srv")],
},
),
&h.ctx,
)
.await,
);
let Response::Started(infos) = started.result.unwrap() else {
panic!("expected started")
};
assert_eq!(infos.len(), 1);
assert_eq!(infos[0].status, ProcStatus::Online);
let roll = h.ctx.registry.roll(&infos, 0);
assert_eq!(roll.apps.len(), 1);
assert_eq!(roll.apps[0].app.script, "./srv");
let listed = reply_of(dispatch(envelope(2, Request::ListFlock), &h.ctx).await);
let Response::Flock(flock) = listed.result.unwrap() else {
panic!("expected flock")
};
assert_eq!(flock.len(), 1);
}
#[tokio::test(start_paused = true)]
async fn start_re_normalizes_untrusted_peer_config() {
let h = harness(vec![]);
let reply = reply_of(
dispatch(
envelope(
1,
Request::Start {
apps: vec![AppConfig::minimal("", "./srv")],
},
),
&h.ctx,
)
.await,
);
assert_eq!(reply.result.unwrap_err().code, RpcErrorCode::InvalidConfig);
}
#[tokio::test(start_paused = true)]
async fn add_registers_a_stopped_member_and_spawns_nothing() {
let h = harness(vec![]);
let added = reply_of(
dispatch(
envelope(
1,
Request::Add {
apps: vec![AppConfig::minimal("web", "./srv")],
},
),
&h.ctx,
)
.await,
);
let Response::Added(infos) = added.result.unwrap() else {
panic!("expected added")
};
assert_eq!(infos.len(), 1);
assert_eq!(infos[0].status, ProcStatus::Stopped);
assert_eq!(infos[0].pid, None, "nothing was spawned");
let roll = h.ctx.registry.roll(&infos, 0);
assert_eq!(roll.apps.len(), 1);
assert_eq!(roll.apps[0].app.script, "./srv");
assert_eq!(roll.apps[0].instances_running, 0);
let listed = reply_of(dispatch(envelope(2, Request::ListFlock), &h.ctx).await);
let Response::Flock(flock) = listed.result.unwrap() else {
panic!("expected flock")
};
assert_eq!(
flock.len(),
1,
"it is a member of the flock, just a still one"
);
}
#[tokio::test(start_paused = true)]
async fn add_re_normalizes_untrusted_peer_config() {
let h = harness(vec![]);
let reply = reply_of(
dispatch(
envelope(
1,
Request::Add {
apps: vec![AppConfig::minimal("", "./srv")],
},
),
&h.ctx,
)
.await,
);
assert_eq!(reply.result.unwrap_err().code, RpcErrorCode::InvalidConfig);
}
#[tokio::test(start_paused = true)]
async fn a_second_add_leaves_a_running_sheep_exactly_as_it_was() {
let h = harness(vec![ProcScript::never_exits()]);
let started = reply_of(
dispatch(
envelope(
1,
Request::Start {
apps: vec![AppConfig::minimal("web", "./srv")],
},
),
&h.ctx,
)
.await,
);
let Response::Started(before) = started.result.unwrap() else {
panic!("expected started")
};
let added = reply_of(
dispatch(
envelope(
2,
Request::Add {
apps: vec![AppConfig::minimal("web", "./srv")],
},
),
&h.ctx,
)
.await,
);
let Response::Added(after) = added.result.unwrap() else {
panic!("expected added")
};
assert_eq!(after.len(), 1);
assert_eq!(
after[0].id, before[0].id,
"the same sheep, not a second one"
);
assert_eq!(after[0].status, ProcStatus::Online, "still running");
assert_eq!(after[0].pid, before[0].pid, "the same process");
let listed = reply_of(dispatch(envelope(3, Request::ListFlock), &h.ctx).await);
let Response::Flock(flock) = listed.result.unwrap() else {
panic!("expected flock")
};
assert_eq!(flock.len(), 1, "one row, not two");
}
fn waiting_on(name: &str, depends_on: &[&str]) -> AppConfig {
let mut app = AppConfig::minimal(name, "./srv");
app.depends_on = depends_on.iter().map(|n| (*n).to_string()).collect();
app
}
#[tokio::test(start_paused = true)]
async fn a_start_runs_its_batch_in_dependency_order() {
let h = harness(vec![ProcScript::never_exits(), ProcScript::never_exits()]);
let started = reply_of(
dispatch(
envelope(
1,
Request::Start {
apps: vec![waiting_on("api", &["db"]), waiting_on("db", &[])],
},
),
&h.ctx,
)
.await,
);
let Response::Started(infos) = started.result.unwrap() else {
panic!("expected started")
};
let order: Vec<&str> = infos.iter().map(|info| info.name.as_str()).collect();
assert_eq!(order, ["db", "api"], "stage order, not request order");
}
#[tokio::test(start_paused = true)]
async fn a_cycle_closing_through_a_registered_sheep_is_refused_too() {
let h = harness(vec![ProcScript::never_exits(), ProcScript::never_exits()]);
let first = reply_of(
dispatch(
envelope(
1,
Request::Start {
apps: vec![waiting_on("db", &["api"])],
},
),
&h.ctx,
)
.await,
);
assert!(
first.result.is_ok(),
"one app, one edge to a name nobody has"
);
let reply = reply_of(
dispatch(
envelope(
2,
Request::Start {
apps: vec![waiting_on("api", &["db"])],
},
),
&h.ctx,
)
.await,
);
let err = reply.result.unwrap_err();
assert_eq!(err.code, RpcErrorCode::InvalidConfig);
assert!(
err.message.contains("api") && err.message.contains("db"),
"both ends of the cycle must be named: {}",
err.message
);
}
#[tokio::test(start_paused = true)]
async fn a_batch_in_a_knot_the_named_path_leaves_out_is_still_refused() {
let h = harness(vec![ProcScript::never_exits(); 3]);
let _started = reply_of(
dispatch(
envelope(
1,
Request::Start {
apps: ["api", "db", "web"]
.iter()
.map(|name| AppConfig::minimal(name, "./srv"))
.collect(),
},
),
&h.ctx,
)
.await,
);
for (name, waits_for) in [
("api", vec!["db", "web"]),
("db", vec!["api"]),
("web", vec!["api"]),
] {
let mut config = AppConfig::minimal(name, "./srv");
config.depends_on = waits_for.iter().map(|n| (*n).to_string()).collect();
let reply = reply_of(
dispatch(
envelope(
2,
Request::ApplyConfig {
apps: vec![DeclaredApp {
config,
declared: ["depends_on"].iter().map(|k| (*k).to_string()).collect(),
declared_env: BTreeSet::new(),
}],
reset: ResetDepth::None,
},
),
&h.ctx,
)
.await,
);
assert!(reply.result.is_ok(), "the load itself draws no cycle");
}
let mut web = AppConfig::minimal("web", "./srv");
web.depends_on = vec!["api".to_string()];
let reply =
reply_of(dispatch(envelope(3, Request::Start { apps: vec![web] }), &h.ctx).await);
let err = reply.result.unwrap_err();
assert_eq!(err.code, RpcErrorCode::InvalidConfig);
assert!(
err.message.starts_with("dependency cycle:"),
"web is in the knot even though the reported path omits it: {}",
err.message
);
assert!(
!err.message.contains("web"),
"the message still renders the representative path, which is what \
an operator breaks: {}",
err.message
);
}
#[tokio::test(start_paused = true)]
async fn a_knot_no_app_in_the_batch_is_in_does_not_refuse_the_batch() {
let h = harness(vec![
ProcScript::never_exits(),
ProcScript::never_exits(),
ProcScript::never_exits(),
]);
let _started = reply_of(
dispatch(
envelope(
1,
Request::Start {
apps: vec![
AppConfig::minimal("api", "./srv"),
AppConfig::minimal("db", "./srv"),
],
},
),
&h.ctx,
)
.await,
);
for (name, waits_for) in [("api", "db"), ("db", "api")] {
let mut config = AppConfig::minimal(name, "./srv");
config.depends_on = vec![waits_for.to_string()];
let reply = reply_of(
dispatch(
envelope(
2,
Request::ApplyConfig {
apps: vec![DeclaredApp {
config,
declared: ["depends_on"].iter().map(|k| (*k).to_string()).collect(),
declared_env: BTreeSet::new(),
}],
reset: ResetDepth::None,
},
),
&h.ctx,
)
.await,
);
assert!(reply.result.is_ok(), "the load itself draws no cycle");
}
let edges = h.ctx.registry.depends_on_by_name();
assert_eq!(edges.get("api"), Some(&vec!["db".to_string()]));
assert_eq!(
edges.get("db"),
Some(&vec!["api".to_string()]),
"the knot is really in the registry, or this test proves nothing"
);
let reply = reply_of(
dispatch(
envelope(
3,
Request::Start {
apps: vec![AppConfig::minimal("web", "./srv")],
},
),
&h.ctx,
)
.await,
);
assert!(
reply.result.is_ok(),
"a batch drawing no cycle must start: {:?}",
reply.result.unwrap_err()
);
}
#[tokio::test(start_paused = true)]
async fn an_add_whose_cycle_closes_through_the_flock_is_refused_too() {
let h = harness(vec![]);
let first = reply_of(
dispatch(
envelope(
1,
Request::Add {
apps: vec![waiting_on("db", &["api"])],
},
),
&h.ctx,
)
.await,
);
assert!(
first.result.is_ok(),
"one app, one edge to a name nobody has"
);
let reply = reply_of(
dispatch(
envelope(
2,
Request::Add {
apps: vec![waiting_on("api", &["db"])],
},
),
&h.ctx,
)
.await,
);
let err = reply.result.unwrap_err();
assert_eq!(err.code, RpcErrorCode::InvalidConfig);
assert!(
err.message.contains(" -> "),
"the cycle must be named as a path: {}",
err.message
);
}
#[tokio::test(start_paused = true)]
async fn a_stage_never_covers_a_sheep_the_request_did_not_carry() {
let h = harness(vec![ProcScript::never_exits(), ProcScript::never_exits()]);
let first = reply_of(
dispatch(
envelope(
1,
Request::Start {
apps: vec![waiting_on("db", &[])],
},
),
&h.ctx,
)
.await,
);
assert!(first.result.is_ok(), "db starts on its own");
let second = reply_of(
dispatch(
envelope(
2,
Request::Start {
apps: vec![waiting_on("api", &["db"])],
},
),
&h.ctx,
)
.await,
);
let Response::Started(infos) = second.result.unwrap() else {
panic!("expected started")
};
let order: Vec<&str> = infos.iter().map(|info| info.name.as_str()).collect();
assert_eq!(order, ["api"], "only what this request carried");
let listed = reply_of(dispatch(envelope(3, Request::ListFlock), &h.ctx).await);
let Response::Flock(flock) = listed.result.unwrap() else {
panic!("expected flock")
};
assert_eq!(flock.len(), 2, "one row each, not a second db");
}
fn declared(name: &str, script: &str, keys: &[&str]) -> DeclaredApp {
DeclaredApp {
config: AppConfig::minimal(name, script),
declared: keys.iter().map(|k| (*k).to_string()).collect(),
declared_env: BTreeSet::new(),
}
}
#[tokio::test(start_paused = true)]
async fn apply_config_refuses_a_request_naming_one_app_twice() {
let h = harness(vec![ProcScript::never_exits()]);
let _started = reply_of(
dispatch(
envelope(
1,
Request::Start {
apps: vec![AppConfig::minimal("web", "./srv")],
},
),
&h.ctx,
)
.await,
);
let reply = reply_of(
dispatch(
envelope(
2,
Request::ApplyConfig {
apps: vec![
declared("web", "./one", &["script"]),
declared("web", "./two", &["script"]),
],
reset: ResetDepth::None,
},
),
&h.ctx,
)
.await,
);
let err = reply.result.unwrap_err();
assert_eq!(err.code, RpcErrorCode::InvalidConfig);
assert!(err.message.contains("web"), "the name is named: {err:?}");
let listed = reply_of(dispatch(envelope(3, Request::ListFlock), &h.ctx).await);
let Response::Flock(flock) = listed.result.unwrap() else {
panic!("expected flock")
};
let roll = h.ctx.registry.roll(&flock, 0);
assert_eq!(roll.apps[0].app.script, "./srv");
}
#[tokio::test(start_paused = true)]
async fn apply_config_records_what_it_applied_in_the_registry() {
let h = harness(vec![ProcScript::never_exits()]);
let _started = reply_of(
dispatch(
envelope(
1,
Request::Start {
apps: vec![AppConfig::minimal("web", "./srv")],
},
),
&h.ctx,
)
.await,
);
let reply = reply_of(
dispatch(
envelope(
2,
Request::ApplyConfig {
apps: vec![DeclaredApp {
config: {
let mut app = AppConfig::minimal("web", "./srv");
app.max_restarts = 99;
app
},
declared: ["max_restarts"].iter().map(|k| (*k).to_string()).collect(),
declared_env: BTreeSet::new(),
}],
reset: ResetDepth::None,
},
),
&h.ctx,
)
.await,
);
let Response::Applied(report) = reply.result.unwrap() else {
panic!("expected applied")
};
assert_eq!(report.len(), 1);
assert_eq!(report[0].name, "web");
assert_eq!(report[0].applied, vec!["max_restarts".to_string()]);
assert_eq!(report[0].refused, None);
let listed = reply_of(dispatch(envelope(3, Request::ListFlock), &h.ctx).await);
let Response::Flock(flock) = listed.result.unwrap() else {
panic!("expected flock")
};
let roll = h.ctx.registry.roll(&flock, 0);
assert_eq!(roll.apps[0].app.max_restarts, 99);
}
#[tokio::test(start_paused = true)]
async fn apply_config_refuses_an_unregistered_app_inside_the_reply() {
let h = harness(vec![]);
let reply = reply_of(
dispatch(
envelope(
1,
Request::ApplyConfig {
apps: vec![declared("ghost", "./srv", &["script"])],
reset: ResetDepth::None,
},
),
&h.ctx,
)
.await,
);
let Response::Applied(report) = reply.result.unwrap() else {
panic!("expected applied")
};
assert_eq!(report.len(), 1);
let refused = report[0].refused.as_deref().unwrap_or_default();
assert!(
refused.contains("ghost"),
"the refusal names the app: {refused}"
);
}
#[tokio::test(start_paused = true)]
async fn a_selector_matching_nothing_is_not_found() {
let h = harness(vec![]);
let reply = reply_of(
dispatch(
envelope(
1,
Request::Stop {
selector: SelectorSpec::Name("ghost".to_string()),
},
),
&h.ctx,
)
.await,
);
assert_eq!(reply.result.unwrap_err().code, RpcErrorCode::NotFound);
}
#[tokio::test(start_paused = true)]
async fn a_bad_peer_regex_is_invalid_config_not_a_panic() {
let h = harness(vec![]);
let reply = reply_of(
dispatch(
envelope(
1,
Request::Describe {
selector: SelectorSpec::Regex("((".to_string()),
},
),
&h.ctx,
)
.await,
);
assert_eq!(reply.result.unwrap_err().code, RpcErrorCode::InvalidConfig);
}
#[tokio::test(start_paused = true)]
async fn describe_filters_by_fold() {
let h = harness(vec![ProcScript::never_exits(), ProcScript::never_exits()]);
let mut api = AppConfig::minimal("api", "./a");
api.fold = Some("backend".to_string());
dispatch(
envelope(
1,
Request::Start {
apps: vec![api, AppConfig::minimal("web", "./w")],
},
),
&h.ctx,
)
.await;
let reply = reply_of(
dispatch(
envelope(
2,
Request::Describe {
selector: SelectorSpec::Fold("backend".to_string()),
},
),
&h.ctx,
)
.await,
);
let Response::Described(hits) = reply.result.unwrap() else {
panic!("expected described")
};
assert_eq!(hits.len(), 1);
assert_eq!(hits[0].name, "api");
}
#[tokio::test(start_paused = true)]
async fn reopen_routes_to_the_supervisor_and_leaves_the_sheep_running() {
let h = harness(vec![ProcScript::never_exits()]);
dispatch(
envelope(
1,
Request::Start {
apps: vec![AppConfig::minimal("web", "./srv")],
},
),
&h.ctx,
)
.await;
let reply = reply_of(
dispatch(
envelope(
2,
Request::Reopen {
selector: SelectorSpec::Name("web".to_string()),
},
),
&h.ctx,
)
.await,
);
let Response::Reopened(infos) = reply.result.unwrap() else {
panic!("expected reopened")
};
assert_eq!(infos.len(), 1);
assert_eq!(infos[0].status, ProcStatus::Online);
let listed = reply_of(dispatch(envelope(3, Request::ListFlock), &h.ctx).await);
let Response::Flock(flock) = listed.result.unwrap() else {
panic!("expected flock")
};
assert_eq!(
flock[0].status,
ProcStatus::Online,
"a reopen must not disturb the sheep it reopens"
);
}
#[tokio::test(start_paused = true)]
async fn reload_routes_to_the_supervisor_and_starts_a_swap() {
let h = harness(vec![ProcScript::never_exits(); 3]);
dispatch(
envelope(
1,
Request::Start {
apps: vec![AppConfig::minimal("web", "./srv")],
},
),
&h.ctx,
)
.await;
let reply = reply_of(
dispatch(
envelope(
2,
Request::Reload {
selector: SelectorSpec::Name("web".to_string()),
},
),
&h.ctx,
)
.await,
);
let Response::Reloading { accepted, .. } = reply.result.unwrap() else {
panic!("expected reloading")
};
assert_eq!(accepted.len(), 1);
assert_eq!(
accepted[0].status,
ProcStatus::Online,
"the answer is the flock as it stood when the reload was accepted"
);
let listed = reply_of(dispatch(envelope(3, Request::ListFlock), &h.ctx).await);
let Response::Flock(flock) = listed.result.unwrap() else {
panic!("expected flock")
};
assert_eq!(
flock.len(),
2,
"a swap in progress is two entries in one instance slot, not one: {flock:?}"
);
assert_eq!(flock[0].id, accepted[0].id);
assert_eq!(flock[0].status, ProcStatus::Stopping);
assert_ne!(
flock[1].id, accepted[0].id,
"the replacement takes a new id"
);
assert_eq!(flock[1].status, ProcStatus::Starting);
}
const DEFAULT_KILL_TIMEOUT: Duration = Duration::from_millis(1600);
async fn start_api_before_the_db_it_waits_for(h: &Harness) {
let mut api = AppConfig::minimal("api", "./api");
api.depends_on = vec!["db".to_string()];
for (id, app) in [(1, api), (2, AppConfig::minimal("db", "./db"))] {
let started =
reply_of(dispatch(envelope(id, Request::Start { apps: vec![app] }), &h.ctx).await);
let Response::Started(infos) = started.result.unwrap() else {
panic!("expected started")
};
assert_eq!(infos.len(), 1, "each app comes up before the next");
}
}
fn names_for(
rx: &mut tokio::sync::broadcast::Receiver<crate::bus::SharedEvent>,
kind: ProcessEventKind,
) -> Vec<String> {
let mut names = Vec::new();
while let Ok(event) = rx.try_recv() {
let shep_core::protocol::BusEvent::Process {
event: seen, info, ..
} = &*event
else {
continue;
};
if *seen == kind {
names.push(info.name.clone());
}
}
names
}
#[tokio::test(start_paused = true)]
async fn a_restart_matching_several_walks_the_stages_forward() {
let h = harness(vec![ProcScript::never_exits(); 4]);
start_api_before_the_db_it_waits_for(&h).await;
let mut rx = h.ctx.events.subscribe();
let reply = reply_of(
dispatch(
envelope(
3,
Request::Restart {
selector: SelectorSpec::All,
},
),
&h.ctx,
)
.await,
);
let Response::Restarted {
accepted: infos, ..
} = reply.result.unwrap()
else {
panic!("expected restarted")
};
assert_eq!(infos.len(), 2, "both sheep restart: {infos:?}");
assert_eq!(names_for(&mut rx, ProcessEventKind::Restart), ["db", "api"]);
}
#[tokio::test(start_paused = true)]
async fn a_reload_matching_several_walks_the_stages_forward() {
let h = harness(vec![ProcScript::never_exits(); 4]);
start_api_before_the_db_it_waits_for(&h).await;
let mut rx = h.ctx.events.subscribe();
let reply = reply_of(
dispatch(
envelope(
3,
Request::Reload {
selector: SelectorSpec::All,
},
),
&h.ctx,
)
.await,
);
let Response::Reloading { accepted, .. } = reply.result.unwrap() else {
panic!("expected reloading")
};
assert_eq!(accepted.len(), 2, "both sheep are accepted: {accepted:?}");
assert_eq!(names_for(&mut rx, ProcessEventKind::Reload), ["db", "api"]);
}
#[tokio::test(start_paused = true)]
async fn a_restart_naming_nothing_the_flock_holds_is_still_not_found() {
let h = harness(vec![ProcScript::never_exits(); 2]);
start_api_before_the_db_it_waits_for(&h).await;
let reply = reply_of(
dispatch(
envelope(
3,
Request::Restart {
selector: SelectorSpec::Name("typo".to_string()),
},
),
&h.ctx,
)
.await,
);
assert_eq!(reply.result.unwrap_err().code, RpcErrorCode::NotFound);
}
#[tokio::test(start_paused = true)]
async fn one_stage_restarts_its_members_at_the_same_time() {
let h = harness(vec![
ProcScript::ignores_signals(),
ProcScript::ignores_signals(),
ProcScript::never_exits(),
ProcScript::never_exits(),
]);
let started = reply_of(
dispatch(
envelope(
1,
Request::Start {
apps: vec![
AppConfig::minimal("alpha", "./a"),
AppConfig::minimal("zulu", "./z"),
],
},
),
&h.ctx,
)
.await,
);
assert!(started.result.is_ok(), "both apps come up");
let began = Instant::now();
let reply = reply_of(
dispatch(
envelope(
2,
Request::Restart {
selector: SelectorSpec::All,
},
),
&h.ctx,
)
.await,
);
let spent = began.elapsed();
let Response::Restarted {
accepted: infos, ..
} = reply.result.unwrap()
else {
panic!("expected restarted")
};
assert_eq!(infos.len(), 2, "both sheep restart: {infos:?}");
assert!(
spent < DEFAULT_KILL_TIMEOUT * 2,
"one stage's ladders must overlap; spent {spent:?} on two of them"
);
}
#[tokio::test(start_paused = true)]
async fn a_matched_sheep_the_registry_does_not_hold_still_restarts() {
let h = harness(vec![ProcScript::never_exits(); 4]);
start_api_before_the_db_it_waits_for(&h).await;
h.ctx.registry.clear();
let mut rx = h.ctx.events.subscribe();
let reply = reply_of(
dispatch(
envelope(
3,
Request::Restart {
selector: SelectorSpec::All,
},
),
&h.ctx,
)
.await,
);
let Response::Restarted {
accepted: infos, ..
} = reply.result.unwrap()
else {
panic!("expected restarted")
};
assert_eq!(infos.len(), 2, "both sheep restart: {infos:?}");
assert_eq!(
names_for(&mut rx, ProcessEventKind::Restart)
.into_iter()
.collect::<BTreeSet<_>>(),
["api".to_string(), "db".to_string()]
.into_iter()
.collect::<BTreeSet<_>>(),
);
}
fn events_in_order(
rx: &mut tokio::sync::broadcast::Receiver<crate::bus::SharedEvent>,
) -> Vec<(ProcessEventKind, String)> {
let mut seen = Vec::new();
while let Ok(event) = rx.try_recv() {
if let shep_core::protocol::BusEvent::Process {
event: kind, info, ..
} = &*event
{
seen.push((*kind, info.name.clone()));
}
}
seen
}
#[tokio::test(start_paused = true)]
async fn a_stage_still_waits_for_an_app_whose_reload_was_refused() {
let h = harness(vec![ProcScript::never_exits(); 4]);
start_api_before_the_db_it_waits_for(&h).await;
let first = reply_of(
dispatch(
envelope(
3,
Request::Reload {
selector: SelectorSpec::Name("db".to_string()),
},
),
&h.ctx,
)
.await,
);
assert!(first.result.is_ok(), "the first reload is accepted");
let mut rx = h.ctx.events.subscribe();
let reply = reply_of(
dispatch(
envelope(
4,
Request::Reload {
selector: SelectorSpec::All,
},
),
&h.ctx,
)
.await,
);
let Response::Reloading { accepted, .. } = reply.result.unwrap() else {
panic!("expected reloading")
};
let names: Vec<&str> = accepted.iter().map(|info| info.name.as_str()).collect();
assert_eq!(names, ["api"], "db is already reloading and is refused");
let seen = events_in_order(&mut rx);
let swapped = seen
.iter()
.position(|(kind, name)| *kind == ProcessEventKind::Reloaded && name == "db")
.unwrap_or_else(|| panic!("db never finished its swap: {seen:?}"));
let dependant = seen
.iter()
.position(|(kind, name)| *kind == ProcessEventKind::Reload && name == "api")
.unwrap_or_else(|| panic!("api never reloaded: {seen:?}"));
assert!(
swapped < dependant,
"api must wait out the refused stage's swap: {seen:?}"
);
}
#[tokio::test(start_paused = true)]
async fn a_staged_restart_names_the_app_it_could_not_restart() {
let h = harness(vec![ProcScript::never_exits(); 3]);
start_api_before_the_db_it_waits_for(&h).await;
let walk = walk_for(&h.ctx, &ProcessSelector::All)
.await
.expect("two sheep are an ordered walk");
let deleted = reply_of(
dispatch(
envelope(
3,
Request::Delete {
selector: SelectorSpec::Name("db".to_string()),
},
),
&h.ctx,
)
.await,
);
assert!(deleted.result.is_ok(), "db leaves the flock");
let (accepted, refused) = restart_in_stages(&h.ctx, &walk)
.await
.expect("api still restarted, so the walk answers Ok");
let names: Vec<&str> = accepted.iter().map(|info| info.name.as_str()).collect();
assert_eq!(names, ["api"], "the rest of the fold still restarts");
let refused_names: Vec<&str> = refused.iter().map(|row| row.name.as_str()).collect();
assert_eq!(refused_names, ["db"], "and the one that did not is named");
assert!(
refused[0].reason.contains("no registered sheep"),
"with the shepherd's own reason: {:?}",
refused[0].reason
);
}
#[tokio::test(start_paused = true)]
async fn a_staged_restart_collects_every_refusal_and_not_just_the_first() {
let h = harness(vec![ProcScript::never_exits(); 4]);
start_api_before_the_db_it_waits_for(&h).await;
let started = reply_of(
dispatch(
envelope(
3,
Request::Start {
apps: vec![AppConfig::minimal("cache", "./cache")],
},
),
&h.ctx,
)
.await,
);
assert!(started.result.is_ok(), "cache comes up beside the pair");
let walk = walk_for(&h.ctx, &ProcessSelector::All)
.await
.expect("three sheep are an ordered walk");
for (id, name) in [(4, "db"), (5, "cache")] {
let deleted = reply_of(
dispatch(
envelope(
id,
Request::Delete {
selector: SelectorSpec::Name(name.to_string()),
},
),
&h.ctx,
)
.await,
);
assert!(deleted.result.is_ok(), "{name} leaves the flock");
}
let (accepted, refused) = restart_in_stages(&h.ctx, &walk)
.await
.expect("api still restarted, so the walk answers Ok");
let names: Vec<&str> = accepted.iter().map(|info| info.name.as_str()).collect();
assert_eq!(names, ["api"], "the one app still in the flock restarts");
let refused_names: BTreeSet<&str> = refused.iter().map(|row| row.name.as_str()).collect();
assert_eq!(
refused_names,
["cache", "db"].into_iter().collect::<BTreeSet<&str>>(),
"both are named, not just whichever was refused first"
);
}
#[tokio::test(start_paused = true)]
async fn a_staged_reload_names_the_app_it_could_not_reload() {
let h = harness(vec![ProcScript::never_exits(); 4]);
start_api_before_the_db_it_waits_for(&h).await;
let first = reply_of(
dispatch(
envelope(
3,
Request::Reload {
selector: SelectorSpec::Name("db".to_string()),
},
),
&h.ctx,
)
.await,
);
assert!(first.result.is_ok(), "the first reload is accepted");
let reply = reply_of(
dispatch(
envelope(
4,
Request::Reload {
selector: SelectorSpec::All,
},
),
&h.ctx,
)
.await,
);
let Response::Reloading { accepted, refused } = reply.result.unwrap() else {
panic!("expected reloading")
};
let names: Vec<&str> = accepted.iter().map(|info| info.name.as_str()).collect();
assert_eq!(names, ["api"], "the rest of the fold still reloads");
let refused_names: Vec<&str> = refused.iter().map(|row| row.name.as_str()).collect();
assert_eq!(refused_names, ["db"], "and the one that did not is named");
assert!(
refused[0].reason.contains("already being reloaded"),
"with the shepherd's own reason: {:?}",
refused[0].reason
);
}
#[tokio::test(start_paused = true)]
async fn a_reload_stage_is_bounded_by_the_swaps_it_is_waiting_for() {
let h = harness(vec![ProcScript::never_exits()]);
let mut web = AppConfig::minimal("web", "./web");
web.listen_timeout = UpDuration::from_millis(4_000);
web.graceful_timeout = UpDuration::from_millis(6_000);
let started =
reply_of(dispatch(envelope(1, Request::Start { apps: vec![web] }), &h.ctx).await);
assert!(started.result.is_ok(), "web comes up: {started:?}");
let one = [("web".to_string(), 1)].into_iter().collect();
let three = [("web".to_string(), 3)].into_iter().collect();
assert_eq!(
reload_stage_bound(&h.ctx, &one),
Duration::from_secs(15),
"one swap is a drain plus a readiness wait, plus the stage slack"
);
assert_eq!(
reload_stage_bound(&h.ctx, &three),
Duration::from_secs(35),
"three swaps are three of each, and the slack is spent once"
);
}
#[tokio::test(start_paused = true)]
async fn a_walk_waits_for_a_dependency_it_reaches_through_an_unmatched_hop() {
let h = harness(vec![ProcScript::never_exits(); 3]);
let mut web = AppConfig::minimal("web", "./web");
web.depends_on = vec!["mid".to_string()];
let mut mid = AppConfig::minimal("mid", "./mid");
mid.depends_on = vec!["db".to_string()];
for (id, app) in [(1, AppConfig::minimal("db", "./db")), (2, mid), (3, web)] {
let started =
reply_of(dispatch(envelope(id, Request::Start { apps: vec![app] }), &h.ctx).await);
assert!(started.result.is_ok(), "the chain comes up: {started:?}");
}
let listed = reply_of(dispatch(envelope(4, Request::ListFlock), &h.ctx).await);
let Response::Flock(flock) = listed.result.unwrap() else {
panic!("expected flock")
};
let selector = selector_of(SelectorSpec::Regex("^(web|db)$".to_string())).unwrap();
let walk = ordered_walk(&h.ctx, &selector, &flock).expect("two names matched");
assert_eq!(
walk.stages,
vec![vec!["db".to_string()], vec!["web".to_string()]],
"the unmatched hop is not restarted, and the ends keep their order"
);
assert_eq!(
walk.depended_on,
["db".to_string()].into_iter().collect::<BTreeSet<_>>(),
"web reaches db through mid, so db's stage has to be held"
);
}
#[test]
fn a_refused_reload_is_internal_and_says_which_refusal_it_was() {
let in_flight = rpc_error(&SupervisorError::ReloadInFlight("web".to_string()));
assert_eq!(in_flight.code, RpcErrorCode::Internal);
assert_eq!(in_flight.message, "web is already being reloaded");
let shutting_down = rpc_error(&SupervisorError::EngineStopped);
assert_eq!(shutting_down.code, RpcErrorCode::Internal);
assert_eq!(shutting_down.message, "the supervisor engine has stopped");
}
#[tokio::test(start_paused = true)]
async fn a_bad_reload_selector_is_invalid_config() {
let h = harness(vec![]);
let reply = reply_of(
dispatch(
envelope(
1,
Request::Reload {
selector: SelectorSpec::Regex("((".to_string()),
},
),
&h.ctx,
)
.await,
);
assert_eq!(reply.result.unwrap_err().code, RpcErrorCode::InvalidConfig);
}
#[tokio::test(start_paused = true)]
async fn a_bad_reopen_selector_is_invalid_config() {
let h = harness(vec![]);
let reply = reply_of(
dispatch(
envelope(
1,
Request::Reopen {
selector: SelectorSpec::Regex("((".to_string()),
},
),
&h.ctx,
)
.await,
);
assert_eq!(reply.result.unwrap_err().code, RpcErrorCode::InvalidConfig);
}
#[tokio::test(start_paused = true)]
async fn trigger_routes_to_the_flock_and_reports_each_match_within_the_budget() {
let h = harness(vec![ProcScript::never_exits(), ProcScript::never_exits()]);
let mut web = AppConfig::minimal("web", "./srv");
web.channel = true;
let started = reply_of(
dispatch(
envelope(
1,
Request::Start {
apps: vec![AppConfig::minimal("other", "./o"), web],
},
),
&h.ctx,
)
.await,
);
let Response::Started(started) = started.result.unwrap() else {
panic!("expected started")
};
let web_id = started
.iter()
.find(|i| i.name == "web")
.expect("web registered")
.id;
assert_ne!(web_id, 0, "the test's own premise: web must not be id 0");
let reply = reply_of(
dispatch(
envelope(
2,
Request::Trigger {
selector: SelectorSpec::Name("web".to_string()),
action: "gc".to_string(),
params: None,
},
),
&h.ctx,
)
.await,
);
let Response::Triggered(rows) = reply.result.unwrap() else {
panic!("expected triggered")
};
assert_eq!(
rows,
vec![ActionReply {
id: web_id,
name: "web".to_string(),
outcome: ActionOutcome::TimedOut,
}]
);
}
#[tokio::test]
async fn a_signal_name_outside_the_grammar_is_refused_with_the_accepted_list() {
let h = harness(vec![]);
let reply = reply_of(
dispatch(
envelope(
1,
Request::Signal {
selector: SelectorSpec::All,
signal: "SIGHUPP".to_string(),
},
),
&h.ctx,
)
.await,
);
let err = reply.result.unwrap_err();
assert_eq!(err.code, RpcErrorCode::InvalidConfig);
assert!(err.message.contains("SIGHUPP"), "{}", err.message);
assert!(err.message.contains("SIGHUP"), "{}", err.message);
assert!(err.message.contains("SIGUSR2"), "{}", err.message);
}
#[tokio::test]
async fn a_line_carrying_a_newline_is_refused_before_it_reaches_the_supervisor() {
let h = harness(vec![]);
let reply = reply_of(
dispatch(
envelope(
1,
Request::SendLine {
selector: SelectorSpec::All,
line: "reload\nrm -rf /".to_string(),
},
),
&h.ctx,
)
.await,
);
let err = reply.result.unwrap_err();
assert_eq!(err.code, RpcErrorCode::InvalidConfig);
assert!(err.message.contains("newline"), "{}", err.message);
}
#[tokio::test(start_paused = true)]
async fn an_oversized_action_timeout_loses_the_race() {
let h = harness(vec![ProcScript::never_exits()]);
let mut web = AppConfig::minimal("web", "./srv");
web.channel = true;
web.action_timeout = UpDuration::from_millis(9_000); reply_of(dispatch(envelope(1, Request::Start { apps: vec![web] }), &h.ctx).await);
let reply = reply_of(
dispatch(
envelope(
2,
Request::Trigger {
selector: SelectorSpec::Name("web".to_string()),
action: "gc".to_string(),
params: None,
},
),
&h.ctx,
)
.await,
);
assert_eq!(
reply.result.unwrap_err().code,
RpcErrorCode::DeadlineExceeded,
"an action_timeout past the caller's default budget must lose that race, not \
report an honest TimedOut row nobody can reach"
);
}
#[tokio::test(start_paused = true)]
async fn a_bad_trigger_selector_is_invalid_config() {
let h = harness(vec![]);
let reply = reply_of(
dispatch(
envelope(
1,
Request::Trigger {
selector: SelectorSpec::Regex("((".to_string()),
action: "gc".to_string(),
params: None,
},
),
&h.ctx,
)
.await,
);
assert_eq!(reply.result.unwrap_err().code, RpcErrorCode::InvalidConfig);
}
#[tokio::test(start_paused = true)]
async fn a_trigger_matching_nothing_is_not_found() {
let h = harness(vec![]);
let reply = reply_of(
dispatch(
envelope(
1,
Request::Trigger {
selector: SelectorSpec::Name("ghost".to_string()),
action: "gc".to_string(),
params: None,
},
),
&h.ctx,
)
.await,
);
assert_eq!(reply.result.unwrap_err().code, RpcErrorCode::NotFound);
}
#[test]
fn a_log_plane_failure_is_internal_and_says_which_half_failed() {
let reopen = rpc_error(&SupervisorError::ReopenFailed(
"web (id 0): could not reopen /logs/web-out.log: Permission denied".to_string(),
));
assert_eq!(reopen.code, RpcErrorCode::Internal);
assert_eq!(
reopen.message,
"log reopen failed: web (id 0): could not reopen \
/logs/web-out.log: Permission denied"
);
let flush = rpc_error(&SupervisorError::FlushFailed(
"/logs/web-out.log: Permission denied".to_string(),
));
assert_eq!(flush.code, RpcErrorCode::Internal);
assert_eq!(
flush.message,
"log flush failed: /logs/web-out.log: Permission denied"
);
}
#[tokio::test(start_paused = true)]
async fn subscribe_hands_back_a_compiled_filter() {
let h = harness(vec![]);
let outcome = dispatch(
envelope(
1,
Request::Subscribe {
topics: vec!["process.*".to_string()],
},
),
&h.ctx,
)
.await;
let Outcome::Subscribe { reply, filter } = outcome else {
panic!("expected subscribe")
};
assert_eq!(reply.result.unwrap(), Response::Subscribed);
assert_eq!(filter.patterns(), ["process.*"]);
let bad = reply_of(
dispatch(
envelope(
2,
Request::Subscribe {
topics: vec!["[".to_string()],
},
),
&h.ctx,
)
.await,
);
assert_eq!(bad.result.unwrap_err().code, RpcErrorCode::InvalidConfig);
}
#[tokio::test(start_paused = true)]
async fn kill_daemon_asks_for_shutdown_without_taking_the_engine_down_itself() {
let mut h = harness(vec![]);
let Outcome::Shutdown(reply) = dispatch(envelope(1, Request::KillDaemon), &h.ctx).await
else {
panic!("expected a shutdown outcome")
};
assert_eq!(reply.result.unwrap(), Response::ShuttingDown);
assert!(!*h.shutdown_rx.borrow_and_update());
h.ctx.shutdown();
assert!(h.shutdown_rx.changed().await.is_ok());
assert!(*h.shutdown_rx.borrow());
}
#[test]
fn budgets_default_and_clamp() {
assert_eq!(budget(None), Duration::from_millis(DEFAULT_DEADLINE_MS));
assert_eq!(budget(Some(250)), Duration::from_millis(250));
assert_eq!(budget(Some(0)), Duration::from_millis(1));
assert_eq!(
budget(Some(u64::MAX)),
Duration::from_millis(MAX_DEADLINE_MS)
);
}
#[tokio::test(start_paused = true)]
async fn envelope_deadline_ms_actually_bounds_the_reply() {
let h = harness(vec![ProcScript::ignores_signals()]);
dispatch(
envelope(
1,
Request::Start {
apps: vec![AppConfig::minimal("web", "./srv")],
},
),
&h.ctx,
)
.await;
let reply = reply_of(
dispatch(
Envelope {
id: 2,
deadline_ms: Some(1),
body: Request::Stop {
selector: SelectorSpec::Name("web".to_string()),
},
},
&h.ctx,
)
.await,
);
let err = reply.result.unwrap_err();
assert_eq!(
err.code,
RpcErrorCode::DeadlineExceeded,
"a 1ms client deadline against a 1600ms kill ladder must expire, not {err:?}"
);
}
#[tokio::test(start_paused = true)]
async fn work_past_its_deadline_answers_deadline_exceeded() {
let outcome = with_deadline(
5,
Duration::from_millis(250),
std::future::pending::<Outcome>(),
)
.await;
let reply = reply_of(outcome);
assert_eq!(reply.id, 5);
let err = reply.result.unwrap_err();
assert_eq!(err.code, RpcErrorCode::DeadlineExceeded);
assert!(err.message.contains("250 ms"), "{}", err.message);
}
#[tokio::test]
async fn save_roll_writes_the_file_it_names_and_counts_what_it_recorded() {
let h = harness(vec![ProcScript::never_exits(), ProcScript::never_exits()]);
reply_of(
dispatch(
envelope(
1,
Request::Start {
apps: vec![
AppConfig::minimal("web", "./srv"),
AppConfig::minimal("worker", "./work"),
],
},
),
&h.ctx,
)
.await,
);
let reply = reply_of(dispatch(envelope(2, Request::SaveRoll), &h.ctx).await);
let Ok(Response::RollSaved { path, apps }) = reply.result else {
panic!("expected RollSaved, got {:?}", reply.result)
};
assert_eq!(apps, 2);
let roll = crate::snapshot::read(std::path::Path::new(&path)).unwrap();
assert_eq!(roll.apps.len(), 2, "the reply's count must match the file");
assert_eq!(path, h.ctx.snapshot_path.display().to_string());
}
#[tokio::test]
async fn a_scale_is_recorded_in_the_roll_the_next_muster_reads() {
let h = harness(vec![ProcScript::never_exits(); 4]);
let mut app = AppConfig::minimal("web", "./srv");
app.instances = 2;
reply_of(dispatch(envelope(1, Request::Start { apps: vec![app] }), &h.ctx).await);
reply_of(
dispatch(
envelope(
2,
Request::Scale {
name: "web".to_string(),
count: 4,
},
),
&h.ctx,
)
.await,
);
let reply = reply_of(dispatch(envelope(3, Request::SaveRoll), &h.ctx).await);
let Ok(Response::RollSaved { path, .. }) = reply.result else {
panic!("expected RollSaved, got {:?}", reply.result)
};
let roll = crate::snapshot::read(std::path::Path::new(&path)).unwrap();
assert_eq!(roll.apps[0].app.instances, 4);
}
#[tokio::test]
async fn a_partial_scale_is_recorded_in_the_roll_and_still_reported_short() {
let h = harness(vec![ProcScript::never_exits(); 3]);
let mut app = AppConfig::minimal("web", "./srv");
app.instances = 2;
reply_of(dispatch(envelope(1, Request::Start { apps: vec![app] }), &h.ctx).await);
let reply = reply_of(
dispatch(
envelope(
2,
Request::Scale {
name: "web".to_string(),
count: 4,
},
),
&h.ctx,
)
.await,
);
let err = reply.result.unwrap_err();
assert_eq!(err.code, RpcErrorCode::SpawnFailed);
assert!(
err.message.contains("3 of 4"),
"the operator has to be told both numbers: {}",
err.message
);
let saved = reply_of(dispatch(envelope(3, Request::SaveRoll), &h.ctx).await);
let Ok(Response::RollSaved { path, .. }) = saved.result else {
panic!("expected RollSaved, got {:?}", saved.result)
};
let roll = crate::snapshot::read(std::path::Path::new(&path)).unwrap();
assert_eq!(
roll.apps[0].app.instances, 3,
"the roll must hold the three instances really running — not the \
pre-scale two, and not the four that were asked for"
);
}
#[tokio::test]
async fn save_roll_against_a_stopped_engine_is_an_error_not_a_silent_success() {
let h = harness(vec![]);
h.ctx.supervisor.shutdown().await;
let reply = reply_of(dispatch(envelope(1, Request::SaveRoll), &h.ctx).await);
let err = reply.result.unwrap_err();
assert_eq!(err.code, RpcErrorCode::Internal);
assert!(
err.message.contains("engine"),
"the operator must be told why nothing was written: {}",
err.message
);
}
#[tokio::test]
async fn a_second_muster_still_reports_the_flock_the_roll_restored() {
let h = harness(vec![ProcScript::never_exits()]);
reply_of(
dispatch(
envelope(
1,
Request::Start {
apps: vec![AppConfig::minimal("web", "./srv")],
},
),
&h.ctx,
)
.await,
);
reply_of(dispatch(envelope(2, Request::SaveRoll), &h.ctx).await);
let reply = reply_of(dispatch(envelope(3, Request::Muster), &h.ctx).await);
let Ok(Response::Mustered(infos)) = reply.result else {
panic!("expected Mustered, got {:?}", reply.result)
};
assert_eq!(
infos.len(),
1,
"the sheep the roll restores, not the ones this call spawned"
);
assert_eq!(infos[0].name, "web");
assert_eq!(infos[0].status, ProcStatus::Online);
}
async fn start_web(h: &Harness) {
reply_of(
dispatch(
envelope(
1,
Request::Start {
apps: vec![AppConfig::minimal("web", "./srv")],
},
),
&h.ctx,
)
.await,
);
}
async fn list_flock(ctx: &RpcContext, id: u64) -> Vec<ProcessInfo> {
let reply = reply_of(dispatch(envelope(id, Request::ListFlock), ctx).await);
let Ok(Response::Flock(infos)) = reply.result else {
panic!("expected Flock, got {:?}", reply.result)
};
infos
}
#[tokio::test]
async fn list_flock_carries_a_live_memory_reading_for_a_running_sheep() {
let h = harness_with_stats(vec![ProcScript::never_exits()]);
start_web(&h).await;
let infos = list_flock(&h.ctx, 2).await;
assert_eq!(infos[0].pid, Some(FIRST_SCRIPTED_PID));
assert_eq!(infos[0].memory_bytes, Some(SCRIPTED_TREE_BYTES));
assert_eq!(
infos[0].cpu_percent, None,
"no periodic baseline has been recorded, and a number invented \
from the read's own window is worse than an empty cell"
);
}
#[tokio::test]
async fn list_flock_measures_cpu_from_the_periodic_baseline_not_from_the_previous_listing() {
let h = harness_with_stats(vec![ProcScript::never_exits()]);
start_web(&h).await;
let last_tick = Instant::now()
.checked_sub(MEMORY_POLL_INTERVAL)
.expect("the monotonic clock is older than one poll interval");
h.stats.record_baseline_now(last_tick);
let first = list_flock(&h.ctx, 2).await[0]
.cpu_percent
.expect("a baseline exists, so a running sheep has a CPU figure");
let second = list_flock(&h.ctx, 3).await[0]
.cpu_percent
.expect("a baseline exists, so a running sheep has a CPU figure");
assert!(
(5.0..=10.05).contains(&first),
"1500 CPU-ms over the 15 s since the baseline is 10%; got {first}"
);
assert!(
(second - first).abs() < 1.0,
"the second listing divided by the gap between the two LISTINGS \
rather than by the window since the tick: {first} then {second}"
);
}
#[tokio::test]
async fn describe_carries_a_live_reading_too() {
let h = harness_with_stats(vec![ProcScript::never_exits()]);
start_web(&h).await;
let described = reply_of(
dispatch(
envelope(
2,
Request::Describe {
selector: SelectorSpec::Name("web".to_string()),
},
),
&h.ctx,
)
.await,
);
let Ok(Response::Described(infos)) = described.result else {
panic!("expected Described, got {:?}", described.result)
};
assert_eq!(infos[0].memory_bytes, Some(SCRIPTED_TREE_BYTES));
}
#[tokio::test]
async fn a_sheep_with_no_pid_reports_no_stats() {
let h = harness_with_stats(vec![ProcScript::never_exits(), ProcScript::never_exits()]);
start_web(&h).await;
reply_of(
dispatch(
envelope(
2,
Request::Start {
apps: vec![AppConfig::minimal("worker", "./srv")],
},
),
&h.ctx,
)
.await,
);
reply_of(
dispatch(
envelope(
3,
Request::Stop {
selector: SelectorSpec::Name("worker".to_string()),
},
),
&h.ctx,
)
.await,
);
let infos = list_flock(&h.ctx, 4).await;
let named = |name: &str| {
infos
.iter()
.find(|info| info.name == name)
.unwrap_or_else(|| panic!("{name} is missing from the listing"))
};
assert_eq!(named("web").pid, Some(FIRST_SCRIPTED_PID));
assert_eq!(named("web").memory_bytes, Some(SCRIPTED_TREE_BYTES));
assert_eq!(named("worker").pid, None);
assert_eq!(named("worker").memory_bytes, None);
assert_eq!(named("worker").cpu_percent, None);
}
#[tokio::test]
async fn a_lifecycle_reply_carries_no_stats() {
let h = harness_with_stats(vec![ProcScript::never_exits()]);
let started = reply_of(
dispatch(
envelope(
1,
Request::Start {
apps: vec![AppConfig::minimal("web", "./srv")],
},
),
&h.ctx,
)
.await,
);
let Ok(Response::Started(infos)) = started.result else {
panic!("expected Started, got {:?}", started.result)
};
assert_eq!(
infos[0].pid,
Some(FIRST_SCRIPTED_PID),
"a row with no pid would report no stats however the verb behaved"
);
assert_eq!(
infos[0].memory_bytes, None,
"only `flock` and `describe` take a live sample"
);
assert_eq!(infos[0].cpu_percent, None);
}
async fn enable_dog(ctx: &RpcContext, id: u64, name: &str) -> ProcessInfo {
let reply = reply_of(
dispatch(
envelope(
id,
Request::EnableDog {
name: name.to_string(),
source: DogSource::BuiltIn,
},
),
ctx,
)
.await,
);
let Ok(Response::DogStarted(info)) = reply.result else {
panic!("expected DogStarted, got {:?}", reply.result)
};
info
}
async fn start_web_with_a_secret(ctx: &RpcContext) {
let mut config = AppConfig::minimal("web", "./srv");
config
.env
.insert("DB_PASS".to_string(), "hunter2".to_string());
let started =
reply_of(dispatch(envelope(1, Request::Start { apps: vec![config] }), ctx).await);
assert!(started.result.is_ok(), "{:?}", started.result);
}
#[tokio::test(start_paused = true)]
async fn a_dog_is_refused_an_env_override_rather_than_given_one() {
let h = harness(vec![ProcScript::never_exits()]);
let dog = enable_dog(&h.ctx, 1, "bark").await;
let reply = reply_of(
dispatch(
envelope(
2,
Request::SetSheepEnv {
name: "bark".to_string(),
key: "PATH".to_string(),
value: Some("/tmp/evil".to_string().into()),
},
),
&h.ctx,
)
.await,
);
let Err(err) = reply.result else {
panic!("a dog was given an env override")
};
assert_eq!(err.code, RpcErrorCode::InvalidConfig);
assert!(err.message.contains("bark is a dog"), "{}", err.message);
assert!(
shep_core::overrides::get(&h.ctx.paths.overrides, "bark")
.unwrap()
.is_none(),
"the refusal still wrote the store"
);
let described = reply_of(
dispatch(
envelope(
3,
Request::Describe {
selector: SelectorSpec::Name("bark".to_string()),
},
),
&h.ctx,
)
.await,
);
let Ok(Response::Described(infos)) = described.result else {
panic!("expected Described")
};
assert_eq!(infos[0].id, dog.id);
assert_eq!(infos[0].pending, None, "the refusal still parked a config");
}
#[tokio::test(start_paused = true)]
async fn a_dogs_config_is_not_readable_through_the_sheep_config_pane() {
let h = harness(vec![ProcScript::never_exits()]);
enable_dog(&h.ctx, 1, "bark").await;
let reply = reply_of(
dispatch(
envelope(
2,
Request::SheepConfig {
name: "bark".to_string(),
},
),
&h.ctx,
)
.await,
);
let Err(err) = reply.result else {
panic!("a dog's config was served to a pane")
};
assert_eq!(err.code, RpcErrorCode::InvalidConfig);
assert!(err.message.contains("bark is a dog"), "{}", err.message);
}
async fn sheep_config_view(
ctx: &RpcContext,
id: u64,
name: &str,
) -> shep_core::protocol::SheepConfigView {
let reply = reply_of(
dispatch(
envelope(
id,
Request::SheepConfig {
name: name.to_string(),
},
),
ctx,
)
.await,
);
match reply.result {
Ok(Response::SheepConfig(view)) => *view,
other => panic!("expected SheepConfig, got {other:?}"),
}
}
#[tokio::test(start_paused = true)]
async fn sheep_config_answers_with_env_emptied_and_its_keys_listed() {
let h = harness(vec![ProcScript::never_exits()]);
start_web_with_a_secret(&h.ctx).await;
let reply = reply_of(
dispatch(
envelope(
2,
Request::SheepConfig {
name: "web".to_string(),
},
),
&h.ctx,
)
.await,
);
let Ok(Response::SheepConfig(view)) = reply.result else {
panic!("expected SheepConfig")
};
assert_eq!(view.name, "web");
assert!(view.config.env.is_empty());
assert_eq!(view.env_keys, ["DB_PASS"]);
}
#[tokio::test(start_paused = true)]
async fn sheep_config_for_an_unknown_name_is_not_found_not_internal() {
let h = harness(vec![]);
let reply = reply_of(
dispatch(
envelope(
1,
Request::SheepConfig {
name: "ghost".to_string(),
},
),
&h.ctx,
)
.await,
);
let Err(err) = reply.result else {
panic!("expected a refusal")
};
assert_eq!(err.code, RpcErrorCode::NotFound);
}
#[tokio::test(start_paused = true)]
async fn set_sheep_env_writes_the_store_and_parks_env_until_a_respawn() {
let h = harness(vec![ProcScript::never_exits()]);
start_web_with_a_secret(&h.ctx).await;
let reply = reply_of(
dispatch(
envelope(
2,
Request::SetSheepEnv {
name: "web".to_string(),
key: "NEW".to_string(),
value: Some("1".to_string().into()),
},
),
&h.ctx,
)
.await,
);
assert!(
matches!(reply.result, Ok(Response::SheepEnvSet { .. })),
"{:?}",
reply.result
);
let stored = shep_core::overrides::get(&h.ctx.paths.overrides, "web")
.unwrap()
.unwrap();
assert_eq!(stored.fields["env"]["NEW"], "1");
let described = reply_of(
dispatch(
envelope(
3,
Request::Describe {
selector: SelectorSpec::Name("web".to_string()),
},
),
&h.ctx,
)
.await,
);
let Ok(Response::Described(infos)) = described.result else {
panic!("expected Described")
};
assert_eq!(
infos[0].pending.as_deref(),
Some(["env".to_string()].as_slice())
);
}
#[tokio::test(start_paused = true)]
async fn removing_the_last_env_override_stops_marking_the_sheep() {
let h = harness(vec![ProcScript::never_exits()]);
start_web_with_a_secret(&h.ctx).await;
for (id, value) in [(2, Some("1".to_string().into())), (3, None)] {
let reply = reply_of(
dispatch(
envelope(
id,
Request::SetSheepEnv {
name: "web".to_string(),
key: "NEW".to_string(),
value,
},
),
&h.ctx,
)
.await,
);
assert!(reply.result.is_ok(), "{:?}", reply.result);
}
let stored = shep_core::overrides::get(&h.ctx.paths.overrides, "web")
.unwrap()
.unwrap();
assert!(!stored.fields.contains_key("env"), "{stored:?}");
let reply = reply_of(
dispatch(
envelope(
4,
Request::SheepConfig {
name: "web".to_string(),
},
),
&h.ctx,
)
.await,
);
let Ok(Response::SheepConfig(view)) = reply.result else {
panic!("expected SheepConfig")
};
assert!(view.overridden.is_empty(), "{view:?}");
}
#[tokio::test(start_paused = true)]
async fn a_set_env_reaches_the_roll_a_cold_restart_would_come_back_on() {
let h = harness(vec![ProcScript::never_exits()]);
start_web_with_a_secret(&h.ctx).await;
let reply = reply_of(
dispatch(
envelope(
2,
Request::SetSheepEnv {
name: "web".to_string(),
key: "NEW".to_string(),
value: Some("1".to_string().into()),
},
),
&h.ctx,
)
.await,
);
assert!(reply.result.is_ok(), "{:?}", reply.result);
let infos = list_flock(&h.ctx, 3).await;
let roll = h.ctx.registry.roll(&infos, 0);
let web = roll
.apps
.iter()
.find(|entry| entry.app.name == "web")
.expect("web is in the roll");
assert_eq!(web.app.env.get("NEW").map(String::as_str), Some("1"));
}
#[tokio::test(start_paused = true)]
async fn a_parked_env_edit_reaches_the_roll_before_a_hard_stop_can_lose_it() {
let h = harness(vec![ProcScript::never_exits()]);
start_web_with_a_secret(&h.ctx).await;
h.ctx.save_roll_now().await.unwrap();
let baseline = crate::snapshot::read(&h.ctx.snapshot_path).unwrap();
assert_eq!(baseline.apps[0].app.env.get("NEW"), None);
let writer = crate::snapshot::spawn_snapshot_writer(
h.ctx.snapshot_path.clone(),
h.ctx.supervisor.clone(),
h.ctx.registry.clone(),
h.ctx.events.subscribe(),
);
let settle = std::time::Duration::from_millis(crate::snapshot::SNAPSHOT_DEBOUNCE_MS * 2);
tokio::time::sleep(settle).await;
let reply = reply_of(
dispatch(
envelope(
2,
Request::SetSheepEnv {
name: "web".to_string(),
key: "NEW".to_string(),
value: Some("1".to_string().into()),
},
),
&h.ctx,
)
.await,
);
assert!(reply.result.is_ok(), "{:?}", reply.result);
tokio::time::sleep(settle).await;
writer.stop().await;
let (events, _rx) = crate::bus::test_bus(64);
let cold = crate::supervisor::spawn_supervisor(
crate::fake::ScriptedRunner::new(vec![ProcScript::never_exits()]),
h.ctx.paths.clone(),
events.clone(),
);
let cold_registry = crate::snapshot::FlockRegistry::new();
let restored = crate::snapshot::muster(
&h.ctx.snapshot_path,
&cold_registry,
&cold,
&events,
&[],
&[],
)
.await
.unwrap();
assert_eq!(restored, vec!["web".to_string()]);
let listed = cold.list().await;
let came_back = cold_registry.roll(&listed, 0);
assert_eq!(
came_back.apps[0].app.env.get("NEW").map(String::as_str),
Some("1"),
"the sheep must come back on the edit its CFG cell claims: {:?}",
listed[0].overridden
);
cold.shutdown().await;
}
#[tokio::test(start_paused = true)]
async fn removing_a_key_the_operator_never_set_is_still_recorded() {
let h = harness(vec![ProcScript::never_exits()]);
start_web_with_a_secret(&h.ctx).await;
let reply = reply_of(
dispatch(
envelope(
2,
Request::SetSheepEnv {
name: "web".to_string(),
key: "DB_PASS".to_string(),
value: None,
},
),
&h.ctx,
)
.await,
);
assert!(reply.result.is_ok(), "{:?}", reply.result);
let stored = shep_core::overrides::get(&h.ctx.paths.overrides, "web")
.unwrap()
.expect("the removal is recorded");
assert_eq!(
stored.fields["env"]["DB_PASS"],
serde_json::Value::Null,
"a removal is a tombstone, not an absence"
);
let view = sheep_config_view(&h.ctx, 3, "web").await;
assert_eq!(view.overridden, ["env"]);
assert!(!view.env_keys.contains(&"DB_PASS".to_string()));
}
#[tokio::test(start_paused = true)]
async fn a_removed_key_stays_removed_and_stays_reported_across_reloads() {
let h = harness(vec![ProcScript::never_exits()]);
start_web_with_a_secret(&h.ctx).await;
let removed = reply_of(
dispatch(
envelope(
2,
Request::SetSheepEnv {
name: "web".to_string(),
key: "DB_PASS".to_string(),
value: None,
},
),
&h.ctx,
)
.await,
);
assert!(removed.result.is_ok(), "{:?}", removed.result);
for id in [3, 4] {
let loaded = reply_of(
dispatch(
envelope(
id,
Request::ApplyConfig {
apps: vec![DeclaredApp {
config: {
let mut app = AppConfig::minimal("web", "./srv");
app.env
.insert("DB_PASS".to_string(), "fromfile".to_string());
app
},
declared: ["name", "script"]
.iter()
.map(|k| (*k).to_string())
.collect(),
declared_env: ["DB_PASS"]
.iter()
.map(|k| (*k).to_string())
.collect(),
}],
reset: ResetDepth::None,
},
),
&h.ctx,
)
.await,
);
let Ok(Response::Applied(report)) = loaded.result else {
panic!("expected Applied")
};
assert_eq!(report[0].refused, None);
let stored = shep_core::overrides::get(&h.ctx.paths.overrides, "web")
.unwrap()
.expect("the removal is still recorded");
assert_eq!(
stored.fields["env"]["DB_PASS"],
serde_json::Value::Null,
"load {id} spent the tombstone"
);
let view = sheep_config_view(&h.ctx, id + 10, "web").await;
assert!(
!view.env_keys.contains(&"DB_PASS".to_string()),
"load {id} put the file's value back"
);
assert_eq!(view.overridden, ["env"], "load {id} stopped reporting it");
}
}
#[tokio::test(start_paused = true)]
async fn an_env_key_normalize_refuses_is_invalid_config_and_writes_nothing() {
let h = harness(vec![ProcScript::never_exits()]);
start_web_with_a_secret(&h.ctx).await;
let reply = reply_of(
dispatch(
envelope(
2,
Request::SetSheepEnv {
name: "web".to_string(),
key: "SHEP_NAME".to_string(),
value: Some("impostor".to_string().into()),
},
),
&h.ctx,
)
.await,
);
let Err(err) = reply.result else {
panic!("a reserved env key was accepted")
};
assert_eq!(err.code, RpcErrorCode::InvalidConfig);
assert!(
shep_core::overrides::get(&h.ctx.paths.overrides, "web")
.unwrap()
.is_none(),
"the refusal still wrote the store"
);
}
#[tokio::test(start_paused = true)]
async fn an_unreadable_override_store_is_internal_not_a_bad_request() {
let h = harness(vec![ProcScript::never_exits()]);
start_web_with_a_secret(&h.ctx).await;
std::fs::write(&h.ctx.paths.overrides, "{ this is not json").unwrap();
let reply = reply_of(
dispatch(
envelope(
2,
Request::SetSheepEnv {
name: "web".to_string(),
key: "NEW".to_string(),
value: Some("1".to_string().into()),
},
),
&h.ctx,
)
.await,
);
let Err(err) = reply.result else {
panic!("an unreadable store was reported as success")
};
assert_eq!(err.code, RpcErrorCode::Internal);
assert!(
err.message.contains("overrides store unusable"),
"{}",
err.message
);
}
async fn set_field(
ctx: &RpcContext,
id: u64,
name: &str,
key: &str,
value: serde_json::Value,
) -> Result<Response, RpcError> {
reply_of(
dispatch(
envelope(
id,
Request::SetSheepField {
name: name.to_string(),
key: key.to_string(),
value,
},
),
ctx,
)
.await,
)
.result
}
#[tokio::test(start_paused = true)]
async fn a_field_edit_is_reported_as_an_operator_override() {
let h = harness(vec![ProcScript::never_exits()]);
start_web_with_a_secret(&h.ctx).await;
let view = sheep_config_view(&h.ctx, 2, "web").await;
assert!(
!view.overridden.contains(&"max_restarts".to_string()),
"nothing is overridden before the edit: {:?}",
view.overridden
);
let reply = set_field(&h.ctx, 3, "web", "max_restarts", serde_json::json!(40)).await;
assert!(
matches!(reply, Ok(Response::SheepFieldSet { .. })),
"{reply:?}"
);
let stored = shep_core::overrides::get(&h.ctx.paths.overrides, "web")
.unwrap()
.expect("the edit is recorded");
assert_eq!(stored.fields["max_restarts"], 40);
let view = sheep_config_view(&h.ctx, 4, "web").await;
assert_eq!(view.config.max_restarts, 40, "the pane shows the new value");
assert!(
view.overridden.contains(&"max_restarts".to_string()),
"the `*` marker reads this: {:?}",
view.overridden
);
let infos = list_flock(&h.ctx, 5).await;
let web = infos
.iter()
.find(|info| info.name == "web")
.expect("web is in the flock");
assert!(
web.overridden
.as_deref()
.is_some_and(|fields| fields.contains(&"max_restarts".to_string())),
"{:?}",
web.overridden
);
}
#[tokio::test(start_paused = true)]
async fn a_live_field_applies_now_and_a_respawn_field_parks() {
let h = harness(vec![ProcScript::never_exits()]);
start_web_with_a_secret(&h.ctx).await;
let reply = set_field(&h.ctx, 2, "web", "max_restarts", serde_json::json!(40)).await;
let Ok(Response::SheepFieldSet { pending, .. }) = reply else {
panic!("{reply:?}")
};
assert!(!pending, "max_restarts is Live and is in force now");
let view = sheep_config_view(&h.ctx, 3, "web").await;
assert!(
!view.pending.contains(&"max_restarts".to_string()),
"{:?}",
view.pending
);
let reply = set_field(&h.ctx, 4, "web", "script", serde_json::json!("./next")).await;
let Ok(Response::SheepFieldSet { pending, .. }) = reply else {
panic!("{reply:?}")
};
assert!(pending, "script needs a respawn");
let view = sheep_config_view(&h.ctx, 5, "web").await;
assert!(
view.pending.contains(&"script".to_string()),
"{:?}",
view.pending
);
assert_eq!(view.config.max_restarts, 40);
assert_eq!(
view.overridden,
["max_restarts", "script"],
"both are the operator's"
);
}
#[tokio::test(start_paused = true)]
async fn a_live_field_whose_subset_will_not_normalize_parks_instead() {
let h = harness(vec![ProcScript::never_exits()]);
let started = reply_of(
dispatch(
envelope(
1,
Request::Start {
apps: vec![AppConfig::minimal("web", "./srv")],
},
),
&h.ctx,
)
.await,
);
assert!(started.result.is_ok(), "{:?}", started.result);
let reply = set_field(&h.ctx, 2, "web", "cwd", serde_json::json!("/srv")).await;
let Ok(Response::SheepFieldSet { pending, .. }) = reply else {
panic!("{reply:?}")
};
assert!(pending, "cwd needs a respawn");
let reply = set_field(&h.ctx, 3, "web", "watch", serde_json::json!(true)).await;
let Ok(Response::SheepFieldSet { pending, .. }) = reply else {
panic!("{reply:?}")
};
assert_eq!(
apply_group("watch"),
ApplyGroup::Live,
"the premise: apply_group predicts this one applies now"
);
assert!(
pending,
"a Live field the running child cannot be given still parks"
);
let view = sheep_config_view(&h.ctx, 4, "web").await;
assert!(
view.pending.contains(&"watch".to_string()),
"{:?}",
view.pending
);
}
#[tokio::test(start_paused = true)]
async fn autostart_reports_in_force_and_its_group_mate_reports_pending() {
let h = harness(vec![ProcScript::never_exits()]);
start_web_with_a_secret(&h.ctx).await;
for key in ["autostart", "kill_signal"] {
assert_eq!(
apply_group(key),
ApplyGroup::NextSpawn,
"the premise: both are the same group"
);
}
let reply = set_field(&h.ctx, 2, "web", "autostart", serde_json::json!(false)).await;
let Ok(Response::SheepFieldSet { pending, .. }) = reply else {
panic!("{reply:?}")
};
assert!(
!pending,
"autostart is read at muster, so a restart would do nothing"
);
let reply = set_field(&h.ctx, 3, "web", "kill_signal", serde_json::json!("SIGINT")).await;
let Ok(Response::SheepFieldSet { pending, .. }) = reply else {
panic!("{reply:?}")
};
assert!(pending, "kill_signal really is read at a spawn");
}
#[tokio::test(start_paused = true)]
async fn depends_on_reports_in_force_the_way_autostart_does() {
let h = harness(vec![ProcScript::never_exits()]);
start_web_with_a_secret(&h.ctx).await;
assert_eq!(
apply_group("depends_on"),
ApplyGroup::NextSpawn,
"the premise: the carve-out is against its own group"
);
let reply = set_field(&h.ctx, 2, "web", "depends_on", serde_json::json!(["db"])).await;
let Ok(Response::SheepFieldSet { pending, .. }) = reply else {
panic!("{reply:?}")
};
assert!(
!pending,
"an ordered walk reads depends_on off the stored spec, so a restart would do nothing"
);
}
#[tokio::test(start_paused = true)]
async fn a_dog_is_refused_a_config_field_rather_than_given_one() {
let h = harness(vec![ProcScript::never_exits()]);
let dog = enable_dog(&h.ctx, 1, "bark").await;
let reply = set_field(&h.ctx, 2, "bark", "script", serde_json::json!("/tmp/evil")).await;
let Err(err) = reply else {
panic!("a dog was given a config field")
};
assert_eq!(err.code, RpcErrorCode::InvalidConfig);
assert!(err.message.contains("bark is a dog"), "{}", err.message);
assert!(
shep_core::overrides::get(&h.ctx.paths.overrides, "bark")
.unwrap()
.is_none(),
"the refusal still wrote the store"
);
drop(dog);
}
#[tokio::test(start_paused = true)]
async fn env_and_the_structural_fields_are_refused_by_this_door() {
let h = harness(vec![ProcScript::never_exits()]);
start_web_with_a_secret(&h.ctx).await;
for (id, key, value) in [
(2, "env", serde_json::json!({ "A": "1" })),
(3, "instances", serde_json::json!(4)),
(4, "name", serde_json::json!("other")),
] {
let reply = set_field(&h.ctx, id, "web", key, value).await;
let Err(err) = reply else {
panic!("{key} was accepted")
};
assert_eq!(err.code, RpcErrorCode::InvalidConfig, "{key}");
assert!(err.message.contains(key), "{key}: {}", err.message);
}
assert!(
shep_core::overrides::get(&h.ctx.paths.overrides, "web")
.unwrap()
.is_none(),
"a refusal still wrote the store"
);
}
#[tokio::test(start_paused = true)]
async fn a_field_this_build_refuses_is_invalid_config_and_writes_nothing() {
let h = harness(vec![ProcScript::never_exits()]);
start_web_with_a_secret(&h.ctx).await;
for (id, key, value) in [
(2, "no_such_field", serde_json::json!(1)),
(3, "max_restarts", serde_json::json!("forty")),
(
4,
"cron_restart",
serde_json::json!("not a cron expression"),
),
] {
let reply = set_field(&h.ctx, id, "web", key, value).await;
let Err(err) = reply else {
panic!("{key} was accepted")
};
assert_eq!(err.code, RpcErrorCode::InvalidConfig, "{key}");
assert!(
shep_core::overrides::get(&h.ctx.paths.overrides, "web")
.unwrap()
.is_none(),
"{key}: the refusal still wrote the store"
);
}
}
#[tokio::test(start_paused = true)]
async fn an_unreadable_store_is_internal_for_a_field_edit_too() {
let h = harness(vec![ProcScript::never_exits()]);
start_web_with_a_secret(&h.ctx).await;
std::fs::write(&h.ctx.paths.overrides, "{ this is not json").unwrap();
let reply = set_field(&h.ctx, 2, "web", "max_restarts", serde_json::json!(40)).await;
let Err(err) = reply else {
panic!("an unreadable store was reported as success")
};
assert_eq!(err.code, RpcErrorCode::Internal);
assert!(
err.message.contains("overrides store unusable"),
"{}",
err.message
);
}
#[tokio::test(start_paused = true)]
async fn a_field_edit_reaches_the_muster_roll() {
let h = harness(vec![ProcScript::never_exits()]);
start_web_with_a_secret(&h.ctx).await;
let reply = set_field(&h.ctx, 2, "web", "max_restarts", serde_json::json!(40)).await;
assert!(reply.is_ok(), "{reply:?}");
let infos = list_flock(&h.ctx, 3).await;
let roll = h.ctx.registry.roll(&infos, 0);
let web = roll
.apps
.iter()
.find(|entry| entry.app.name == "web")
.expect("web is in the roll");
assert_eq!(web.app.max_restarts, 40);
}
#[tokio::test(start_paused = true)]
async fn every_new_variant_reaches_an_arm_and_not_the_wildcard() {
let h = harness(vec![]);
let requests = [
Request::SheepConfig {
name: "ghost".to_string(),
},
Request::SetSheepEnv {
name: "ghost".to_string(),
key: "K".to_string(),
value: None,
},
Request::SetSheepField {
name: "ghost".to_string(),
key: "max_restarts".to_string(),
value: serde_json::json!(1),
},
Request::SetDogConfig {
name: "ghost".to_string(),
toml: String::new().into(),
},
];
for (id, request) in requests.into_iter().enumerate() {
let reply = reply_of(
dispatch(
envelope(
u64::try_from(id).expect("three requests fit a u64"),
request,
),
&h.ctx,
)
.await,
);
let Err(err) = reply.result else {
continue;
};
assert_ne!(
err.message, "this daemon does not implement that request",
"a config-pane variant fell through to the wildcard"
);
}
}
#[tokio::test(start_paused = true)]
async fn disabling_a_dog_stops_it_and_takes_it_off_the_listing() {
let h = harness(vec![ProcScript::never_exits()]);
let info = enable_dog(&h.ctx, 1, "bark").await;
assert_eq!(info.dog, Some(DogSource::BuiltIn));
let disabled = reply_of(
dispatch(
envelope(
2,
Request::DisableDog {
name: "bark".to_string(),
},
),
&h.ctx,
)
.await,
);
assert_eq!(disabled.result.unwrap(), Response::Deleted(vec![info.id]));
assert!(h.ctx.supervisor.list().await.is_empty());
}
#[tokio::test(start_paused = true)]
async fn a_dog_config_request_reads_the_file_as_it_stands_now() {
let h = harness(vec![]);
std::fs::write(&h.ctx.dogs_config, "[bark]\ndebounce = \"30s\"\n").unwrap();
let reply = reply_of(
dispatch(
envelope(
1,
Request::DogConfig {
name: "bark".to_string(),
},
),
&h.ctx,
)
.await,
);
let Ok(Response::DogSection { toml }) = reply.result else {
panic!("expected DogSection, got {:?}", reply.result)
};
assert!(toml.as_str().contains("30s"));
}
#[tokio::test(start_paused = true)]
async fn set_dog_config_writes_the_file_and_a_subscriber_hears_about_it() {
let h = harness(vec![ProcScript::never_exits()]);
enable_dog(&h.ctx, 1, "bark").await;
let mut sub = h.ctx.events.subscribe();
let reply = reply_of(
dispatch(
envelope(
2,
Request::SetDogConfig {
name: "bark".to_string(),
toml: "poll = \"30s\"\n".to_string().into(),
},
),
&h.ctx,
)
.await,
);
assert_eq!(
reply.result.unwrap(),
Response::DogConfigSet {
name: "bark".to_string()
}
);
let text = std::fs::read_to_string(&h.ctx.dogs_config).unwrap();
assert!(text.contains("poll = \"30s\""), "{text}");
let mut topics = Vec::new();
while let Ok(Ok(event)) = tokio::time::timeout(Duration::from_secs(5), sub.recv()).await {
topics.push(event.to_event().topic().into_owned());
if topics
.last()
.is_some_and(|topic| topic == "config.dog.bark")
{
break;
}
}
assert!(
topics.iter().any(|topic| topic == "config.dog.bark"),
"{topics:?}"
);
}
#[tokio::test(start_paused = true)]
async fn a_dog_that_is_adopted_and_never_started_can_still_be_configured() {
let mut h = harness(vec![]);
h.ctx.known_dogs = KnownDogs::new(["otel".to_string()].into_iter().collect());
let reply = reply_of(
dispatch(
envelope(
1,
Request::SetDogConfig {
name: "otel".to_string(),
toml: "endpoint = \"http://127.0.0.1:4317\"\n".to_string().into(),
},
),
&h.ctx,
)
.await,
);
assert_eq!(
reply.result.unwrap(),
Response::DogConfigSet {
name: "otel".to_string()
}
);
let text = std::fs::read_to_string(&h.ctx.dogs_config).unwrap();
assert!(text.contains("4317"), "{text}");
}
#[tokio::test(start_paused = true)]
async fn a_dog_that_is_enabled_but_not_running_can_still_be_configured() {
let h = harness(vec![]);
assert!(h.ctx.supervisor.list().await.is_empty());
let reply = reply_of(
dispatch(
envelope(
1,
Request::SetDogConfig {
name: "bark".to_string(),
toml: "poll = \"30s\"\n".to_string().into(),
},
),
&h.ctx,
)
.await,
);
assert_eq!(
reply.result.unwrap(),
Response::DogConfigSet {
name: "bark".to_string()
}
);
}
#[tokio::test(start_paused = true)]
async fn a_dog_adopted_since_boot_is_reached_through_the_running_flock() {
let mut h = harness(vec![ProcScript::never_exits()]);
h.ctx.known_dogs = KnownDogs::new(BTreeSet::new());
enable_dog(&h.ctx, 1, "bark").await;
let reply = reply_of(
dispatch(
envelope(
2,
Request::SetDogConfig {
name: "bark".to_string(),
toml: "poll = \"30s\"\n".to_string().into(),
},
),
&h.ctx,
)
.await,
);
assert_eq!(
reply.result.unwrap(),
Response::DogConfigSet {
name: "bark".to_string()
}
);
}
#[tokio::test(start_paused = true)]
async fn a_dog_adopted_since_boot_stays_configurable_once_it_is_disabled() {
let mut h = harness(vec![ProcScript::never_exits()]);
h.ctx.known_dogs = KnownDogs::new(BTreeSet::new());
enable_dog(&h.ctx, 1, "bark").await;
let disabled = reply_of(
dispatch(
envelope(
2,
Request::DisableDog {
name: "bark".to_string(),
},
),
&h.ctx,
)
.await,
);
assert!(disabled.result.is_ok(), "{:?}", disabled.result);
assert!(
!h.ctx
.supervisor
.list()
.await
.iter()
.any(|info| info.name == "bark"),
"the flock must not hold it, or the widening answers and the test means nothing"
);
let reply = reply_of(
dispatch(
envelope(
3,
Request::SetDogConfig {
name: "bark".to_string(),
toml: "poll = \"30s\"\n".to_string().into(),
},
),
&h.ctx,
)
.await,
);
assert_eq!(
reply.result.unwrap(),
Response::DogConfigSet {
name: "bark".to_string()
}
);
}
#[tokio::test(start_paused = true)]
async fn setting_a_dogs_config_over_a_sheeps_name_is_refused_and_writes_nothing() {
let h = harness(vec![ProcScript::never_exits()]);
let started = reply_of(
dispatch(
envelope(
1,
Request::Start {
apps: vec![AppConfig::minimal("web", "./srv")],
},
),
&h.ctx,
)
.await,
);
assert!(started.result.is_ok(), "{:?}", started.result);
std::fs::write(&h.ctx.dogs_config, "[bark]\npoll = \"60s\"\n").unwrap();
for (id, name) in [(2, "web"), (3, "ghost")] {
let reply = reply_of(
dispatch(
envelope(
id,
Request::SetDogConfig {
name: name.to_string(),
toml: "poll = \"30s\"\n".to_string().into(),
},
),
&h.ctx,
)
.await,
);
let Err(err) = reply.result else {
panic!("{name} is not a dog and must be refused")
};
assert_eq!(err.code, RpcErrorCode::NotFound, "{err:?}");
assert!(err.message.contains(name), "{err:?}");
}
assert_eq!(
std::fs::read_to_string(&h.ctx.dogs_config).unwrap(),
"[bark]\npoll = \"60s\"\n"
);
}
#[tokio::test]
async fn setting_a_dogs_config_against_a_stopped_engine_is_refused_not_a_panic() {
let h = harness(vec![]);
h.ctx.supervisor.shutdown().await;
let reply = reply_of(
dispatch(
envelope(
1,
Request::SetDogConfig {
name: "otel".to_string(),
toml: "poll = \"30s\"\n".to_string().into(),
},
),
&h.ctx,
)
.await,
);
let Err(err) = reply.result else {
panic!("otel is not a known dog and must be refused")
};
assert_eq!(err.code, RpcErrorCode::NotFound, "{err:?}");
}
#[tokio::test(start_paused = true)]
async fn describe_sweeps_past_a_dog_and_still_answers_when_one_is_named() {
let h = harness(vec![ProcScript::never_exits(), ProcScript::never_exits()]);
let started = reply_of(
dispatch(
envelope(
1,
Request::Start {
apps: vec![AppConfig::minimal("web", "./srv")],
},
),
&h.ctx,
)
.await,
);
assert!(started.result.is_ok(), "{:?}", started.result);
let dog = enable_dog(&h.ctx, 2, "bark").await;
let swept = reply_of(
dispatch(
envelope(
3,
Request::Describe {
selector: SelectorSpec::All,
},
),
&h.ctx,
)
.await,
);
let Ok(Response::Described(hits)) = swept.result else {
panic!("expected Described, got {:?}", swept.result)
};
assert_eq!(
hits.iter().map(|i| i.name.as_str()).collect::<Vec<_>>(),
vec!["web"],
"`all` is the flock, not the kennel"
);
let named = reply_of(
dispatch(
envelope(
4,
Request::Describe {
selector: SelectorSpec::Name("bark".to_string()),
},
),
&h.ctx,
)
.await,
);
let Ok(Response::Described(hits)) = named.result else {
panic!("expected Described, got {:?}", named.result)
};
assert_eq!(hits.len(), 1);
assert_eq!(hits[0].id, dog.id);
}
#[tokio::test(start_paused = true)]
async fn enabling_a_dog_over_a_sheeps_name_is_refused_rather_than_faked() {
let h = harness(vec![ProcScript::never_exits()]);
let started = reply_of(
dispatch(
envelope(
1,
Request::Start {
apps: vec![AppConfig::minimal("bark", "./srv")],
},
),
&h.ctx,
)
.await,
);
assert!(started.result.is_ok(), "{:?}", started.result);
let reply = reply_of(
dispatch(
envelope(
2,
Request::EnableDog {
name: "bark".to_string(),
source: DogSource::BuiltIn,
},
),
&h.ctx,
)
.await,
);
let Err(err) = reply.result else {
panic!("expected a refusal, got {:?}", reply.result)
};
assert_eq!(err.code, RpcErrorCode::InvalidConfig);
assert!(
err.message.contains("bark"),
"the refusal names the collision: {}",
err.message
);
let listed = h.ctx.supervisor.list().await;
assert_eq!(listed.len(), 1, "nothing was started: {listed:?}");
assert_eq!(listed[0].dog, None);
}
#[tokio::test(start_paused = true)]
async fn only_describe_carries_a_lamb_tree() {
let h = harness_identifying(
vec![ProcScript::never_exits()],
vec![
identity(FIRST_SCRIPTED_PID, None, "srv"),
identity(FIRST_SCRIPTED_PID + 1, Some(FIRST_SCRIPTED_PID), "node"),
],
);
reply_of(
dispatch(
envelope(
1,
Request::Start {
apps: vec![AppConfig::minimal("web", "./srv")],
},
),
&h.ctx,
)
.await,
);
let listed = reply_of(dispatch(envelope(2, Request::ListFlock), &h.ctx).await);
let Ok(Response::Flock(rows)) = listed.result else {
panic!("expected a flock listing");
};
assert!(
rows.iter().all(|row| row.lambs.is_none()),
"ListFlock must not walk the process table"
);
let described = reply_of(
dispatch(
envelope(
3,
Request::Describe {
selector: SelectorSpec::Name("web".to_string()),
},
),
&h.ctx,
)
.await,
);
let Ok(Response::Described(rows)) = described.result else {
panic!("expected a describe listing");
};
assert_eq!(
rows[0].lambs,
Some(vec![Lamb::new(FIRST_SCRIPTED_PID + 1, "node")])
);
}
async fn start_dog(ctx: &RpcContext, name: &str) -> ProcessInfo {
let spec = DogSpec {
name: name.to_string(),
source: DogSource::BuiltIn,
};
let app = crate::dogs::dog_app(&spec, &ctx.paths).expect("the dog fixture must assemble");
ctx.supervisor
.start_dog(app, DogSource::BuiltIn)
.await
.expect("the dog fixture must start")
}
async fn staleness(ctx: &RpcContext) -> (Vec<String>, Vec<String>) {
let reply = reply_of(dispatch(envelope(1, Request::DogStaleness), ctx).await);
let Ok(Response::DogStaleness { stale, pending }) = reply.result else {
panic!("expected a dog staleness answer");
};
(stale, pending)
}
#[tokio::test]
async fn a_flock_of_ordinary_sheep_has_nothing_stale_and_nothing_pending() {
let h = harness(vec![ProcScript::never_exits()]);
let started = reply_of(
dispatch(
envelope(
1,
Request::Start {
apps: vec![AppConfig::minimal("web", "./srv")],
},
),
&h.ctx,
)
.await,
);
started.result.expect("the sheep must start");
assert_eq!(staleness(&h.ctx).await, (Vec::new(), Vec::new()));
}
#[tokio::test]
async fn a_listing_says_which_dogs_have_answered_this_shepherd() {
let h = harness(vec![ProcScript::never_exits(), ProcScript::never_exits()]);
start_dog(&h.ctx, "metrics").await;
let silent = list_flock(&h.ctx, 1).await;
let dog = silent
.iter()
.find(|info| info.name == "metrics")
.expect("the dog must be listed");
assert_eq!(dog.handshook, Some(false));
assert_eq!(
dog.status,
ProcStatus::Online,
"the process is up, and the listing still says so"
);
h.ctx.dog_refusals.handshook("metrics");
let talking = list_flock(&h.ctx, 2).await;
assert_eq!(
talking
.iter()
.find(|info| info.name == "metrics")
.expect("still listed")
.handshook,
Some(true)
);
}
#[tokio::test]
async fn a_sheep_carries_no_handshake_fact_at_all() {
let h = harness(vec![ProcScript::never_exits()]);
start_web(&h).await;
let infos = list_flock(&h.ctx, 1).await;
assert_eq!(infos[0].name, "web");
assert_eq!(infos[0].handshook, None);
assert_eq!(
infos[0].dog_stale, None,
"a sheep is never given up on, because it was never asked to answer"
);
}
#[tokio::test]
async fn a_listing_says_which_silent_dogs_this_shepherd_gave_up_on() {
let h = harness(vec![ProcScript::never_exits()]);
start_dog(&h.ctx, "metrics").await;
let waiting = list_flock(&h.ctx, 1).await;
let dog = waiting
.iter()
.find(|info| info.name == "metrics")
.expect("the dog must be listed");
assert_eq!(dog.handshook, Some(false));
assert_eq!(
dog.dog_stale,
Some(false),
"a dog that has not answered YET is not one this shepherd gave up on"
);
h.ctx.dog_refusals.refused("metrics");
h.ctx.dog_refusals.refused("metrics");
let given_up = list_flock(&h.ctx, 2).await;
let dog = given_up
.iter()
.find(|info| info.name == "metrics")
.expect("still listed");
assert_eq!(dog.dog_stale, Some(true));
assert_eq!(
dog.status,
ProcStatus::Online,
"the process is still up, and the listing still says so"
);
h.ctx.dog_refusals.handshook("metrics");
let talking = list_flock(&h.ctx, 3).await;
let dog = talking
.iter()
.find(|info| info.name == "metrics")
.expect("still listed");
assert_eq!(dog.handshook, Some(true));
assert_eq!(dog.dog_stale, Some(false));
}
#[tokio::test]
async fn describe_carries_the_handshake_fact_too() {
let h = harness(vec![ProcScript::never_exits()]);
start_dog(&h.ctx, "metrics").await;
let described = reply_of(
dispatch(
envelope(
1,
Request::Describe {
selector: SelectorSpec::Name("metrics".to_string()),
},
),
&h.ctx,
)
.await,
);
let Ok(Response::Described(rows)) = described.result else {
panic!("expected a describe listing");
};
assert_eq!(rows[0].handshook, Some(false));
}
#[tokio::test]
async fn a_dog_that_has_not_handshaken_is_pending() {
let h = harness(vec![ProcScript::never_exits()]);
start_dog(&h.ctx, "metrics").await;
assert_eq!(
staleness(&h.ctx).await,
(Vec::new(), vec!["metrics".to_string()])
);
h.ctx.dog_refusals.handshook("metrics");
assert_eq!(
staleness(&h.ctx).await,
(Vec::new(), Vec::new()),
"a dog talking to this shepherd is settled and is not worth reporting"
);
}
#[tokio::test]
async fn a_dog_being_restarted_is_pending_and_then_stale() {
let h = harness(vec![ProcScript::never_exits()]);
start_dog(&h.ctx, "metrics").await;
h.ctx.dog_refusals.handshook("metrics");
h.ctx.dog_refusals.refused("metrics");
assert_eq!(
staleness(&h.ctx).await,
(Vec::new(), vec!["metrics".to_string()]),
"one refusal buys a restart; it does not condemn the dog"
);
h.ctx.dog_refusals.refused("metrics");
assert_eq!(
staleness(&h.ctx).await,
(vec!["metrics".to_string()], Vec::new()),
"a stale dog is a finding, not something still to wait on"
);
}
#[tokio::test]
async fn a_dog_that_has_stopped_running_is_not_waited_on() {
let h = harness(vec![ProcScript::never_exits()]);
start_dog(&h.ctx, "metrics").await;
assert_eq!(staleness(&h.ctx).await.1, vec!["metrics".to_string()]);
h.ctx
.supervisor
.stop(ProcessSelector::Name("metrics".to_string()))
.await
.expect("the dog must stop");
assert_eq!(
staleness(&h.ctx).await,
(Vec::new(), Vec::new()),
"a dog that is not running has nothing to answer with"
);
}
}