use core::future::Future;
use core::time::Duration;
use std::collections::{BTreeSet, HashMap};
use std::path::PathBuf;
use std::sync::Arc;
use tokio::sync::watch;
use shep_core::config::{DeclaredApp, NormalizeError, ResolvedApp, normalize_all};
use shep_core::paths::ShepPaths;
use shep_core::protocol::{
Envelope, Lamb, ProcessInfo, Reply, Request, Response, RpcError, RpcErrorCode, SelectorSpec,
SheepApplied,
};
use shep_core::selector::ProcessSelector;
use shep_core::signals::OperatorSignal;
use crate::bus::{Bus, TopicFilter};
use crate::dogs::DogSpec;
use crate::limits::stats::StatsState;
use crate::snapshot::{FlockRegistry, SnapshotError, write_atomic};
use crate::supervisor::{Applied, ConnId, SupervisorError, SupervisorHandle};
pub(crate) const DEFAULT_DEADLINE_MS: u64 = 5_000;
pub(crate) const MAX_DEADLINE_MS: u64 = 60_000;
#[derive(Clone, Debug)]
pub struct RpcContext {
pub(crate) supervisor: SupervisorHandle,
pub(crate) events: Bus,
pub(crate) registry: FlockRegistry,
pub(crate) snapshot_path: PathBuf,
pub(crate) dogs_config: PathBuf,
pub(crate) paths: ShepPaths,
pub(crate) daemon_version: String,
pub(crate) dog_refusals: crate::dogs::DogRefusals,
pub(crate) peer_contacts: crate::dogs::PeerContacts,
pub(crate) pid: u32,
pub(crate) shutdown: Arc<watch::Sender<bool>>,
pub(crate) stats: Arc<StatsState>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct SavedRoll {
pub path: PathBuf,
pub apps: u32,
}
impl RpcContext {
pub fn shutdown(&self) {
let _ = self.shutdown.send(true);
}
pub fn announce_dog_config(&self, dogs: &[String]) {
crate::bus::publish_dog_config_changed(&self.events, dogs);
}
pub async fn save_roll_now(&self) -> Result<Option<SavedRoll>, SnapshotError> {
let Ok(infos) = self.supervisor.list_checked().await else {
return Ok(None);
};
let roll = self.registry.roll(&infos, crate::now_ms());
write_atomic(&self.snapshot_path, &roll)?;
Ok(Some(SavedRoll {
path: self.snapshot_path.clone(),
apps: u32::try_from(roll.apps.len()).unwrap_or(u32::MAX),
}))
}
pub async fn snapshot_now(&self) -> Result<(), SnapshotError> {
self.save_roll_now().await.map(|_| ())
}
}
#[derive(Debug)]
pub(crate) enum Outcome {
Reply(Reply),
Subscribe {
reply: Reply,
filter: TopicFilter,
},
Shutdown(Reply),
}
#[must_use]
pub(crate) fn budget(deadline_ms: Option<u64>) -> Duration {
Duration::from_millis(
deadline_ms
.unwrap_or(DEFAULT_DEADLINE_MS)
.clamp(1, MAX_DEADLINE_MS),
)
}
pub(crate) async fn dispatch(envelope: Envelope, conn: ConnId, ctx: &RpcContext) -> Outcome {
let id = envelope.id;
with_deadline(
id,
budget(envelope.deadline_ms),
run(id, conn, envelope.body, ctx),
)
.await
}
async fn with_deadline<F: Future<Output = Outcome> + Send>(
id: u64,
budget: Duration,
work: F,
) -> Outcome {
match tokio::time::timeout(budget, work).await {
Ok(outcome) => outcome,
Err(_) => Outcome::Reply(Reply {
id,
result: Err(RpcError {
code: RpcErrorCode::DeadlineExceeded,
message: format!(
"the request deadline of {} ms expired before the daemon finished",
budget.as_millis()
),
daemon_version: None,
}),
}),
}
}
async fn run(id: u64, conn: ConnId, request: Request, ctx: &RpcContext) -> Outcome {
let reply = |result| Outcome::Reply(Reply { id, result });
match request {
Request::Ping => reply(Ok(Response::Pong)),
Request::ListFlock => match ctx.supervisor.list_checked().await {
Ok(infos) => reply(Ok(Response::Flock(with_dog_contact(
&ctx.dog_refusals,
with_live_stats(&ctx.stats, infos).await,
)))),
Err(err) => reply(Err(rpc_error(&err))),
},
Request::Describe { selector } => match selector_of(selector) {
Err(err) => reply(Err(err)),
Ok(selector) => match ctx.supervisor.list_checked().await {
Err(err) => reply(Err(rpc_error(&err))),
Ok(infos) => {
let exact = selector.is_exact();
let hits: Vec<_> = infos
.into_iter()
.filter(|i| exact || i.dog.is_none())
.filter(|i| selector.matches(&i.name, i.id, i.fold.as_deref(), i.instance))
.collect();
if hits.is_empty() {
reply(Err(not_found()))
} else {
let hits = with_live_stats(&ctx.stats, hits).await;
let hits = with_lambs(&ctx.stats, hits).await;
reply(Ok(Response::Described(with_dog_contact(
&ctx.dog_refusals,
hits,
))))
}
}
},
},
Request::Start { apps } => match normalize_all(apps) {
Err(err) => reply(Err(RpcError {
code: RpcErrorCode::InvalidConfig,
message: err.to_string(),
daemon_version: None,
})),
Ok(resolved) => {
ctx.registry.record(&resolved);
match ctx.supervisor.start(resolved).await {
Ok(infos) => reply(Ok(Response::Started(infos))),
Err(err) => reply(Err(rpc_error(&err))),
}
}
},
Request::Add { apps } => match normalize_all(apps) {
Err(err) => reply(Err(RpcError {
code: RpcErrorCode::InvalidConfig,
message: err.to_string(),
daemon_version: None,
})),
Ok(resolved) => {
ctx.registry.record(&resolved);
match ctx.supervisor.register_at_rest(resolved).await {
Ok(infos) => reply(Ok(Response::Added(infos))),
Err(err) => reply(Err(rpc_error(&err))),
}
}
},
Request::ConfigDrift { apps } => match normalize_all(apps) {
Err(err) => reply(Err(RpcError {
code: RpcErrorCode::InvalidConfig,
message: err.to_string(),
daemon_version: None,
})),
Ok(resolved) => match ctx.supervisor.config_drift(resolved).await {
Ok(drifted) => reply(Ok(Response::Drifted(drifted))),
Err(err) => reply(Err(rpc_error(&err))),
},
},
Request::Stop { selector } => {
selector_call(id, selector, |s| ctx.supervisor.stop(s), Response::Stopped).await
}
Request::Restart { selector } => {
selector_call(
id,
selector,
|s| ctx.supervisor.restart(s),
Response::Restarted,
)
.await
}
Request::Reload { selector } => {
selector_call(
id,
selector,
|s| ctx.supervisor.reload(s),
Response::Reloading,
)
.await
}
Request::Reopen { selector } => {
selector_call(
id,
selector,
|s| ctx.supervisor.reopen(s),
Response::Reopened,
)
.await
}
Request::Flush { selector } => {
selector_call(id, selector, |s| ctx.supervisor.flush(s), Response::Flushed).await
}
Request::Trigger {
selector,
action,
params,
} => trigger(id, selector, action, params, ctx).await,
Request::Signal { selector, signal } => signal_request(id, selector, signal, ctx).await,
Request::SendLine { selector, line } => {
if line.contains(['\n', '\r']) {
return reply(Err(RpcError {
code: RpcErrorCode::InvalidConfig,
message: "a line may not contain a newline or a carriage return; \
send one line per request"
.to_string(),
daemon_version: None,
}));
}
match selector_of(selector) {
Ok(selector) => match ctx.supervisor.send_line(selector, line).await {
Ok(rows) => reply(Ok(Response::SentLine(rows))),
Err(err) => reply(Err(rpc_error(&err))),
},
Err(err) => reply(Err(err)),
}
}
Request::Delete { selector } => match selector_of(selector) {
Err(err) => reply(Err(err)),
Ok(selector) => match ctx.supervisor.delete(selector).await {
Ok(ids) => reply(Ok(Response::Deleted(ids))),
Err(err) => reply(Err(rpc_error(&err))),
},
},
Request::Scale { name, count } => match ctx.supervisor.scale(&name, count).await {
Ok(scaled) => {
let achieved = scaled.achieved();
let requested = scaled.requested;
ctx.registry.record(&[scaled.app]);
match scaled.shortfall {
None => reply(Ok(Response::Scaled(scaled.instances))),
Some(message) => reply(Err(RpcError {
code: RpcErrorCode::SpawnFailed,
message: format!(
"scaled {name} to {achieved} of {requested} requested; \
the next instance would not spawn: {message}"
),
daemon_version: None,
})),
}
}
Err(err) => reply(Err(rpc_error(&err))),
},
Request::SetSmit { sheep, smit } => {
match ctx.supervisor.set_smit(conn, &sheep, smit).await {
Ok(infos) => reply(Ok(Response::SmitPainted(infos))),
Err(err) => reply(Err(rpc_error(&err))),
}
}
Request::SaveRoll => match ctx.save_roll_now().await {
Ok(Some(saved)) => reply(Ok(Response::RollSaved {
path: saved.path.to_string_lossy().into_owned(),
apps: saved.apps,
})),
Ok(None) => reply(Err(RpcError {
code: RpcErrorCode::Internal,
message: "the supervisor engine has stopped; no roll was written".to_string(),
daemon_version: None,
})),
Err(err) => reply(Err(RpcError {
code: RpcErrorCode::Internal,
message: err.to_string(),
daemon_version: None,
})),
},
Request::Muster => {
match crate::snapshot::muster(&ctx.snapshot_path, &ctx.registry, &ctx.supervisor).await
{
Err(err) => reply(Err(RpcError {
code: RpcErrorCode::Internal,
message: err.to_string(),
daemon_version: None,
})),
Ok(names) => match ctx.supervisor.list_checked().await {
Err(err) => reply(Err(rpc_error(&err))),
Ok(infos) => reply(Ok(Response::Mustered(
infos
.into_iter()
.filter(|info| names.contains(&info.name))
.collect(),
))),
},
}
}
Request::DogConfig { name } => match crate::dogs::dog_section(&ctx.dogs_config, &name) {
Ok(toml) => reply(Ok(Response::DogSection { toml: toml.into() })),
Err(err) => reply(Err(RpcError {
code: RpcErrorCode::InvalidConfig,
message: err.to_string(),
daemon_version: None,
})),
},
Request::EnableDog { name, source } => {
let spec = DogSpec { name, source };
match crate::dogs::dog_app(&spec, &ctx.paths) {
Err(err) => reply(Err(RpcError {
code: RpcErrorCode::InvalidConfig,
message: err.to_string(),
daemon_version: None,
})),
Ok(app) => {
let script = app.config().script.clone();
match ctx.supervisor.start_dog(app, spec.source).await {
Ok(info) if info.dog.is_none() => reply(Err(RpcError {
code: RpcErrorCode::InvalidConfig,
message: format!(
"a sheep is already registered as `{}`; rename it or give the dog another name",
spec.name
),
daemon_version: None,
})),
Ok(info) => {
crate::dogs::narrate(
&ctx.events,
&info,
&format!(
"shep has this dog enabled, running the binary at {script}"
),
)
.await;
reply(Ok(Response::DogStarted(info)))
}
Err(err) => reply(Err(rpc_error(&err))),
}
}
}
}
Request::DisableDog { name } => {
match ctx.supervisor.delete(ProcessSelector::Name(name)).await {
Ok(ids) => reply(Ok(Response::Deleted(ids))),
Err(err) => reply(Err(rpc_error(&err))),
}
}
Request::Subscribe { topics } => match TopicFilter::new(&topics) {
Ok(filter) => Outcome::Subscribe {
reply: Reply {
id,
result: Ok(Response::Subscribed),
},
filter,
},
Err(err) => reply(Err(RpcError {
code: RpcErrorCode::InvalidConfig,
message: err.to_string(),
daemon_version: None,
})),
},
Request::DogStaleness => {
let (stale, pending) = dog_staleness(ctx).await;
reply(Ok(Response::DogStaleness { stale, pending }))
}
Request::HandoverFitness => reply(Ok(Response::HandoverFitness {
refusal: handover_refusal(ctx).await,
})),
Request::KillDaemon => Outcome::Shutdown(Reply {
id,
result: Ok(Response::ShuttingDown),
}),
Request::ApplyConfig { apps, reset } => match duplicate_name(&apps) {
Some(name) => reply(Err(RpcError {
code: RpcErrorCode::InvalidConfig,
message: NormalizeError::DuplicateName(name).to_string(),
daemon_version: None,
})),
None => match ctx.supervisor.apply_config(apps, reset).await {
Ok(applied) => {
let recorded: Vec<ResolvedApp> =
applied.iter().filter_map(|a| a.app.clone()).collect();
ctx.registry.record(&recorded);
reply(Ok(Response::Applied(
applied.into_iter().map(SheepApplied::from).collect(),
)))
}
Err(err) => reply(Err(rpc_error(&err))),
},
},
_ => reply(Err(RpcError {
code: RpcErrorCode::Internal,
message: "this daemon does not implement that request".to_string(),
daemon_version: None,
})),
}
}
fn duplicate_name(apps: &[DeclaredApp]) -> Option<String> {
let mut seen = BTreeSet::new();
apps.iter()
.find(|app| !seen.insert(app.config.name.as_str()))
.map(|app| app.config.name.clone())
}
impl From<Applied> for SheepApplied {
fn from(applied: Applied) -> Self {
Self::new(
applied.name,
applied.applied,
applied.pending,
applied.refused,
)
}
}
async fn dog_staleness(ctx: &RpcContext) -> (Vec<String>, Vec<String>) {
let stale = ctx.dog_refusals.stale();
let mut pending = ctx.dog_refusals.restarting();
if let Ok(infos) = ctx.supervisor.list_checked().await {
pending.extend(crate::dogs::silent_dogs(&infos, &ctx.dog_refusals));
}
pending.sort();
pending.dedup();
(stale, pending)
}
#[cfg(unix)]
async fn handover_refusal(ctx: &RpcContext) -> Option<String> {
match ctx.supervisor.handover_fitness().await {
Ok(crate::handover::Fitness::Carryable) => None,
Ok(crate::handover::Fitness::Refused(reason)) => Some(reason.to_string()),
Err(err) => Some(format!(
"this shepherd could not check whether its flock can be handed over ({err})"
)),
}
}
#[cfg(windows)]
#[expect(
clippy::unused_async,
reason = "one signature for both platforms; the unix arm awaits the supervisor"
)]
async fn handover_refusal(_ctx: &RpcContext) -> Option<String> {
Some(
"this shepherd runs on Windows, which has no `execve`, so its flock cannot be handed to \
a successor in place"
.to_string(),
)
}
async fn with_live_stats(stats: &Arc<StatsState>, mut infos: Vec<ProcessInfo>) -> Vec<ProcessInfo> {
let stats = Arc::clone(stats);
let Ok(sample) = tokio::task::spawn_blocking(move || stats.sample_now()).await else {
return infos;
};
for info in &mut infos {
if let Some(reading) = info.pid.and_then(|pid| sample.get(&pid)) {
info.cpu_percent = reading.cpu_percent;
info.memory_bytes = Some(reading.memory_bytes);
}
}
infos
}
fn with_dog_contact(
refusals: &crate::dogs::DogRefusals,
mut infos: Vec<ProcessInfo>,
) -> Vec<ProcessInfo> {
let stale = refusals.stale();
for info in &mut infos {
if info.dog.is_some() {
info.handshook = Some(refusals.has_handshook(&info.name));
info.dog_stale = Some(stale.contains(&info.name));
}
}
infos
}
async fn with_lambs(stats: &Arc<StatsState>, mut infos: Vec<ProcessInfo>) -> Vec<ProcessInfo> {
if infos.iter().all(|info| info.pid.is_none()) {
return infos;
}
let stats = Arc::clone(stats);
let pids: Vec<u32> = infos.iter().filter_map(|info| info.pid).collect();
let Ok(walked) = tokio::task::spawn_blocking(move || {
let index = stats.lamb_index();
pids.into_iter()
.map(|pid| (pid, stats.lambs_of(&index, pid)))
.collect::<HashMap<u32, Vec<Lamb>>>()
})
.await
else {
return infos;
};
for info in &mut infos {
if let Some(lambs) = info.pid.and_then(|pid| walked.get(&pid)) {
info.lambs = Some(lambs.clone());
}
}
infos
}
fn rpc_error(err: &SupervisorError) -> RpcError {
match err {
SupervisorError::NotFound => not_found(),
SupervisorError::SpawnFailed(msg) => RpcError {
code: RpcErrorCode::SpawnFailed,
message: msg.clone(),
daemon_version: None,
},
SupervisorError::CannotStart(msg) => RpcError {
code: RpcErrorCode::SpawnFailed,
message: msg.clone(),
daemon_version: None,
},
SupervisorError::ReopenFailed(_) | SupervisorError::FlushFailed(_) => RpcError {
code: RpcErrorCode::Internal,
message: err.to_string(),
daemon_version: None,
},
SupervisorError::ReloadInFlight(_) => RpcError {
code: RpcErrorCode::Internal,
message: err.to_string(),
daemon_version: None,
},
SupervisorError::InvalidScale(msg) => RpcError {
code: RpcErrorCode::InvalidConfig,
message: msg.clone(),
daemon_version: None,
},
SupervisorError::EngineStopped => RpcError {
code: RpcErrorCode::Internal,
message: "the supervisor engine has stopped".to_string(),
daemon_version: None,
},
}
}
fn not_found() -> RpcError {
RpcError {
code: RpcErrorCode::NotFound,
message: "selector matched no registered sheep".to_string(),
daemon_version: None,
}
}
fn selector_of(spec: SelectorSpec) -> Result<ProcessSelector, RpcError> {
ProcessSelector::try_from(spec).map_err(|err| RpcError {
code: RpcErrorCode::InvalidConfig,
message: err.to_string(),
daemon_version: None,
})
}
async fn selector_call<F, Fut>(
id: u64,
spec: SelectorSpec,
call: F,
ok: fn(Vec<ProcessInfo>) -> Response,
) -> Outcome
where
F: FnOnce(ProcessSelector) -> Fut + Send,
Fut: Future<Output = Result<Vec<ProcessInfo>, SupervisorError>> + Send,
{
let result = match selector_of(spec) {
Ok(selector) => call(selector).await.map(ok).map_err(|err| rpc_error(&err)),
Err(err) => Err(err),
};
Outcome::Reply(Reply { id, result })
}
async fn trigger(
id: u64,
spec: SelectorSpec,
action: String,
params: Option<String>,
ctx: &RpcContext,
) -> Outcome {
let result = match selector_of(spec) {
Err(err) => Err(err),
Ok(selector) => ctx
.supervisor
.trigger(selector, action, params)
.await
.map(Response::Triggered)
.map_err(|err| rpc_error(&err)),
};
Outcome::Reply(Reply { id, result })
}
async fn signal_request(id: u64, spec: SelectorSpec, signal: String, ctx: &RpcContext) -> Outcome {
let result = match OperatorSignal::parse(&signal) {
None => Err(RpcError {
code: RpcErrorCode::InvalidConfig,
message: format!(
"`{signal}` is not a signal shep will send; accepted: {}",
OperatorSignal::ACCEPTED.join(", ")
),
daemon_version: None,
}),
Some(sig) => match selector_of(spec) {
Err(err) => Err(err),
Ok(selector) => ctx
.supervisor
.signal(selector, sig)
.await
.map(Response::Signalled)
.map_err(|err| rpc_error(&err)),
},
};
Outcome::Reply(Reply { id, result })
}
#[cfg(test)]
mod tests {
use super::*;
use crate::fake::{FIRST_SCRIPTED_PID, ProcScript};
use crate::limits::MEMORY_POLL_INTERVAL;
use crate::testing::{
Harness, SCRIPTED_TREE_BYTES, harness, harness_identifying, harness_with_stats, identity,
};
use shep_core::config::{AppConfig, DeclaredApp, ResetDepth};
use shep_core::protocol::{
ActionOutcome, ActionReply, DogSource, Request, Response, RpcErrorCode, SelectorSpec,
};
use shep_core::status::ProcStatus;
use shep_core::values::UpDuration;
use std::collections::BTreeSet;
use tokio::time::Instant;
async fn dispatch(envelope: Envelope, ctx: &RpcContext) -> Outcome {
super::dispatch(envelope, ConnId::next(), ctx).await
}
fn envelope(id: u64, body: Request) -> Envelope {
Envelope {
id,
deadline_ms: None,
body,
}
}
fn reply_of(outcome: Outcome) -> Reply {
match outcome {
Outcome::Reply(reply) | Outcome::Subscribe { reply, .. } | Outcome::Shutdown(reply) => {
reply
}
}
}
#[tokio::test(start_paused = true)]
async fn ping_answers_pong_on_the_same_envelope_id() {
let h = harness(vec![]);
let reply = reply_of(dispatch(envelope(9, Request::Ping), &h.ctx).await);
assert_eq!(reply.id, 9);
assert_eq!(reply.result.unwrap(), Response::Pong);
}
#[tokio::test(start_paused = true)]
async fn start_registers_the_config_and_lists_it() {
let h = harness(vec![ProcScript::never_exits()]);
let started = reply_of(
dispatch(
envelope(
1,
Request::Start {
apps: vec![AppConfig::minimal("web", "./srv")],
},
),
&h.ctx,
)
.await,
);
let Response::Started(infos) = started.result.unwrap() else {
panic!("expected started")
};
assert_eq!(infos.len(), 1);
assert_eq!(infos[0].status, ProcStatus::Online);
let roll = h.ctx.registry.roll(&infos, 0);
assert_eq!(roll.apps.len(), 1);
assert_eq!(roll.apps[0].app.script, "./srv");
let listed = reply_of(dispatch(envelope(2, Request::ListFlock), &h.ctx).await);
let Response::Flock(flock) = listed.result.unwrap() else {
panic!("expected flock")
};
assert_eq!(flock.len(), 1);
}
#[tokio::test(start_paused = true)]
async fn start_re_normalizes_untrusted_peer_config() {
let h = harness(vec![]);
let reply = reply_of(
dispatch(
envelope(
1,
Request::Start {
apps: vec![AppConfig::minimal("", "./srv")],
},
),
&h.ctx,
)
.await,
);
assert_eq!(reply.result.unwrap_err().code, RpcErrorCode::InvalidConfig);
}
#[tokio::test(start_paused = true)]
async fn add_registers_a_stopped_member_and_spawns_nothing() {
let h = harness(vec![]);
let added = reply_of(
dispatch(
envelope(
1,
Request::Add {
apps: vec![AppConfig::minimal("web", "./srv")],
},
),
&h.ctx,
)
.await,
);
let Response::Added(infos) = added.result.unwrap() else {
panic!("expected added")
};
assert_eq!(infos.len(), 1);
assert_eq!(infos[0].status, ProcStatus::Stopped);
assert_eq!(infos[0].pid, None, "nothing was spawned");
let roll = h.ctx.registry.roll(&infos, 0);
assert_eq!(roll.apps.len(), 1);
assert_eq!(roll.apps[0].app.script, "./srv");
assert_eq!(roll.apps[0].instances_running, 0);
let listed = reply_of(dispatch(envelope(2, Request::ListFlock), &h.ctx).await);
let Response::Flock(flock) = listed.result.unwrap() else {
panic!("expected flock")
};
assert_eq!(
flock.len(),
1,
"it is a member of the flock, just a still one"
);
}
#[tokio::test(start_paused = true)]
async fn add_re_normalizes_untrusted_peer_config() {
let h = harness(vec![]);
let reply = reply_of(
dispatch(
envelope(
1,
Request::Add {
apps: vec![AppConfig::minimal("", "./srv")],
},
),
&h.ctx,
)
.await,
);
assert_eq!(reply.result.unwrap_err().code, RpcErrorCode::InvalidConfig);
}
#[tokio::test(start_paused = true)]
async fn a_second_add_leaves_a_running_sheep_exactly_as_it_was() {
let h = harness(vec![ProcScript::never_exits()]);
let started = reply_of(
dispatch(
envelope(
1,
Request::Start {
apps: vec![AppConfig::minimal("web", "./srv")],
},
),
&h.ctx,
)
.await,
);
let Response::Started(before) = started.result.unwrap() else {
panic!("expected started")
};
let added = reply_of(
dispatch(
envelope(
2,
Request::Add {
apps: vec![AppConfig::minimal("web", "./srv")],
},
),
&h.ctx,
)
.await,
);
let Response::Added(after) = added.result.unwrap() else {
panic!("expected added")
};
assert_eq!(after.len(), 1);
assert_eq!(
after[0].id, before[0].id,
"the same sheep, not a second one"
);
assert_eq!(after[0].status, ProcStatus::Online, "still running");
assert_eq!(after[0].pid, before[0].pid, "the same process");
let listed = reply_of(dispatch(envelope(3, Request::ListFlock), &h.ctx).await);
let Response::Flock(flock) = listed.result.unwrap() else {
panic!("expected flock")
};
assert_eq!(flock.len(), 1, "one row, not two");
}
fn declared(name: &str, script: &str, keys: &[&str]) -> DeclaredApp {
DeclaredApp {
config: AppConfig::minimal(name, script),
declared: keys.iter().map(|k| (*k).to_string()).collect(),
declared_env: BTreeSet::new(),
}
}
#[tokio::test(start_paused = true)]
async fn apply_config_refuses_a_request_naming_one_app_twice() {
let h = harness(vec![ProcScript::never_exits()]);
let _started = reply_of(
dispatch(
envelope(
1,
Request::Start {
apps: vec![AppConfig::minimal("web", "./srv")],
},
),
&h.ctx,
)
.await,
);
let reply = reply_of(
dispatch(
envelope(
2,
Request::ApplyConfig {
apps: vec![
declared("web", "./one", &["script"]),
declared("web", "./two", &["script"]),
],
reset: ResetDepth::None,
},
),
&h.ctx,
)
.await,
);
let err = reply.result.unwrap_err();
assert_eq!(err.code, RpcErrorCode::InvalidConfig);
assert!(err.message.contains("web"), "the name is named: {err:?}");
let listed = reply_of(dispatch(envelope(3, Request::ListFlock), &h.ctx).await);
let Response::Flock(flock) = listed.result.unwrap() else {
panic!("expected flock")
};
let roll = h.ctx.registry.roll(&flock, 0);
assert_eq!(roll.apps[0].app.script, "./srv");
}
#[tokio::test(start_paused = true)]
async fn apply_config_records_what_it_applied_in_the_registry() {
let h = harness(vec![ProcScript::never_exits()]);
let _started = reply_of(
dispatch(
envelope(
1,
Request::Start {
apps: vec![AppConfig::minimal("web", "./srv")],
},
),
&h.ctx,
)
.await,
);
let reply = reply_of(
dispatch(
envelope(
2,
Request::ApplyConfig {
apps: vec![DeclaredApp {
config: {
let mut app = AppConfig::minimal("web", "./srv");
app.max_restarts = 99;
app
},
declared: ["max_restarts"].iter().map(|k| (*k).to_string()).collect(),
declared_env: BTreeSet::new(),
}],
reset: ResetDepth::None,
},
),
&h.ctx,
)
.await,
);
let Response::Applied(report) = reply.result.unwrap() else {
panic!("expected applied")
};
assert_eq!(report.len(), 1);
assert_eq!(report[0].name, "web");
assert_eq!(report[0].applied, vec!["max_restarts".to_string()]);
assert_eq!(report[0].refused, None);
let listed = reply_of(dispatch(envelope(3, Request::ListFlock), &h.ctx).await);
let Response::Flock(flock) = listed.result.unwrap() else {
panic!("expected flock")
};
let roll = h.ctx.registry.roll(&flock, 0);
assert_eq!(roll.apps[0].app.max_restarts, 99);
}
#[tokio::test(start_paused = true)]
async fn apply_config_refuses_an_unregistered_app_inside_the_reply() {
let h = harness(vec![]);
let reply = reply_of(
dispatch(
envelope(
1,
Request::ApplyConfig {
apps: vec![declared("ghost", "./srv", &["script"])],
reset: ResetDepth::None,
},
),
&h.ctx,
)
.await,
);
let Response::Applied(report) = reply.result.unwrap() else {
panic!("expected applied")
};
assert_eq!(report.len(), 1);
let refused = report[0].refused.as_deref().unwrap_or_default();
assert!(
refused.contains("ghost"),
"the refusal names the app: {refused}"
);
}
#[tokio::test(start_paused = true)]
async fn a_selector_matching_nothing_is_not_found() {
let h = harness(vec![]);
let reply = reply_of(
dispatch(
envelope(
1,
Request::Stop {
selector: SelectorSpec::Name("ghost".to_string()),
},
),
&h.ctx,
)
.await,
);
assert_eq!(reply.result.unwrap_err().code, RpcErrorCode::NotFound);
}
#[tokio::test(start_paused = true)]
async fn a_bad_peer_regex_is_invalid_config_not_a_panic() {
let h = harness(vec![]);
let reply = reply_of(
dispatch(
envelope(
1,
Request::Describe {
selector: SelectorSpec::Regex("((".to_string()),
},
),
&h.ctx,
)
.await,
);
assert_eq!(reply.result.unwrap_err().code, RpcErrorCode::InvalidConfig);
}
#[tokio::test(start_paused = true)]
async fn describe_filters_by_fold() {
let h = harness(vec![ProcScript::never_exits(), ProcScript::never_exits()]);
let mut api = AppConfig::minimal("api", "./a");
api.fold = Some("backend".to_string());
dispatch(
envelope(
1,
Request::Start {
apps: vec![api, AppConfig::minimal("web", "./w")],
},
),
&h.ctx,
)
.await;
let reply = reply_of(
dispatch(
envelope(
2,
Request::Describe {
selector: SelectorSpec::Fold("backend".to_string()),
},
),
&h.ctx,
)
.await,
);
let Response::Described(hits) = reply.result.unwrap() else {
panic!("expected described")
};
assert_eq!(hits.len(), 1);
assert_eq!(hits[0].name, "api");
}
#[tokio::test(start_paused = true)]
async fn reopen_routes_to_the_supervisor_and_leaves_the_sheep_running() {
let h = harness(vec![ProcScript::never_exits()]);
dispatch(
envelope(
1,
Request::Start {
apps: vec![AppConfig::minimal("web", "./srv")],
},
),
&h.ctx,
)
.await;
let reply = reply_of(
dispatch(
envelope(
2,
Request::Reopen {
selector: SelectorSpec::Name("web".to_string()),
},
),
&h.ctx,
)
.await,
);
let Response::Reopened(infos) = reply.result.unwrap() else {
panic!("expected reopened")
};
assert_eq!(infos.len(), 1);
assert_eq!(infos[0].status, ProcStatus::Online);
let listed = reply_of(dispatch(envelope(3, Request::ListFlock), &h.ctx).await);
let Response::Flock(flock) = listed.result.unwrap() else {
panic!("expected flock")
};
assert_eq!(
flock[0].status,
ProcStatus::Online,
"a reopen must not disturb the sheep it reopens"
);
}
#[tokio::test(start_paused = true)]
async fn reload_routes_to_the_supervisor_and_starts_a_swap() {
let h = harness(vec![ProcScript::never_exits(); 3]);
dispatch(
envelope(
1,
Request::Start {
apps: vec![AppConfig::minimal("web", "./srv")],
},
),
&h.ctx,
)
.await;
let reply = reply_of(
dispatch(
envelope(
2,
Request::Reload {
selector: SelectorSpec::Name("web".to_string()),
},
),
&h.ctx,
)
.await,
);
let Response::Reloading(accepted) = reply.result.unwrap() else {
panic!("expected reloading")
};
assert_eq!(accepted.len(), 1);
assert_eq!(
accepted[0].status,
ProcStatus::Online,
"the answer is the flock as it stood when the reload was accepted"
);
let listed = reply_of(dispatch(envelope(3, Request::ListFlock), &h.ctx).await);
let Response::Flock(flock) = listed.result.unwrap() else {
panic!("expected flock")
};
assert_eq!(
flock.len(),
2,
"a swap in progress is two entries in one instance slot, not one: {flock:?}"
);
assert_eq!(flock[0].id, accepted[0].id);
assert_eq!(flock[0].status, ProcStatus::Stopping);
assert_ne!(
flock[1].id, accepted[0].id,
"the replacement takes a new id"
);
assert_eq!(flock[1].status, ProcStatus::Starting);
}
#[test]
fn a_refused_reload_is_internal_and_says_which_refusal_it_was() {
let in_flight = rpc_error(&SupervisorError::ReloadInFlight("web".to_string()));
assert_eq!(in_flight.code, RpcErrorCode::Internal);
assert_eq!(in_flight.message, "web is already being reloaded");
let shutting_down = rpc_error(&SupervisorError::EngineStopped);
assert_eq!(shutting_down.code, RpcErrorCode::Internal);
assert_eq!(shutting_down.message, "the supervisor engine has stopped");
}
#[tokio::test(start_paused = true)]
async fn a_bad_reload_selector_is_invalid_config() {
let h = harness(vec![]);
let reply = reply_of(
dispatch(
envelope(
1,
Request::Reload {
selector: SelectorSpec::Regex("((".to_string()),
},
),
&h.ctx,
)
.await,
);
assert_eq!(reply.result.unwrap_err().code, RpcErrorCode::InvalidConfig);
}
#[tokio::test(start_paused = true)]
async fn a_bad_reopen_selector_is_invalid_config() {
let h = harness(vec![]);
let reply = reply_of(
dispatch(
envelope(
1,
Request::Reopen {
selector: SelectorSpec::Regex("((".to_string()),
},
),
&h.ctx,
)
.await,
);
assert_eq!(reply.result.unwrap_err().code, RpcErrorCode::InvalidConfig);
}
#[tokio::test(start_paused = true)]
async fn trigger_routes_to_the_flock_and_reports_each_match_within_the_budget() {
let h = harness(vec![ProcScript::never_exits(), ProcScript::never_exits()]);
let mut web = AppConfig::minimal("web", "./srv");
web.channel = true;
let started = reply_of(
dispatch(
envelope(
1,
Request::Start {
apps: vec![AppConfig::minimal("other", "./o"), web],
},
),
&h.ctx,
)
.await,
);
let Response::Started(started) = started.result.unwrap() else {
panic!("expected started")
};
let web_id = started
.iter()
.find(|i| i.name == "web")
.expect("web registered")
.id;
assert_ne!(web_id, 0, "the test's own premise: web must not be id 0");
let reply = reply_of(
dispatch(
envelope(
2,
Request::Trigger {
selector: SelectorSpec::Name("web".to_string()),
action: "gc".to_string(),
params: None,
},
),
&h.ctx,
)
.await,
);
let Response::Triggered(rows) = reply.result.unwrap() else {
panic!("expected triggered")
};
assert_eq!(
rows,
vec![ActionReply {
id: web_id,
name: "web".to_string(),
outcome: ActionOutcome::TimedOut,
}]
);
}
#[tokio::test]
async fn a_signal_name_outside_the_grammar_is_refused_with_the_accepted_list() {
let h = harness(vec![]);
let reply = reply_of(
dispatch(
envelope(
1,
Request::Signal {
selector: SelectorSpec::All,
signal: "SIGHUPP".to_string(),
},
),
&h.ctx,
)
.await,
);
let err = reply.result.unwrap_err();
assert_eq!(err.code, RpcErrorCode::InvalidConfig);
assert!(err.message.contains("SIGHUPP"), "{}", err.message);
assert!(err.message.contains("SIGHUP"), "{}", err.message);
assert!(err.message.contains("SIGUSR2"), "{}", err.message);
}
#[tokio::test]
async fn a_line_carrying_a_newline_is_refused_before_it_reaches_the_supervisor() {
let h = harness(vec![]);
let reply = reply_of(
dispatch(
envelope(
1,
Request::SendLine {
selector: SelectorSpec::All,
line: "reload\nrm -rf /".to_string(),
},
),
&h.ctx,
)
.await,
);
let err = reply.result.unwrap_err();
assert_eq!(err.code, RpcErrorCode::InvalidConfig);
assert!(err.message.contains("newline"), "{}", err.message);
}
#[tokio::test(start_paused = true)]
async fn an_oversized_action_timeout_loses_the_race() {
let h = harness(vec![ProcScript::never_exits()]);
let mut web = AppConfig::minimal("web", "./srv");
web.channel = true;
web.action_timeout = UpDuration::from_millis(9_000); reply_of(dispatch(envelope(1, Request::Start { apps: vec![web] }), &h.ctx).await);
let reply = reply_of(
dispatch(
envelope(
2,
Request::Trigger {
selector: SelectorSpec::Name("web".to_string()),
action: "gc".to_string(),
params: None,
},
),
&h.ctx,
)
.await,
);
assert_eq!(
reply.result.unwrap_err().code,
RpcErrorCode::DeadlineExceeded,
"an action_timeout past the caller's default budget must lose that race, not \
report an honest TimedOut row nobody can reach"
);
}
#[tokio::test(start_paused = true)]
async fn a_bad_trigger_selector_is_invalid_config() {
let h = harness(vec![]);
let reply = reply_of(
dispatch(
envelope(
1,
Request::Trigger {
selector: SelectorSpec::Regex("((".to_string()),
action: "gc".to_string(),
params: None,
},
),
&h.ctx,
)
.await,
);
assert_eq!(reply.result.unwrap_err().code, RpcErrorCode::InvalidConfig);
}
#[tokio::test(start_paused = true)]
async fn a_trigger_matching_nothing_is_not_found() {
let h = harness(vec![]);
let reply = reply_of(
dispatch(
envelope(
1,
Request::Trigger {
selector: SelectorSpec::Name("ghost".to_string()),
action: "gc".to_string(),
params: None,
},
),
&h.ctx,
)
.await,
);
assert_eq!(reply.result.unwrap_err().code, RpcErrorCode::NotFound);
}
#[test]
fn a_log_plane_failure_is_internal_and_says_which_half_failed() {
let reopen = rpc_error(&SupervisorError::ReopenFailed(
"web (id 0): could not reopen /logs/web-out.log: Permission denied".to_string(),
));
assert_eq!(reopen.code, RpcErrorCode::Internal);
assert_eq!(
reopen.message,
"log reopen failed: web (id 0): could not reopen \
/logs/web-out.log: Permission denied"
);
let flush = rpc_error(&SupervisorError::FlushFailed(
"/logs/web-out.log: Permission denied".to_string(),
));
assert_eq!(flush.code, RpcErrorCode::Internal);
assert_eq!(
flush.message,
"log flush failed: /logs/web-out.log: Permission denied"
);
}
#[tokio::test(start_paused = true)]
async fn subscribe_hands_back_a_compiled_filter() {
let h = harness(vec![]);
let outcome = dispatch(
envelope(
1,
Request::Subscribe {
topics: vec!["process.*".to_string()],
},
),
&h.ctx,
)
.await;
let Outcome::Subscribe { reply, filter } = outcome else {
panic!("expected subscribe")
};
assert_eq!(reply.result.unwrap(), Response::Subscribed);
assert_eq!(filter.patterns(), ["process.*"]);
let bad = reply_of(
dispatch(
envelope(
2,
Request::Subscribe {
topics: vec!["[".to_string()],
},
),
&h.ctx,
)
.await,
);
assert_eq!(bad.result.unwrap_err().code, RpcErrorCode::InvalidConfig);
}
#[tokio::test(start_paused = true)]
async fn kill_daemon_asks_for_shutdown_without_taking_the_engine_down_itself() {
let mut h = harness(vec![]);
let Outcome::Shutdown(reply) = dispatch(envelope(1, Request::KillDaemon), &h.ctx).await
else {
panic!("expected a shutdown outcome")
};
assert_eq!(reply.result.unwrap(), Response::ShuttingDown);
assert!(!*h.shutdown_rx.borrow_and_update());
h.ctx.shutdown();
assert!(h.shutdown_rx.changed().await.is_ok());
assert!(*h.shutdown_rx.borrow());
}
#[test]
fn budgets_default_and_clamp() {
assert_eq!(budget(None), Duration::from_millis(DEFAULT_DEADLINE_MS));
assert_eq!(budget(Some(250)), Duration::from_millis(250));
assert_eq!(budget(Some(0)), Duration::from_millis(1));
assert_eq!(
budget(Some(u64::MAX)),
Duration::from_millis(MAX_DEADLINE_MS)
);
}
#[tokio::test(start_paused = true)]
async fn envelope_deadline_ms_actually_bounds_the_reply() {
let h = harness(vec![ProcScript::ignores_signals()]);
dispatch(
envelope(
1,
Request::Start {
apps: vec![AppConfig::minimal("web", "./srv")],
},
),
&h.ctx,
)
.await;
let reply = reply_of(
dispatch(
Envelope {
id: 2,
deadline_ms: Some(1),
body: Request::Stop {
selector: SelectorSpec::Name("web".to_string()),
},
},
&h.ctx,
)
.await,
);
let err = reply.result.unwrap_err();
assert_eq!(
err.code,
RpcErrorCode::DeadlineExceeded,
"a 1ms client deadline against a 1600ms kill ladder must expire, not {err:?}"
);
}
#[tokio::test(start_paused = true)]
async fn work_past_its_deadline_answers_deadline_exceeded() {
let outcome = with_deadline(
5,
Duration::from_millis(250),
std::future::pending::<Outcome>(),
)
.await;
let reply = reply_of(outcome);
assert_eq!(reply.id, 5);
let err = reply.result.unwrap_err();
assert_eq!(err.code, RpcErrorCode::DeadlineExceeded);
assert!(err.message.contains("250 ms"), "{}", err.message);
}
#[tokio::test]
async fn save_roll_writes_the_file_it_names_and_counts_what_it_recorded() {
let h = harness(vec![ProcScript::never_exits(), ProcScript::never_exits()]);
reply_of(
dispatch(
envelope(
1,
Request::Start {
apps: vec![
AppConfig::minimal("web", "./srv"),
AppConfig::minimal("worker", "./work"),
],
},
),
&h.ctx,
)
.await,
);
let reply = reply_of(dispatch(envelope(2, Request::SaveRoll), &h.ctx).await);
let Ok(Response::RollSaved { path, apps }) = reply.result else {
panic!("expected RollSaved, got {:?}", reply.result)
};
assert_eq!(apps, 2);
let roll = crate::snapshot::read(std::path::Path::new(&path)).unwrap();
assert_eq!(roll.apps.len(), 2, "the reply's count must match the file");
assert_eq!(path, h.ctx.snapshot_path.display().to_string());
}
#[tokio::test]
async fn a_scale_is_recorded_in_the_roll_the_next_muster_reads() {
let h = harness(vec![ProcScript::never_exits(); 4]);
let mut app = AppConfig::minimal("web", "./srv");
app.instances = 2;
reply_of(dispatch(envelope(1, Request::Start { apps: vec![app] }), &h.ctx).await);
reply_of(
dispatch(
envelope(
2,
Request::Scale {
name: "web".to_string(),
count: 4,
},
),
&h.ctx,
)
.await,
);
let reply = reply_of(dispatch(envelope(3, Request::SaveRoll), &h.ctx).await);
let Ok(Response::RollSaved { path, .. }) = reply.result else {
panic!("expected RollSaved, got {:?}", reply.result)
};
let roll = crate::snapshot::read(std::path::Path::new(&path)).unwrap();
assert_eq!(roll.apps[0].app.instances, 4);
}
#[tokio::test]
async fn a_partial_scale_is_recorded_in_the_roll_and_still_reported_short() {
let h = harness(vec![ProcScript::never_exits(); 3]);
let mut app = AppConfig::minimal("web", "./srv");
app.instances = 2;
reply_of(dispatch(envelope(1, Request::Start { apps: vec![app] }), &h.ctx).await);
let reply = reply_of(
dispatch(
envelope(
2,
Request::Scale {
name: "web".to_string(),
count: 4,
},
),
&h.ctx,
)
.await,
);
let err = reply.result.unwrap_err();
assert_eq!(err.code, RpcErrorCode::SpawnFailed);
assert!(
err.message.contains("3 of 4"),
"the operator has to be told both numbers: {}",
err.message
);
let saved = reply_of(dispatch(envelope(3, Request::SaveRoll), &h.ctx).await);
let Ok(Response::RollSaved { path, .. }) = saved.result else {
panic!("expected RollSaved, got {:?}", saved.result)
};
let roll = crate::snapshot::read(std::path::Path::new(&path)).unwrap();
assert_eq!(
roll.apps[0].app.instances, 3,
"the roll must hold the three instances really running — not the \
pre-scale two, and not the four that were asked for"
);
}
#[tokio::test]
async fn save_roll_against_a_stopped_engine_is_an_error_not_a_silent_success() {
let h = harness(vec![]);
h.ctx.supervisor.shutdown().await;
let reply = reply_of(dispatch(envelope(1, Request::SaveRoll), &h.ctx).await);
let err = reply.result.unwrap_err();
assert_eq!(err.code, RpcErrorCode::Internal);
assert!(
err.message.contains("engine"),
"the operator must be told why nothing was written: {}",
err.message
);
}
#[tokio::test]
async fn a_second_muster_still_reports_the_flock_the_roll_restored() {
let h = harness(vec![ProcScript::never_exits()]);
reply_of(
dispatch(
envelope(
1,
Request::Start {
apps: vec![AppConfig::minimal("web", "./srv")],
},
),
&h.ctx,
)
.await,
);
reply_of(dispatch(envelope(2, Request::SaveRoll), &h.ctx).await);
let reply = reply_of(dispatch(envelope(3, Request::Muster), &h.ctx).await);
let Ok(Response::Mustered(infos)) = reply.result else {
panic!("expected Mustered, got {:?}", reply.result)
};
assert_eq!(
infos.len(),
1,
"the sheep the roll restores, not the ones this call spawned"
);
assert_eq!(infos[0].name, "web");
assert_eq!(infos[0].status, ProcStatus::Online);
}
async fn start_web(h: &Harness) {
reply_of(
dispatch(
envelope(
1,
Request::Start {
apps: vec![AppConfig::minimal("web", "./srv")],
},
),
&h.ctx,
)
.await,
);
}
async fn list_flock(ctx: &RpcContext, id: u64) -> Vec<ProcessInfo> {
let reply = reply_of(dispatch(envelope(id, Request::ListFlock), ctx).await);
let Ok(Response::Flock(infos)) = reply.result else {
panic!("expected Flock, got {:?}", reply.result)
};
infos
}
#[tokio::test]
async fn list_flock_carries_a_live_memory_reading_for_a_running_sheep() {
let h = harness_with_stats(vec![ProcScript::never_exits()]);
start_web(&h).await;
let infos = list_flock(&h.ctx, 2).await;
assert_eq!(infos[0].pid, Some(FIRST_SCRIPTED_PID));
assert_eq!(infos[0].memory_bytes, Some(SCRIPTED_TREE_BYTES));
assert_eq!(
infos[0].cpu_percent, None,
"no periodic baseline has been recorded, and a number invented \
from the read's own window is worse than an empty cell"
);
}
#[tokio::test]
async fn list_flock_measures_cpu_from_the_periodic_baseline_not_from_the_previous_listing() {
let h = harness_with_stats(vec![ProcScript::never_exits()]);
start_web(&h).await;
let last_tick = Instant::now()
.checked_sub(MEMORY_POLL_INTERVAL)
.expect("the monotonic clock is older than one poll interval");
h.stats.record_baseline_now(last_tick);
let first = list_flock(&h.ctx, 2).await[0]
.cpu_percent
.expect("a baseline exists, so a running sheep has a CPU figure");
let second = list_flock(&h.ctx, 3).await[0]
.cpu_percent
.expect("a baseline exists, so a running sheep has a CPU figure");
assert!(
(5.0..=10.05).contains(&first),
"1500 CPU-ms over the 15 s since the baseline is 10%; got {first}"
);
assert!(
(second - first).abs() < 1.0,
"the second listing divided by the gap between the two LISTINGS \
rather than by the window since the tick: {first} then {second}"
);
}
#[tokio::test]
async fn describe_carries_a_live_reading_too() {
let h = harness_with_stats(vec![ProcScript::never_exits()]);
start_web(&h).await;
let described = reply_of(
dispatch(
envelope(
2,
Request::Describe {
selector: SelectorSpec::Name("web".to_string()),
},
),
&h.ctx,
)
.await,
);
let Ok(Response::Described(infos)) = described.result else {
panic!("expected Described, got {:?}", described.result)
};
assert_eq!(infos[0].memory_bytes, Some(SCRIPTED_TREE_BYTES));
}
#[tokio::test]
async fn a_sheep_with_no_pid_reports_no_stats() {
let h = harness_with_stats(vec![ProcScript::never_exits(), ProcScript::never_exits()]);
start_web(&h).await;
reply_of(
dispatch(
envelope(
2,
Request::Start {
apps: vec![AppConfig::minimal("worker", "./srv")],
},
),
&h.ctx,
)
.await,
);
reply_of(
dispatch(
envelope(
3,
Request::Stop {
selector: SelectorSpec::Name("worker".to_string()),
},
),
&h.ctx,
)
.await,
);
let infos = list_flock(&h.ctx, 4).await;
let named = |name: &str| {
infos
.iter()
.find(|info| info.name == name)
.unwrap_or_else(|| panic!("{name} is missing from the listing"))
};
assert_eq!(named("web").pid, Some(FIRST_SCRIPTED_PID));
assert_eq!(named("web").memory_bytes, Some(SCRIPTED_TREE_BYTES));
assert_eq!(named("worker").pid, None);
assert_eq!(named("worker").memory_bytes, None);
assert_eq!(named("worker").cpu_percent, None);
}
#[tokio::test]
async fn a_lifecycle_reply_carries_no_stats() {
let h = harness_with_stats(vec![ProcScript::never_exits()]);
let started = reply_of(
dispatch(
envelope(
1,
Request::Start {
apps: vec![AppConfig::minimal("web", "./srv")],
},
),
&h.ctx,
)
.await,
);
let Ok(Response::Started(infos)) = started.result else {
panic!("expected Started, got {:?}", started.result)
};
assert_eq!(
infos[0].pid,
Some(FIRST_SCRIPTED_PID),
"a row with no pid would report no stats however the verb behaved"
);
assert_eq!(
infos[0].memory_bytes, None,
"only `flock` and `describe` take a live sample"
);
assert_eq!(infos[0].cpu_percent, None);
}
async fn enable_dog(ctx: &RpcContext, id: u64, name: &str) -> ProcessInfo {
let reply = reply_of(
dispatch(
envelope(
id,
Request::EnableDog {
name: name.to_string(),
source: DogSource::BuiltIn,
},
),
ctx,
)
.await,
);
let Ok(Response::DogStarted(info)) = reply.result else {
panic!("expected DogStarted, got {:?}", reply.result)
};
info
}
#[tokio::test(start_paused = true)]
async fn disabling_a_dog_stops_it_and_takes_it_off_the_listing() {
let h = harness(vec![ProcScript::never_exits()]);
let info = enable_dog(&h.ctx, 1, "bark").await;
assert_eq!(info.dog, Some(DogSource::BuiltIn));
let disabled = reply_of(
dispatch(
envelope(
2,
Request::DisableDog {
name: "bark".to_string(),
},
),
&h.ctx,
)
.await,
);
assert_eq!(disabled.result.unwrap(), Response::Deleted(vec![info.id]));
assert!(h.ctx.supervisor.list().await.is_empty());
}
#[tokio::test(start_paused = true)]
async fn a_dog_config_request_reads_the_file_as_it_stands_now() {
let h = harness(vec![]);
std::fs::write(&h.ctx.dogs_config, "[bark]\ndebounce = \"30s\"\n").unwrap();
let reply = reply_of(
dispatch(
envelope(
1,
Request::DogConfig {
name: "bark".to_string(),
},
),
&h.ctx,
)
.await,
);
let Ok(Response::DogSection { toml }) = reply.result else {
panic!("expected DogSection, got {:?}", reply.result)
};
assert!(toml.contains("30s"));
}
#[tokio::test(start_paused = true)]
async fn describe_sweeps_past_a_dog_and_still_answers_when_one_is_named() {
let h = harness(vec![ProcScript::never_exits(), ProcScript::never_exits()]);
let started = reply_of(
dispatch(
envelope(
1,
Request::Start {
apps: vec![AppConfig::minimal("web", "./srv")],
},
),
&h.ctx,
)
.await,
);
assert!(started.result.is_ok(), "{:?}", started.result);
let dog = enable_dog(&h.ctx, 2, "bark").await;
let swept = reply_of(
dispatch(
envelope(
3,
Request::Describe {
selector: SelectorSpec::All,
},
),
&h.ctx,
)
.await,
);
let Ok(Response::Described(hits)) = swept.result else {
panic!("expected Described, got {:?}", swept.result)
};
assert_eq!(
hits.iter().map(|i| i.name.as_str()).collect::<Vec<_>>(),
vec!["web"],
"`all` is the flock, not the kennel"
);
let named = reply_of(
dispatch(
envelope(
4,
Request::Describe {
selector: SelectorSpec::Name("bark".to_string()),
},
),
&h.ctx,
)
.await,
);
let Ok(Response::Described(hits)) = named.result else {
panic!("expected Described, got {:?}", named.result)
};
assert_eq!(hits.len(), 1);
assert_eq!(hits[0].id, dog.id);
}
#[tokio::test(start_paused = true)]
async fn enabling_a_dog_over_a_sheeps_name_is_refused_rather_than_faked() {
let h = harness(vec![ProcScript::never_exits()]);
let started = reply_of(
dispatch(
envelope(
1,
Request::Start {
apps: vec![AppConfig::minimal("bark", "./srv")],
},
),
&h.ctx,
)
.await,
);
assert!(started.result.is_ok(), "{:?}", started.result);
let reply = reply_of(
dispatch(
envelope(
2,
Request::EnableDog {
name: "bark".to_string(),
source: DogSource::BuiltIn,
},
),
&h.ctx,
)
.await,
);
let Err(err) = reply.result else {
panic!("expected a refusal, got {:?}", reply.result)
};
assert_eq!(err.code, RpcErrorCode::InvalidConfig);
assert!(
err.message.contains("bark"),
"the refusal names the collision: {}",
err.message
);
let listed = h.ctx.supervisor.list().await;
assert_eq!(listed.len(), 1, "nothing was started: {listed:?}");
assert_eq!(listed[0].dog, None);
}
#[tokio::test(start_paused = true)]
async fn only_describe_carries_a_lamb_tree() {
let h = harness_identifying(
vec![ProcScript::never_exits()],
vec![
identity(FIRST_SCRIPTED_PID, None, "srv"),
identity(FIRST_SCRIPTED_PID + 1, Some(FIRST_SCRIPTED_PID), "node"),
],
);
reply_of(
dispatch(
envelope(
1,
Request::Start {
apps: vec![AppConfig::minimal("web", "./srv")],
},
),
&h.ctx,
)
.await,
);
let listed = reply_of(dispatch(envelope(2, Request::ListFlock), &h.ctx).await);
let Ok(Response::Flock(rows)) = listed.result else {
panic!("expected a flock listing");
};
assert!(
rows.iter().all(|row| row.lambs.is_none()),
"ListFlock must not walk the process table"
);
let described = reply_of(
dispatch(
envelope(
3,
Request::Describe {
selector: SelectorSpec::Name("web".to_string()),
},
),
&h.ctx,
)
.await,
);
let Ok(Response::Described(rows)) = described.result else {
panic!("expected a describe listing");
};
assert_eq!(
rows[0].lambs,
Some(vec![Lamb::new(FIRST_SCRIPTED_PID + 1, "node")])
);
}
async fn start_dog(ctx: &RpcContext, name: &str) -> ProcessInfo {
let spec = DogSpec {
name: name.to_string(),
source: DogSource::BuiltIn,
};
let app = crate::dogs::dog_app(&spec, &ctx.paths).expect("the dog fixture must assemble");
ctx.supervisor
.start_dog(app, DogSource::BuiltIn)
.await
.expect("the dog fixture must start")
}
async fn staleness(ctx: &RpcContext) -> (Vec<String>, Vec<String>) {
let reply = reply_of(dispatch(envelope(1, Request::DogStaleness), ctx).await);
let Ok(Response::DogStaleness { stale, pending }) = reply.result else {
panic!("expected a dog staleness answer");
};
(stale, pending)
}
#[tokio::test]
async fn a_flock_of_ordinary_sheep_has_nothing_stale_and_nothing_pending() {
let h = harness(vec![ProcScript::never_exits()]);
let started = reply_of(
dispatch(
envelope(
1,
Request::Start {
apps: vec![AppConfig::minimal("web", "./srv")],
},
),
&h.ctx,
)
.await,
);
started.result.expect("the sheep must start");
assert_eq!(staleness(&h.ctx).await, (Vec::new(), Vec::new()));
}
#[tokio::test]
async fn a_listing_says_which_dogs_have_answered_this_shepherd() {
let h = harness(vec![ProcScript::never_exits(), ProcScript::never_exits()]);
start_dog(&h.ctx, "metrics").await;
let silent = list_flock(&h.ctx, 1).await;
let dog = silent
.iter()
.find(|info| info.name == "metrics")
.expect("the dog must be listed");
assert_eq!(dog.handshook, Some(false));
assert_eq!(
dog.status,
ProcStatus::Online,
"the process is up, and the listing still says so"
);
h.ctx.dog_refusals.handshook("metrics");
let talking = list_flock(&h.ctx, 2).await;
assert_eq!(
talking
.iter()
.find(|info| info.name == "metrics")
.expect("still listed")
.handshook,
Some(true)
);
}
#[tokio::test]
async fn a_sheep_carries_no_handshake_fact_at_all() {
let h = harness(vec![ProcScript::never_exits()]);
start_web(&h).await;
let infos = list_flock(&h.ctx, 1).await;
assert_eq!(infos[0].name, "web");
assert_eq!(infos[0].handshook, None);
assert_eq!(
infos[0].dog_stale, None,
"a sheep is never given up on, because it was never asked to answer"
);
}
#[tokio::test]
async fn a_listing_says_which_silent_dogs_this_shepherd_gave_up_on() {
let h = harness(vec![ProcScript::never_exits()]);
start_dog(&h.ctx, "metrics").await;
let waiting = list_flock(&h.ctx, 1).await;
let dog = waiting
.iter()
.find(|info| info.name == "metrics")
.expect("the dog must be listed");
assert_eq!(dog.handshook, Some(false));
assert_eq!(
dog.dog_stale,
Some(false),
"a dog that has not answered YET is not one this shepherd gave up on"
);
h.ctx.dog_refusals.refused("metrics");
h.ctx.dog_refusals.refused("metrics");
let given_up = list_flock(&h.ctx, 2).await;
let dog = given_up
.iter()
.find(|info| info.name == "metrics")
.expect("still listed");
assert_eq!(dog.dog_stale, Some(true));
assert_eq!(
dog.status,
ProcStatus::Online,
"the process is still up, and the listing still says so"
);
h.ctx.dog_refusals.handshook("metrics");
let talking = list_flock(&h.ctx, 3).await;
let dog = talking
.iter()
.find(|info| info.name == "metrics")
.expect("still listed");
assert_eq!(dog.handshook, Some(true));
assert_eq!(dog.dog_stale, Some(false));
}
#[tokio::test]
async fn describe_carries_the_handshake_fact_too() {
let h = harness(vec![ProcScript::never_exits()]);
start_dog(&h.ctx, "metrics").await;
let described = reply_of(
dispatch(
envelope(
1,
Request::Describe {
selector: SelectorSpec::Name("metrics".to_string()),
},
),
&h.ctx,
)
.await,
);
let Ok(Response::Described(rows)) = described.result else {
panic!("expected a describe listing");
};
assert_eq!(rows[0].handshook, Some(false));
}
#[tokio::test]
async fn a_dog_that_has_not_handshaken_is_pending() {
let h = harness(vec![ProcScript::never_exits()]);
start_dog(&h.ctx, "metrics").await;
assert_eq!(
staleness(&h.ctx).await,
(Vec::new(), vec!["metrics".to_string()])
);
h.ctx.dog_refusals.handshook("metrics");
assert_eq!(
staleness(&h.ctx).await,
(Vec::new(), Vec::new()),
"a dog talking to this shepherd is settled and is not worth reporting"
);
}
#[tokio::test]
async fn a_dog_being_restarted_is_pending_and_then_stale() {
let h = harness(vec![ProcScript::never_exits()]);
start_dog(&h.ctx, "metrics").await;
h.ctx.dog_refusals.handshook("metrics");
h.ctx.dog_refusals.refused("metrics");
assert_eq!(
staleness(&h.ctx).await,
(Vec::new(), vec!["metrics".to_string()]),
"one refusal buys a restart; it does not condemn the dog"
);
h.ctx.dog_refusals.refused("metrics");
assert_eq!(
staleness(&h.ctx).await,
(vec!["metrics".to_string()], Vec::new()),
"a stale dog is a finding, not something still to wait on"
);
}
#[tokio::test]
async fn a_dog_that_has_stopped_running_is_not_waited_on() {
let h = harness(vec![ProcScript::never_exits()]);
start_dog(&h.ctx, "metrics").await;
assert_eq!(staleness(&h.ctx).await.1, vec!["metrics".to_string()]);
h.ctx
.supervisor
.stop(ProcessSelector::Name("metrics".to_string()))
.await
.expect("the dog must stop");
assert_eq!(
staleness(&h.ctx).await,
(Vec::new(), Vec::new()),
"a dog that is not running has nothing to answer with"
);
}
}