use core::future::Future;
use core::time::Duration;
use std::collections::HashMap;
use std::path::PathBuf;
use std::sync::Arc;
use tokio::sync::{broadcast, watch};
use shep_core::config::normalize_all;
use shep_core::paths::ShepPaths;
use shep_core::protocol::{
BusEvent, Envelope, Lamb, ProcessInfo, Reply, Request, Response, RpcError, RpcErrorCode,
SelectorSpec,
};
use shep_core::selector::ProcessSelector;
use shep_core::signals::OperatorSignal;
use crate::bus::TopicFilter;
use crate::dogs::DogSpec;
use crate::limits::stats::StatsState;
use crate::snapshot::{FlockRegistry, SnapshotError, write_atomic};
use crate::supervisor::{ConnId, SupervisorError, SupervisorHandle};
pub(crate) const DEFAULT_DEADLINE_MS: u64 = 5_000;
pub(crate) const MAX_DEADLINE_MS: u64 = 60_000;
#[derive(Clone, Debug)]
pub struct RpcContext {
pub(crate) supervisor: SupervisorHandle,
pub(crate) events: broadcast::Sender<BusEvent>,
pub(crate) registry: FlockRegistry,
pub(crate) snapshot_path: PathBuf,
pub(crate) daemon_config: PathBuf,
pub(crate) paths: ShepPaths,
pub(crate) daemon_version: String,
pub(crate) pid: u32,
pub(crate) shutdown: Arc<watch::Sender<bool>>,
pub(crate) stats: Arc<StatsState>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct SavedRoll {
pub path: PathBuf,
pub apps: u32,
}
impl RpcContext {
pub fn shutdown(&self) {
let _ = self.shutdown.send(true);
}
pub async fn save_roll_now(&self) -> Result<Option<SavedRoll>, SnapshotError> {
let Ok(infos) = self.supervisor.list_checked().await else {
return Ok(None);
};
let roll = self.registry.roll(&infos, crate::now_ms());
write_atomic(&self.snapshot_path, &roll)?;
Ok(Some(SavedRoll {
path: self.snapshot_path.clone(),
apps: u32::try_from(roll.apps.len()).unwrap_or(u32::MAX),
}))
}
pub async fn snapshot_now(&self) -> Result<(), SnapshotError> {
self.save_roll_now().await.map(|_| ())
}
}
#[derive(Debug)]
pub(crate) enum Outcome {
Reply(Reply),
Subscribe {
reply: Reply,
filter: TopicFilter,
},
Shutdown(Reply),
}
#[must_use]
pub(crate) fn budget(deadline_ms: Option<u64>) -> Duration {
Duration::from_millis(
deadline_ms
.unwrap_or(DEFAULT_DEADLINE_MS)
.clamp(1, MAX_DEADLINE_MS),
)
}
pub(crate) async fn dispatch(envelope: Envelope, conn: ConnId, ctx: &RpcContext) -> Outcome {
let id = envelope.id;
with_deadline(
id,
budget(envelope.deadline_ms),
run(id, conn, envelope.body, ctx),
)
.await
}
async fn with_deadline<F: Future<Output = Outcome> + Send>(
id: u64,
budget: Duration,
work: F,
) -> Outcome {
match tokio::time::timeout(budget, work).await {
Ok(outcome) => outcome,
Err(_) => Outcome::Reply(Reply {
id,
result: Err(RpcError {
code: RpcErrorCode::DeadlineExceeded,
message: format!(
"the request deadline of {} ms expired before the daemon finished",
budget.as_millis()
),
}),
}),
}
}
async fn run(id: u64, conn: ConnId, request: Request, ctx: &RpcContext) -> Outcome {
let reply = |result| Outcome::Reply(Reply { id, result });
match request {
Request::Ping => reply(Ok(Response::Pong)),
Request::ListFlock => match ctx.supervisor.list_checked().await {
Ok(infos) => reply(Ok(Response::Flock(
with_live_stats(&ctx.stats, infos).await,
))),
Err(err) => reply(Err(rpc_error(&err))),
},
Request::Describe { selector } => match selector_of(selector) {
Err(err) => reply(Err(err)),
Ok(selector) => match ctx.supervisor.list_checked().await {
Err(err) => reply(Err(rpc_error(&err))),
Ok(infos) => {
let exact = selector.is_exact();
let hits: Vec<_> = infos
.into_iter()
.filter(|i| exact || i.dog.is_none())
.filter(|i| selector.matches(&i.name, i.id, i.fold.as_deref()))
.collect();
if hits.is_empty() {
reply(Err(not_found()))
} else {
let hits = with_live_stats(&ctx.stats, hits).await;
reply(Ok(Response::Described(with_lambs(&ctx.stats, hits).await)))
}
}
},
},
Request::Start { apps } => match normalize_all(apps) {
Err(err) => reply(Err(RpcError {
code: RpcErrorCode::InvalidConfig,
message: err.to_string(),
})),
Ok(resolved) => {
ctx.registry.record(&resolved);
match ctx.supervisor.start(resolved).await {
Ok(infos) => reply(Ok(Response::Started(infos))),
Err(err) => reply(Err(rpc_error(&err))),
}
}
},
Request::ConfigDrift { apps } => match normalize_all(apps) {
Err(err) => reply(Err(RpcError {
code: RpcErrorCode::InvalidConfig,
message: err.to_string(),
})),
Ok(resolved) => match ctx.supervisor.config_drift(resolved).await {
Ok(drifted) => reply(Ok(Response::Drifted(drifted))),
Err(err) => reply(Err(rpc_error(&err))),
},
},
Request::Stop { selector } => {
selector_call(id, selector, |s| ctx.supervisor.stop(s), Response::Stopped).await
}
Request::Restart { selector } => {
selector_call(
id,
selector,
|s| ctx.supervisor.restart(s),
Response::Restarted,
)
.await
}
Request::Reload { selector } => {
selector_call(
id,
selector,
|s| ctx.supervisor.reload(s),
Response::Reloading,
)
.await
}
Request::Reopen { selector } => {
selector_call(
id,
selector,
|s| ctx.supervisor.reopen(s),
Response::Reopened,
)
.await
}
Request::Flush { selector } => {
selector_call(id, selector, |s| ctx.supervisor.flush(s), Response::Flushed).await
}
Request::Trigger {
selector,
action,
params,
} => trigger(id, selector, action, params, ctx).await,
Request::Signal { selector, signal } => signal_request(id, selector, signal, ctx).await,
Request::SendLine { selector, line } => {
if line.contains(['\n', '\r']) {
return reply(Err(RpcError {
code: RpcErrorCode::InvalidConfig,
message: "a line may not contain a newline or a carriage return; \
send one line per request"
.to_string(),
}));
}
match selector_of(selector) {
Ok(selector) => match ctx.supervisor.send_line(selector, line).await {
Ok(rows) => reply(Ok(Response::SentLine(rows))),
Err(err) => reply(Err(rpc_error(&err))),
},
Err(err) => reply(Err(err)),
}
}
Request::Delete { selector } => match selector_of(selector) {
Err(err) => reply(Err(err)),
Ok(selector) => match ctx.supervisor.delete(selector).await {
Ok(ids) => reply(Ok(Response::Deleted(ids))),
Err(err) => reply(Err(rpc_error(&err))),
},
},
Request::Scale { name, count } => match ctx.supervisor.scale(&name, count).await {
Ok(scaled) => {
let achieved = scaled.achieved();
let requested = scaled.requested;
ctx.registry.record(&[scaled.app]);
match scaled.shortfall {
None => reply(Ok(Response::Scaled(scaled.instances))),
Some(message) => reply(Err(RpcError {
code: RpcErrorCode::SpawnFailed,
message: format!(
"scaled {name} to {achieved} of {requested} requested; \
the next instance would not spawn: {message}"
),
})),
}
}
Err(err) => reply(Err(rpc_error(&err))),
},
Request::SetSmit { sheep, smit } => {
match ctx.supervisor.set_smit(conn, &sheep, smit).await {
Ok(infos) => reply(Ok(Response::SmitPainted(infos))),
Err(err) => reply(Err(rpc_error(&err))),
}
}
Request::SaveRoll => match ctx.save_roll_now().await {
Ok(Some(saved)) => reply(Ok(Response::RollSaved {
path: saved.path.to_string_lossy().into_owned(),
apps: saved.apps,
})),
Ok(None) => reply(Err(RpcError {
code: RpcErrorCode::Internal,
message: "the supervisor engine has stopped; no roll was written".to_string(),
})),
Err(err) => reply(Err(RpcError {
code: RpcErrorCode::Internal,
message: err.to_string(),
})),
},
Request::Muster => {
match crate::snapshot::muster(&ctx.snapshot_path, &ctx.registry, &ctx.supervisor).await
{
Err(err) => reply(Err(RpcError {
code: RpcErrorCode::Internal,
message: err.to_string(),
})),
Ok(names) => match ctx.supervisor.list_checked().await {
Err(err) => reply(Err(rpc_error(&err))),
Ok(infos) => reply(Ok(Response::Mustered(
infos
.into_iter()
.filter(|info| names.contains(&info.name))
.collect(),
))),
},
}
}
Request::DogConfig { name } => match crate::dogs::dog_section(&ctx.daemon_config, &name) {
Ok(toml) => reply(Ok(Response::DogSection { toml: toml.into() })),
Err(err) => reply(Err(RpcError {
code: RpcErrorCode::InvalidConfig,
message: err.to_string(),
})),
},
Request::EnableDog { name, source } => {
let spec = DogSpec { name, source };
match crate::dogs::dog_app(&spec, &ctx.paths) {
Err(err) => reply(Err(RpcError {
code: RpcErrorCode::InvalidConfig,
message: err.to_string(),
})),
Ok(app) => match ctx.supervisor.start_dog(app, spec.source).await {
Ok(info) if info.dog.is_none() => reply(Err(RpcError {
code: RpcErrorCode::InvalidConfig,
message: format!(
"a sheep is already registered as `{}`; rename it or give the dog another name",
spec.name
),
})),
Ok(info) => reply(Ok(Response::DogStarted(info))),
Err(err) => reply(Err(rpc_error(&err))),
},
}
}
Request::DisableDog { name } => {
match ctx.supervisor.delete(ProcessSelector::Name(name)).await {
Ok(ids) => reply(Ok(Response::Deleted(ids))),
Err(err) => reply(Err(rpc_error(&err))),
}
}
Request::Subscribe { topics } => match TopicFilter::new(&topics) {
Ok(filter) => Outcome::Subscribe {
reply: Reply {
id,
result: Ok(Response::Subscribed),
},
filter,
},
Err(err) => reply(Err(RpcError {
code: RpcErrorCode::InvalidConfig,
message: err.to_string(),
})),
},
Request::KillDaemon => Outcome::Shutdown(Reply {
id,
result: Ok(Response::ShuttingDown),
}),
_ => reply(Err(RpcError {
code: RpcErrorCode::Internal,
message: "this daemon does not implement that request".to_string(),
})),
}
}
async fn with_live_stats(stats: &Arc<StatsState>, mut infos: Vec<ProcessInfo>) -> Vec<ProcessInfo> {
let stats = Arc::clone(stats);
let Ok(sample) = tokio::task::spawn_blocking(move || stats.sample_now()).await else {
return infos;
};
for info in &mut infos {
if let Some(reading) = info.pid.and_then(|pid| sample.get(&pid)) {
info.cpu_percent = reading.cpu_percent;
info.memory_bytes = Some(reading.memory_bytes);
}
}
infos
}
async fn with_lambs(stats: &Arc<StatsState>, mut infos: Vec<ProcessInfo>) -> Vec<ProcessInfo> {
if infos.iter().all(|info| info.pid.is_none()) {
return infos;
}
let stats = Arc::clone(stats);
let pids: Vec<u32> = infos.iter().filter_map(|info| info.pid).collect();
let Ok(walked) = tokio::task::spawn_blocking(move || {
let index = stats.lamb_index();
pids.into_iter()
.map(|pid| (pid, stats.lambs_of(&index, pid)))
.collect::<HashMap<u32, Vec<Lamb>>>()
})
.await
else {
return infos;
};
for info in &mut infos {
if let Some(lambs) = info.pid.and_then(|pid| walked.get(&pid)) {
info.lambs = Some(lambs.clone());
}
}
infos
}
fn rpc_error(err: &SupervisorError) -> RpcError {
match err {
SupervisorError::NotFound => not_found(),
SupervisorError::SpawnFailed(msg) => RpcError {
code: RpcErrorCode::SpawnFailed,
message: msg.clone(),
},
SupervisorError::CannotStart(msg) => RpcError {
code: RpcErrorCode::SpawnFailed,
message: msg.clone(),
},
SupervisorError::ReopenFailed(_) | SupervisorError::FlushFailed(_) => RpcError {
code: RpcErrorCode::Internal,
message: err.to_string(),
},
SupervisorError::ReloadInFlight(_) => RpcError {
code: RpcErrorCode::Internal,
message: err.to_string(),
},
SupervisorError::InvalidScale(msg) => RpcError {
code: RpcErrorCode::InvalidConfig,
message: msg.clone(),
},
SupervisorError::EngineStopped => RpcError {
code: RpcErrorCode::Internal,
message: "the supervisor engine has stopped".to_string(),
},
}
}
fn not_found() -> RpcError {
RpcError {
code: RpcErrorCode::NotFound,
message: "selector matched no registered sheep".to_string(),
}
}
fn selector_of(spec: SelectorSpec) -> Result<ProcessSelector, RpcError> {
ProcessSelector::try_from(spec).map_err(|err| RpcError {
code: RpcErrorCode::InvalidConfig,
message: err.to_string(),
})
}
async fn selector_call<F, Fut>(
id: u64,
spec: SelectorSpec,
call: F,
ok: fn(Vec<ProcessInfo>) -> Response,
) -> Outcome
where
F: FnOnce(ProcessSelector) -> Fut + Send,
Fut: Future<Output = Result<Vec<ProcessInfo>, SupervisorError>> + Send,
{
let result = match selector_of(spec) {
Ok(selector) => call(selector).await.map(ok).map_err(|err| rpc_error(&err)),
Err(err) => Err(err),
};
Outcome::Reply(Reply { id, result })
}
async fn trigger(
id: u64,
spec: SelectorSpec,
action: String,
params: Option<String>,
ctx: &RpcContext,
) -> Outcome {
let result = match selector_of(spec) {
Err(err) => Err(err),
Ok(selector) => ctx
.supervisor
.trigger(selector, action, params)
.await
.map(Response::Triggered)
.map_err(|err| rpc_error(&err)),
};
Outcome::Reply(Reply { id, result })
}
async fn signal_request(id: u64, spec: SelectorSpec, signal: String, ctx: &RpcContext) -> Outcome {
let result = match OperatorSignal::parse(&signal) {
None => Err(RpcError {
code: RpcErrorCode::InvalidConfig,
message: format!(
"`{signal}` is not a signal shep will send; accepted: {}",
OperatorSignal::ACCEPTED.join(", ")
),
}),
Some(sig) => match selector_of(spec) {
Err(err) => Err(err),
Ok(selector) => ctx
.supervisor
.signal(selector, sig)
.await
.map(Response::Signalled)
.map_err(|err| rpc_error(&err)),
},
};
Outcome::Reply(Reply { id, result })
}
#[cfg(test)]
mod tests {
use super::*;
use crate::fake::{FIRST_SCRIPTED_PID, ProcScript};
use crate::limits::MEMORY_POLL_INTERVAL;
use crate::testing::{
Harness, SCRIPTED_TREE_BYTES, harness, harness_identifying, harness_with_stats, identity,
};
use shep_core::config::AppConfig;
use shep_core::protocol::{
ActionOutcome, ActionReply, DogSource, Request, Response, RpcErrorCode, SelectorSpec,
};
use shep_core::status::ProcStatus;
use shep_core::values::UpDuration;
use tokio::time::Instant;
async fn dispatch(envelope: Envelope, ctx: &RpcContext) -> Outcome {
super::dispatch(envelope, ConnId::next(), ctx).await
}
fn envelope(id: u64, body: Request) -> Envelope {
Envelope {
id,
deadline_ms: None,
body,
}
}
fn reply_of(outcome: Outcome) -> Reply {
match outcome {
Outcome::Reply(reply) | Outcome::Subscribe { reply, .. } | Outcome::Shutdown(reply) => {
reply
}
}
}
#[tokio::test(start_paused = true)]
async fn ping_answers_pong_on_the_same_envelope_id() {
let h = harness(vec![]);
let reply = reply_of(dispatch(envelope(9, Request::Ping), &h.ctx).await);
assert_eq!(reply.id, 9);
assert_eq!(reply.result.unwrap(), Response::Pong);
}
#[tokio::test(start_paused = true)]
async fn start_registers_the_config_and_lists_it() {
let h = harness(vec![ProcScript::never_exits()]);
let started = reply_of(
dispatch(
envelope(
1,
Request::Start {
apps: vec![AppConfig::minimal("web", "./srv")],
},
),
&h.ctx,
)
.await,
);
let Response::Started(infos) = started.result.unwrap() else {
panic!("expected started")
};
assert_eq!(infos.len(), 1);
assert_eq!(infos[0].status, ProcStatus::Online);
let roll = h.ctx.registry.roll(&infos, 0);
assert_eq!(roll.apps.len(), 1);
assert_eq!(roll.apps[0].app.script, "./srv");
let listed = reply_of(dispatch(envelope(2, Request::ListFlock), &h.ctx).await);
let Response::Flock(flock) = listed.result.unwrap() else {
panic!("expected flock")
};
assert_eq!(flock.len(), 1);
}
#[tokio::test(start_paused = true)]
async fn start_re_normalizes_untrusted_peer_config() {
let h = harness(vec![]);
let reply = reply_of(
dispatch(
envelope(
1,
Request::Start {
apps: vec![AppConfig::minimal("", "./srv")],
},
),
&h.ctx,
)
.await,
);
assert_eq!(reply.result.unwrap_err().code, RpcErrorCode::InvalidConfig);
}
#[tokio::test(start_paused = true)]
async fn a_selector_matching_nothing_is_not_found() {
let h = harness(vec![]);
let reply = reply_of(
dispatch(
envelope(
1,
Request::Stop {
selector: SelectorSpec::Name("ghost".to_string()),
},
),
&h.ctx,
)
.await,
);
assert_eq!(reply.result.unwrap_err().code, RpcErrorCode::NotFound);
}
#[tokio::test(start_paused = true)]
async fn a_bad_peer_regex_is_invalid_config_not_a_panic() {
let h = harness(vec![]);
let reply = reply_of(
dispatch(
envelope(
1,
Request::Describe {
selector: SelectorSpec::Regex("((".to_string()),
},
),
&h.ctx,
)
.await,
);
assert_eq!(reply.result.unwrap_err().code, RpcErrorCode::InvalidConfig);
}
#[tokio::test(start_paused = true)]
async fn describe_filters_by_fold() {
let h = harness(vec![ProcScript::never_exits(), ProcScript::never_exits()]);
let mut api = AppConfig::minimal("api", "./a");
api.fold = Some("backend".to_string());
dispatch(
envelope(
1,
Request::Start {
apps: vec![api, AppConfig::minimal("web", "./w")],
},
),
&h.ctx,
)
.await;
let reply = reply_of(
dispatch(
envelope(
2,
Request::Describe {
selector: SelectorSpec::Fold("backend".to_string()),
},
),
&h.ctx,
)
.await,
);
let Response::Described(hits) = reply.result.unwrap() else {
panic!("expected described")
};
assert_eq!(hits.len(), 1);
assert_eq!(hits[0].name, "api");
}
#[tokio::test(start_paused = true)]
async fn reopen_routes_to_the_supervisor_and_leaves_the_sheep_running() {
let h = harness(vec![ProcScript::never_exits()]);
dispatch(
envelope(
1,
Request::Start {
apps: vec![AppConfig::minimal("web", "./srv")],
},
),
&h.ctx,
)
.await;
let reply = reply_of(
dispatch(
envelope(
2,
Request::Reopen {
selector: SelectorSpec::Name("web".to_string()),
},
),
&h.ctx,
)
.await,
);
let Response::Reopened(infos) = reply.result.unwrap() else {
panic!("expected reopened")
};
assert_eq!(infos.len(), 1);
assert_eq!(infos[0].status, ProcStatus::Online);
let listed = reply_of(dispatch(envelope(3, Request::ListFlock), &h.ctx).await);
let Response::Flock(flock) = listed.result.unwrap() else {
panic!("expected flock")
};
assert_eq!(
flock[0].status,
ProcStatus::Online,
"a reopen must not disturb the sheep it reopens"
);
}
#[tokio::test(start_paused = true)]
async fn reload_routes_to_the_supervisor_and_starts_a_swap() {
let h = harness(vec![ProcScript::never_exits(); 3]);
dispatch(
envelope(
1,
Request::Start {
apps: vec![AppConfig::minimal("web", "./srv")],
},
),
&h.ctx,
)
.await;
let reply = reply_of(
dispatch(
envelope(
2,
Request::Reload {
selector: SelectorSpec::Name("web".to_string()),
},
),
&h.ctx,
)
.await,
);
let Response::Reloading(accepted) = reply.result.unwrap() else {
panic!("expected reloading")
};
assert_eq!(accepted.len(), 1);
assert_eq!(
accepted[0].status,
ProcStatus::Online,
"the answer is the flock as it stood when the reload was accepted"
);
let listed = reply_of(dispatch(envelope(3, Request::ListFlock), &h.ctx).await);
let Response::Flock(flock) = listed.result.unwrap() else {
panic!("expected flock")
};
assert_eq!(
flock.len(),
2,
"a swap in progress is two entries in one instance slot, not one: {flock:?}"
);
assert_eq!(flock[0].id, accepted[0].id);
assert_eq!(flock[0].status, ProcStatus::Stopping);
assert_ne!(
flock[1].id, accepted[0].id,
"the replacement takes a new id"
);
assert_eq!(flock[1].status, ProcStatus::Starting);
}
#[test]
fn a_refused_reload_is_internal_and_says_which_refusal_it_was() {
let in_flight = rpc_error(&SupervisorError::ReloadInFlight("web".to_string()));
assert_eq!(in_flight.code, RpcErrorCode::Internal);
assert_eq!(in_flight.message, "web is already being reloaded");
let shutting_down = rpc_error(&SupervisorError::EngineStopped);
assert_eq!(shutting_down.code, RpcErrorCode::Internal);
assert_eq!(shutting_down.message, "the supervisor engine has stopped");
}
#[tokio::test(start_paused = true)]
async fn a_bad_reload_selector_is_invalid_config() {
let h = harness(vec![]);
let reply = reply_of(
dispatch(
envelope(
1,
Request::Reload {
selector: SelectorSpec::Regex("((".to_string()),
},
),
&h.ctx,
)
.await,
);
assert_eq!(reply.result.unwrap_err().code, RpcErrorCode::InvalidConfig);
}
#[tokio::test(start_paused = true)]
async fn a_bad_reopen_selector_is_invalid_config() {
let h = harness(vec![]);
let reply = reply_of(
dispatch(
envelope(
1,
Request::Reopen {
selector: SelectorSpec::Regex("((".to_string()),
},
),
&h.ctx,
)
.await,
);
assert_eq!(reply.result.unwrap_err().code, RpcErrorCode::InvalidConfig);
}
#[tokio::test(start_paused = true)]
async fn trigger_routes_to_the_flock_and_reports_each_match_within_the_budget() {
let h = harness(vec![ProcScript::never_exits(), ProcScript::never_exits()]);
let mut web = AppConfig::minimal("web", "./srv");
web.channel = true;
let started = reply_of(
dispatch(
envelope(
1,
Request::Start {
apps: vec![AppConfig::minimal("other", "./o"), web],
},
),
&h.ctx,
)
.await,
);
let Response::Started(started) = started.result.unwrap() else {
panic!("expected started")
};
let web_id = started
.iter()
.find(|i| i.name == "web")
.expect("web registered")
.id;
assert_ne!(web_id, 0, "the test's own premise: web must not be id 0");
let reply = reply_of(
dispatch(
envelope(
2,
Request::Trigger {
selector: SelectorSpec::Name("web".to_string()),
action: "gc".to_string(),
params: None,
},
),
&h.ctx,
)
.await,
);
let Response::Triggered(rows) = reply.result.unwrap() else {
panic!("expected triggered")
};
assert_eq!(
rows,
vec![ActionReply {
id: web_id,
name: "web".to_string(),
outcome: ActionOutcome::TimedOut,
}]
);
}
#[tokio::test]
async fn a_signal_name_outside_the_grammar_is_refused_with_the_accepted_list() {
let h = harness(vec![]);
let reply = reply_of(
dispatch(
envelope(
1,
Request::Signal {
selector: SelectorSpec::All,
signal: "SIGHUPP".to_string(),
},
),
&h.ctx,
)
.await,
);
let err = reply.result.unwrap_err();
assert_eq!(err.code, RpcErrorCode::InvalidConfig);
assert!(err.message.contains("SIGHUPP"), "{}", err.message);
assert!(err.message.contains("SIGHUP"), "{}", err.message);
assert!(err.message.contains("SIGUSR2"), "{}", err.message);
}
#[tokio::test]
async fn a_line_carrying_a_newline_is_refused_before_it_reaches_the_supervisor() {
let h = harness(vec![]);
let reply = reply_of(
dispatch(
envelope(
1,
Request::SendLine {
selector: SelectorSpec::All,
line: "reload\nrm -rf /".to_string(),
},
),
&h.ctx,
)
.await,
);
let err = reply.result.unwrap_err();
assert_eq!(err.code, RpcErrorCode::InvalidConfig);
assert!(err.message.contains("newline"), "{}", err.message);
}
#[tokio::test(start_paused = true)]
async fn an_oversized_action_timeout_loses_the_race() {
let h = harness(vec![ProcScript::never_exits()]);
let mut web = AppConfig::minimal("web", "./srv");
web.channel = true;
web.action_timeout = UpDuration::from_millis(9_000); reply_of(dispatch(envelope(1, Request::Start { apps: vec![web] }), &h.ctx).await);
let reply = reply_of(
dispatch(
envelope(
2,
Request::Trigger {
selector: SelectorSpec::Name("web".to_string()),
action: "gc".to_string(),
params: None,
},
),
&h.ctx,
)
.await,
);
assert_eq!(
reply.result.unwrap_err().code,
RpcErrorCode::DeadlineExceeded,
"an action_timeout past the caller's default budget must lose that race, not \
report an honest TimedOut row nobody can reach"
);
}
#[tokio::test(start_paused = true)]
async fn a_bad_trigger_selector_is_invalid_config() {
let h = harness(vec![]);
let reply = reply_of(
dispatch(
envelope(
1,
Request::Trigger {
selector: SelectorSpec::Regex("((".to_string()),
action: "gc".to_string(),
params: None,
},
),
&h.ctx,
)
.await,
);
assert_eq!(reply.result.unwrap_err().code, RpcErrorCode::InvalidConfig);
}
#[tokio::test(start_paused = true)]
async fn a_trigger_matching_nothing_is_not_found() {
let h = harness(vec![]);
let reply = reply_of(
dispatch(
envelope(
1,
Request::Trigger {
selector: SelectorSpec::Name("ghost".to_string()),
action: "gc".to_string(),
params: None,
},
),
&h.ctx,
)
.await,
);
assert_eq!(reply.result.unwrap_err().code, RpcErrorCode::NotFound);
}
#[test]
fn a_log_plane_failure_is_internal_and_says_which_half_failed() {
let reopen = rpc_error(&SupervisorError::ReopenFailed(
"web (id 0): could not reopen /logs/web-out.log: Permission denied".to_string(),
));
assert_eq!(reopen.code, RpcErrorCode::Internal);
assert_eq!(
reopen.message,
"log reopen failed: web (id 0): could not reopen \
/logs/web-out.log: Permission denied"
);
let flush = rpc_error(&SupervisorError::FlushFailed(
"/logs/web-out.log: Permission denied".to_string(),
));
assert_eq!(flush.code, RpcErrorCode::Internal);
assert_eq!(
flush.message,
"log flush failed: /logs/web-out.log: Permission denied"
);
}
#[tokio::test(start_paused = true)]
async fn subscribe_hands_back_a_compiled_filter() {
let h = harness(vec![]);
let outcome = dispatch(
envelope(
1,
Request::Subscribe {
topics: vec!["process.*".to_string()],
},
),
&h.ctx,
)
.await;
let Outcome::Subscribe { reply, filter } = outcome else {
panic!("expected subscribe")
};
assert_eq!(reply.result.unwrap(), Response::Subscribed);
assert_eq!(filter.patterns(), ["process.*"]);
let bad = reply_of(
dispatch(
envelope(
2,
Request::Subscribe {
topics: vec!["[".to_string()],
},
),
&h.ctx,
)
.await,
);
assert_eq!(bad.result.unwrap_err().code, RpcErrorCode::InvalidConfig);
}
#[tokio::test(start_paused = true)]
async fn kill_daemon_asks_for_shutdown_without_taking_the_engine_down_itself() {
let mut h = harness(vec![]);
let Outcome::Shutdown(reply) = dispatch(envelope(1, Request::KillDaemon), &h.ctx).await
else {
panic!("expected a shutdown outcome")
};
assert_eq!(reply.result.unwrap(), Response::ShuttingDown);
assert!(!*h.shutdown_rx.borrow_and_update());
h.ctx.shutdown();
assert!(h.shutdown_rx.changed().await.is_ok());
assert!(*h.shutdown_rx.borrow());
}
#[test]
fn budgets_default_and_clamp() {
assert_eq!(budget(None), Duration::from_millis(DEFAULT_DEADLINE_MS));
assert_eq!(budget(Some(250)), Duration::from_millis(250));
assert_eq!(budget(Some(0)), Duration::from_millis(1));
assert_eq!(
budget(Some(u64::MAX)),
Duration::from_millis(MAX_DEADLINE_MS)
);
}
#[tokio::test(start_paused = true)]
async fn envelope_deadline_ms_actually_bounds_the_reply() {
let h = harness(vec![ProcScript::ignores_signals()]);
dispatch(
envelope(
1,
Request::Start {
apps: vec![AppConfig::minimal("web", "./srv")],
},
),
&h.ctx,
)
.await;
let reply = reply_of(
dispatch(
Envelope {
id: 2,
deadline_ms: Some(1),
body: Request::Stop {
selector: SelectorSpec::Name("web".to_string()),
},
},
&h.ctx,
)
.await,
);
let err = reply.result.unwrap_err();
assert_eq!(
err.code,
RpcErrorCode::DeadlineExceeded,
"a 1ms client deadline against a 1600ms kill ladder must expire, not {err:?}"
);
}
#[tokio::test(start_paused = true)]
async fn work_past_its_deadline_answers_deadline_exceeded() {
let outcome = with_deadline(
5,
Duration::from_millis(250),
std::future::pending::<Outcome>(),
)
.await;
let reply = reply_of(outcome);
assert_eq!(reply.id, 5);
let err = reply.result.unwrap_err();
assert_eq!(err.code, RpcErrorCode::DeadlineExceeded);
assert!(err.message.contains("250 ms"), "{}", err.message);
}
#[tokio::test]
async fn save_roll_writes_the_file_it_names_and_counts_what_it_recorded() {
let h = harness(vec![ProcScript::never_exits(), ProcScript::never_exits()]);
reply_of(
dispatch(
envelope(
1,
Request::Start {
apps: vec![
AppConfig::minimal("web", "./srv"),
AppConfig::minimal("worker", "./work"),
],
},
),
&h.ctx,
)
.await,
);
let reply = reply_of(dispatch(envelope(2, Request::SaveRoll), &h.ctx).await);
let Ok(Response::RollSaved { path, apps }) = reply.result else {
panic!("expected RollSaved, got {:?}", reply.result)
};
assert_eq!(apps, 2);
let roll = crate::snapshot::read(std::path::Path::new(&path)).unwrap();
assert_eq!(roll.apps.len(), 2, "the reply's count must match the file");
assert_eq!(path, h.ctx.snapshot_path.display().to_string());
}
#[tokio::test]
async fn a_scale_is_recorded_in_the_roll_the_next_muster_reads() {
let h = harness(vec![ProcScript::never_exits(); 4]);
let mut app = AppConfig::minimal("web", "./srv");
app.instances = 2;
reply_of(dispatch(envelope(1, Request::Start { apps: vec![app] }), &h.ctx).await);
reply_of(
dispatch(
envelope(
2,
Request::Scale {
name: "web".to_string(),
count: 4,
},
),
&h.ctx,
)
.await,
);
let reply = reply_of(dispatch(envelope(3, Request::SaveRoll), &h.ctx).await);
let Ok(Response::RollSaved { path, .. }) = reply.result else {
panic!("expected RollSaved, got {:?}", reply.result)
};
let roll = crate::snapshot::read(std::path::Path::new(&path)).unwrap();
assert_eq!(roll.apps[0].app.instances, 4);
}
#[tokio::test]
async fn a_partial_scale_is_recorded_in_the_roll_and_still_reported_short() {
let h = harness(vec![ProcScript::never_exits(); 3]);
let mut app = AppConfig::minimal("web", "./srv");
app.instances = 2;
reply_of(dispatch(envelope(1, Request::Start { apps: vec![app] }), &h.ctx).await);
let reply = reply_of(
dispatch(
envelope(
2,
Request::Scale {
name: "web".to_string(),
count: 4,
},
),
&h.ctx,
)
.await,
);
let err = reply.result.unwrap_err();
assert_eq!(err.code, RpcErrorCode::SpawnFailed);
assert!(
err.message.contains("3 of 4"),
"the operator has to be told both numbers: {}",
err.message
);
let saved = reply_of(dispatch(envelope(3, Request::SaveRoll), &h.ctx).await);
let Ok(Response::RollSaved { path, .. }) = saved.result else {
panic!("expected RollSaved, got {:?}", saved.result)
};
let roll = crate::snapshot::read(std::path::Path::new(&path)).unwrap();
assert_eq!(
roll.apps[0].app.instances, 3,
"the roll must hold the three instances really running — not the \
pre-scale two, and not the four that were asked for"
);
}
#[tokio::test]
async fn save_roll_against_a_stopped_engine_is_an_error_not_a_silent_success() {
let h = harness(vec![]);
h.ctx.supervisor.shutdown().await;
let reply = reply_of(dispatch(envelope(1, Request::SaveRoll), &h.ctx).await);
let err = reply.result.unwrap_err();
assert_eq!(err.code, RpcErrorCode::Internal);
assert!(
err.message.contains("engine"),
"the operator must be told why nothing was written: {}",
err.message
);
}
#[tokio::test]
async fn a_second_muster_still_reports_the_flock_the_roll_restored() {
let h = harness(vec![ProcScript::never_exits()]);
reply_of(
dispatch(
envelope(
1,
Request::Start {
apps: vec![AppConfig::minimal("web", "./srv")],
},
),
&h.ctx,
)
.await,
);
reply_of(dispatch(envelope(2, Request::SaveRoll), &h.ctx).await);
let reply = reply_of(dispatch(envelope(3, Request::Muster), &h.ctx).await);
let Ok(Response::Mustered(infos)) = reply.result else {
panic!("expected Mustered, got {:?}", reply.result)
};
assert_eq!(
infos.len(),
1,
"the sheep the roll restores, not the ones this call spawned"
);
assert_eq!(infos[0].name, "web");
assert_eq!(infos[0].status, ProcStatus::Online);
}
async fn start_web(h: &Harness) {
reply_of(
dispatch(
envelope(
1,
Request::Start {
apps: vec![AppConfig::minimal("web", "./srv")],
},
),
&h.ctx,
)
.await,
);
}
async fn list_flock(ctx: &RpcContext, id: u64) -> Vec<ProcessInfo> {
let reply = reply_of(dispatch(envelope(id, Request::ListFlock), ctx).await);
let Ok(Response::Flock(infos)) = reply.result else {
panic!("expected Flock, got {:?}", reply.result)
};
infos
}
#[tokio::test]
async fn list_flock_carries_a_live_memory_reading_for_a_running_sheep() {
let h = harness_with_stats(vec![ProcScript::never_exits()]);
start_web(&h).await;
let infos = list_flock(&h.ctx, 2).await;
assert_eq!(infos[0].pid, Some(FIRST_SCRIPTED_PID));
assert_eq!(infos[0].memory_bytes, Some(SCRIPTED_TREE_BYTES));
assert_eq!(
infos[0].cpu_percent, None,
"no periodic baseline has been recorded, and a number invented \
from the read's own window is worse than an empty cell"
);
}
#[tokio::test]
async fn list_flock_measures_cpu_from_the_periodic_baseline_not_from_the_previous_listing() {
let h = harness_with_stats(vec![ProcScript::never_exits()]);
start_web(&h).await;
let last_tick = Instant::now()
.checked_sub(MEMORY_POLL_INTERVAL)
.expect("the monotonic clock is older than one poll interval");
h.stats.record_baseline_now(last_tick);
let first = list_flock(&h.ctx, 2).await[0]
.cpu_percent
.expect("a baseline exists, so a running sheep has a CPU figure");
let second = list_flock(&h.ctx, 3).await[0]
.cpu_percent
.expect("a baseline exists, so a running sheep has a CPU figure");
assert!(
(5.0..=10.05).contains(&first),
"1500 CPU-ms over the 15 s since the baseline is 10%; got {first}"
);
assert!(
(second - first).abs() < 1.0,
"the second listing divided by the gap between the two LISTINGS \
rather than by the window since the tick: {first} then {second}"
);
}
#[tokio::test]
async fn describe_carries_a_live_reading_too() {
let h = harness_with_stats(vec![ProcScript::never_exits()]);
start_web(&h).await;
let described = reply_of(
dispatch(
envelope(
2,
Request::Describe {
selector: SelectorSpec::Name("web".to_string()),
},
),
&h.ctx,
)
.await,
);
let Ok(Response::Described(infos)) = described.result else {
panic!("expected Described, got {:?}", described.result)
};
assert_eq!(infos[0].memory_bytes, Some(SCRIPTED_TREE_BYTES));
}
#[tokio::test]
async fn a_sheep_with_no_pid_reports_no_stats() {
let h = harness_with_stats(vec![ProcScript::never_exits(), ProcScript::never_exits()]);
start_web(&h).await;
reply_of(
dispatch(
envelope(
2,
Request::Start {
apps: vec![AppConfig::minimal("worker", "./srv")],
},
),
&h.ctx,
)
.await,
);
reply_of(
dispatch(
envelope(
3,
Request::Stop {
selector: SelectorSpec::Name("worker".to_string()),
},
),
&h.ctx,
)
.await,
);
let infos = list_flock(&h.ctx, 4).await;
let named = |name: &str| {
infos
.iter()
.find(|info| info.name == name)
.unwrap_or_else(|| panic!("{name} is missing from the listing"))
};
assert_eq!(named("web").pid, Some(FIRST_SCRIPTED_PID));
assert_eq!(named("web").memory_bytes, Some(SCRIPTED_TREE_BYTES));
assert_eq!(named("worker").pid, None);
assert_eq!(named("worker").memory_bytes, None);
assert_eq!(named("worker").cpu_percent, None);
}
#[tokio::test]
async fn a_lifecycle_reply_carries_no_stats() {
let h = harness_with_stats(vec![ProcScript::never_exits()]);
let started = reply_of(
dispatch(
envelope(
1,
Request::Start {
apps: vec![AppConfig::minimal("web", "./srv")],
},
),
&h.ctx,
)
.await,
);
let Ok(Response::Started(infos)) = started.result else {
panic!("expected Started, got {:?}", started.result)
};
assert_eq!(
infos[0].pid,
Some(FIRST_SCRIPTED_PID),
"a row with no pid would report no stats however the verb behaved"
);
assert_eq!(
infos[0].memory_bytes, None,
"only `flock` and `describe` take a live sample"
);
assert_eq!(infos[0].cpu_percent, None);
}
async fn enable_dog(ctx: &RpcContext, id: u64, name: &str) -> ProcessInfo {
let reply = reply_of(
dispatch(
envelope(
id,
Request::EnableDog {
name: name.to_string(),
source: DogSource::BuiltIn,
},
),
ctx,
)
.await,
);
let Ok(Response::DogStarted(info)) = reply.result else {
panic!("expected DogStarted, got {:?}", reply.result)
};
info
}
#[tokio::test(start_paused = true)]
async fn disabling_a_dog_stops_it_and_takes_it_off_the_listing() {
let h = harness(vec![ProcScript::never_exits()]);
let info = enable_dog(&h.ctx, 1, "bark").await;
assert_eq!(info.dog, Some(DogSource::BuiltIn));
let disabled = reply_of(
dispatch(
envelope(
2,
Request::DisableDog {
name: "bark".to_string(),
},
),
&h.ctx,
)
.await,
);
assert_eq!(disabled.result.unwrap(), Response::Deleted(vec![info.id]));
assert!(h.ctx.supervisor.list().await.is_empty());
}
#[tokio::test(start_paused = true)]
async fn a_dog_config_request_reads_the_file_as_it_stands_now() {
let h = harness(vec![]);
std::fs::write(&h.ctx.daemon_config, "[dog.bark]\ndebounce = \"30s\"\n").unwrap();
let reply = reply_of(
dispatch(
envelope(
1,
Request::DogConfig {
name: "bark".to_string(),
},
),
&h.ctx,
)
.await,
);
let Ok(Response::DogSection { toml }) = reply.result else {
panic!("expected DogSection, got {:?}", reply.result)
};
assert!(toml.contains("30s"));
}
#[tokio::test(start_paused = true)]
async fn describe_sweeps_past_a_dog_and_still_answers_when_one_is_named() {
let h = harness(vec![ProcScript::never_exits(), ProcScript::never_exits()]);
let started = reply_of(
dispatch(
envelope(
1,
Request::Start {
apps: vec![AppConfig::minimal("web", "./srv")],
},
),
&h.ctx,
)
.await,
);
assert!(started.result.is_ok(), "{:?}", started.result);
let dog = enable_dog(&h.ctx, 2, "bark").await;
let swept = reply_of(
dispatch(
envelope(
3,
Request::Describe {
selector: SelectorSpec::All,
},
),
&h.ctx,
)
.await,
);
let Ok(Response::Described(hits)) = swept.result else {
panic!("expected Described, got {:?}", swept.result)
};
assert_eq!(
hits.iter().map(|i| i.name.as_str()).collect::<Vec<_>>(),
vec!["web"],
"`all` is the flock, not the kennel"
);
let named = reply_of(
dispatch(
envelope(
4,
Request::Describe {
selector: SelectorSpec::Name("bark".to_string()),
},
),
&h.ctx,
)
.await,
);
let Ok(Response::Described(hits)) = named.result else {
panic!("expected Described, got {:?}", named.result)
};
assert_eq!(hits.len(), 1);
assert_eq!(hits[0].id, dog.id);
}
#[tokio::test(start_paused = true)]
async fn enabling_a_dog_over_a_sheeps_name_is_refused_rather_than_faked() {
let h = harness(vec![ProcScript::never_exits()]);
let started = reply_of(
dispatch(
envelope(
1,
Request::Start {
apps: vec![AppConfig::minimal("bark", "./srv")],
},
),
&h.ctx,
)
.await,
);
assert!(started.result.is_ok(), "{:?}", started.result);
let reply = reply_of(
dispatch(
envelope(
2,
Request::EnableDog {
name: "bark".to_string(),
source: DogSource::BuiltIn,
},
),
&h.ctx,
)
.await,
);
let Err(err) = reply.result else {
panic!("expected a refusal, got {:?}", reply.result)
};
assert_eq!(err.code, RpcErrorCode::InvalidConfig);
assert!(
err.message.contains("bark"),
"the refusal names the collision: {}",
err.message
);
let listed = h.ctx.supervisor.list().await;
assert_eq!(listed.len(), 1, "nothing was started: {listed:?}");
assert_eq!(listed[0].dog, None);
}
#[tokio::test(start_paused = true)]
async fn only_describe_carries_a_lamb_tree() {
let h = harness_identifying(
vec![ProcScript::never_exits()],
vec![
identity(FIRST_SCRIPTED_PID, None, "srv"),
identity(FIRST_SCRIPTED_PID + 1, Some(FIRST_SCRIPTED_PID), "node"),
],
);
reply_of(
dispatch(
envelope(
1,
Request::Start {
apps: vec![AppConfig::minimal("web", "./srv")],
},
),
&h.ctx,
)
.await,
);
let listed = reply_of(dispatch(envelope(2, Request::ListFlock), &h.ctx).await);
let Ok(Response::Flock(rows)) = listed.result else {
panic!("expected a flock listing");
};
assert!(
rows.iter().all(|row| row.lambs.is_none()),
"ListFlock must not walk the process table"
);
let described = reply_of(
dispatch(
envelope(
3,
Request::Describe {
selector: SelectorSpec::Name("web".to_string()),
},
),
&h.ctx,
)
.await,
);
let Ok(Response::Described(rows)) = described.result else {
panic!("expected a describe listing");
};
assert_eq!(
rows[0].lambs,
Some(vec![Lamb::new(FIRST_SCRIPTED_PID + 1, "node")])
);
}
}