use core::future::Future;
use core::time::Duration;
use std::collections::{BTreeSet, HashMap};
use std::path::PathBuf;
use std::sync::{Arc, Mutex, MutexGuard, PoisonError};
use tokio::sync::watch;
use shep_core::config::{DeclaredApp, NormalizeError, ResolvedApp, normalize_all};
use shep_core::paths::ShepPaths;
use shep_core::protocol::{
Envelope, Lamb, ProcessInfo, Reply, Request, Response, RpcError, RpcErrorCode, SelectorSpec,
SheepApplied,
};
use shep_core::selector::ProcessSelector;
use shep_core::signals::OperatorSignal;
use crate::bus::{Bus, TopicFilter};
use crate::dogs::DogSpec;
use crate::limits::stats::StatsState;
use crate::snapshot::{FlockRegistry, SnapshotError, write_atomic};
use crate::supervisor::{Applied, ConnId, SupervisorError, SupervisorHandle};
pub(crate) const DEFAULT_DEADLINE_MS: u64 = 5_000;
pub(crate) const MAX_DEADLINE_MS: u64 = 60_000;
#[derive(Debug, Clone)]
pub(crate) struct KnownDogs {
names: Arc<Mutex<BTreeSet<String>>>,
}
impl KnownDogs {
pub(crate) fn new(names: BTreeSet<String>) -> Self {
Self {
names: Arc::new(Mutex::new(names)),
}
}
pub(crate) fn contains(&self, name: &str) -> bool {
self.lock().contains(name)
}
pub(crate) fn insert(&self, name: &str) {
self.lock().insert(name.to_owned());
}
fn lock(&self) -> MutexGuard<'_, BTreeSet<String>> {
self.names.lock().unwrap_or_else(PoisonError::into_inner)
}
}
#[derive(Clone, Debug)]
pub struct RpcContext {
pub(crate) supervisor: SupervisorHandle,
pub(crate) events: Bus,
pub(crate) registry: FlockRegistry,
pub(crate) snapshot_path: PathBuf,
pub(crate) dogs_config: PathBuf,
pub(crate) known_dogs: KnownDogs,
pub(crate) paths: ShepPaths,
pub(crate) daemon_version: String,
pub(crate) dog_refusals: crate::dogs::DogRefusals,
pub(crate) peer_contacts: crate::dogs::PeerContacts,
pub(crate) pid: u32,
pub(crate) shutdown: Arc<watch::Sender<bool>>,
pub(crate) stats: Arc<StatsState>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct SavedRoll {
pub path: PathBuf,
pub apps: u32,
}
impl RpcContext {
pub fn shutdown(&self) {
let _ = self.shutdown.send(true);
}
pub fn announce_dog_config(&self, dogs: &[String]) {
crate::bus::publish_dog_config_changed(&self.events, dogs);
}
pub async fn save_roll_now(&self) -> Result<Option<SavedRoll>, SnapshotError> {
let Ok(infos) = self.supervisor.list_checked().await else {
return Ok(None);
};
let roll = self.registry.roll(&infos, crate::now_ms());
write_atomic(&self.snapshot_path, &roll)?;
Ok(Some(SavedRoll {
path: self.snapshot_path.clone(),
apps: u32::try_from(roll.apps.len()).unwrap_or(u32::MAX),
}))
}
pub async fn snapshot_now(&self) -> Result<(), SnapshotError> {
self.save_roll_now().await.map(|_| ())
}
}
#[derive(Debug)]
pub(crate) enum Outcome {
Reply(Reply),
Subscribe {
reply: Reply,
filter: TopicFilter,
},
Shutdown(Reply),
}
#[must_use]
pub(crate) fn budget(deadline_ms: Option<u64>) -> Duration {
Duration::from_millis(
deadline_ms
.unwrap_or(DEFAULT_DEADLINE_MS)
.clamp(1, MAX_DEADLINE_MS),
)
}
pub(crate) async fn dispatch(envelope: Envelope, conn: ConnId, ctx: &RpcContext) -> Outcome {
let id = envelope.id;
with_deadline(
id,
budget(envelope.deadline_ms),
run(id, conn, envelope.body, ctx),
)
.await
}
async fn with_deadline<F: Future<Output = Outcome> + Send>(
id: u64,
budget: Duration,
work: F,
) -> Outcome {
match tokio::time::timeout(budget, work).await {
Ok(outcome) => outcome,
Err(_) => Outcome::Reply(Reply {
id,
result: Err(RpcError {
code: RpcErrorCode::DeadlineExceeded,
message: format!(
"the request deadline of {} ms expired before the daemon finished",
budget.as_millis()
),
daemon_version: None,
}),
}),
}
}
async fn run(id: u64, conn: ConnId, request: Request, ctx: &RpcContext) -> Outcome {
let reply = |result| Outcome::Reply(Reply { id, result });
match request {
Request::Ping => reply(Ok(Response::Pong)),
Request::ListFlock => match ctx.supervisor.list_checked().await {
Ok(infos) => reply(Ok(Response::Flock(with_dog_contact(
&ctx.dog_refusals,
with_live_stats(&ctx.stats, infos).await,
)))),
Err(err) => reply(Err(rpc_error(&err))),
},
Request::Describe { selector } => match selector_of(selector) {
Err(err) => reply(Err(err)),
Ok(selector) => match ctx.supervisor.list_checked().await {
Err(err) => reply(Err(rpc_error(&err))),
Ok(infos) => {
let exact = selector.is_exact();
let hits: Vec<_> = infos
.into_iter()
.filter(|i| exact || i.dog.is_none())
.filter(|i| selector.matches(&i.name, i.id, i.fold.as_deref(), i.instance))
.collect();
if hits.is_empty() {
reply(Err(not_found()))
} else {
let hits = with_live_stats(&ctx.stats, hits).await;
let hits = with_lambs(&ctx.stats, hits).await;
reply(Ok(Response::Described(with_dog_contact(
&ctx.dog_refusals,
hits,
))))
}
}
},
},
Request::Start { apps } => match normalize_all(apps) {
Err(err) => reply(Err(RpcError {
code: RpcErrorCode::InvalidConfig,
message: err.to_string(),
daemon_version: None,
})),
Ok(resolved) => {
ctx.registry.record(&resolved);
match ctx.supervisor.start(resolved).await {
Ok(infos) => reply(Ok(Response::Started(infos))),
Err(err) => reply(Err(rpc_error(&err))),
}
}
},
Request::Add { apps } => match normalize_all(apps) {
Err(err) => reply(Err(RpcError {
code: RpcErrorCode::InvalidConfig,
message: err.to_string(),
daemon_version: None,
})),
Ok(resolved) => {
ctx.registry.record(&resolved);
match ctx.supervisor.register_at_rest(resolved).await {
Ok(infos) => reply(Ok(Response::Added(infos))),
Err(err) => reply(Err(rpc_error(&err))),
}
}
},
Request::ConfigDrift { apps } => match normalize_all(apps) {
Err(err) => reply(Err(RpcError {
code: RpcErrorCode::InvalidConfig,
message: err.to_string(),
daemon_version: None,
})),
Ok(resolved) => match ctx.supervisor.config_drift(resolved).await {
Ok(drifted) => reply(Ok(Response::Drifted(drifted))),
Err(err) => reply(Err(rpc_error(&err))),
},
},
Request::Stop { selector } => {
selector_call(id, selector, |s| ctx.supervisor.stop(s), Response::Stopped).await
}
Request::Restart { selector } => {
selector_call(
id,
selector,
|s| ctx.supervisor.restart(s),
Response::Restarted,
)
.await
}
Request::Reload { selector } => {
selector_call(
id,
selector,
|s| ctx.supervisor.reload(s),
Response::Reloading,
)
.await
}
Request::Reopen { selector } => {
selector_call(
id,
selector,
|s| ctx.supervisor.reopen(s),
Response::Reopened,
)
.await
}
Request::Flush { selector } => {
selector_call(id, selector, |s| ctx.supervisor.flush(s), Response::Flushed).await
}
Request::Trigger {
selector,
action,
params,
} => trigger(id, selector, action, params, ctx).await,
Request::Signal { selector, signal } => signal_request(id, selector, signal, ctx).await,
Request::SendLine { selector, line } => {
if line.contains(['\n', '\r']) {
return reply(Err(RpcError {
code: RpcErrorCode::InvalidConfig,
message: "a line may not contain a newline or a carriage return; \
send one line per request"
.to_string(),
daemon_version: None,
}));
}
match selector_of(selector) {
Ok(selector) => match ctx.supervisor.send_line(selector, line).await {
Ok(rows) => reply(Ok(Response::SentLine(rows))),
Err(err) => reply(Err(rpc_error(&err))),
},
Err(err) => reply(Err(err)),
}
}
Request::Delete { selector } => match selector_of(selector) {
Err(err) => reply(Err(err)),
Ok(selector) => match ctx.supervisor.delete(selector).await {
Ok(ids) => reply(Ok(Response::Deleted(ids))),
Err(err) => reply(Err(rpc_error(&err))),
},
},
Request::Scale { name, count } => match ctx.supervisor.scale(&name, count).await {
Ok(scaled) => {
let achieved = scaled.achieved();
let requested = scaled.requested;
ctx.registry.record(&[scaled.app]);
match scaled.shortfall {
None => reply(Ok(Response::Scaled(scaled.instances))),
Some(message) => reply(Err(RpcError {
code: RpcErrorCode::SpawnFailed,
message: format!(
"scaled {name} to {achieved} of {requested} requested; \
the next instance would not spawn: {message}"
),
daemon_version: None,
})),
}
}
Err(err) => reply(Err(rpc_error(&err))),
},
Request::SetSmit { sheep, smit } => {
match ctx.supervisor.set_smit(conn, &sheep, smit).await {
Ok(infos) => reply(Ok(Response::SmitPainted(infos))),
Err(err) => reply(Err(rpc_error(&err))),
}
}
Request::SaveRoll => match ctx.save_roll_now().await {
Ok(Some(saved)) => reply(Ok(Response::RollSaved {
path: saved.path.to_string_lossy().into_owned(),
apps: saved.apps,
})),
Ok(None) => reply(Err(RpcError {
code: RpcErrorCode::Internal,
message: "the supervisor engine has stopped; no roll was written".to_string(),
daemon_version: None,
})),
Err(err) => reply(Err(RpcError {
code: RpcErrorCode::Internal,
message: err.to_string(),
daemon_version: None,
})),
},
Request::Muster => {
match crate::snapshot::muster(&ctx.snapshot_path, &ctx.registry, &ctx.supervisor).await
{
Err(err) => reply(Err(RpcError {
code: RpcErrorCode::Internal,
message: err.to_string(),
daemon_version: None,
})),
Ok(names) => match ctx.supervisor.list_checked().await {
Err(err) => reply(Err(rpc_error(&err))),
Ok(infos) => reply(Ok(Response::Mustered(
infos
.into_iter()
.filter(|info| names.contains(&info.name))
.collect(),
))),
},
}
}
Request::DogConfig { name } => match crate::dogs::dog_section(&ctx.dogs_config, &name) {
Ok(toml) => reply(Ok(Response::DogSection { toml: toml.into() })),
Err(err) => reply(Err(RpcError {
code: RpcErrorCode::InvalidConfig,
message: err.to_string(),
daemon_version: None,
})),
},
Request::EnableDog { name, source } => {
let spec = DogSpec { name, source };
match crate::dogs::dog_app(&spec, &ctx.paths) {
Err(err) => reply(Err(RpcError {
code: RpcErrorCode::InvalidConfig,
message: err.to_string(),
daemon_version: None,
})),
Ok(app) => {
let script = app.config().script.clone();
match ctx.supervisor.start_dog(app, spec.source).await {
Ok(info) if info.dog.is_none() => reply(Err(RpcError {
code: RpcErrorCode::InvalidConfig,
message: format!(
"a sheep is already registered as `{}`; rename it or give the dog another name",
spec.name
),
daemon_version: None,
})),
Ok(info) => {
ctx.known_dogs.insert(&info.name);
crate::dogs::narrate(
&ctx.events,
&info,
&format!(
"shep has this dog enabled, running the binary at {script}"
),
)
.await;
reply(Ok(Response::DogStarted(info)))
}
Err(err) => reply(Err(rpc_error(&err))),
}
}
}
}
Request::DisableDog { name } => {
match ctx.supervisor.delete(ProcessSelector::Name(name)).await {
Ok(ids) => reply(Ok(Response::Deleted(ids))),
Err(err) => reply(Err(rpc_error(&err))),
}
}
Request::Subscribe { topics } => match TopicFilter::new(&topics) {
Ok(filter) => Outcome::Subscribe {
reply: Reply {
id,
result: Ok(Response::Subscribed),
},
filter,
},
Err(err) => reply(Err(RpcError {
code: RpcErrorCode::InvalidConfig,
message: err.to_string(),
daemon_version: None,
})),
},
Request::DogStaleness => {
let (stale, pending) = dog_staleness(ctx).await;
reply(Ok(Response::DogStaleness { stale, pending }))
}
Request::HandoverFitness => reply(Ok(Response::HandoverFitness {
refusal: handover_refusal(ctx).await,
})),
Request::KillDaemon => Outcome::Shutdown(Reply {
id,
result: Ok(Response::ShuttingDown),
}),
Request::ApplyConfig { apps, reset } => match duplicate_name(&apps) {
Some(name) => reply(Err(RpcError {
code: RpcErrorCode::InvalidConfig,
message: NormalizeError::DuplicateName(name).to_string(),
daemon_version: None,
})),
None => match ctx.supervisor.apply_config(apps, reset).await {
Ok(applied) => {
let recorded: Vec<ResolvedApp> =
applied.iter().filter_map(|a| a.app.clone()).collect();
ctx.registry.record(&recorded);
reply(Ok(Response::Applied(
applied.into_iter().map(SheepApplied::from).collect(),
)))
}
Err(err) => reply(Err(rpc_error(&err))),
},
},
Request::SheepConfig { name } => match ctx.supervisor.sheep_config(name.clone()).await {
Ok(Some(view)) => reply(Ok(Response::SheepConfig(Box::new(view)))),
Ok(None) => reply(Err(RpcError {
code: RpcErrorCode::NotFound,
message: format!("no sheep named {name}"),
daemon_version: None,
})),
Err(err) => reply(Err(rpc_error(&err))),
},
Request::SetSheepEnv { name, key, value } => {
match ctx
.supervisor
.set_sheep_env(name.clone(), key.clone(), value)
.await
{
Ok(Some(app)) => {
ctx.registry.record(&[app]);
reply(Ok(Response::SheepEnvSet { name, key }))
}
Ok(None) => reply(Err(RpcError {
code: RpcErrorCode::NotFound,
message: format!("no sheep named {name}"),
daemon_version: None,
})),
Err(err) => reply(Err(rpc_error(&err))),
}
}
Request::SetSheepField { name, key, value } => {
match ctx
.supervisor
.set_sheep_field(name.clone(), key.clone(), value)
.await
{
Ok(Some(set)) => {
ctx.registry.record(&[set.app]);
reply(Ok(Response::SheepFieldSet {
name,
key,
pending: set.pending,
}))
}
Ok(None) => reply(Err(RpcError {
code: RpcErrorCode::NotFound,
message: format!("no sheep named {name}"),
daemon_version: None,
})),
Err(err) => reply(Err(rpc_error(&err))),
}
}
Request::SetDogConfig { name, toml } => {
let running_dog = || async {
ctx.supervisor
.list_checked()
.await
.unwrap_or_default()
.iter()
.any(|info| info.name == name && info.dog.is_some())
};
if !ctx.known_dogs.contains(&name) && !running_dog().await {
return reply(Err(RpcError {
code: RpcErrorCode::NotFound,
message: format!(
"no dog named {name}; `shep adopt` or `shep enable` makes one known \
to this shepherd"
),
daemon_version: None,
}));
}
match crate::dogs::set_dog_section(&ctx.dogs_config, &name, toml.as_str()) {
Ok(()) => {
crate::bus::publish_dog_config_changed(
&ctx.events,
std::slice::from_ref(&name),
);
reply(Ok(Response::DogConfigSet { name }))
}
Err(err) => reply(Err(RpcError {
code: RpcErrorCode::InvalidConfig,
message: err.to_string(),
daemon_version: None,
})),
}
}
_ => reply(Err(RpcError {
code: RpcErrorCode::Internal,
message: "this daemon does not implement that request".to_string(),
daemon_version: None,
})),
}
}
fn duplicate_name(apps: &[DeclaredApp]) -> Option<String> {
let mut seen = BTreeSet::new();
apps.iter()
.find(|app| !seen.insert(app.config.name.as_str()))
.map(|app| app.config.name.clone())
}
impl From<Applied> for SheepApplied {
fn from(applied: Applied) -> Self {
Self::new(
applied.name,
applied.applied,
applied.pending,
applied.refused,
)
}
}
async fn dog_staleness(ctx: &RpcContext) -> (Vec<String>, Vec<String>) {
let stale = ctx.dog_refusals.stale();
let mut pending = ctx.dog_refusals.restarting();
if let Ok(infos) = ctx.supervisor.list_checked().await {
pending.extend(crate::dogs::silent_dogs(&infos, &ctx.dog_refusals));
}
pending.sort();
pending.dedup();
(stale, pending)
}
#[cfg(unix)]
async fn handover_refusal(ctx: &RpcContext) -> Option<String> {
match ctx.supervisor.handover_fitness().await {
Ok(crate::handover::Fitness::Carryable) => None,
Ok(crate::handover::Fitness::Refused(reason)) => Some(reason.to_string()),
Err(err) => Some(format!(
"this shepherd could not check whether its flock can be handed over ({err})"
)),
}
}
#[cfg(windows)]
#[expect(
clippy::unused_async,
reason = "one signature for both platforms; the unix arm awaits the supervisor"
)]
async fn handover_refusal(_ctx: &RpcContext) -> Option<String> {
Some(
"this shepherd runs on Windows, which has no `execve`, so its flock cannot be handed to \
a successor in place"
.to_string(),
)
}
async fn with_live_stats(stats: &Arc<StatsState>, mut infos: Vec<ProcessInfo>) -> Vec<ProcessInfo> {
let stats = Arc::clone(stats);
let Ok(sample) = tokio::task::spawn_blocking(move || stats.sample_now()).await else {
return infos;
};
for info in &mut infos {
if let Some(reading) = info.pid.and_then(|pid| sample.get(&pid)) {
info.cpu_percent = reading.cpu_percent;
info.memory_bytes = Some(reading.memory_bytes);
}
}
infos
}
fn with_dog_contact(
refusals: &crate::dogs::DogRefusals,
mut infos: Vec<ProcessInfo>,
) -> Vec<ProcessInfo> {
let stale = refusals.stale();
for info in &mut infos {
if info.dog.is_some() {
info.handshook = Some(refusals.has_handshook(&info.name));
info.dog_stale = Some(stale.contains(&info.name));
}
}
infos
}
async fn with_lambs(stats: &Arc<StatsState>, mut infos: Vec<ProcessInfo>) -> Vec<ProcessInfo> {
if infos.iter().all(|info| info.pid.is_none()) {
return infos;
}
let stats = Arc::clone(stats);
let pids: Vec<u32> = infos.iter().filter_map(|info| info.pid).collect();
let Ok(walked) = tokio::task::spawn_blocking(move || {
let index = stats.lamb_index();
pids.into_iter()
.map(|pid| (pid, stats.lambs_of(&index, pid)))
.collect::<HashMap<u32, Vec<Lamb>>>()
})
.await
else {
return infos;
};
for info in &mut infos {
if let Some(lambs) = info.pid.and_then(|pid| walked.get(&pid)) {
info.lambs = Some(lambs.clone());
}
}
infos
}
fn rpc_error(err: &SupervisorError) -> RpcError {
match err {
SupervisorError::NotFound => not_found(),
SupervisorError::SpawnFailed(msg) => RpcError {
code: RpcErrorCode::SpawnFailed,
message: msg.clone(),
daemon_version: None,
},
SupervisorError::CannotStart(msg) => RpcError {
code: RpcErrorCode::SpawnFailed,
message: msg.clone(),
daemon_version: None,
},
SupervisorError::ReopenFailed(_) | SupervisorError::FlushFailed(_) => RpcError {
code: RpcErrorCode::Internal,
message: err.to_string(),
daemon_version: None,
},
SupervisorError::ReloadInFlight(_) => RpcError {
code: RpcErrorCode::Internal,
message: err.to_string(),
daemon_version: None,
},
SupervisorError::InvalidScale(msg) => RpcError {
code: RpcErrorCode::InvalidConfig,
message: msg.clone(),
daemon_version: None,
},
SupervisorError::IsADog(msg) => RpcError {
code: RpcErrorCode::InvalidConfig,
message: msg.clone(),
daemon_version: None,
},
SupervisorError::InvalidEnv(msg) => RpcError {
code: RpcErrorCode::InvalidConfig,
message: msg.clone(),
daemon_version: None,
},
SupervisorError::InvalidField(msg) => RpcError {
code: RpcErrorCode::InvalidConfig,
message: msg.clone(),
daemon_version: None,
},
SupervisorError::Overrides(_) => RpcError {
code: RpcErrorCode::Internal,
message: err.to_string(),
daemon_version: None,
},
SupervisorError::EngineStopped => RpcError {
code: RpcErrorCode::Internal,
message: "the supervisor engine has stopped".to_string(),
daemon_version: None,
},
}
}
fn not_found() -> RpcError {
RpcError {
code: RpcErrorCode::NotFound,
message: "selector matched no registered sheep".to_string(),
daemon_version: None,
}
}
fn selector_of(spec: SelectorSpec) -> Result<ProcessSelector, RpcError> {
ProcessSelector::try_from(spec).map_err(|err| RpcError {
code: RpcErrorCode::InvalidConfig,
message: err.to_string(),
daemon_version: None,
})
}
async fn selector_call<F, Fut>(
id: u64,
spec: SelectorSpec,
call: F,
ok: fn(Vec<ProcessInfo>) -> Response,
) -> Outcome
where
F: FnOnce(ProcessSelector) -> Fut + Send,
Fut: Future<Output = Result<Vec<ProcessInfo>, SupervisorError>> + Send,
{
let result = match selector_of(spec) {
Ok(selector) => call(selector).await.map(ok).map_err(|err| rpc_error(&err)),
Err(err) => Err(err),
};
Outcome::Reply(Reply { id, result })
}
async fn trigger(
id: u64,
spec: SelectorSpec,
action: String,
params: Option<String>,
ctx: &RpcContext,
) -> Outcome {
let result = match selector_of(spec) {
Err(err) => Err(err),
Ok(selector) => ctx
.supervisor
.trigger(selector, action, params)
.await
.map(Response::Triggered)
.map_err(|err| rpc_error(&err)),
};
Outcome::Reply(Reply { id, result })
}
async fn signal_request(id: u64, spec: SelectorSpec, signal: String, ctx: &RpcContext) -> Outcome {
let result = match OperatorSignal::parse(&signal) {
None => Err(RpcError {
code: RpcErrorCode::InvalidConfig,
message: format!(
"`{signal}` is not a signal shep will send; accepted: {}",
OperatorSignal::ACCEPTED.join(", ")
),
daemon_version: None,
}),
Some(sig) => match selector_of(spec) {
Err(err) => Err(err),
Ok(selector) => ctx
.supervisor
.signal(selector, sig)
.await
.map(Response::Signalled)
.map_err(|err| rpc_error(&err)),
},
};
Outcome::Reply(Reply { id, result })
}
#[cfg(test)]
mod tests {
use super::*;
use crate::fake::{FIRST_SCRIPTED_PID, ProcScript};
use crate::limits::MEMORY_POLL_INTERVAL;
use crate::testing::{
Harness, SCRIPTED_TREE_BYTES, harness, harness_identifying, harness_with_stats, identity,
};
use shep_core::config::{AppConfig, ApplyGroup, DeclaredApp, ResetDepth, apply_group};
use shep_core::protocol::{
ActionOutcome, ActionReply, DogSource, Request, Response, RpcErrorCode, SelectorSpec,
};
use shep_core::status::ProcStatus;
use shep_core::values::UpDuration;
use std::collections::BTreeSet;
use tokio::time::Instant;
async fn dispatch(envelope: Envelope, ctx: &RpcContext) -> Outcome {
super::dispatch(envelope, ConnId::next(), ctx).await
}
fn envelope(id: u64, body: Request) -> Envelope {
Envelope {
id,
deadline_ms: None,
body,
}
}
fn reply_of(outcome: Outcome) -> Reply {
match outcome {
Outcome::Reply(reply) | Outcome::Subscribe { reply, .. } | Outcome::Shutdown(reply) => {
reply
}
}
}
#[tokio::test(start_paused = true)]
async fn ping_answers_pong_on_the_same_envelope_id() {
let h = harness(vec![]);
let reply = reply_of(dispatch(envelope(9, Request::Ping), &h.ctx).await);
assert_eq!(reply.id, 9);
assert_eq!(reply.result.unwrap(), Response::Pong);
}
#[tokio::test(start_paused = true)]
async fn start_registers_the_config_and_lists_it() {
let h = harness(vec![ProcScript::never_exits()]);
let started = reply_of(
dispatch(
envelope(
1,
Request::Start {
apps: vec![AppConfig::minimal("web", "./srv")],
},
),
&h.ctx,
)
.await,
);
let Response::Started(infos) = started.result.unwrap() else {
panic!("expected started")
};
assert_eq!(infos.len(), 1);
assert_eq!(infos[0].status, ProcStatus::Online);
let roll = h.ctx.registry.roll(&infos, 0);
assert_eq!(roll.apps.len(), 1);
assert_eq!(roll.apps[0].app.script, "./srv");
let listed = reply_of(dispatch(envelope(2, Request::ListFlock), &h.ctx).await);
let Response::Flock(flock) = listed.result.unwrap() else {
panic!("expected flock")
};
assert_eq!(flock.len(), 1);
}
#[tokio::test(start_paused = true)]
async fn start_re_normalizes_untrusted_peer_config() {
let h = harness(vec![]);
let reply = reply_of(
dispatch(
envelope(
1,
Request::Start {
apps: vec![AppConfig::minimal("", "./srv")],
},
),
&h.ctx,
)
.await,
);
assert_eq!(reply.result.unwrap_err().code, RpcErrorCode::InvalidConfig);
}
#[tokio::test(start_paused = true)]
async fn add_registers_a_stopped_member_and_spawns_nothing() {
let h = harness(vec![]);
let added = reply_of(
dispatch(
envelope(
1,
Request::Add {
apps: vec![AppConfig::minimal("web", "./srv")],
},
),
&h.ctx,
)
.await,
);
let Response::Added(infos) = added.result.unwrap() else {
panic!("expected added")
};
assert_eq!(infos.len(), 1);
assert_eq!(infos[0].status, ProcStatus::Stopped);
assert_eq!(infos[0].pid, None, "nothing was spawned");
let roll = h.ctx.registry.roll(&infos, 0);
assert_eq!(roll.apps.len(), 1);
assert_eq!(roll.apps[0].app.script, "./srv");
assert_eq!(roll.apps[0].instances_running, 0);
let listed = reply_of(dispatch(envelope(2, Request::ListFlock), &h.ctx).await);
let Response::Flock(flock) = listed.result.unwrap() else {
panic!("expected flock")
};
assert_eq!(
flock.len(),
1,
"it is a member of the flock, just a still one"
);
}
#[tokio::test(start_paused = true)]
async fn add_re_normalizes_untrusted_peer_config() {
let h = harness(vec![]);
let reply = reply_of(
dispatch(
envelope(
1,
Request::Add {
apps: vec![AppConfig::minimal("", "./srv")],
},
),
&h.ctx,
)
.await,
);
assert_eq!(reply.result.unwrap_err().code, RpcErrorCode::InvalidConfig);
}
#[tokio::test(start_paused = true)]
async fn a_second_add_leaves_a_running_sheep_exactly_as_it_was() {
let h = harness(vec![ProcScript::never_exits()]);
let started = reply_of(
dispatch(
envelope(
1,
Request::Start {
apps: vec![AppConfig::minimal("web", "./srv")],
},
),
&h.ctx,
)
.await,
);
let Response::Started(before) = started.result.unwrap() else {
panic!("expected started")
};
let added = reply_of(
dispatch(
envelope(
2,
Request::Add {
apps: vec![AppConfig::minimal("web", "./srv")],
},
),
&h.ctx,
)
.await,
);
let Response::Added(after) = added.result.unwrap() else {
panic!("expected added")
};
assert_eq!(after.len(), 1);
assert_eq!(
after[0].id, before[0].id,
"the same sheep, not a second one"
);
assert_eq!(after[0].status, ProcStatus::Online, "still running");
assert_eq!(after[0].pid, before[0].pid, "the same process");
let listed = reply_of(dispatch(envelope(3, Request::ListFlock), &h.ctx).await);
let Response::Flock(flock) = listed.result.unwrap() else {
panic!("expected flock")
};
assert_eq!(flock.len(), 1, "one row, not two");
}
fn declared(name: &str, script: &str, keys: &[&str]) -> DeclaredApp {
DeclaredApp {
config: AppConfig::minimal(name, script),
declared: keys.iter().map(|k| (*k).to_string()).collect(),
declared_env: BTreeSet::new(),
}
}
#[tokio::test(start_paused = true)]
async fn apply_config_refuses_a_request_naming_one_app_twice() {
let h = harness(vec![ProcScript::never_exits()]);
let _started = reply_of(
dispatch(
envelope(
1,
Request::Start {
apps: vec![AppConfig::minimal("web", "./srv")],
},
),
&h.ctx,
)
.await,
);
let reply = reply_of(
dispatch(
envelope(
2,
Request::ApplyConfig {
apps: vec![
declared("web", "./one", &["script"]),
declared("web", "./two", &["script"]),
],
reset: ResetDepth::None,
},
),
&h.ctx,
)
.await,
);
let err = reply.result.unwrap_err();
assert_eq!(err.code, RpcErrorCode::InvalidConfig);
assert!(err.message.contains("web"), "the name is named: {err:?}");
let listed = reply_of(dispatch(envelope(3, Request::ListFlock), &h.ctx).await);
let Response::Flock(flock) = listed.result.unwrap() else {
panic!("expected flock")
};
let roll = h.ctx.registry.roll(&flock, 0);
assert_eq!(roll.apps[0].app.script, "./srv");
}
#[tokio::test(start_paused = true)]
async fn apply_config_records_what_it_applied_in_the_registry() {
let h = harness(vec![ProcScript::never_exits()]);
let _started = reply_of(
dispatch(
envelope(
1,
Request::Start {
apps: vec![AppConfig::minimal("web", "./srv")],
},
),
&h.ctx,
)
.await,
);
let reply = reply_of(
dispatch(
envelope(
2,
Request::ApplyConfig {
apps: vec![DeclaredApp {
config: {
let mut app = AppConfig::minimal("web", "./srv");
app.max_restarts = 99;
app
},
declared: ["max_restarts"].iter().map(|k| (*k).to_string()).collect(),
declared_env: BTreeSet::new(),
}],
reset: ResetDepth::None,
},
),
&h.ctx,
)
.await,
);
let Response::Applied(report) = reply.result.unwrap() else {
panic!("expected applied")
};
assert_eq!(report.len(), 1);
assert_eq!(report[0].name, "web");
assert_eq!(report[0].applied, vec!["max_restarts".to_string()]);
assert_eq!(report[0].refused, None);
let listed = reply_of(dispatch(envelope(3, Request::ListFlock), &h.ctx).await);
let Response::Flock(flock) = listed.result.unwrap() else {
panic!("expected flock")
};
let roll = h.ctx.registry.roll(&flock, 0);
assert_eq!(roll.apps[0].app.max_restarts, 99);
}
#[tokio::test(start_paused = true)]
async fn apply_config_refuses_an_unregistered_app_inside_the_reply() {
let h = harness(vec![]);
let reply = reply_of(
dispatch(
envelope(
1,
Request::ApplyConfig {
apps: vec![declared("ghost", "./srv", &["script"])],
reset: ResetDepth::None,
},
),
&h.ctx,
)
.await,
);
let Response::Applied(report) = reply.result.unwrap() else {
panic!("expected applied")
};
assert_eq!(report.len(), 1);
let refused = report[0].refused.as_deref().unwrap_or_default();
assert!(
refused.contains("ghost"),
"the refusal names the app: {refused}"
);
}
#[tokio::test(start_paused = true)]
async fn a_selector_matching_nothing_is_not_found() {
let h = harness(vec![]);
let reply = reply_of(
dispatch(
envelope(
1,
Request::Stop {
selector: SelectorSpec::Name("ghost".to_string()),
},
),
&h.ctx,
)
.await,
);
assert_eq!(reply.result.unwrap_err().code, RpcErrorCode::NotFound);
}
#[tokio::test(start_paused = true)]
async fn a_bad_peer_regex_is_invalid_config_not_a_panic() {
let h = harness(vec![]);
let reply = reply_of(
dispatch(
envelope(
1,
Request::Describe {
selector: SelectorSpec::Regex("((".to_string()),
},
),
&h.ctx,
)
.await,
);
assert_eq!(reply.result.unwrap_err().code, RpcErrorCode::InvalidConfig);
}
#[tokio::test(start_paused = true)]
async fn describe_filters_by_fold() {
let h = harness(vec![ProcScript::never_exits(), ProcScript::never_exits()]);
let mut api = AppConfig::minimal("api", "./a");
api.fold = Some("backend".to_string());
dispatch(
envelope(
1,
Request::Start {
apps: vec![api, AppConfig::minimal("web", "./w")],
},
),
&h.ctx,
)
.await;
let reply = reply_of(
dispatch(
envelope(
2,
Request::Describe {
selector: SelectorSpec::Fold("backend".to_string()),
},
),
&h.ctx,
)
.await,
);
let Response::Described(hits) = reply.result.unwrap() else {
panic!("expected described")
};
assert_eq!(hits.len(), 1);
assert_eq!(hits[0].name, "api");
}
#[tokio::test(start_paused = true)]
async fn reopen_routes_to_the_supervisor_and_leaves_the_sheep_running() {
let h = harness(vec![ProcScript::never_exits()]);
dispatch(
envelope(
1,
Request::Start {
apps: vec![AppConfig::minimal("web", "./srv")],
},
),
&h.ctx,
)
.await;
let reply = reply_of(
dispatch(
envelope(
2,
Request::Reopen {
selector: SelectorSpec::Name("web".to_string()),
},
),
&h.ctx,
)
.await,
);
let Response::Reopened(infos) = reply.result.unwrap() else {
panic!("expected reopened")
};
assert_eq!(infos.len(), 1);
assert_eq!(infos[0].status, ProcStatus::Online);
let listed = reply_of(dispatch(envelope(3, Request::ListFlock), &h.ctx).await);
let Response::Flock(flock) = listed.result.unwrap() else {
panic!("expected flock")
};
assert_eq!(
flock[0].status,
ProcStatus::Online,
"a reopen must not disturb the sheep it reopens"
);
}
#[tokio::test(start_paused = true)]
async fn reload_routes_to_the_supervisor_and_starts_a_swap() {
let h = harness(vec![ProcScript::never_exits(); 3]);
dispatch(
envelope(
1,
Request::Start {
apps: vec![AppConfig::minimal("web", "./srv")],
},
),
&h.ctx,
)
.await;
let reply = reply_of(
dispatch(
envelope(
2,
Request::Reload {
selector: SelectorSpec::Name("web".to_string()),
},
),
&h.ctx,
)
.await,
);
let Response::Reloading(accepted) = reply.result.unwrap() else {
panic!("expected reloading")
};
assert_eq!(accepted.len(), 1);
assert_eq!(
accepted[0].status,
ProcStatus::Online,
"the answer is the flock as it stood when the reload was accepted"
);
let listed = reply_of(dispatch(envelope(3, Request::ListFlock), &h.ctx).await);
let Response::Flock(flock) = listed.result.unwrap() else {
panic!("expected flock")
};
assert_eq!(
flock.len(),
2,
"a swap in progress is two entries in one instance slot, not one: {flock:?}"
);
assert_eq!(flock[0].id, accepted[0].id);
assert_eq!(flock[0].status, ProcStatus::Stopping);
assert_ne!(
flock[1].id, accepted[0].id,
"the replacement takes a new id"
);
assert_eq!(flock[1].status, ProcStatus::Starting);
}
#[test]
fn a_refused_reload_is_internal_and_says_which_refusal_it_was() {
let in_flight = rpc_error(&SupervisorError::ReloadInFlight("web".to_string()));
assert_eq!(in_flight.code, RpcErrorCode::Internal);
assert_eq!(in_flight.message, "web is already being reloaded");
let shutting_down = rpc_error(&SupervisorError::EngineStopped);
assert_eq!(shutting_down.code, RpcErrorCode::Internal);
assert_eq!(shutting_down.message, "the supervisor engine has stopped");
}
#[tokio::test(start_paused = true)]
async fn a_bad_reload_selector_is_invalid_config() {
let h = harness(vec![]);
let reply = reply_of(
dispatch(
envelope(
1,
Request::Reload {
selector: SelectorSpec::Regex("((".to_string()),
},
),
&h.ctx,
)
.await,
);
assert_eq!(reply.result.unwrap_err().code, RpcErrorCode::InvalidConfig);
}
#[tokio::test(start_paused = true)]
async fn a_bad_reopen_selector_is_invalid_config() {
let h = harness(vec![]);
let reply = reply_of(
dispatch(
envelope(
1,
Request::Reopen {
selector: SelectorSpec::Regex("((".to_string()),
},
),
&h.ctx,
)
.await,
);
assert_eq!(reply.result.unwrap_err().code, RpcErrorCode::InvalidConfig);
}
#[tokio::test(start_paused = true)]
async fn trigger_routes_to_the_flock_and_reports_each_match_within_the_budget() {
let h = harness(vec![ProcScript::never_exits(), ProcScript::never_exits()]);
let mut web = AppConfig::minimal("web", "./srv");
web.channel = true;
let started = reply_of(
dispatch(
envelope(
1,
Request::Start {
apps: vec![AppConfig::minimal("other", "./o"), web],
},
),
&h.ctx,
)
.await,
);
let Response::Started(started) = started.result.unwrap() else {
panic!("expected started")
};
let web_id = started
.iter()
.find(|i| i.name == "web")
.expect("web registered")
.id;
assert_ne!(web_id, 0, "the test's own premise: web must not be id 0");
let reply = reply_of(
dispatch(
envelope(
2,
Request::Trigger {
selector: SelectorSpec::Name("web".to_string()),
action: "gc".to_string(),
params: None,
},
),
&h.ctx,
)
.await,
);
let Response::Triggered(rows) = reply.result.unwrap() else {
panic!("expected triggered")
};
assert_eq!(
rows,
vec![ActionReply {
id: web_id,
name: "web".to_string(),
outcome: ActionOutcome::TimedOut,
}]
);
}
#[tokio::test]
async fn a_signal_name_outside_the_grammar_is_refused_with_the_accepted_list() {
let h = harness(vec![]);
let reply = reply_of(
dispatch(
envelope(
1,
Request::Signal {
selector: SelectorSpec::All,
signal: "SIGHUPP".to_string(),
},
),
&h.ctx,
)
.await,
);
let err = reply.result.unwrap_err();
assert_eq!(err.code, RpcErrorCode::InvalidConfig);
assert!(err.message.contains("SIGHUPP"), "{}", err.message);
assert!(err.message.contains("SIGHUP"), "{}", err.message);
assert!(err.message.contains("SIGUSR2"), "{}", err.message);
}
#[tokio::test]
async fn a_line_carrying_a_newline_is_refused_before_it_reaches_the_supervisor() {
let h = harness(vec![]);
let reply = reply_of(
dispatch(
envelope(
1,
Request::SendLine {
selector: SelectorSpec::All,
line: "reload\nrm -rf /".to_string(),
},
),
&h.ctx,
)
.await,
);
let err = reply.result.unwrap_err();
assert_eq!(err.code, RpcErrorCode::InvalidConfig);
assert!(err.message.contains("newline"), "{}", err.message);
}
#[tokio::test(start_paused = true)]
async fn an_oversized_action_timeout_loses_the_race() {
let h = harness(vec![ProcScript::never_exits()]);
let mut web = AppConfig::minimal("web", "./srv");
web.channel = true;
web.action_timeout = UpDuration::from_millis(9_000); reply_of(dispatch(envelope(1, Request::Start { apps: vec![web] }), &h.ctx).await);
let reply = reply_of(
dispatch(
envelope(
2,
Request::Trigger {
selector: SelectorSpec::Name("web".to_string()),
action: "gc".to_string(),
params: None,
},
),
&h.ctx,
)
.await,
);
assert_eq!(
reply.result.unwrap_err().code,
RpcErrorCode::DeadlineExceeded,
"an action_timeout past the caller's default budget must lose that race, not \
report an honest TimedOut row nobody can reach"
);
}
#[tokio::test(start_paused = true)]
async fn a_bad_trigger_selector_is_invalid_config() {
let h = harness(vec![]);
let reply = reply_of(
dispatch(
envelope(
1,
Request::Trigger {
selector: SelectorSpec::Regex("((".to_string()),
action: "gc".to_string(),
params: None,
},
),
&h.ctx,
)
.await,
);
assert_eq!(reply.result.unwrap_err().code, RpcErrorCode::InvalidConfig);
}
#[tokio::test(start_paused = true)]
async fn a_trigger_matching_nothing_is_not_found() {
let h = harness(vec![]);
let reply = reply_of(
dispatch(
envelope(
1,
Request::Trigger {
selector: SelectorSpec::Name("ghost".to_string()),
action: "gc".to_string(),
params: None,
},
),
&h.ctx,
)
.await,
);
assert_eq!(reply.result.unwrap_err().code, RpcErrorCode::NotFound);
}
#[test]
fn a_log_plane_failure_is_internal_and_says_which_half_failed() {
let reopen = rpc_error(&SupervisorError::ReopenFailed(
"web (id 0): could not reopen /logs/web-out.log: Permission denied".to_string(),
));
assert_eq!(reopen.code, RpcErrorCode::Internal);
assert_eq!(
reopen.message,
"log reopen failed: web (id 0): could not reopen \
/logs/web-out.log: Permission denied"
);
let flush = rpc_error(&SupervisorError::FlushFailed(
"/logs/web-out.log: Permission denied".to_string(),
));
assert_eq!(flush.code, RpcErrorCode::Internal);
assert_eq!(
flush.message,
"log flush failed: /logs/web-out.log: Permission denied"
);
}
#[tokio::test(start_paused = true)]
async fn subscribe_hands_back_a_compiled_filter() {
let h = harness(vec![]);
let outcome = dispatch(
envelope(
1,
Request::Subscribe {
topics: vec!["process.*".to_string()],
},
),
&h.ctx,
)
.await;
let Outcome::Subscribe { reply, filter } = outcome else {
panic!("expected subscribe")
};
assert_eq!(reply.result.unwrap(), Response::Subscribed);
assert_eq!(filter.patterns(), ["process.*"]);
let bad = reply_of(
dispatch(
envelope(
2,
Request::Subscribe {
topics: vec!["[".to_string()],
},
),
&h.ctx,
)
.await,
);
assert_eq!(bad.result.unwrap_err().code, RpcErrorCode::InvalidConfig);
}
#[tokio::test(start_paused = true)]
async fn kill_daemon_asks_for_shutdown_without_taking_the_engine_down_itself() {
let mut h = harness(vec![]);
let Outcome::Shutdown(reply) = dispatch(envelope(1, Request::KillDaemon), &h.ctx).await
else {
panic!("expected a shutdown outcome")
};
assert_eq!(reply.result.unwrap(), Response::ShuttingDown);
assert!(!*h.shutdown_rx.borrow_and_update());
h.ctx.shutdown();
assert!(h.shutdown_rx.changed().await.is_ok());
assert!(*h.shutdown_rx.borrow());
}
#[test]
fn budgets_default_and_clamp() {
assert_eq!(budget(None), Duration::from_millis(DEFAULT_DEADLINE_MS));
assert_eq!(budget(Some(250)), Duration::from_millis(250));
assert_eq!(budget(Some(0)), Duration::from_millis(1));
assert_eq!(
budget(Some(u64::MAX)),
Duration::from_millis(MAX_DEADLINE_MS)
);
}
#[tokio::test(start_paused = true)]
async fn envelope_deadline_ms_actually_bounds_the_reply() {
let h = harness(vec![ProcScript::ignores_signals()]);
dispatch(
envelope(
1,
Request::Start {
apps: vec![AppConfig::minimal("web", "./srv")],
},
),
&h.ctx,
)
.await;
let reply = reply_of(
dispatch(
Envelope {
id: 2,
deadline_ms: Some(1),
body: Request::Stop {
selector: SelectorSpec::Name("web".to_string()),
},
},
&h.ctx,
)
.await,
);
let err = reply.result.unwrap_err();
assert_eq!(
err.code,
RpcErrorCode::DeadlineExceeded,
"a 1ms client deadline against a 1600ms kill ladder must expire, not {err:?}"
);
}
#[tokio::test(start_paused = true)]
async fn work_past_its_deadline_answers_deadline_exceeded() {
let outcome = with_deadline(
5,
Duration::from_millis(250),
std::future::pending::<Outcome>(),
)
.await;
let reply = reply_of(outcome);
assert_eq!(reply.id, 5);
let err = reply.result.unwrap_err();
assert_eq!(err.code, RpcErrorCode::DeadlineExceeded);
assert!(err.message.contains("250 ms"), "{}", err.message);
}
#[tokio::test]
async fn save_roll_writes_the_file_it_names_and_counts_what_it_recorded() {
let h = harness(vec![ProcScript::never_exits(), ProcScript::never_exits()]);
reply_of(
dispatch(
envelope(
1,
Request::Start {
apps: vec![
AppConfig::minimal("web", "./srv"),
AppConfig::minimal("worker", "./work"),
],
},
),
&h.ctx,
)
.await,
);
let reply = reply_of(dispatch(envelope(2, Request::SaveRoll), &h.ctx).await);
let Ok(Response::RollSaved { path, apps }) = reply.result else {
panic!("expected RollSaved, got {:?}", reply.result)
};
assert_eq!(apps, 2);
let roll = crate::snapshot::read(std::path::Path::new(&path)).unwrap();
assert_eq!(roll.apps.len(), 2, "the reply's count must match the file");
assert_eq!(path, h.ctx.snapshot_path.display().to_string());
}
#[tokio::test]
async fn a_scale_is_recorded_in_the_roll_the_next_muster_reads() {
let h = harness(vec![ProcScript::never_exits(); 4]);
let mut app = AppConfig::minimal("web", "./srv");
app.instances = 2;
reply_of(dispatch(envelope(1, Request::Start { apps: vec![app] }), &h.ctx).await);
reply_of(
dispatch(
envelope(
2,
Request::Scale {
name: "web".to_string(),
count: 4,
},
),
&h.ctx,
)
.await,
);
let reply = reply_of(dispatch(envelope(3, Request::SaveRoll), &h.ctx).await);
let Ok(Response::RollSaved { path, .. }) = reply.result else {
panic!("expected RollSaved, got {:?}", reply.result)
};
let roll = crate::snapshot::read(std::path::Path::new(&path)).unwrap();
assert_eq!(roll.apps[0].app.instances, 4);
}
#[tokio::test]
async fn a_partial_scale_is_recorded_in_the_roll_and_still_reported_short() {
let h = harness(vec![ProcScript::never_exits(); 3]);
let mut app = AppConfig::minimal("web", "./srv");
app.instances = 2;
reply_of(dispatch(envelope(1, Request::Start { apps: vec![app] }), &h.ctx).await);
let reply = reply_of(
dispatch(
envelope(
2,
Request::Scale {
name: "web".to_string(),
count: 4,
},
),
&h.ctx,
)
.await,
);
let err = reply.result.unwrap_err();
assert_eq!(err.code, RpcErrorCode::SpawnFailed);
assert!(
err.message.contains("3 of 4"),
"the operator has to be told both numbers: {}",
err.message
);
let saved = reply_of(dispatch(envelope(3, Request::SaveRoll), &h.ctx).await);
let Ok(Response::RollSaved { path, .. }) = saved.result else {
panic!("expected RollSaved, got {:?}", saved.result)
};
let roll = crate::snapshot::read(std::path::Path::new(&path)).unwrap();
assert_eq!(
roll.apps[0].app.instances, 3,
"the roll must hold the three instances really running — not the \
pre-scale two, and not the four that were asked for"
);
}
#[tokio::test]
async fn save_roll_against_a_stopped_engine_is_an_error_not_a_silent_success() {
let h = harness(vec![]);
h.ctx.supervisor.shutdown().await;
let reply = reply_of(dispatch(envelope(1, Request::SaveRoll), &h.ctx).await);
let err = reply.result.unwrap_err();
assert_eq!(err.code, RpcErrorCode::Internal);
assert!(
err.message.contains("engine"),
"the operator must be told why nothing was written: {}",
err.message
);
}
#[tokio::test]
async fn a_second_muster_still_reports_the_flock_the_roll_restored() {
let h = harness(vec![ProcScript::never_exits()]);
reply_of(
dispatch(
envelope(
1,
Request::Start {
apps: vec![AppConfig::minimal("web", "./srv")],
},
),
&h.ctx,
)
.await,
);
reply_of(dispatch(envelope(2, Request::SaveRoll), &h.ctx).await);
let reply = reply_of(dispatch(envelope(3, Request::Muster), &h.ctx).await);
let Ok(Response::Mustered(infos)) = reply.result else {
panic!("expected Mustered, got {:?}", reply.result)
};
assert_eq!(
infos.len(),
1,
"the sheep the roll restores, not the ones this call spawned"
);
assert_eq!(infos[0].name, "web");
assert_eq!(infos[0].status, ProcStatus::Online);
}
async fn start_web(h: &Harness) {
reply_of(
dispatch(
envelope(
1,
Request::Start {
apps: vec![AppConfig::minimal("web", "./srv")],
},
),
&h.ctx,
)
.await,
);
}
async fn list_flock(ctx: &RpcContext, id: u64) -> Vec<ProcessInfo> {
let reply = reply_of(dispatch(envelope(id, Request::ListFlock), ctx).await);
let Ok(Response::Flock(infos)) = reply.result else {
panic!("expected Flock, got {:?}", reply.result)
};
infos
}
#[tokio::test]
async fn list_flock_carries_a_live_memory_reading_for_a_running_sheep() {
let h = harness_with_stats(vec![ProcScript::never_exits()]);
start_web(&h).await;
let infos = list_flock(&h.ctx, 2).await;
assert_eq!(infos[0].pid, Some(FIRST_SCRIPTED_PID));
assert_eq!(infos[0].memory_bytes, Some(SCRIPTED_TREE_BYTES));
assert_eq!(
infos[0].cpu_percent, None,
"no periodic baseline has been recorded, and a number invented \
from the read's own window is worse than an empty cell"
);
}
#[tokio::test]
async fn list_flock_measures_cpu_from_the_periodic_baseline_not_from_the_previous_listing() {
let h = harness_with_stats(vec![ProcScript::never_exits()]);
start_web(&h).await;
let last_tick = Instant::now()
.checked_sub(MEMORY_POLL_INTERVAL)
.expect("the monotonic clock is older than one poll interval");
h.stats.record_baseline_now(last_tick);
let first = list_flock(&h.ctx, 2).await[0]
.cpu_percent
.expect("a baseline exists, so a running sheep has a CPU figure");
let second = list_flock(&h.ctx, 3).await[0]
.cpu_percent
.expect("a baseline exists, so a running sheep has a CPU figure");
assert!(
(5.0..=10.05).contains(&first),
"1500 CPU-ms over the 15 s since the baseline is 10%; got {first}"
);
assert!(
(second - first).abs() < 1.0,
"the second listing divided by the gap between the two LISTINGS \
rather than by the window since the tick: {first} then {second}"
);
}
#[tokio::test]
async fn describe_carries_a_live_reading_too() {
let h = harness_with_stats(vec![ProcScript::never_exits()]);
start_web(&h).await;
let described = reply_of(
dispatch(
envelope(
2,
Request::Describe {
selector: SelectorSpec::Name("web".to_string()),
},
),
&h.ctx,
)
.await,
);
let Ok(Response::Described(infos)) = described.result else {
panic!("expected Described, got {:?}", described.result)
};
assert_eq!(infos[0].memory_bytes, Some(SCRIPTED_TREE_BYTES));
}
#[tokio::test]
async fn a_sheep_with_no_pid_reports_no_stats() {
let h = harness_with_stats(vec![ProcScript::never_exits(), ProcScript::never_exits()]);
start_web(&h).await;
reply_of(
dispatch(
envelope(
2,
Request::Start {
apps: vec![AppConfig::minimal("worker", "./srv")],
},
),
&h.ctx,
)
.await,
);
reply_of(
dispatch(
envelope(
3,
Request::Stop {
selector: SelectorSpec::Name("worker".to_string()),
},
),
&h.ctx,
)
.await,
);
let infos = list_flock(&h.ctx, 4).await;
let named = |name: &str| {
infos
.iter()
.find(|info| info.name == name)
.unwrap_or_else(|| panic!("{name} is missing from the listing"))
};
assert_eq!(named("web").pid, Some(FIRST_SCRIPTED_PID));
assert_eq!(named("web").memory_bytes, Some(SCRIPTED_TREE_BYTES));
assert_eq!(named("worker").pid, None);
assert_eq!(named("worker").memory_bytes, None);
assert_eq!(named("worker").cpu_percent, None);
}
#[tokio::test]
async fn a_lifecycle_reply_carries_no_stats() {
let h = harness_with_stats(vec![ProcScript::never_exits()]);
let started = reply_of(
dispatch(
envelope(
1,
Request::Start {
apps: vec![AppConfig::minimal("web", "./srv")],
},
),
&h.ctx,
)
.await,
);
let Ok(Response::Started(infos)) = started.result else {
panic!("expected Started, got {:?}", started.result)
};
assert_eq!(
infos[0].pid,
Some(FIRST_SCRIPTED_PID),
"a row with no pid would report no stats however the verb behaved"
);
assert_eq!(
infos[0].memory_bytes, None,
"only `flock` and `describe` take a live sample"
);
assert_eq!(infos[0].cpu_percent, None);
}
async fn enable_dog(ctx: &RpcContext, id: u64, name: &str) -> ProcessInfo {
let reply = reply_of(
dispatch(
envelope(
id,
Request::EnableDog {
name: name.to_string(),
source: DogSource::BuiltIn,
},
),
ctx,
)
.await,
);
let Ok(Response::DogStarted(info)) = reply.result else {
panic!("expected DogStarted, got {:?}", reply.result)
};
info
}
async fn start_web_with_a_secret(ctx: &RpcContext) {
let mut config = AppConfig::minimal("web", "./srv");
config
.env
.insert("DB_PASS".to_string(), "hunter2".to_string());
let started =
reply_of(dispatch(envelope(1, Request::Start { apps: vec![config] }), ctx).await);
assert!(started.result.is_ok(), "{:?}", started.result);
}
#[tokio::test(start_paused = true)]
async fn a_dog_is_refused_an_env_override_rather_than_given_one() {
let h = harness(vec![ProcScript::never_exits()]);
let dog = enable_dog(&h.ctx, 1, "bark").await;
let reply = reply_of(
dispatch(
envelope(
2,
Request::SetSheepEnv {
name: "bark".to_string(),
key: "PATH".to_string(),
value: Some("/tmp/evil".to_string().into()),
},
),
&h.ctx,
)
.await,
);
let Err(err) = reply.result else {
panic!("a dog was given an env override")
};
assert_eq!(err.code, RpcErrorCode::InvalidConfig);
assert!(err.message.contains("bark is a dog"), "{}", err.message);
assert!(
shep_core::overrides::get(&h.ctx.paths.overrides, "bark")
.unwrap()
.is_none(),
"the refusal still wrote the store"
);
let described = reply_of(
dispatch(
envelope(
3,
Request::Describe {
selector: SelectorSpec::Name("bark".to_string()),
},
),
&h.ctx,
)
.await,
);
let Ok(Response::Described(infos)) = described.result else {
panic!("expected Described")
};
assert_eq!(infos[0].id, dog.id);
assert_eq!(infos[0].pending, None, "the refusal still parked a config");
}
#[tokio::test(start_paused = true)]
async fn a_dogs_config_is_not_readable_through_the_sheep_config_pane() {
let h = harness(vec![ProcScript::never_exits()]);
enable_dog(&h.ctx, 1, "bark").await;
let reply = reply_of(
dispatch(
envelope(
2,
Request::SheepConfig {
name: "bark".to_string(),
},
),
&h.ctx,
)
.await,
);
let Err(err) = reply.result else {
panic!("a dog's config was served to a pane")
};
assert_eq!(err.code, RpcErrorCode::InvalidConfig);
assert!(err.message.contains("bark is a dog"), "{}", err.message);
}
async fn sheep_config_view(
ctx: &RpcContext,
id: u64,
name: &str,
) -> shep_core::protocol::SheepConfigView {
let reply = reply_of(
dispatch(
envelope(
id,
Request::SheepConfig {
name: name.to_string(),
},
),
ctx,
)
.await,
);
match reply.result {
Ok(Response::SheepConfig(view)) => *view,
other => panic!("expected SheepConfig, got {other:?}"),
}
}
#[tokio::test(start_paused = true)]
async fn sheep_config_answers_with_env_emptied_and_its_keys_listed() {
let h = harness(vec![ProcScript::never_exits()]);
start_web_with_a_secret(&h.ctx).await;
let reply = reply_of(
dispatch(
envelope(
2,
Request::SheepConfig {
name: "web".to_string(),
},
),
&h.ctx,
)
.await,
);
let Ok(Response::SheepConfig(view)) = reply.result else {
panic!("expected SheepConfig")
};
assert_eq!(view.name, "web");
assert!(view.config.env.is_empty());
assert_eq!(view.env_keys, ["DB_PASS"]);
}
#[tokio::test(start_paused = true)]
async fn sheep_config_for_an_unknown_name_is_not_found_not_internal() {
let h = harness(vec![]);
let reply = reply_of(
dispatch(
envelope(
1,
Request::SheepConfig {
name: "ghost".to_string(),
},
),
&h.ctx,
)
.await,
);
let Err(err) = reply.result else {
panic!("expected a refusal")
};
assert_eq!(err.code, RpcErrorCode::NotFound);
}
#[tokio::test(start_paused = true)]
async fn set_sheep_env_writes_the_store_and_parks_env_until_a_respawn() {
let h = harness(vec![ProcScript::never_exits()]);
start_web_with_a_secret(&h.ctx).await;
let reply = reply_of(
dispatch(
envelope(
2,
Request::SetSheepEnv {
name: "web".to_string(),
key: "NEW".to_string(),
value: Some("1".to_string().into()),
},
),
&h.ctx,
)
.await,
);
assert!(
matches!(reply.result, Ok(Response::SheepEnvSet { .. })),
"{:?}",
reply.result
);
let stored = shep_core::overrides::get(&h.ctx.paths.overrides, "web")
.unwrap()
.unwrap();
assert_eq!(stored.fields["env"]["NEW"], "1");
let described = reply_of(
dispatch(
envelope(
3,
Request::Describe {
selector: SelectorSpec::Name("web".to_string()),
},
),
&h.ctx,
)
.await,
);
let Ok(Response::Described(infos)) = described.result else {
panic!("expected Described")
};
assert_eq!(
infos[0].pending.as_deref(),
Some(["env".to_string()].as_slice())
);
}
#[tokio::test(start_paused = true)]
async fn removing_the_last_env_override_stops_marking_the_sheep() {
let h = harness(vec![ProcScript::never_exits()]);
start_web_with_a_secret(&h.ctx).await;
for (id, value) in [(2, Some("1".to_string().into())), (3, None)] {
let reply = reply_of(
dispatch(
envelope(
id,
Request::SetSheepEnv {
name: "web".to_string(),
key: "NEW".to_string(),
value,
},
),
&h.ctx,
)
.await,
);
assert!(reply.result.is_ok(), "{:?}", reply.result);
}
let stored = shep_core::overrides::get(&h.ctx.paths.overrides, "web")
.unwrap()
.unwrap();
assert!(!stored.fields.contains_key("env"), "{stored:?}");
let reply = reply_of(
dispatch(
envelope(
4,
Request::SheepConfig {
name: "web".to_string(),
},
),
&h.ctx,
)
.await,
);
let Ok(Response::SheepConfig(view)) = reply.result else {
panic!("expected SheepConfig")
};
assert!(view.overridden.is_empty(), "{view:?}");
}
#[tokio::test(start_paused = true)]
async fn a_set_env_reaches_the_roll_a_cold_restart_would_come_back_on() {
let h = harness(vec![ProcScript::never_exits()]);
start_web_with_a_secret(&h.ctx).await;
let reply = reply_of(
dispatch(
envelope(
2,
Request::SetSheepEnv {
name: "web".to_string(),
key: "NEW".to_string(),
value: Some("1".to_string().into()),
},
),
&h.ctx,
)
.await,
);
assert!(reply.result.is_ok(), "{:?}", reply.result);
let infos = list_flock(&h.ctx, 3).await;
let roll = h.ctx.registry.roll(&infos, 0);
let web = roll
.apps
.iter()
.find(|entry| entry.app.name == "web")
.expect("web is in the roll");
assert_eq!(web.app.env.get("NEW").map(String::as_str), Some("1"));
}
#[tokio::test(start_paused = true)]
async fn a_parked_env_edit_reaches_the_roll_before_a_hard_stop_can_lose_it() {
let h = harness(vec![ProcScript::never_exits()]);
start_web_with_a_secret(&h.ctx).await;
h.ctx.save_roll_now().await.unwrap();
let baseline = crate::snapshot::read(&h.ctx.snapshot_path).unwrap();
assert_eq!(baseline.apps[0].app.env.get("NEW"), None);
let writer = crate::snapshot::spawn_snapshot_writer(
h.ctx.snapshot_path.clone(),
h.ctx.supervisor.clone(),
h.ctx.registry.clone(),
h.ctx.events.subscribe(),
);
let settle = std::time::Duration::from_millis(crate::snapshot::SNAPSHOT_DEBOUNCE_MS * 2);
tokio::time::sleep(settle).await;
let reply = reply_of(
dispatch(
envelope(
2,
Request::SetSheepEnv {
name: "web".to_string(),
key: "NEW".to_string(),
value: Some("1".to_string().into()),
},
),
&h.ctx,
)
.await,
);
assert!(reply.result.is_ok(), "{:?}", reply.result);
tokio::time::sleep(settle).await;
writer.stop().await;
let (events, _rx) = crate::bus::test_bus(64);
let cold = crate::supervisor::spawn_supervisor(
crate::fake::ScriptedRunner::new(vec![ProcScript::never_exits()]),
h.ctx.paths.clone(),
events,
);
let cold_registry = crate::snapshot::FlockRegistry::new();
let restored = crate::snapshot::muster(&h.ctx.snapshot_path, &cold_registry, &cold)
.await
.unwrap();
assert_eq!(restored, vec!["web".to_string()]);
let listed = cold.list().await;
let came_back = cold_registry.roll(&listed, 0);
assert_eq!(
came_back.apps[0].app.env.get("NEW").map(String::as_str),
Some("1"),
"the sheep must come back on the edit its CFG cell claims: {:?}",
listed[0].overridden
);
cold.shutdown().await;
}
#[tokio::test(start_paused = true)]
async fn removing_a_key_the_operator_never_set_is_still_recorded() {
let h = harness(vec![ProcScript::never_exits()]);
start_web_with_a_secret(&h.ctx).await;
let reply = reply_of(
dispatch(
envelope(
2,
Request::SetSheepEnv {
name: "web".to_string(),
key: "DB_PASS".to_string(),
value: None,
},
),
&h.ctx,
)
.await,
);
assert!(reply.result.is_ok(), "{:?}", reply.result);
let stored = shep_core::overrides::get(&h.ctx.paths.overrides, "web")
.unwrap()
.expect("the removal is recorded");
assert_eq!(
stored.fields["env"]["DB_PASS"],
serde_json::Value::Null,
"a removal is a tombstone, not an absence"
);
let view = sheep_config_view(&h.ctx, 3, "web").await;
assert_eq!(view.overridden, ["env"]);
assert!(!view.env_keys.contains(&"DB_PASS".to_string()));
}
#[tokio::test(start_paused = true)]
async fn a_removed_key_stays_removed_and_stays_reported_across_reloads() {
let h = harness(vec![ProcScript::never_exits()]);
start_web_with_a_secret(&h.ctx).await;
let removed = reply_of(
dispatch(
envelope(
2,
Request::SetSheepEnv {
name: "web".to_string(),
key: "DB_PASS".to_string(),
value: None,
},
),
&h.ctx,
)
.await,
);
assert!(removed.result.is_ok(), "{:?}", removed.result);
for id in [3, 4] {
let loaded = reply_of(
dispatch(
envelope(
id,
Request::ApplyConfig {
apps: vec![DeclaredApp {
config: {
let mut app = AppConfig::minimal("web", "./srv");
app.env
.insert("DB_PASS".to_string(), "fromfile".to_string());
app
},
declared: ["name", "script"]
.iter()
.map(|k| (*k).to_string())
.collect(),
declared_env: ["DB_PASS"]
.iter()
.map(|k| (*k).to_string())
.collect(),
}],
reset: ResetDepth::None,
},
),
&h.ctx,
)
.await,
);
let Ok(Response::Applied(report)) = loaded.result else {
panic!("expected Applied")
};
assert_eq!(report[0].refused, None);
let stored = shep_core::overrides::get(&h.ctx.paths.overrides, "web")
.unwrap()
.expect("the removal is still recorded");
assert_eq!(
stored.fields["env"]["DB_PASS"],
serde_json::Value::Null,
"load {id} spent the tombstone"
);
let view = sheep_config_view(&h.ctx, id + 10, "web").await;
assert!(
!view.env_keys.contains(&"DB_PASS".to_string()),
"load {id} put the file's value back"
);
assert_eq!(view.overridden, ["env"], "load {id} stopped reporting it");
}
}
#[tokio::test(start_paused = true)]
async fn an_env_key_normalize_refuses_is_invalid_config_and_writes_nothing() {
let h = harness(vec![ProcScript::never_exits()]);
start_web_with_a_secret(&h.ctx).await;
let reply = reply_of(
dispatch(
envelope(
2,
Request::SetSheepEnv {
name: "web".to_string(),
key: "SHEP_NAME".to_string(),
value: Some("impostor".to_string().into()),
},
),
&h.ctx,
)
.await,
);
let Err(err) = reply.result else {
panic!("a reserved env key was accepted")
};
assert_eq!(err.code, RpcErrorCode::InvalidConfig);
assert!(
shep_core::overrides::get(&h.ctx.paths.overrides, "web")
.unwrap()
.is_none(),
"the refusal still wrote the store"
);
}
#[tokio::test(start_paused = true)]
async fn an_unreadable_override_store_is_internal_not_a_bad_request() {
let h = harness(vec![ProcScript::never_exits()]);
start_web_with_a_secret(&h.ctx).await;
std::fs::write(&h.ctx.paths.overrides, "{ this is not json").unwrap();
let reply = reply_of(
dispatch(
envelope(
2,
Request::SetSheepEnv {
name: "web".to_string(),
key: "NEW".to_string(),
value: Some("1".to_string().into()),
},
),
&h.ctx,
)
.await,
);
let Err(err) = reply.result else {
panic!("an unreadable store was reported as success")
};
assert_eq!(err.code, RpcErrorCode::Internal);
assert!(
err.message.contains("overrides store unusable"),
"{}",
err.message
);
}
async fn set_field(
ctx: &RpcContext,
id: u64,
name: &str,
key: &str,
value: serde_json::Value,
) -> Result<Response, RpcError> {
reply_of(
dispatch(
envelope(
id,
Request::SetSheepField {
name: name.to_string(),
key: key.to_string(),
value,
},
),
ctx,
)
.await,
)
.result
}
#[tokio::test(start_paused = true)]
async fn a_field_edit_is_reported_as_an_operator_override() {
let h = harness(vec![ProcScript::never_exits()]);
start_web_with_a_secret(&h.ctx).await;
let view = sheep_config_view(&h.ctx, 2, "web").await;
assert!(
!view.overridden.contains(&"max_restarts".to_string()),
"nothing is overridden before the edit: {:?}",
view.overridden
);
let reply = set_field(&h.ctx, 3, "web", "max_restarts", serde_json::json!(40)).await;
assert!(
matches!(reply, Ok(Response::SheepFieldSet { .. })),
"{reply:?}"
);
let stored = shep_core::overrides::get(&h.ctx.paths.overrides, "web")
.unwrap()
.expect("the edit is recorded");
assert_eq!(stored.fields["max_restarts"], 40);
let view = sheep_config_view(&h.ctx, 4, "web").await;
assert_eq!(view.config.max_restarts, 40, "the pane shows the new value");
assert!(
view.overridden.contains(&"max_restarts".to_string()),
"the `*` marker reads this: {:?}",
view.overridden
);
let infos = list_flock(&h.ctx, 5).await;
let web = infos
.iter()
.find(|info| info.name == "web")
.expect("web is in the flock");
assert!(
web.overridden
.as_deref()
.is_some_and(|fields| fields.contains(&"max_restarts".to_string())),
"{:?}",
web.overridden
);
}
#[tokio::test(start_paused = true)]
async fn a_live_field_applies_now_and_a_respawn_field_parks() {
let h = harness(vec![ProcScript::never_exits()]);
start_web_with_a_secret(&h.ctx).await;
let reply = set_field(&h.ctx, 2, "web", "max_restarts", serde_json::json!(40)).await;
let Ok(Response::SheepFieldSet { pending, .. }) = reply else {
panic!("{reply:?}")
};
assert!(!pending, "max_restarts is Live and is in force now");
let view = sheep_config_view(&h.ctx, 3, "web").await;
assert!(
!view.pending.contains(&"max_restarts".to_string()),
"{:?}",
view.pending
);
let reply = set_field(&h.ctx, 4, "web", "script", serde_json::json!("./next")).await;
let Ok(Response::SheepFieldSet { pending, .. }) = reply else {
panic!("{reply:?}")
};
assert!(pending, "script needs a respawn");
let view = sheep_config_view(&h.ctx, 5, "web").await;
assert!(
view.pending.contains(&"script".to_string()),
"{:?}",
view.pending
);
assert_eq!(view.config.max_restarts, 40);
assert_eq!(
view.overridden,
["max_restarts", "script"],
"both are the operator's"
);
}
#[tokio::test(start_paused = true)]
async fn a_live_field_whose_subset_will_not_normalize_parks_instead() {
let h = harness(vec![ProcScript::never_exits()]);
let started = reply_of(
dispatch(
envelope(
1,
Request::Start {
apps: vec![AppConfig::minimal("web", "./srv")],
},
),
&h.ctx,
)
.await,
);
assert!(started.result.is_ok(), "{:?}", started.result);
let reply = set_field(&h.ctx, 2, "web", "cwd", serde_json::json!("/srv")).await;
let Ok(Response::SheepFieldSet { pending, .. }) = reply else {
panic!("{reply:?}")
};
assert!(pending, "cwd needs a respawn");
let reply = set_field(&h.ctx, 3, "web", "watch", serde_json::json!(true)).await;
let Ok(Response::SheepFieldSet { pending, .. }) = reply else {
panic!("{reply:?}")
};
assert_eq!(
apply_group("watch"),
ApplyGroup::Live,
"the premise: apply_group predicts this one applies now"
);
assert!(
pending,
"a Live field the running child cannot be given still parks"
);
let view = sheep_config_view(&h.ctx, 4, "web").await;
assert!(
view.pending.contains(&"watch".to_string()),
"{:?}",
view.pending
);
}
#[tokio::test(start_paused = true)]
async fn autostart_reports_in_force_and_its_group_mate_reports_pending() {
let h = harness(vec![ProcScript::never_exits()]);
start_web_with_a_secret(&h.ctx).await;
for key in ["autostart", "kill_signal"] {
assert_eq!(
apply_group(key),
ApplyGroup::NextSpawn,
"the premise: both are the same group"
);
}
let reply = set_field(&h.ctx, 2, "web", "autostart", serde_json::json!(false)).await;
let Ok(Response::SheepFieldSet { pending, .. }) = reply else {
panic!("{reply:?}")
};
assert!(
!pending,
"autostart is read at muster, so a restart would do nothing"
);
let reply = set_field(&h.ctx, 3, "web", "kill_signal", serde_json::json!("SIGINT")).await;
let Ok(Response::SheepFieldSet { pending, .. }) = reply else {
panic!("{reply:?}")
};
assert!(pending, "kill_signal really is read at a spawn");
}
#[tokio::test(start_paused = true)]
async fn a_dog_is_refused_a_config_field_rather_than_given_one() {
let h = harness(vec![ProcScript::never_exits()]);
let dog = enable_dog(&h.ctx, 1, "bark").await;
let reply = set_field(&h.ctx, 2, "bark", "script", serde_json::json!("/tmp/evil")).await;
let Err(err) = reply else {
panic!("a dog was given a config field")
};
assert_eq!(err.code, RpcErrorCode::InvalidConfig);
assert!(err.message.contains("bark is a dog"), "{}", err.message);
assert!(
shep_core::overrides::get(&h.ctx.paths.overrides, "bark")
.unwrap()
.is_none(),
"the refusal still wrote the store"
);
drop(dog);
}
#[tokio::test(start_paused = true)]
async fn env_and_the_structural_fields_are_refused_by_this_door() {
let h = harness(vec![ProcScript::never_exits()]);
start_web_with_a_secret(&h.ctx).await;
for (id, key, value) in [
(2, "env", serde_json::json!({ "A": "1" })),
(3, "instances", serde_json::json!(4)),
(4, "name", serde_json::json!("other")),
] {
let reply = set_field(&h.ctx, id, "web", key, value).await;
let Err(err) = reply else {
panic!("{key} was accepted")
};
assert_eq!(err.code, RpcErrorCode::InvalidConfig, "{key}");
assert!(err.message.contains(key), "{key}: {}", err.message);
}
assert!(
shep_core::overrides::get(&h.ctx.paths.overrides, "web")
.unwrap()
.is_none(),
"a refusal still wrote the store"
);
}
#[tokio::test(start_paused = true)]
async fn a_field_this_build_refuses_is_invalid_config_and_writes_nothing() {
let h = harness(vec![ProcScript::never_exits()]);
start_web_with_a_secret(&h.ctx).await;
for (id, key, value) in [
(2, "no_such_field", serde_json::json!(1)),
(3, "max_restarts", serde_json::json!("forty")),
(
4,
"cron_restart",
serde_json::json!("not a cron expression"),
),
] {
let reply = set_field(&h.ctx, id, "web", key, value).await;
let Err(err) = reply else {
panic!("{key} was accepted")
};
assert_eq!(err.code, RpcErrorCode::InvalidConfig, "{key}");
assert!(
shep_core::overrides::get(&h.ctx.paths.overrides, "web")
.unwrap()
.is_none(),
"{key}: the refusal still wrote the store"
);
}
}
#[tokio::test(start_paused = true)]
async fn an_unreadable_store_is_internal_for_a_field_edit_too() {
let h = harness(vec![ProcScript::never_exits()]);
start_web_with_a_secret(&h.ctx).await;
std::fs::write(&h.ctx.paths.overrides, "{ this is not json").unwrap();
let reply = set_field(&h.ctx, 2, "web", "max_restarts", serde_json::json!(40)).await;
let Err(err) = reply else {
panic!("an unreadable store was reported as success")
};
assert_eq!(err.code, RpcErrorCode::Internal);
assert!(
err.message.contains("overrides store unusable"),
"{}",
err.message
);
}
#[tokio::test(start_paused = true)]
async fn a_field_edit_reaches_the_muster_roll() {
let h = harness(vec![ProcScript::never_exits()]);
start_web_with_a_secret(&h.ctx).await;
let reply = set_field(&h.ctx, 2, "web", "max_restarts", serde_json::json!(40)).await;
assert!(reply.is_ok(), "{reply:?}");
let infos = list_flock(&h.ctx, 3).await;
let roll = h.ctx.registry.roll(&infos, 0);
let web = roll
.apps
.iter()
.find(|entry| entry.app.name == "web")
.expect("web is in the roll");
assert_eq!(web.app.max_restarts, 40);
}
#[tokio::test(start_paused = true)]
async fn every_new_variant_reaches_an_arm_and_not_the_wildcard() {
let h = harness(vec![]);
let requests = [
Request::SheepConfig {
name: "ghost".to_string(),
},
Request::SetSheepEnv {
name: "ghost".to_string(),
key: "K".to_string(),
value: None,
},
Request::SetSheepField {
name: "ghost".to_string(),
key: "max_restarts".to_string(),
value: serde_json::json!(1),
},
Request::SetDogConfig {
name: "ghost".to_string(),
toml: String::new().into(),
},
];
for (id, request) in requests.into_iter().enumerate() {
let reply = reply_of(
dispatch(
envelope(
u64::try_from(id).expect("three requests fit a u64"),
request,
),
&h.ctx,
)
.await,
);
let Err(err) = reply.result else {
continue;
};
assert_ne!(
err.message, "this daemon does not implement that request",
"a config-pane variant fell through to the wildcard"
);
}
}
#[tokio::test(start_paused = true)]
async fn disabling_a_dog_stops_it_and_takes_it_off_the_listing() {
let h = harness(vec![ProcScript::never_exits()]);
let info = enable_dog(&h.ctx, 1, "bark").await;
assert_eq!(info.dog, Some(DogSource::BuiltIn));
let disabled = reply_of(
dispatch(
envelope(
2,
Request::DisableDog {
name: "bark".to_string(),
},
),
&h.ctx,
)
.await,
);
assert_eq!(disabled.result.unwrap(), Response::Deleted(vec![info.id]));
assert!(h.ctx.supervisor.list().await.is_empty());
}
#[tokio::test(start_paused = true)]
async fn a_dog_config_request_reads_the_file_as_it_stands_now() {
let h = harness(vec![]);
std::fs::write(&h.ctx.dogs_config, "[bark]\ndebounce = \"30s\"\n").unwrap();
let reply = reply_of(
dispatch(
envelope(
1,
Request::DogConfig {
name: "bark".to_string(),
},
),
&h.ctx,
)
.await,
);
let Ok(Response::DogSection { toml }) = reply.result else {
panic!("expected DogSection, got {:?}", reply.result)
};
assert!(toml.as_str().contains("30s"));
}
#[tokio::test(start_paused = true)]
async fn set_dog_config_writes_the_file_and_a_subscriber_hears_about_it() {
let h = harness(vec![ProcScript::never_exits()]);
enable_dog(&h.ctx, 1, "bark").await;
let mut sub = h.ctx.events.subscribe();
let reply = reply_of(
dispatch(
envelope(
2,
Request::SetDogConfig {
name: "bark".to_string(),
toml: "poll = \"30s\"\n".to_string().into(),
},
),
&h.ctx,
)
.await,
);
assert_eq!(
reply.result.unwrap(),
Response::DogConfigSet {
name: "bark".to_string()
}
);
let text = std::fs::read_to_string(&h.ctx.dogs_config).unwrap();
assert!(text.contains("poll = \"30s\""), "{text}");
let mut topics = Vec::new();
while let Ok(Ok(event)) = tokio::time::timeout(Duration::from_secs(5), sub.recv()).await {
topics.push(event.to_event().topic().into_owned());
if topics
.last()
.is_some_and(|topic| topic == "config.dog.bark")
{
break;
}
}
assert!(
topics.iter().any(|topic| topic == "config.dog.bark"),
"{topics:?}"
);
}
#[tokio::test(start_paused = true)]
async fn a_dog_that_is_adopted_and_never_started_can_still_be_configured() {
let mut h = harness(vec![]);
h.ctx.known_dogs = KnownDogs::new(["otel".to_string()].into_iter().collect());
let reply = reply_of(
dispatch(
envelope(
1,
Request::SetDogConfig {
name: "otel".to_string(),
toml: "endpoint = \"http://127.0.0.1:4317\"\n".to_string().into(),
},
),
&h.ctx,
)
.await,
);
assert_eq!(
reply.result.unwrap(),
Response::DogConfigSet {
name: "otel".to_string()
}
);
let text = std::fs::read_to_string(&h.ctx.dogs_config).unwrap();
assert!(text.contains("4317"), "{text}");
}
#[tokio::test(start_paused = true)]
async fn a_dog_that_is_enabled_but_not_running_can_still_be_configured() {
let h = harness(vec![]);
assert!(h.ctx.supervisor.list().await.is_empty());
let reply = reply_of(
dispatch(
envelope(
1,
Request::SetDogConfig {
name: "bark".to_string(),
toml: "poll = \"30s\"\n".to_string().into(),
},
),
&h.ctx,
)
.await,
);
assert_eq!(
reply.result.unwrap(),
Response::DogConfigSet {
name: "bark".to_string()
}
);
}
#[tokio::test(start_paused = true)]
async fn a_dog_adopted_since_boot_is_reached_through_the_running_flock() {
let mut h = harness(vec![ProcScript::never_exits()]);
h.ctx.known_dogs = KnownDogs::new(BTreeSet::new());
enable_dog(&h.ctx, 1, "bark").await;
let reply = reply_of(
dispatch(
envelope(
2,
Request::SetDogConfig {
name: "bark".to_string(),
toml: "poll = \"30s\"\n".to_string().into(),
},
),
&h.ctx,
)
.await,
);
assert_eq!(
reply.result.unwrap(),
Response::DogConfigSet {
name: "bark".to_string()
}
);
}
#[tokio::test(start_paused = true)]
async fn a_dog_adopted_since_boot_stays_configurable_once_it_is_disabled() {
let mut h = harness(vec![ProcScript::never_exits()]);
h.ctx.known_dogs = KnownDogs::new(BTreeSet::new());
enable_dog(&h.ctx, 1, "bark").await;
let disabled = reply_of(
dispatch(
envelope(
2,
Request::DisableDog {
name: "bark".to_string(),
},
),
&h.ctx,
)
.await,
);
assert!(disabled.result.is_ok(), "{:?}", disabled.result);
assert!(
!h.ctx
.supervisor
.list()
.await
.iter()
.any(|info| info.name == "bark"),
"the flock must not hold it, or the widening answers and the test means nothing"
);
let reply = reply_of(
dispatch(
envelope(
3,
Request::SetDogConfig {
name: "bark".to_string(),
toml: "poll = \"30s\"\n".to_string().into(),
},
),
&h.ctx,
)
.await,
);
assert_eq!(
reply.result.unwrap(),
Response::DogConfigSet {
name: "bark".to_string()
}
);
}
#[tokio::test(start_paused = true)]
async fn setting_a_dogs_config_over_a_sheeps_name_is_refused_and_writes_nothing() {
let h = harness(vec![ProcScript::never_exits()]);
let started = reply_of(
dispatch(
envelope(
1,
Request::Start {
apps: vec![AppConfig::minimal("web", "./srv")],
},
),
&h.ctx,
)
.await,
);
assert!(started.result.is_ok(), "{:?}", started.result);
std::fs::write(&h.ctx.dogs_config, "[bark]\npoll = \"60s\"\n").unwrap();
for (id, name) in [(2, "web"), (3, "ghost")] {
let reply = reply_of(
dispatch(
envelope(
id,
Request::SetDogConfig {
name: name.to_string(),
toml: "poll = \"30s\"\n".to_string().into(),
},
),
&h.ctx,
)
.await,
);
let Err(err) = reply.result else {
panic!("{name} is not a dog and must be refused")
};
assert_eq!(err.code, RpcErrorCode::NotFound, "{err:?}");
assert!(err.message.contains(name), "{err:?}");
}
assert_eq!(
std::fs::read_to_string(&h.ctx.dogs_config).unwrap(),
"[bark]\npoll = \"60s\"\n"
);
}
#[tokio::test]
async fn setting_a_dogs_config_against_a_stopped_engine_is_refused_not_a_panic() {
let h = harness(vec![]);
h.ctx.supervisor.shutdown().await;
let reply = reply_of(
dispatch(
envelope(
1,
Request::SetDogConfig {
name: "otel".to_string(),
toml: "poll = \"30s\"\n".to_string().into(),
},
),
&h.ctx,
)
.await,
);
let Err(err) = reply.result else {
panic!("otel is not a known dog and must be refused")
};
assert_eq!(err.code, RpcErrorCode::NotFound, "{err:?}");
}
#[tokio::test(start_paused = true)]
async fn describe_sweeps_past_a_dog_and_still_answers_when_one_is_named() {
let h = harness(vec![ProcScript::never_exits(), ProcScript::never_exits()]);
let started = reply_of(
dispatch(
envelope(
1,
Request::Start {
apps: vec![AppConfig::minimal("web", "./srv")],
},
),
&h.ctx,
)
.await,
);
assert!(started.result.is_ok(), "{:?}", started.result);
let dog = enable_dog(&h.ctx, 2, "bark").await;
let swept = reply_of(
dispatch(
envelope(
3,
Request::Describe {
selector: SelectorSpec::All,
},
),
&h.ctx,
)
.await,
);
let Ok(Response::Described(hits)) = swept.result else {
panic!("expected Described, got {:?}", swept.result)
};
assert_eq!(
hits.iter().map(|i| i.name.as_str()).collect::<Vec<_>>(),
vec!["web"],
"`all` is the flock, not the kennel"
);
let named = reply_of(
dispatch(
envelope(
4,
Request::Describe {
selector: SelectorSpec::Name("bark".to_string()),
},
),
&h.ctx,
)
.await,
);
let Ok(Response::Described(hits)) = named.result else {
panic!("expected Described, got {:?}", named.result)
};
assert_eq!(hits.len(), 1);
assert_eq!(hits[0].id, dog.id);
}
#[tokio::test(start_paused = true)]
async fn enabling_a_dog_over_a_sheeps_name_is_refused_rather_than_faked() {
let h = harness(vec![ProcScript::never_exits()]);
let started = reply_of(
dispatch(
envelope(
1,
Request::Start {
apps: vec![AppConfig::minimal("bark", "./srv")],
},
),
&h.ctx,
)
.await,
);
assert!(started.result.is_ok(), "{:?}", started.result);
let reply = reply_of(
dispatch(
envelope(
2,
Request::EnableDog {
name: "bark".to_string(),
source: DogSource::BuiltIn,
},
),
&h.ctx,
)
.await,
);
let Err(err) = reply.result else {
panic!("expected a refusal, got {:?}", reply.result)
};
assert_eq!(err.code, RpcErrorCode::InvalidConfig);
assert!(
err.message.contains("bark"),
"the refusal names the collision: {}",
err.message
);
let listed = h.ctx.supervisor.list().await;
assert_eq!(listed.len(), 1, "nothing was started: {listed:?}");
assert_eq!(listed[0].dog, None);
}
#[tokio::test(start_paused = true)]
async fn only_describe_carries_a_lamb_tree() {
let h = harness_identifying(
vec![ProcScript::never_exits()],
vec![
identity(FIRST_SCRIPTED_PID, None, "srv"),
identity(FIRST_SCRIPTED_PID + 1, Some(FIRST_SCRIPTED_PID), "node"),
],
);
reply_of(
dispatch(
envelope(
1,
Request::Start {
apps: vec![AppConfig::minimal("web", "./srv")],
},
),
&h.ctx,
)
.await,
);
let listed = reply_of(dispatch(envelope(2, Request::ListFlock), &h.ctx).await);
let Ok(Response::Flock(rows)) = listed.result else {
panic!("expected a flock listing");
};
assert!(
rows.iter().all(|row| row.lambs.is_none()),
"ListFlock must not walk the process table"
);
let described = reply_of(
dispatch(
envelope(
3,
Request::Describe {
selector: SelectorSpec::Name("web".to_string()),
},
),
&h.ctx,
)
.await,
);
let Ok(Response::Described(rows)) = described.result else {
panic!("expected a describe listing");
};
assert_eq!(
rows[0].lambs,
Some(vec![Lamb::new(FIRST_SCRIPTED_PID + 1, "node")])
);
}
async fn start_dog(ctx: &RpcContext, name: &str) -> ProcessInfo {
let spec = DogSpec {
name: name.to_string(),
source: DogSource::BuiltIn,
};
let app = crate::dogs::dog_app(&spec, &ctx.paths).expect("the dog fixture must assemble");
ctx.supervisor
.start_dog(app, DogSource::BuiltIn)
.await
.expect("the dog fixture must start")
}
async fn staleness(ctx: &RpcContext) -> (Vec<String>, Vec<String>) {
let reply = reply_of(dispatch(envelope(1, Request::DogStaleness), ctx).await);
let Ok(Response::DogStaleness { stale, pending }) = reply.result else {
panic!("expected a dog staleness answer");
};
(stale, pending)
}
#[tokio::test]
async fn a_flock_of_ordinary_sheep_has_nothing_stale_and_nothing_pending() {
let h = harness(vec![ProcScript::never_exits()]);
let started = reply_of(
dispatch(
envelope(
1,
Request::Start {
apps: vec![AppConfig::minimal("web", "./srv")],
},
),
&h.ctx,
)
.await,
);
started.result.expect("the sheep must start");
assert_eq!(staleness(&h.ctx).await, (Vec::new(), Vec::new()));
}
#[tokio::test]
async fn a_listing_says_which_dogs_have_answered_this_shepherd() {
let h = harness(vec![ProcScript::never_exits(), ProcScript::never_exits()]);
start_dog(&h.ctx, "metrics").await;
let silent = list_flock(&h.ctx, 1).await;
let dog = silent
.iter()
.find(|info| info.name == "metrics")
.expect("the dog must be listed");
assert_eq!(dog.handshook, Some(false));
assert_eq!(
dog.status,
ProcStatus::Online,
"the process is up, and the listing still says so"
);
h.ctx.dog_refusals.handshook("metrics");
let talking = list_flock(&h.ctx, 2).await;
assert_eq!(
talking
.iter()
.find(|info| info.name == "metrics")
.expect("still listed")
.handshook,
Some(true)
);
}
#[tokio::test]
async fn a_sheep_carries_no_handshake_fact_at_all() {
let h = harness(vec![ProcScript::never_exits()]);
start_web(&h).await;
let infos = list_flock(&h.ctx, 1).await;
assert_eq!(infos[0].name, "web");
assert_eq!(infos[0].handshook, None);
assert_eq!(
infos[0].dog_stale, None,
"a sheep is never given up on, because it was never asked to answer"
);
}
#[tokio::test]
async fn a_listing_says_which_silent_dogs_this_shepherd_gave_up_on() {
let h = harness(vec![ProcScript::never_exits()]);
start_dog(&h.ctx, "metrics").await;
let waiting = list_flock(&h.ctx, 1).await;
let dog = waiting
.iter()
.find(|info| info.name == "metrics")
.expect("the dog must be listed");
assert_eq!(dog.handshook, Some(false));
assert_eq!(
dog.dog_stale,
Some(false),
"a dog that has not answered YET is not one this shepherd gave up on"
);
h.ctx.dog_refusals.refused("metrics");
h.ctx.dog_refusals.refused("metrics");
let given_up = list_flock(&h.ctx, 2).await;
let dog = given_up
.iter()
.find(|info| info.name == "metrics")
.expect("still listed");
assert_eq!(dog.dog_stale, Some(true));
assert_eq!(
dog.status,
ProcStatus::Online,
"the process is still up, and the listing still says so"
);
h.ctx.dog_refusals.handshook("metrics");
let talking = list_flock(&h.ctx, 3).await;
let dog = talking
.iter()
.find(|info| info.name == "metrics")
.expect("still listed");
assert_eq!(dog.handshook, Some(true));
assert_eq!(dog.dog_stale, Some(false));
}
#[tokio::test]
async fn describe_carries_the_handshake_fact_too() {
let h = harness(vec![ProcScript::never_exits()]);
start_dog(&h.ctx, "metrics").await;
let described = reply_of(
dispatch(
envelope(
1,
Request::Describe {
selector: SelectorSpec::Name("metrics".to_string()),
},
),
&h.ctx,
)
.await,
);
let Ok(Response::Described(rows)) = described.result else {
panic!("expected a describe listing");
};
assert_eq!(rows[0].handshook, Some(false));
}
#[tokio::test]
async fn a_dog_that_has_not_handshaken_is_pending() {
let h = harness(vec![ProcScript::never_exits()]);
start_dog(&h.ctx, "metrics").await;
assert_eq!(
staleness(&h.ctx).await,
(Vec::new(), vec!["metrics".to_string()])
);
h.ctx.dog_refusals.handshook("metrics");
assert_eq!(
staleness(&h.ctx).await,
(Vec::new(), Vec::new()),
"a dog talking to this shepherd is settled and is not worth reporting"
);
}
#[tokio::test]
async fn a_dog_being_restarted_is_pending_and_then_stale() {
let h = harness(vec![ProcScript::never_exits()]);
start_dog(&h.ctx, "metrics").await;
h.ctx.dog_refusals.handshook("metrics");
h.ctx.dog_refusals.refused("metrics");
assert_eq!(
staleness(&h.ctx).await,
(Vec::new(), vec!["metrics".to_string()]),
"one refusal buys a restart; it does not condemn the dog"
);
h.ctx.dog_refusals.refused("metrics");
assert_eq!(
staleness(&h.ctx).await,
(vec!["metrics".to_string()], Vec::new()),
"a stale dog is a finding, not something still to wait on"
);
}
#[tokio::test]
async fn a_dog_that_has_stopped_running_is_not_waited_on() {
let h = harness(vec![ProcScript::never_exits()]);
start_dog(&h.ctx, "metrics").await;
assert_eq!(staleness(&h.ctx).await.1, vec!["metrics".to_string()]);
h.ctx
.supervisor
.stop(ProcessSelector::Name("metrics".to_string()))
.await
.expect("the dog must stop");
assert_eq!(
staleness(&h.ctx).await,
(Vec::new(), Vec::new()),
"a dog that is not running has nothing to answer with"
);
}
}