use core::cmp::Ordering;
use core::fmt;
use core::sync::atomic::{self, AtomicU64};
use core::time::Duration;
use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet, VecDeque};
use std::io;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::time::SystemTime;
use serde::{Deserialize, Serialize};
use tokio::sync::{mpsc, oneshot};
use shep_core::config::{
AppConfig, ApplyGroup, DeclaredApp, ResetDepth, ResolvedApp, apply_group, normalize,
};
use shep_core::overrides::{self, AppOverrides};
use shep_core::paths::ShepPaths;
use shep_core::protocol::{
ActionOutcome, ActionReply, BusEvent, DogSource, EnvValue, ExitInfo, LineOutcome, LineReply,
ProcessEventKind, ProcessInfo, SheepConfigView, SheepDrift, SignalOutcome, SignalReply, Smit,
sort_flock,
};
use shep_core::secrets::SecretView;
use shep_core::selector::ProcessSelector;
use shep_core::signals::OperatorSignal;
use shep_core::status::ProcStatus;
use shep_core::values::MemSize;
use crate::assemble::{AssembleError, assemble, describe, instance_slots};
use crate::backoff::restart_delay;
use crate::brain::{Decision, decide_on_exit};
use crate::bus::{Bus, SharedEvent};
use crate::channel::{ChildMessage, ShepherdMessage};
use crate::entry::{ProcessEntry, ReloadState, RestartBudget};
use crate::extras::{Extras, ExtrasRegistry};
#[cfg(unix)]
use crate::handover::adopt::AdoptedSheep;
#[cfg(unix)]
use crate::handover::reap::AdoptedReaper;
#[cfg(unix)]
use crate::handover::{
Candidate, CarriedFds, CarriedSheep, Counters, DaemonFds, Fitness, Handover, OwnedCandidate,
fitness,
};
use crate::kill::kill_process;
use crate::privilege::{self, Credentials, PrivilegeError, SpawnIdentity};
use crate::probes::Prober;
use crate::probes::os::OsProber;
use crate::probes::ready::{Readiness, ReadinessSource, await_ready};
#[cfg(unix)]
use crate::runner::AdoptSpec;
use crate::runner::{
ExitOutcome, FlushError, LogCtl, Preflight, ProcIo, ProcessRunner, ReopenError, RunnerError,
RunningProcess, SpawnSpec, StdinWrite, check_log_ancestry, open_log_path,
};
use crate::secrets::ProviderSecrets;
const MAILBOX_CAPACITY: usize = 256;
const SHEEP_CTL_CAPACITY: usize = 4;
const SIGNAL_CAPACITY: usize = 16;
const RELOAD_DEADLINE_SLACK: Duration = Duration::from_secs(5);
const STDIN_WRITE_TIMEOUT: Duration = Duration::from_secs(2);
const MAX_ABANDONED_ACTION_REPLIES: usize = 64;
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub(crate) struct ConnId(u64);
impl ConnId {
pub(crate) fn next() -> Self {
static NEXT: AtomicU64 = AtomicU64::new(0);
Self(NEXT.fetch_add(1, atomic::Ordering::Relaxed))
}
}
type Smits = HashMap<String, (ConnId, String)>;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum BatchPolicy {
AllOrNothing,
PerApp,
}
#[derive(Debug)]
enum Registration {
Fresh(ProcessInfo),
AlreadyKnown(ProcessInfo),
}
impl Registration {
fn into_info(self) -> ProcessInfo {
match self {
Self::Fresh(info) | Self::AlreadyKnown(info) => info,
}
}
}
#[derive(Debug)]
pub(crate) enum Command {
Start {
apps: Vec<ResolvedApp>,
policy: BatchPolicy,
gate: BTreeSet<String>,
reply: oneshot::Sender<Result<Vec<ProcessInfo>, SupervisorError>>,
},
RegisterAtRest {
apps: Vec<ResolvedApp>,
reply: oneshot::Sender<Result<Vec<ProcessInfo>, SupervisorError>>,
},
ConfigDrift {
apps: Vec<ResolvedApp>,
reply: oneshot::Sender<Result<Vec<SheepDrift>, SupervisorError>>,
},
StartDog {
app: Box<ResolvedApp>,
source: DogSource,
reply: oneshot::Sender<Result<ProcessInfo, SupervisorError>>,
},
Stop {
selector: ProcessSelector,
reply: oneshot::Sender<Result<Vec<ProcessInfo>, SupervisorError>>,
},
Restart {
selector: ProcessSelector,
origin: CommandOrigin,
reply: oneshot::Sender<Result<Vec<ProcessInfo>, SupervisorError>>,
},
ExtraRestart {
id: u32,
pid: u32,
epoch: Option<u64>,
observed: Option<MemSize>,
},
Reload {
selector: ProcessSelector,
reply: oneshot::Sender<Result<Vec<ProcessInfo>, SupervisorError>>,
},
Delete {
selector: ProcessSelector,
reply: oneshot::Sender<Result<Vec<u32>, SupervisorError>>,
},
Scale {
name: String,
count: u32,
reply: oneshot::Sender<Result<Scaled, SupervisorError>>,
},
ApplyConfig {
apps: Vec<DeclaredApp>,
reset: ResetDepth,
reply: oneshot::Sender<Result<Vec<Applied>, SupervisorError>>,
},
SheepConfig {
name: String,
reply: oneshot::Sender<Result<Option<SheepConfigView>, SupervisorError>>,
},
SetSheepEnv {
name: String,
key: String,
value: Option<EnvValue>,
reply: oneshot::Sender<Result<Option<ResolvedApp>, SupervisorError>>,
},
SetSheepEnvBatch {
name: String,
entries: BTreeMap<String, EnvValue>,
force: bool,
dry_run: bool,
reply: oneshot::Sender<Result<Option<EnvBatch>, SupervisorError>>,
},
SetSheepField {
name: String,
key: String,
value: serde_json::Value,
reply: oneshot::Sender<Result<Option<FieldSet>, SupervisorError>>,
},
SetSmit {
conn: ConnId,
sheep: String,
smit: Option<Smit>,
reply: oneshot::Sender<Result<Vec<ProcessInfo>, SupervisorError>>,
},
ForgetSmits {
conn: ConnId,
reply: oneshot::Sender<()>,
},
List {
reply: oneshot::Sender<Vec<ProcessInfo>>,
},
Reopen {
selector: ProcessSelector,
reply: oneshot::Sender<Result<Vec<ProcessInfo>, SupervisorError>>,
},
Flush {
selector: ProcessSelector,
reply: oneshot::Sender<Result<Vec<ProcessInfo>, SupervisorError>>,
},
#[cfg(unix)]
HandoverSnapshot {
fds: DaemonFds,
reply: oneshot::Sender<Result<Snapshot, SupervisorError>>,
},
#[cfg(unix)]
HandoverFitness {
reply: oneshot::Sender<Result<Fitness, SupervisorError>>,
},
Trigger {
selector: ProcessSelector,
action: String,
params: Option<String>,
reply: oneshot::Sender<Result<Vec<ActionReply>, SupervisorError>>,
},
Signal {
selector: ProcessSelector,
sig: OperatorSignal,
reply: oneshot::Sender<Result<Vec<SignalReply>, SupervisorError>>,
},
SendLine {
selector: ProcessSelector,
line: String,
reply: oneshot::Sender<Result<Vec<LineReply>, SupervisorError>>,
},
Shutdown {
reply: oneshot::Sender<()>,
},
}
#[derive(Debug)]
pub(crate) enum Msg {
Command(Command),
Exited {
id: u32,
outcome: ExitOutcome,
},
RestartDue {
id: u32,
epoch: u64,
},
Ready {
id: u32,
},
ReloadDeadline {
name: String,
stamp: u64,
},
ReloadVerified {
name: String,
new_id: u32,
readiness: Readiness,
},
ActionReply {
id: u32,
action: String,
body: String,
stamp: Option<u64>,
},
ActionResult {
id: u32,
stamp: u64,
outcome: ActionOutcome,
},
ReadyResult {
id: u32,
epoch: u64,
manually: bool,
readiness: Readiness,
},
}
#[non_exhaustive]
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum SupervisorError {
NotFound,
SpawnFailed(String),
CannotStart(String),
ReloadInFlight(String),
InvalidScale(String),
ReopenFailed(String),
FlushFailed(String),
IsADog(String),
InvalidEnv(String),
InvalidField(String),
Overrides(String),
EngineStopped,
}
impl fmt::Display for SupervisorError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::NotFound => f.write_str("selector matched no registered sheep"),
Self::SpawnFailed(msg) => write!(f, "spawn failed: {msg}"),
Self::CannotStart(msg) => write!(f, "start refused: {msg}"),
Self::ReloadInFlight(name) => write!(f, "{name} is already being reloaded"),
Self::InvalidScale(msg) => write!(f, "cannot scale: {msg}"),
Self::ReopenFailed(msg) => write!(f, "log reopen failed: {msg}"),
Self::FlushFailed(msg) => write!(f, "log flush failed: {msg}"),
Self::IsADog(msg) => write!(f, "refused: {msg}"),
Self::InvalidEnv(msg) => write!(f, "cannot set that env key: {msg}"),
Self::InvalidField(msg) => write!(f, "cannot set that field: {msg}"),
Self::Overrides(msg) => write!(f, "overrides store unusable: {msg}"),
Self::EngineStopped => f.write_str("supervisor engine has shut down"),
}
}
}
impl core::error::Error for SupervisorError {}
#[derive(Debug, Clone)]
pub(crate) struct FieldSet {
pub(crate) app: ResolvedApp,
pub(crate) pending: bool,
}
#[derive(Debug, Clone)]
pub(crate) struct EnvBatch {
pub(crate) app: Option<ResolvedApp>,
pub(crate) set: Vec<String>,
pub(crate) unchanged: Vec<String>,
pub(crate) collisions: Vec<String>,
}
#[derive(Debug, Clone)]
pub struct SupervisorHandle {
tx: mpsc::Sender<Msg>,
}
impl SupervisorHandle {
pub async fn start(&self, apps: Vec<ResolvedApp>) -> Result<Vec<ProcessInfo>, SupervisorError> {
self.start_staged(apps, BTreeSet::new(), BatchPolicy::AllOrNothing)
.await
}
pub(crate) async fn start_staged(
&self,
apps: Vec<ResolvedApp>,
gate: BTreeSet<String>,
policy: BatchPolicy,
) -> Result<Vec<ProcessInfo>, SupervisorError> {
let (reply, rx) = oneshot::channel();
self.tx
.send(Msg::Command(Command::Start {
apps,
policy,
gate,
reply,
}))
.await
.map_err(|_| SupervisorError::EngineStopped)?;
rx.await.map_err(|_| SupervisorError::EngineStopped)?
}
pub(crate) async fn register_at_rest(
&self,
apps: Vec<ResolvedApp>,
) -> Result<Vec<ProcessInfo>, SupervisorError> {
let (reply, rx) = oneshot::channel();
self.tx
.send(Msg::Command(Command::RegisterAtRest { apps, reply }))
.await
.map_err(|_| SupervisorError::EngineStopped)?;
rx.await.map_err(|_| SupervisorError::EngineStopped)?
}
pub(crate) async fn config_drift(
&self,
apps: Vec<ResolvedApp>,
) -> Result<Vec<SheepDrift>, SupervisorError> {
let (reply, rx) = oneshot::channel();
self.tx
.send(Msg::Command(Command::ConfigDrift { apps, reply }))
.await
.map_err(|_| SupervisorError::EngineStopped)?;
rx.await.map_err(|_| SupervisorError::EngineStopped)?
}
pub(crate) async fn apply_config(
&self,
apps: Vec<DeclaredApp>,
reset: ResetDepth,
) -> Result<Vec<Applied>, SupervisorError> {
let (reply, rx) = oneshot::channel();
self.tx
.send(Msg::Command(Command::ApplyConfig { apps, reset, reply }))
.await
.map_err(|_| SupervisorError::EngineStopped)?;
rx.await.map_err(|_| SupervisorError::EngineStopped)?
}
pub(crate) async fn sheep_config(
&self,
name: String,
) -> Result<Option<SheepConfigView>, SupervisorError> {
let (reply, rx) = oneshot::channel();
self.tx
.send(Msg::Command(Command::SheepConfig { name, reply }))
.await
.map_err(|_| SupervisorError::EngineStopped)?;
rx.await.map_err(|_| SupervisorError::EngineStopped)?
}
pub(crate) async fn set_sheep_env(
&self,
name: String,
key: String,
value: Option<EnvValue>,
) -> Result<Option<ResolvedApp>, SupervisorError> {
let (reply, rx) = oneshot::channel();
self.tx
.send(Msg::Command(Command::SetSheepEnv {
name,
key,
value,
reply,
}))
.await
.map_err(|_| SupervisorError::EngineStopped)?;
rx.await.map_err(|_| SupervisorError::EngineStopped)?
}
pub(crate) async fn set_sheep_env_batch(
&self,
name: String,
entries: BTreeMap<String, String>,
force: bool,
dry_run: bool,
) -> Result<Option<EnvBatch>, SupervisorError> {
let entries = entries
.into_iter()
.map(|(key, value)| (key, EnvValue::from(value)))
.collect();
let (reply, rx) = oneshot::channel();
self.tx
.send(Msg::Command(Command::SetSheepEnvBatch {
name,
entries,
force,
dry_run,
reply,
}))
.await
.map_err(|_| SupervisorError::EngineStopped)?;
rx.await.map_err(|_| SupervisorError::EngineStopped)?
}
pub(crate) async fn set_sheep_field(
&self,
name: String,
key: String,
value: serde_json::Value,
) -> Result<Option<FieldSet>, SupervisorError> {
let (reply, rx) = oneshot::channel();
self.tx
.send(Msg::Command(Command::SetSheepField {
name,
key,
value,
reply,
}))
.await
.map_err(|_| SupervisorError::EngineStopped)?;
rx.await.map_err(|_| SupervisorError::EngineStopped)?
}
pub async fn start_dog(
&self,
app: ResolvedApp,
source: DogSource,
) -> Result<ProcessInfo, SupervisorError> {
let (reply, rx) = oneshot::channel();
self.tx
.send(Msg::Command(Command::StartDog {
app: Box::new(app),
source,
reply,
}))
.await
.map_err(|_| SupervisorError::EngineStopped)?;
rx.await.map_err(|_| SupervisorError::EngineStopped)?
}
pub(crate) async fn stop(
&self,
selector: ProcessSelector,
) -> Result<Vec<ProcessInfo>, SupervisorError> {
let (reply, rx) = oneshot::channel();
self.tx
.send(Msg::Command(Command::Stop { selector, reply }))
.await
.map_err(|_| SupervisorError::EngineStopped)?;
rx.await.map_err(|_| SupervisorError::EngineStopped)?
}
pub(crate) async fn restart(
&self,
selector: ProcessSelector,
) -> Result<Vec<ProcessInfo>, SupervisorError> {
self.restart_with_origin(selector, CommandOrigin::Operator)
.await
}
pub(crate) async fn restart_automatic(
&self,
selector: ProcessSelector,
) -> Result<Vec<ProcessInfo>, SupervisorError> {
self.restart_with_origin(selector, CommandOrigin::Automatic)
.await
}
async fn restart_with_origin(
&self,
selector: ProcessSelector,
origin: CommandOrigin,
) -> Result<Vec<ProcessInfo>, SupervisorError> {
let (reply, rx) = oneshot::channel();
self.tx
.send(Msg::Command(Command::Restart {
selector,
origin,
reply,
}))
.await
.map_err(|_| SupervisorError::EngineStopped)?;
rx.await.map_err(|_| SupervisorError::EngineStopped)?
}
pub(crate) async fn extra_restart(
&self,
id: u32,
pid: u32,
epoch: Option<u64>,
observed: Option<MemSize>,
) {
let _ = self
.tx
.send(Msg::Command(Command::ExtraRestart {
id,
pid,
epoch,
observed,
}))
.await;
}
pub(crate) async fn reload(
&self,
selector: ProcessSelector,
) -> Result<Vec<ProcessInfo>, SupervisorError> {
let (reply, rx) = oneshot::channel();
self.tx
.send(Msg::Command(Command::Reload { selector, reply }))
.await
.map_err(|_| SupervisorError::EngineStopped)?;
rx.await.map_err(|_| SupervisorError::EngineStopped)?
}
pub(crate) async fn delete(
&self,
selector: ProcessSelector,
) -> Result<Vec<u32>, SupervisorError> {
let (reply, rx) = oneshot::channel();
self.tx
.send(Msg::Command(Command::Delete { selector, reply }))
.await
.map_err(|_| SupervisorError::EngineStopped)?;
rx.await.map_err(|_| SupervisorError::EngineStopped)?
}
pub(crate) async fn scale(&self, name: &str, count: u32) -> Result<Scaled, SupervisorError> {
let (reply, rx) = oneshot::channel();
self.tx
.send(Msg::Command(Command::Scale {
name: name.to_string(),
count,
reply,
}))
.await
.map_err(|_| SupervisorError::EngineStopped)?;
rx.await.map_err(|_| SupervisorError::EngineStopped)?
}
pub(crate) async fn set_smit(
&self,
conn: ConnId,
sheep: &str,
smit: Option<Smit>,
) -> Result<Vec<ProcessInfo>, SupervisorError> {
let (reply, rx) = oneshot::channel();
self.tx
.send(Msg::Command(Command::SetSmit {
conn,
sheep: sheep.to_string(),
smit,
reply,
}))
.await
.map_err(|_| SupervisorError::EngineStopped)?;
rx.await.map_err(|_| SupervisorError::EngineStopped)?
}
pub(crate) async fn forget_smits(&self, conn: ConnId) {
let (reply, rx) = oneshot::channel();
if self
.tx
.send(Msg::Command(Command::ForgetSmits { conn, reply }))
.await
.is_ok()
{
let _ = rx.await;
}
}
pub(crate) async fn reopen(
&self,
selector: ProcessSelector,
) -> Result<Vec<ProcessInfo>, SupervisorError> {
let (reply, rx) = oneshot::channel();
self.tx
.send(Msg::Command(Command::Reopen { selector, reply }))
.await
.map_err(|_| SupervisorError::EngineStopped)?;
rx.await.map_err(|_| SupervisorError::EngineStopped)?
}
#[cfg(unix)]
pub(crate) async fn handover_snapshot(
&self,
fds: DaemonFds,
) -> Result<Snapshot, SupervisorError> {
let (reply, rx) = oneshot::channel();
self.tx
.send(Msg::Command(Command::HandoverSnapshot { fds, reply }))
.await
.map_err(|_| SupervisorError::EngineStopped)?;
rx.await.map_err(|_| SupervisorError::EngineStopped)?
}
#[cfg(unix)]
pub(crate) async fn handover_fitness(&self) -> Result<Fitness, SupervisorError> {
let (reply, rx) = oneshot::channel();
self.tx
.send(Msg::Command(Command::HandoverFitness { reply }))
.await
.map_err(|_| SupervisorError::EngineStopped)?;
rx.await.map_err(|_| SupervisorError::EngineStopped)?
}
pub(crate) async fn flush(
&self,
selector: ProcessSelector,
) -> Result<Vec<ProcessInfo>, SupervisorError> {
let (reply, rx) = oneshot::channel();
self.tx
.send(Msg::Command(Command::Flush { selector, reply }))
.await
.map_err(|_| SupervisorError::EngineStopped)?;
rx.await.map_err(|_| SupervisorError::EngineStopped)?
}
pub(crate) async fn trigger(
&self,
selector: ProcessSelector,
action: String,
params: Option<String>,
) -> Result<Vec<ActionReply>, SupervisorError> {
let (reply, rx) = oneshot::channel();
self.tx
.send(Msg::Command(Command::Trigger {
selector,
action,
params,
reply,
}))
.await
.map_err(|_| SupervisorError::EngineStopped)?;
rx.await.map_err(|_| SupervisorError::EngineStopped)?
}
pub(crate) async fn signal(
&self,
selector: ProcessSelector,
sig: OperatorSignal,
) -> Result<Vec<SignalReply>, SupervisorError> {
let (reply, rx) = oneshot::channel();
self.tx
.send(Msg::Command(Command::Signal {
selector,
sig,
reply,
}))
.await
.map_err(|_| SupervisorError::EngineStopped)?;
rx.await.map_err(|_| SupervisorError::EngineStopped)?
}
pub(crate) async fn send_line(
&self,
selector: ProcessSelector,
line: String,
) -> Result<Vec<LineReply>, SupervisorError> {
let (reply, rx) = oneshot::channel();
self.tx
.send(Msg::Command(Command::SendLine {
selector,
line,
reply,
}))
.await
.map_err(|_| SupervisorError::EngineStopped)?;
rx.await.map_err(|_| SupervisorError::EngineStopped)?
}
pub(crate) async fn list_checked(&self) -> Result<Vec<ProcessInfo>, SupervisorError> {
let (reply, rx) = oneshot::channel();
self.tx
.send(Msg::Command(Command::List { reply }))
.await
.map_err(|_| SupervisorError::EngineStopped)?;
rx.await.map_err(|_| SupervisorError::EngineStopped)
}
#[must_use]
pub async fn list(&self) -> Vec<ProcessInfo> {
self.list_checked()
.await
.expect("supervisor actor is no longer running")
}
pub async fn shutdown(&self) {
let (reply, rx) = oneshot::channel();
if self
.tx
.send(Msg::Command(Command::Shutdown { reply }))
.await
.is_ok()
{
let _ = rx.await;
}
}
}
#[derive(Debug)]
pub(crate) struct SupervisorBuilder<R: ProcessRunner> {
runner: R,
paths: ShepPaths,
events: Bus,
extras: Option<Extras>,
environment: String,
provider_secrets: Option<Arc<ProviderSecrets>>,
}
impl<R: ProcessRunner> SupervisorBuilder<R> {
pub(crate) fn new(runner: R, paths: ShepPaths, events: Bus) -> Self {
Self {
runner,
paths,
events,
extras: None,
environment: DEFAULT_ENVIRONMENT.to_string(),
provider_secrets: None,
}
}
#[must_use]
pub(crate) fn extras(mut self, extras: Extras) -> Self {
self.extras = Some(extras);
self
}
#[must_use]
pub(crate) fn environment(mut self, environment: String) -> Self {
self.environment = environment;
self
}
#[must_use]
pub(crate) fn provider_secrets(mut self, secrets: Arc<ProviderSecrets>) -> Self {
self.provider_secrets = Some(secrets);
self
}
pub(crate) fn spawn(self) -> SupervisorHandle {
let (tx, rx) = mpsc::channel(MAILBOX_CAPACITY);
let actor = self.build(tx.clone());
tokio::spawn(actor.run(rx));
SupervisorHandle { tx }
}
#[cfg(unix)]
pub(crate) fn spawn_adopted(
self,
flock: Vec<AdoptedSheep>,
counters: Counters,
reloads: Vec<CarriedReload>,
) -> Result<SupervisorHandle, AdoptError> {
let (tx, rx) = mpsc::channel(MAILBOX_CAPACITY);
let mut actor = self.build(tx.clone());
actor.next_id = counters.next_id;
actor.next_deadline = counters.next_deadline;
actor.next_action_stamp = counters.next_action_stamp;
let reaper = Arc::new(AdoptedReaper::new());
for sheep in flock {
actor.install_adopted(sheep, &reaper)?;
}
actor.install_carried_reloads(reloads);
tokio::spawn(actor.run(rx));
Ok(SupervisorHandle { tx })
}
fn build(self, tx: mpsc::Sender<Msg>) -> Actor<R> {
let provider_secrets = self
.provider_secrets
.unwrap_or_else(|| Arc::new(ProviderSecrets::load(&self.paths.secrets_cache)));
Actor {
runner: self.runner,
paths: self.paths,
events: self.events,
host_environment: self.environment,
provider_secrets,
tx,
sheep: HashMap::new(),
next_id: 0,
next_deadline: 0,
next_action_stamp: 0,
pending: Vec::new(),
shutting_down: false,
extras: self.extras,
registry: ExtrasRegistry::default(),
reloads: HashMap::new(),
smits: Smits::new(),
}
}
}
#[cfg(unix)]
#[derive(Debug)]
pub(crate) enum AdoptError {
Spec {
sheep: String,
source: shep_core::config::NormalizeError,
},
Runner {
sheep: String,
source: RunnerError,
},
}
#[cfg(unix)]
impl fmt::Display for AdoptError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::Spec { sheep, source } => {
write!(
f,
"sheep '{sheep}' carried a config that no longer validates: {source}"
)
}
Self::Runner { sheep, source } => {
write!(f, "sheep '{sheep}' could not be adopted: {source}")
}
}
}
}
#[cfg(unix)]
impl core::error::Error for AdoptError {
fn source(&self) -> Option<&(dyn core::error::Error + 'static)> {
match self {
Self::Spec { source, .. } => Some(source),
Self::Runner { source, .. } => Some(source),
}
}
}
pub(crate) const DEFAULT_ENVIRONMENT: &str = "production";
pub fn spawn_supervisor<R: ProcessRunner>(
runner: R,
paths: ShepPaths,
events: Bus,
) -> SupervisorHandle {
SupervisorBuilder::new(runner, paths, events).spawn()
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum SheepCtl {
Kill {
grace: Duration,
},
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum LadderCap {
Stop,
Drain,
}
impl LadderCap {
fn of(self, app: &AppConfig) -> Duration {
match self {
Self::Stop => app.kill_timeout,
Self::Drain => app.graceful_timeout,
}
.as_duration()
}
}
#[derive(Debug)]
struct ReloadJob {
queue: VecDeque<u32>,
mode: ReloadMode,
swap: ReloadSwap,
deadline: u64,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[cfg_attr(
not(unix),
expect(dead_code, reason = "no target without a handover ever builds one")
)]
pub(crate) struct CarriedReload {
pub(crate) app: String,
pub(crate) queue: Vec<u32>,
pub(crate) mode: ReloadMode,
pub(crate) swap: ReloadSwap,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub(crate) enum ReloadMode {
Overlap,
Serial,
}
impl ReloadMode {
fn of(config: &AppConfig, source: &ReadinessSource) -> Self {
match source {
ReadinessSource::Probe(..) if !config.reuse_port => Self::Serial,
_ => Self::Overlap,
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub(crate) struct ReloadSwap {
pub(crate) old_id: u32,
pub(crate) new_id: Option<u32>,
pub(crate) phase: ReloadPhase,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub(crate) enum ReloadPhase {
DrainFirst,
AwaitReady,
DrainOld,
Verify,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub(crate) enum ManualKind {
Stop,
Restart,
Delete,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub(crate) enum CommandOrigin {
Operator,
Automatic,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
pub(crate) struct PendingManual {
pub(crate) kind: ManualKind,
pub(crate) origin: CommandOrigin,
}
#[derive(Debug)]
struct PendingAction {
stamp: u64,
action: String,
waiter: Option<oneshot::Sender<String>>,
reply: oneshot::Sender<ActionOutcome>,
}
#[derive(Debug)]
struct AbandonedReply {
stamp: u64,
action: String,
}
#[derive(Debug, Default)]
struct ActionWaits {
live: Vec<PendingAction>,
abandoned: VecDeque<AbandonedReply>,
}
impl ActionWaits {
fn arm(&mut self, pending: PendingAction) {
self.live.push(pending);
}
fn answer(&mut self, action: &str, stamp: Option<u64>) -> Option<oneshot::Sender<String>> {
if let Some(stamp) = stamp {
if let Some(pending) = self
.live
.iter_mut()
.find(|pending| pending.stamp == stamp && pending.waiter.is_some())
{
return pending.waiter.take();
}
if let Some(owed) = self.abandoned.iter().position(|debt| debt.stamp == stamp) {
self.abandoned.remove(owed);
}
return None;
}
if let Some(owed) = self.abandoned.iter().position(|debt| debt.action == action) {
self.abandoned.remove(owed);
return None;
}
self.live
.iter_mut()
.find(|pending| pending.action == action && pending.waiter.is_some())
.and_then(|pending| pending.waiter.take())
}
fn resolve(&mut self, stamp: u64) -> Option<oneshot::Sender<ActionOutcome>> {
let at = self
.live
.iter()
.position(|pending| pending.stamp == stamp)?;
let pending = self.live.remove(at);
if pending.waiter.is_some() {
self.abandoned.push_back(AbandonedReply {
stamp: pending.stamp,
action: pending.action,
});
if self.abandoned.len() > MAX_ABANDONED_ACTION_REPLIES {
self.abandoned.pop_front();
}
}
Some(pending.reply)
}
fn abandon_all(&mut self) {
for pending in self.live.drain(..) {
let _ = pending.reply.send(ActionOutcome::NoChannel);
}
self.abandoned.clear();
}
}
#[derive(Debug)]
struct SheepSlot {
entry: ProcessEntry,
ctl: Option<mpsc::Sender<SheepCtl>>,
log_ctl: Option<mpsc::Sender<LogCtl>>,
to_child: Option<mpsc::Sender<ShepherdMessage>>,
signals: Option<mpsc::Sender<SignalRequest>>,
to_stdin: Option<mpsc::Sender<StdinWrite>>,
manual: Option<PendingManual>,
pending_delete: bool,
epoch: u64,
ready_tx: Option<oneshot::Sender<()>>,
actions: ActionWaits,
ready_failed: bool,
restart_due: Option<SystemTime>,
}
impl SheepSlot {
fn new(entry: ProcessEntry) -> Self {
Self {
entry,
ctl: None,
log_ctl: None,
to_child: None,
signals: None,
to_stdin: None,
manual: None,
pending_delete: false,
epoch: 0,
ready_tx: None,
actions: ActionWaits::default(),
ready_failed: false,
restart_due: None,
}
}
fn open_channel(&self) -> Option<&mpsc::Sender<ShepherdMessage>> {
self.to_child
.as_ref()
.filter(|to_child| !to_child.is_closed())
}
fn open_stdin(&self) -> Option<&mpsc::Sender<StdinWrite>> {
self.to_stdin
.as_ref()
.filter(|to_stdin| !to_stdin.is_closed())
}
}
#[derive(Debug)]
enum ReplyKind {
Info(oneshot::Sender<Result<Vec<ProcessInfo>, SupervisorError>>),
Ids(oneshot::Sender<Result<Vec<u32>, SupervisorError>>),
Shutdown(oneshot::Sender<()>),
}
#[derive(Debug)]
pub(crate) struct Scaled {
pub(crate) instances: Vec<ProcessInfo>,
pub(crate) app: ResolvedApp,
pub(crate) requested: u32,
pub(crate) shortfall: Option<String>,
}
impl Scaled {
pub(crate) fn achieved(&self) -> u32 {
u32::try_from(self.instances.len()).unwrap_or(u32::MAX)
}
}
#[derive(Debug)]
pub(crate) struct Applied {
pub(crate) name: String,
pub(crate) applied: Vec<String>,
pub(crate) pending: Vec<String>,
pub(crate) refused: Option<String>,
pub(crate) app: Option<ResolvedApp>,
}
const EXTRAS_FIELDS: &[&str] = &[
"max_memory",
"watch",
"ignore_watch",
"watch_delay",
"watch_options",
"cron_restart",
"cron_timezone",
"liveness_probe",
];
fn establish_env(next: &mut AppOverrides, incoming: &DeclaredApp) {
if let Some(serde_json::Value::Object(env)) = next.fields.get_mut("env") {
for key in &incoming.declared_env {
if env.get(key).is_some_and(serde_json::Value::is_null) {
continue;
}
env.remove(key);
}
if env.is_empty() {
next.fields.remove("env");
}
}
next.declared_env
.extend(incoming.declared_env.iter().cloned());
}
fn merge_declared(
stored: &AppConfig,
incoming: &DeclaredApp,
overrides: &AppOverrides,
reset: ResetDepth,
) -> Result<(AppConfig, AppOverrides), String> {
let object = |config: &AppConfig| match serde_json::to_value(config) {
Ok(serde_json::Value::Object(map)) => Ok(map),
Ok(_) => Err("an app config must serialize as an object".to_string()),
Err(err) => Err(err.to_string()),
};
let mut merged = object(stored)?;
let file = object(&incoming.config)?;
let defaults = object(&AppConfig::default())?;
let mut next = overrides.clone();
let established: BTreeSet<&String> = overrides
.declared
.iter()
.chain(overrides.fields.keys())
.collect();
for key in defaults.keys() {
if key == "env" {
continue;
}
let in_scope = match reset {
ResetDepth::Policy | ResetDepth::All => true,
ResetDepth::File => incoming.declared.contains(key),
_ => {
key != "instances" && incoming.declared.contains(key) && !established.contains(key)
}
};
if !in_scope {
continue;
}
next.fields.remove(key);
if incoming.declared.contains(key) {
next.declared.insert(key.clone());
}
if let Some(value) = file.get(key) {
merged.insert(key.clone(), value.clone());
}
}
match reset {
ResetDepth::Policy | ResetDepth::File => {}
ResetDepth::All | ResetDepth::Env => {
next.fields.remove("env");
merged.insert(
"env".to_string(),
file.get("env")
.cloned()
.unwrap_or_else(|| serde_json::Value::Object(serde_json::Map::new())),
);
establish_env(&mut next, incoming);
}
_ => {
let overridden_env: BTreeSet<&String> = overrides
.fields
.get("env")
.and_then(serde_json::Value::as_object)
.map(|env| env.keys().collect())
.unwrap_or_default();
let mut env = stored.env.clone();
for key in &incoming.declared_env {
if overrides.declared_env.contains(key) || overridden_env.contains(key) {
continue;
}
if let Some(value) = incoming.config.env.get(key) {
env.insert(key.clone(), value.clone());
}
}
merged.insert(
"env".to_string(),
serde_json::to_value(env).map_err(|err| err.to_string())?,
);
establish_env(&mut next, incoming);
}
}
let merged: AppConfig =
serde_json::from_value(serde_json::Value::Object(merged)).map_err(|err| err.to_string())?;
Ok((merged, next))
}
fn reached_spec(
stored: &AppConfig,
merged: &AppConfig,
reaching: &[String],
instances: u32,
) -> Result<ResolvedApp, String> {
let object = |config: &AppConfig| match serde_json::to_value(config) {
Ok(serde_json::Value::Object(map)) => Ok(map),
Ok(_) => Err("an app config must serialize as an object".to_string()),
Err(err) => Err(err.to_string()),
};
let mut next = object(stored)?;
let source = object(merged)?;
for field in reaching {
if let Some(value) = source.get(field) {
next.insert(field.clone(), value.clone());
}
}
let mut config: AppConfig =
serde_json::from_value(serde_json::Value::Object(next)).map_err(|err| err.to_string())?;
config.instances = instances;
normalize(config).map_err(|err| err.to_string())
}
fn pending_fields(entry: &ProcessEntry) -> Vec<String> {
entry.pending.as_ref().map_or_else(Vec::new, |parked| {
entry.spec.config().drifted_fields(parked.config())
})
}
fn dog_config_refusal(name: &str) -> String {
format!(
"{name} is a dog, and a dog's config comes from `shep adopt` rather than \
from a Flockfile"
)
}
fn env_override_map<'a>(
record: &'a mut AppOverrides,
name: &str,
) -> Result<&'a mut serde_json::Map<String, serde_json::Value>, SupervisorError> {
record
.fields
.entry("env".to_string())
.or_insert_with(|| serde_json::Value::Object(serde_json::Map::new()))
.as_object_mut()
.ok_or_else(|| {
SupervisorError::Overrides(format!("{name}'s stored `env` override is not an object"))
})
}
fn with_count(app: &ResolvedApp, instances: u32) -> Option<ResolvedApp> {
let mut config = app.config().clone();
config.instances = instances;
normalize(config).ok()
}
#[derive(Debug)]
struct PendingReply {
remaining: HashSet<u32>,
results: Vec<ProcessInfo>,
reply: ReplyKind,
}
struct Actor<R: ProcessRunner> {
runner: R,
paths: ShepPaths,
host_environment: String,
provider_secrets: Arc<ProviderSecrets>,
events: Bus,
tx: mpsc::Sender<Msg>,
sheep: HashMap<u32, SheepSlot>,
next_id: u32,
next_deadline: u64,
next_action_stamp: u64,
pending: Vec<PendingReply>,
shutting_down: bool,
extras: Option<Extras>,
registry: ExtrasRegistry,
smits: Smits,
reloads: HashMap<String, ReloadJob>,
}
impl<R: ProcessRunner> Actor<R> {
async fn run(mut self, mut rx: mpsc::Receiver<Msg>) {
while let Some(msg) = rx.recv().await {
let should_break = match msg {
Msg::Command(cmd) => self.handle_command(cmd),
Msg::Exited { id, outcome } => self.handle_exited(id, outcome),
Msg::RestartDue { id, epoch } => {
self.handle_restart_due(id, epoch);
false
}
Msg::Ready { id } => {
self.handle_ready_signal(id);
false
}
Msg::ReloadDeadline { name, stamp } => {
self.handle_reload_deadline(&name, stamp);
false
}
Msg::ReloadVerified {
name,
new_id,
readiness,
} => {
self.handle_reload_verified(&name, new_id, readiness);
false
}
Msg::ActionReply {
id,
action,
body,
stamp,
} => {
self.handle_action_reply(id, &action, body, stamp);
false
}
Msg::ActionResult { id, stamp, outcome } => {
self.handle_action_result(id, stamp, outcome);
false
}
Msg::ReadyResult {
id,
epoch,
manually,
readiness,
} => {
self.handle_ready_result(id, epoch, manually, readiness);
false
}
};
if should_break {
break;
}
}
}
fn handle_command(&mut self, cmd: Command) -> bool {
match cmd {
Command::Start {
apps,
policy,
gate,
reply,
} => {
let result = if self.shutting_down {
Err(SupervisorError::EngineStopped)
} else {
self.do_start(apps, None, policy, &gate)
};
let _ = reply.send(result);
false
}
Command::RegisterAtRest { apps, reply } => {
let registered = apps.iter().map(|app| self.register_at_rest(app)).collect();
let _ = reply.send(Ok(registered));
false
}
Command::ConfigDrift { apps, reply } => {
let _ = reply.send(Ok(self.config_drift(&apps)));
false
}
Command::StartDog { app, source, reply } => {
let result = self.do_start_dog(*app, source);
let _ = reply.send(result);
false
}
Command::Scale { name, count, reply } => {
self.handle_scale(&name, count, reply);
false
}
Command::ApplyConfig { apps, reset, reply } => {
let report = self.handle_apply_config(apps, reset);
let _ = reply.send(Ok(report));
false
}
Command::SheepConfig { name, reply } => {
let _ = reply.send(self.handle_sheep_config(&name));
false
}
Command::SetSheepEnv {
name,
key,
value,
reply,
} => {
let _ = reply.send(self.handle_set_sheep_env(
&name,
&key,
value.as_ref().map(EnvValue::as_str),
));
false
}
Command::SetSheepEnvBatch {
name,
entries,
force,
dry_run,
reply,
} => {
let entries: BTreeMap<String, String> = entries
.into_iter()
.map(|(key, value)| (key, value.as_str().to_string()))
.collect();
let _ =
reply.send(self.handle_set_sheep_env_batch(&name, &entries, force, dry_run));
false
}
Command::SetSheepField {
name,
key,
value,
reply,
} => {
let _ = reply.send(self.handle_set_sheep_field(&name, &key, &value));
false
}
Command::SetSmit {
conn,
sheep,
smit,
reply,
} => {
let _ = reply.send(self.handle_set_smit(conn, &sheep, smit));
false
}
Command::ForgetSmits { conn, reply } => {
self.smits.retain(|_, (painter, _)| *painter != conn);
let _ = reply.send(());
false
}
Command::List { reply } => {
let _ = reply.send(self.snapshot_all());
false
}
Command::Reopen { selector, reply } => {
self.handle_reopen(&selector, reply);
false
}
Command::Flush { selector, reply } => {
self.handle_flush(&selector, reply);
false
}
#[cfg(unix)]
Command::HandoverSnapshot { fds, reply } => {
self.handle_handover_snapshot(fds, reply);
false
}
#[cfg(unix)]
Command::HandoverFitness { reply } => {
self.handle_handover_fitness(reply);
false
}
Command::Trigger {
selector,
action,
params,
reply,
} => {
self.begin_action(&selector, action, params, reply);
false
}
Command::Signal {
selector,
sig,
reply,
} => {
self.begin_signal(&selector, sig, reply);
false
}
Command::SendLine {
selector,
line,
reply,
} => {
self.begin_send_line(&selector, line, reply);
false
}
Command::Stop { selector, reply } => {
self.begin_manual(
selector,
ManualKind::Stop,
CommandOrigin::Operator,
ReplyKind::Info(reply),
);
false
}
Command::Restart {
selector,
origin,
reply,
} => {
if self.shutting_down {
send_reply(ReplyKind::Info(reply), Err(SupervisorError::EngineStopped));
} else {
self.begin_manual(
selector,
ManualKind::Restart,
origin,
ReplyKind::Info(reply),
);
}
false
}
Command::ExtraRestart {
id,
pid,
epoch,
observed,
} => {
self.handle_extra_restart(id, pid, epoch, observed);
false
}
Command::Reload { selector, reply } => {
if self.shutting_down {
let _ = reply.send(Err(SupervisorError::EngineStopped));
} else {
self.handle_reload(&selector, reply);
}
false
}
Command::Delete { selector, reply } => {
self.begin_manual(
selector,
ManualKind::Delete,
CommandOrigin::Operator,
ReplyKind::Ids(reply),
);
false
}
Command::Shutdown { reply } => self.begin_shutdown(reply),
}
}
fn do_start_dog(
&mut self,
app: ResolvedApp,
source: DogSource,
) -> Result<ProcessInfo, SupervisorError> {
if self.shutting_down {
return Err(SupervisorError::EngineStopped);
}
if let Some(slot) = self
.sheep
.values()
.find(|slot| slot.entry.spec.config().name == app.config().name)
{
return Ok(to_info(&slot.entry, &self.smits));
}
let started = self.do_start(
vec![app],
Some(source),
BatchPolicy::PerApp,
&BTreeSet::new(),
)?;
started
.into_iter()
.next()
.ok_or_else(|| SupervisorError::SpawnFailed("the dog registered no instance".into()))
}
fn do_start(
&mut self,
apps: Vec<ResolvedApp>,
dog: Option<DogSource>,
policy: BatchPolicy,
gate: &BTreeSet<String>,
) -> Result<Vec<ProcessInfo>, SupervisorError> {
let mut ready: Vec<(ResolvedApp, Option<Credentials>)> = Vec::with_capacity(apps.len());
let mut refusals = Vec::new();
let mut failures = Vec::new();
let total = apps.len();
for app in apps {
let name = app.config().name.clone();
let credentials = match privilege::resolve(app.config()) {
Ok(resolved) => resolved,
Err(err) => {
match policy {
BatchPolicy::AllOrNothing => refusals.push(format!("{name}: {err}")),
BatchPolicy::PerApp => {
failures.push(format!("{name}: {err}"));
if let Registration::Fresh(info) = self.register_without_spawning(
&app,
ProcStatus::Errored,
dog.clone(),
) {
self.emit(ProcessEventKind::Errored, info, true);
}
}
}
continue;
}
};
let view = self.secret_view(&app);
let described = match assemble(&app, 0, &self.paths, None, &view) {
Ok(spec) => spec,
Err(err) if !err.is_retriable() => match policy {
BatchPolicy::AllOrNothing => {
refusals.push(format!("{name}: {err}"));
continue;
}
BatchPolicy::PerApp => {
tracing::warn!(sheep = %name, "{err}");
describe(&app, 0, &self.paths, None, &view)
}
},
Err(_) => describe(&app, 0, &self.paths, None, &view),
};
match self.runner.preflight(&described) {
Preflight::Unknown => {}
Preflight::Impossible(reason) if policy == BatchPolicy::AllOrNothing => {
refusals.push(format!("{name}: {reason}"));
continue;
}
Preflight::Impossible(reason) | Preflight::Doubtful(reason) => {
tracing::warn!(sheep = %name, "{reason}");
}
}
ready.push((app, credentials));
}
if !refusals.is_empty() {
return Err(SupervisorError::CannotStart(format!(
"nothing in this batch was registered; {} of {} apps cannot start: {}",
refusals.len(),
total,
refusals.join("; "),
)));
}
let mut results = Vec::new();
'apps: for (app, credentials) in ready {
let name = app.config().name.clone();
let mut existing: Vec<u32> = self
.sheep
.values()
.filter(|slot| slot.entry.spec.config().name == name)
.map(|slot| slot.entry.instance)
.collect();
existing.sort_unstable();
let slots = instance_slots(&existing, app.config().instances);
for instance in slots {
match self.spawn_fresh(&app, instance, credentials, dog.clone(), gate) {
Ok(info) => results.push(info),
Err(message) => {
let failure = format!("{name}: {message}");
match policy {
BatchPolicy::AllOrNothing => {
return Err(SupervisorError::SpawnFailed(failure));
}
BatchPolicy::PerApp => {
failures.push(failure);
continue 'apps;
}
}
}
}
}
}
if !failures.is_empty() {
return Err(SupervisorError::SpawnFailed(failures.join("; ")));
}
sort_flock(&mut results);
Ok(results)
}
fn credentials_for_spawn(&mut self, id: u32) -> Result<Option<Credentials>, PrivilegeError> {
let slot = self
.sheep
.get(&id)
.expect("credentials_for_spawn: unknown id");
match slot.entry.credentials {
SpawnIdentity::Resolved(credentials) => Ok(credentials),
SpawnIdentity::Unresolved => {
let credentials = privilege::resolve(slot.entry.spec.config())?;
self.sheep
.get_mut(&id)
.expect("credentials_for_spawn: the entry was read a moment ago")
.entry
.credentials = SpawnIdentity::Resolved(credentials);
Ok(credentials)
}
}
}
fn register_at_rest(&mut self, app: &ResolvedApp) -> ProcessInfo {
self.register_without_spawning(app, ProcStatus::Stopped, None)
.into_info()
}
fn secret_view(&self, app: &ResolvedApp) -> SecretView {
let environment = app
.config()
.environment
.clone()
.unwrap_or_else(|| self.host_environment.clone());
let store = shep_core::secrets::all(&self.paths.secrets).unwrap_or_else(|error| {
tracing::warn!(
path = %self.paths.secrets.display(),
%error,
"the secret store could not be read; every reference will refuse"
);
BTreeMap::new()
});
SecretView::new(environment, store, self.provider_secrets.snapshot())
}
fn overridden_for(&self, name: &str) -> Vec<String> {
if let Some(slot) = self
.sheep
.values()
.find(|slot| slot.entry.spec.config().name == name)
{
return slot.entry.overridden.clone();
}
overrides::get(&self.paths.overrides, name)
.ok()
.flatten()
.map(|record| record.fields.keys().cloned().collect())
.unwrap_or_default()
}
fn register_without_spawning(
&mut self,
app: &ResolvedApp,
status: ProcStatus,
dog: Option<DogSource>,
) -> Registration {
let name = &app.config().name;
if let Some(slot) = self
.sheep
.values()
.find(|slot| &slot.entry.spec.config().name == name)
{
return Registration::AlreadyKnown(to_info(&slot.entry, &self.smits));
}
let overridden = self.overridden_for(name);
let id = self.next_id;
self.next_id += 1;
let described = describe(app, 0, &self.paths, None, &self.secret_view(app));
let entry = ProcessEntry {
id,
spec: app.clone(),
pending: None,
pending_reidentifies: false,
overridden,
instance: 0,
status,
pid: None,
restarts: 0,
started_at: None,
budget: RestartBudget::default(),
reload: ReloadState::None,
credentials: SpawnIdentity::Unresolved,
out_file: described.out_file,
err_file: described.err_file,
dog,
last_exit: None,
};
let info = to_info(&entry, &self.smits);
self.sheep.insert(id, SheepSlot::new(entry));
Registration::Fresh(info)
}
fn config_drift(&self, apps: &[ResolvedApp]) -> Vec<SheepDrift> {
apps.iter()
.filter_map(|app| {
let incoming = app.config();
let stored = self
.sheep
.values()
.find(|slot| slot.entry.spec.config().name == incoming.name)?
.entry
.spec
.config();
let fields = stored.drifted_fields(incoming);
(!fields.is_empty()).then(|| SheepDrift::new(&incoming.name, fields))
})
.collect()
}
fn spawn_fresh(
&mut self,
app: &ResolvedApp,
instance: u32,
credentials: Option<Credentials>,
dog: Option<DogSource>,
gate: &BTreeSet<String>,
) -> Result<ProcessInfo, String> {
let overridden = self.overridden_for(&app.config().name);
let secrets = self.secret_view(app);
let id = self.next_id;
self.next_id += 1;
let spec = match assemble(app, instance, &self.paths, credentials, &secrets) {
Ok(spec) => spec,
Err(err) => {
let described = describe(app, instance, &self.paths, credentials, &secrets);
let entry = ProcessEntry {
id,
spec: app.clone(),
pending: None,
pending_reidentifies: false,
overridden,
instance,
status: ProcStatus::Errored,
pid: None,
restarts: 0,
started_at: None,
budget: RestartBudget::default(),
reload: ReloadState::None,
credentials: SpawnIdentity::Resolved(credentials),
out_file: described.out_file,
err_file: described.err_file,
dog,
last_exit: None,
};
self.sheep.insert(id, SheepSlot::new(entry));
let info = self.refuse_spawn(id, true, &err);
return if err.is_retriable() {
Ok(info)
} else {
Err(err.to_string())
};
}
};
let out_file = spec.out_file.clone();
let err_file = spec.err_file.clone();
let source = ReadinessSource::of(app.config())
.expect("ResolvedApp already passed ProbeTarget::parse in normalize");
let gated =
!matches!(source, ReadinessSource::Heuristic) || gate.contains(&app.config().name);
match self.runner.spawn(&spec) {
Ok((proc, io)) => {
let pid = proc.pid();
let status = if gated {
ProcStatus::Starting
} else {
ProcStatus::Online
};
let entry = ProcessEntry {
id,
spec: app.clone(),
pending: None,
pending_reidentifies: false,
overridden,
instance,
status,
pid: Some(pid),
restarts: 0,
started_at: Some(tokio::time::Instant::now()),
budget: RestartBudget::default(),
reload: ReloadState::None,
credentials: SpawnIdentity::Resolved(credentials),
out_file,
err_file,
dog,
last_exit: None,
};
let info = to_info(&entry, &self.smits);
let log_ctl = io.log_ctl.clone();
let to_child = io.to_child.clone();
let to_stdin = io.to_stdin.clone();
let handles = spawn_sheep_task::<R::Proc>(
id,
proc,
io,
app.clone(),
self.events.clone(),
self.tx.clone(),
);
let ready_tx = if gated {
Some(spawn_readiness_task(
id,
0,
true,
source,
app.config().listen_timeout.as_duration(),
spec_prober(&spec),
self.tx.clone(),
))
} else {
None
};
self.sheep.insert(
id,
SheepSlot {
ctl: Some(handles.ctl),
log_ctl: Some(log_ctl),
to_child: Some(to_child),
signals: Some(handles.signals),
to_stdin: Some(to_stdin),
ready_tx,
..SheepSlot::new(entry)
},
);
self.emit(ProcessEventKind::Start, info.clone(), true);
if !gated {
self.went_online(id, info.clone(), true);
}
Ok(info)
}
Err(error) => {
let entry = ProcessEntry {
id,
spec: app.clone(),
pending: None,
pending_reidentifies: false,
overridden,
instance,
status: ProcStatus::Errored,
pid: None,
restarts: 0,
started_at: None,
budget: RestartBudget::default(),
reload: ReloadState::None,
credentials: SpawnIdentity::Resolved(credentials),
out_file,
err_file,
dog,
last_exit: None,
};
let info = to_info(&entry, &self.smits);
self.sheep.insert(id, SheepSlot::new(entry));
self.emit(ProcessEventKind::Errored, info, true);
let attempted = match self.runner.preflight(&spec) {
Preflight::Doubtful(reason) => reason,
_ => match &spec.cwd {
Some(cwd) => format!("tried `{}` in {}", spec.program, cwd.display()),
None => format!("tried `{}`", spec.program),
},
};
Err(format!("{error}; {attempted}"))
}
}
}
#[cfg(unix)]
fn install_adopted(
&mut self,
sheep: AdoptedSheep,
reaper: &Arc<AdoptedReaper>,
) -> Result<(), AdoptError> {
let AdoptedSheep {
carried,
out_pipe,
err_pipe,
out_log,
err_log,
stdin_pipe,
channel,
} = sheep;
let app = normalize(carried.app().clone()).map_err(|source| AdoptError::Spec {
sheep: carried.name().to_string(),
source,
})?;
let credentials = match carried.credentials() {
SpawnIdentity::Resolved(credentials) => credentials,
SpawnIdentity::Unresolved => None,
};
let described = describe(
&app,
carried.instance(),
&self.paths,
credentials,
&self.secret_view(&app),
);
let id = carried.id();
let status = carried.status();
let reload = carried.reload().unwrap_or(ReloadState::None);
let ready_failed = carried.ready_failed().unwrap_or(false);
let pending = carried
.pending()
.and_then(|parked| match normalize(parked.clone()) {
Ok(app) => Some(app),
Err(source) => {
tracing::warn!(
sheep = carried.name(),
%source,
"a config this sheep was owed did not survive the handover: this daemon \
will not accept it, so the change is gone and the file must be loaded \
again"
);
None
}
});
let pending_reidentifies =
pending.is_some() && carried.pending_reidentifies().unwrap_or(false);
let mut entry = ProcessEntry {
id,
spec: app.clone(),
pending,
pending_reidentifies,
instance: carried.instance(),
status,
pid: carried.pid(),
restarts: carried.restarts(),
started_at: None,
budget: RestartBudget::default(),
reload,
credentials: carried.credentials(),
out_file: described.out_file.clone(),
err_file: described.err_file.clone(),
dog: carried.dog().cloned(),
last_exit: carried.last_exit(),
overridden: self.overridden_for(&app.config().name),
};
let Some(pid) = carried.pid() else {
self.sheep.insert(
id,
SheepSlot {
manual: None,
pending_delete: carried.pending_delete().unwrap_or(false),
epoch: carried.epoch(),
ready_failed,
restart_due: carried.restart_due(),
..SheepSlot::new(entry)
},
);
if status == ProcStatus::WaitingRestart {
let delay = crate::backoff::adopted_restart_delay(
app.config(),
carried.restart_due(),
SystemTime::now(),
);
self.schedule_restart(id, carried.epoch(), delay);
}
return Ok(());
};
let (proc, io) = self
.runner
.adopt(AdoptSpec {
pid,
out_file: described.out_file.clone(),
err_file: described.err_file.clone(),
out_pipe,
err_pipe,
out_log,
err_log,
stdin_pipe,
channel,
reaper: Arc::clone(reaper),
})
.map_err(|source| AdoptError::Runner {
sheep: carried.name().to_string(),
source,
})?;
entry.pid = Some(proc.pid());
entry.started_at = Some(crate::handover::uptime::started_at_of(proc.pid()));
let manually = matches!(reload, ReloadState::Replacement);
let ready_tx = (status == ProcStatus::Starting && !ready_failed).then(|| {
let source = ReadinessSource::of(app.config())
.expect("ResolvedApp already passed ProbeTarget::parse in normalize");
spawn_readiness_task(
id,
carried.epoch(),
manually,
source,
app.config().listen_timeout.as_duration(),
spec_prober(&described),
self.tx.clone(),
)
});
let log_ctl = io.log_ctl.clone();
let to_child = io.to_child.clone();
let to_stdin = io.to_stdin.clone();
let handles = spawn_sheep_task::<R::Proc>(
id,
proc,
io,
app.clone(),
self.events.clone(),
self.tx.clone(),
);
self.sheep.insert(
id,
SheepSlot {
ctl: Some(handles.ctl),
log_ctl: Some(log_ctl),
to_child: Some(to_child),
signals: Some(handles.signals),
to_stdin: Some(to_stdin),
manual: None,
pending_delete: carried.pending_delete().unwrap_or(false),
epoch: carried.epoch(),
ready_tx,
ready_failed,
restart_due: carried.restart_due(),
..SheepSlot::new(entry)
},
);
if let Some(manual) = carried.manual() {
let cap = match reload {
ReloadState::Drainee { .. } => LadderCap::Drain,
ReloadState::None | ReloadState::Replacement => LadderCap::Stop,
};
self.claim_manual(id, manual, cap);
}
if status == ProcStatus::Online {
self.arm_extras(id);
}
Ok(())
}
#[cfg(unix)]
fn install_carried_reloads(&mut self, reloads: Vec<CarriedReload>) {
for carried in reloads {
let CarriedReload {
app,
queue,
mode,
swap,
} = carried;
let anchor = swap
.new_id
.filter(|id| self.sheep.contains_key(id))
.or_else(|| self.sheep.contains_key(&swap.old_id).then_some(swap.old_id));
let Some(anchor) = anchor else {
tracing::warn!(
name = app,
old_id = swap.old_id,
new_id = swap.new_id,
"a carried reload named no instance this shepherd was given, so it was \
dropped rather than left unfinishable"
);
continue;
};
self.reloads.insert(
app.clone(),
ReloadJob {
queue: queue.into_iter().collect(),
mode,
swap,
deadline: 0,
},
);
self.arm_reload_deadline(&app, anchor);
if swap.phase == ReloadPhase::Verify
&& let Some(new_id) = swap.new_id
&& let Some(source) = self.post_drain_probe(new_id, mode)
{
self.spawn_verify_task(&app, new_id, source);
}
}
}
fn intended_spec(&self, id: u32) -> Option<&ResolvedApp> {
let slot = self.sheep.get(&id)?;
Some(slot.entry.pending.as_ref().unwrap_or(&slot.entry.spec))
}
fn intended_credentials(&self, id: u32) -> Result<Option<Credentials>, PrivilegeError> {
let slot = self
.sheep
.get(&id)
.expect("intended_credentials: the instance was found replaceable a moment ago");
match slot.entry.credentials {
SpawnIdentity::Resolved(credentials) if !slot.entry.pending_reidentifies => {
Ok(credentials)
}
SpawnIdentity::Resolved(_) | SpawnIdentity::Unresolved => privilege::resolve(
self.intended_spec(id)
.expect("intended_credentials: the slot was read a moment ago")
.config(),
),
}
}
fn promote_pending(&mut self, id: u32) {
let Some(slot) = self.sheep.get_mut(&id) else {
return;
};
let Some(pending) = slot.entry.pending.take() else {
return;
};
if core::mem::take(&mut slot.entry.pending_reidentifies) {
slot.entry.credentials = SpawnIdentity::Unresolved;
}
slot.entry.spec = pending;
}
fn respawn(&mut self, id: u32, manually: bool) -> ProcessInfo {
self.promote_pending(id);
let credentials = match self.credentials_for_spawn(id) {
Ok(credentials) => credentials,
Err(err) => return self.respawn_failed(id, manually, &err),
};
let slot = self.sheep.get(&id).expect("respawn: unknown id");
let app = slot.entry.spec.clone();
let instance = slot.entry.instance;
let next_epoch = slot.epoch + 1;
let spec = match assemble(
&app,
instance,
&self.paths,
credentials,
&self.secret_view(&app),
) {
Ok(spec) => spec,
Err(err) => return self.refuse_spawn(id, manually, &err),
};
let source = ReadinessSource::of(app.config())
.expect("ResolvedApp already passed ProbeTarget::parse in normalize");
let gated = !matches!(source, ReadinessSource::Heuristic);
match self.runner.spawn(&spec) {
Ok((proc, io)) => {
let pid = proc.pid();
let log_ctl = io.log_ctl.clone();
let to_child = io.to_child.clone();
let to_stdin = io.to_stdin.clone();
let handles = spawn_sheep_task::<R::Proc>(
id,
proc,
io,
app.clone(),
self.events.clone(),
self.tx.clone(),
);
let ready_tx = if gated {
Some(spawn_readiness_task(
id,
next_epoch,
manually,
source,
app.config().listen_timeout.as_duration(),
spec_prober(&spec),
self.tx.clone(),
))
} else {
None
};
let slot = self
.sheep
.get_mut(&id)
.expect("respawn: entry vanished mid-respawn");
slot.entry.status = if gated {
ProcStatus::Starting
} else {
ProcStatus::Online
};
slot.entry.pid = Some(pid);
slot.entry.started_at = Some(tokio::time::Instant::now());
slot.entry.restarts += 1;
slot.ready_failed = false;
slot.ctl = Some(handles.ctl);
slot.log_ctl = Some(log_ctl);
slot.to_child = Some(to_child);
slot.signals = Some(handles.signals);
slot.to_stdin = Some(to_stdin);
slot.epoch += 1;
debug_assert_eq!(slot.epoch, next_epoch);
slot.ready_tx = ready_tx;
let info = to_info(&slot.entry, &self.smits);
self.emit(ProcessEventKind::Restart, info.clone(), manually);
if !gated {
self.went_online(id, info.clone(), manually);
}
info
}
Err(error) => self.respawn_failed(id, manually, &error),
}
}
fn respawn_failed(
&mut self,
id: u32,
manually: bool,
reason: &dyn fmt::Display,
) -> ProcessInfo {
tracing::warn!(id, %reason, "no process was started");
let slot = self
.sheep
.get_mut(&id)
.expect("respawn: entry vanished mid-respawn");
slot.entry.status = ProcStatus::Errored;
slot.entry.pid = None;
slot.entry.started_at = None;
slot.entry.last_exit = None;
slot.ctl = None;
slot.to_child = None;
slot.signals = None;
slot.to_stdin = None;
slot.ready_tx = None;
let info = to_info(&slot.entry, &self.smits);
self.emit(ProcessEventKind::Errored, info.clone(), manually);
self.disarm_extras(id, &info.name);
info
}
fn refuse_spawn(&mut self, id: u32, manually: bool, err: &AssembleError) -> ProcessInfo {
if !err.is_retriable() {
return self.respawn_failed(id, manually, err);
}
let slot = self
.sheep
.get_mut(&id)
.expect("refuse_spawn: the slot was registered a moment ago");
slot.entry.budget.note_failed_start();
let (max_restarts, delay) = {
let config = slot.entry.spec.config();
(
config.max_restarts,
restart_delay(config, slot.entry.budget.unstable_count()),
)
};
if slot.entry.budget.exhausted(max_restarts) {
return self.respawn_failed(id, manually, err);
}
tracing::warn!(id, %err, "spawn refused; waiting to try again");
let epoch = slot.epoch;
slot.entry.status = ProcStatus::WaitingRestart;
slot.entry.pid = None;
slot.entry.started_at = None;
slot.entry.last_exit = None;
slot.ctl = None;
slot.to_child = None;
slot.signals = None;
slot.to_stdin = None;
slot.ready_tx = None;
slot.restart_due = SystemTime::now()
.checked_add(delay.unwrap_or(Duration::ZERO))
.filter(|due| due.duration_since(SystemTime::UNIX_EPOCH).is_ok());
let info = to_info(&slot.entry, &self.smits);
self.emit(ProcessEventKind::Exit, info.clone(), manually);
self.schedule_restart(id, epoch, delay);
info
}
fn claim_manual(&mut self, id: u32, manual: PendingManual, cap: LadderCap) {
let Some(slot) = self.sheep.get_mut(&id) else {
return;
};
match slot.manual.map(|in_flight| in_flight.origin) {
None => {
slot.manual = Some(manual);
if let Some(ctl) = &slot.ctl {
let grace = cap.of(slot.entry.spec.config());
let _ = ctl.try_send(SheepCtl::Kill { grace });
}
}
Some(CommandOrigin::Automatic) if manual.origin == CommandOrigin::Operator => {
slot.manual = Some(manual);
}
Some(CommandOrigin::Operator | CommandOrigin::Automatic) => {}
}
}
fn matching_ids(&self, selector: &ProcessSelector) -> Vec<u32> {
let exact = selector.is_exact();
let mut ids: Vec<u32> = self
.sheep
.iter()
.filter(|(_, slot)| exact || slot.entry.dog.is_none())
.filter_map(|(id, slot)| {
let config = slot.entry.spec.config();
selector
.matches(
&config.name,
*id,
config.fold.as_deref(),
Some(slot.entry.instance),
)
.then_some(*id)
})
.collect();
ids.sort_unstable();
ids
}
fn begin_manual(
&mut self,
selector: ProcessSelector,
kind: ManualKind,
origin: CommandOrigin,
reply: ReplyKind,
) {
let matched = self.matching_ids(&selector);
if matched.is_empty() {
send_reply(reply, Err(SupervisorError::NotFound));
return;
}
self.begin_manual_ids(matched, kind, origin, reply);
}
fn begin_manual_ids(
&mut self,
matched: Vec<u32>,
kind: ManualKind,
origin: CommandOrigin,
reply: ReplyKind,
) {
let mut remaining = HashSet::new();
let mut results = Vec::new();
for id in matched {
let held_off_by_a_swap =
origin == CommandOrigin::Automatic && self.in_an_uncommitted_swap(id);
if held_off_by_a_swap {
tracing::debug!(
id,
?kind,
"automatic command dropped: this sheep is half of a swap that has not \
committed"
);
continue;
}
let is_running = self.sheep.get(&id).is_some_and(|slot| slot.ctl.is_some());
if is_running {
self.claim_manual(id, PendingManual { kind, origin }, LadderCap::Stop);
if kind == ManualKind::Delete {
if let Some(slot) = self.sheep.get_mut(&id) {
slot.pending_delete = true;
}
}
remaining.insert(id);
} else if let Some(info) = self.apply_immediate(id, kind, origin) {
results.push(info);
}
}
if remaining.is_empty() {
sort_flock(&mut results);
send_reply(reply, Ok(results));
return;
}
self.pending.push(PendingReply {
remaining,
results,
reply,
});
}
fn apply_immediate(
&mut self,
id: u32,
kind: ManualKind,
origin: CommandOrigin,
) -> Option<ProcessInfo> {
let manually = origin == CommandOrigin::Operator;
match kind {
ManualKind::Stop => {
let slot = self.sheep.get_mut(&id)?;
match slot.entry.status {
ProcStatus::WaitingRestart | ProcStatus::Errored => {
slot.entry.status = ProcStatus::Stopped;
let info = to_info(&slot.entry, &self.smits);
self.emit(ProcessEventKind::Stop, info.clone(), manually);
self.disarm_extras(id, &info.name);
Some(info)
}
_ => Some(to_info(&slot.entry, &self.smits)),
}
}
ManualKind::Delete => {
let slot = self.sheep.remove(&id)?;
let info = to_info(&slot.entry, &self.smits);
self.emit(ProcessEventKind::Delete, info.clone(), manually);
self.disarm_extras(id, &info.name);
Some(info)
}
ManualKind::Restart => {
self.sheep.get_mut(&id)?.entry.budget.reset();
Some(self.respawn(id, manually))
}
}
}
fn handle_scale(
&mut self,
name: &str,
count: u32,
reply: oneshot::Sender<Result<Scaled, SupervisorError>>,
) {
let mut slots: Vec<(u32, u32)> = self
.sheep
.iter()
.filter(|(_, slot)| slot.entry.spec.config().name == name)
.map(|(id, slot)| (slot.entry.instance, *id))
.collect();
if slots.is_empty() {
let _ = reply.send(Err(SupervisorError::NotFound));
return;
}
slots.sort_unstable();
if count == 0 {
let _ = reply.send(Err(SupervisorError::InvalidScale(format!(
"an app runs at least one instance; use `shep delete {name}` to remove it"
))));
return;
}
if self
.sheep
.get(&slots[0].1)
.is_some_and(|slot| slot.entry.dog.is_some())
{
let _ = reply.send(Err(SupervisorError::InvalidScale(format!(
"{name} is a dog, and a dog runs one process"
))));
return;
}
if self.reloads.contains_key(name) {
let _ = reply.send(Err(SupervisorError::ReloadInFlight(name.to_string())));
return;
}
let leaving = slots
.iter()
.filter(|(_, id)| self.sheep.get(id).is_some_and(|slot| slot.pending_delete))
.count();
if leaving > 0 {
let _ = reply.send(Err(SupervisorError::InvalidScale(format!(
"{name} has {leaving} instance(s) still shutting down from an \
earlier command; wait for them to leave `shep flock` and scale \
again"
))));
return;
}
let mut config = self
.sheep
.get(&slots[0].1)
.expect("handle_scale: id read off this map a moment ago")
.entry
.spec
.config()
.clone();
config.instances = count;
let rescaled = match normalize(config) {
Ok(app) => app,
Err(err) => {
let _ = reply.send(Err(SupervisorError::InvalidScale(err.to_string())));
return;
}
};
let current = u32::try_from(slots.len()).unwrap_or(u32::MAX);
let mut failure = None;
let mut orphaned_by_failed_spawn = None;
let survivors: Vec<u32> = match count.cmp(¤t) {
Ordering::Equal => slots.iter().map(|(_, id)| *id).collect(),
Ordering::Greater => {
let credentials = match self.credentials_for_spawn(slots[0].1) {
Ok(credentials) => credentials,
Err(err) => {
let _ =
reply.send(Err(SupervisorError::CannotStart(format!("{name}: {err}"))));
return;
}
};
let existing: Vec<u32> = slots.iter().map(|(instance, _)| *instance).collect();
let mut ids: Vec<u32> = slots.iter().map(|(_, id)| *id).collect();
for instance in instance_slots(&existing, count - current) {
let attempted_id = self.next_id;
match self.spawn_fresh(&rescaled, instance, credentials, None, &BTreeSet::new())
{
Ok(info) => ids.push(info.id),
Err(message) => {
orphaned_by_failed_spawn = Some(attempted_id);
failure = Some(message);
break;
}
}
}
ids
}
Ordering::Less => {
let cut = usize::try_from(count).unwrap_or(usize::MAX);
let (keep, remove) = slots.split_at(cut);
let removed: Vec<u32> = remove.iter().map(|(_, id)| *id).collect();
self.begin_manual_ids(
removed,
ManualKind::Delete,
CommandOrigin::Operator,
ReplyKind::Ids(oneshot::channel().0),
);
keep.iter().map(|(_, id)| *id).collect()
}
};
let achieved = u32::try_from(survivors.len()).unwrap_or(u32::MAX);
let stored = if achieved == count {
rescaled
} else {
let mut config = rescaled.config().clone();
config.instances = achieved;
match normalize(config) {
Ok(app) => app,
Err(err) => {
let _ = reply.send(Err(SupervisorError::InvalidScale(err.to_string())));
return;
}
}
};
let owed = self.sheep.get(&slots[0].1).and_then(|slot| {
let parked = slot.entry.pending.clone()?;
let parked = with_count(&parked, achieved).unwrap_or(parked);
Some((parked, slot.entry.pending_reidentifies))
});
for id in survivors.iter().chain(orphaned_by_failed_spawn.iter()) {
if let Some(slot) = self.sheep.get_mut(id) {
slot.entry.spec = stored.clone();
match &mut slot.entry.pending {
Some(parked) => {
if let Some(recounted) = with_count(parked, achieved) {
*parked = recounted;
}
}
None => {
if let Some((parked, reidentifies)) = &owed {
slot.entry.pending = Some(parked.clone());
slot.entry.pending_reidentifies = *reidentifies;
}
}
}
}
}
let mut instances: Vec<ProcessInfo> = survivors
.iter()
.filter_map(|id| {
self.sheep
.get(id)
.map(|slot| to_info(&slot.entry, &self.smits))
})
.collect();
sort_flock(&mut instances);
let _ = reply.send(Ok(Scaled {
instances,
app: stored,
requested: count,
shortfall: failure,
}));
}
fn ids_of_name(&self, name: &str) -> Vec<u32> {
let mut slots: Vec<(u32, u32)> = self
.sheep
.iter()
.filter(|(_, slot)| slot.entry.spec.config().name == name)
.map(|(id, slot)| (slot.entry.instance, *id))
.collect();
slots.sort_unstable();
slots.into_iter().map(|(_, id)| id).collect()
}
fn handle_apply_config(&mut self, apps: Vec<DeclaredApp>, reset: ResetDepth) -> Vec<Applied> {
let store = match overrides::all(&self.paths.overrides) {
Ok(store) => store,
Err(err) => {
let message = format!("overrides could not be read: {err}");
return apps
.iter()
.map(|incoming| Applied {
name: incoming.config.name.clone(),
applied: Vec::new(),
pending: Vec::new(),
refused: Some(message.clone()),
app: None,
})
.collect();
}
};
let mut changes = BTreeMap::new();
let mut report = Vec::with_capacity(apps.len());
for incoming in &apps {
let overrides = store
.get(&incoming.config.name)
.cloned()
.unwrap_or_default();
report.push(self.apply_one(incoming, reset, &overrides, &mut changes));
}
if let Err(err) = overrides::update(&self.paths.overrides, &changes) {
let note = format!("overrides could not be written: {err}");
for applied in &mut report {
if !changes.contains_key(&applied.name) {
continue;
}
applied.refused = Some(match applied.refused.take() {
Some(existing) => format!("{existing}; {note}"),
None => note.clone(),
});
}
}
report
}
fn representative_id(&self, name: &str) -> Option<u32> {
let ids = self.ids_of_name(name);
ids.iter()
.copied()
.find(|id| {
self.sheep
.get(id)
.is_some_and(|slot| !matches!(slot.entry.reload, ReloadState::Drainee { .. }))
})
.or_else(|| ids.first().copied())
}
fn handle_sheep_config(&self, name: &str) -> Result<Option<SheepConfigView>, SupervisorError> {
let Some(id) = self.representative_id(name) else {
return Ok(None);
};
let Some(slot) = self.sheep.get(&id) else {
return Ok(None);
};
if slot.entry.dog.is_some() {
return Err(SupervisorError::IsADog(dog_config_refusal(name)));
}
let Some(config) = self.intended_spec(id).map(|spec| spec.config().clone()) else {
return Ok(None);
};
Ok(Some(SheepConfigView::new(
config,
self.overridden_for(name),
pending_fields(&slot.entry),
)))
}
fn handle_set_sheep_env(
&mut self,
name: &str,
key: &str,
value: Option<&str>,
) -> Result<Option<ResolvedApp>, SupervisorError> {
let Some(id) = self.representative_id(name) else {
return Ok(None);
};
if self
.sheep
.get(&id)
.is_some_and(|slot| slot.entry.dog.is_some())
{
return Err(SupervisorError::IsADog(dog_config_refusal(name)));
}
let Some(mut intended) = self.intended_spec(id).map(|spec| spec.config().clone()) else {
return Ok(None);
};
let was_in_config = intended.env.contains_key(key);
match value {
Some(value) => intended.env.insert(key.to_string(), value.to_string()),
None => intended.env.remove(key),
};
let parked =
normalize(intended).map_err(|err| SupervisorError::InvalidEnv(err.to_string()))?;
let mut record = overrides::get(&self.paths.overrides, name)
.map_err(|err| SupervisorError::Overrides(err.to_string()))?
.unwrap_or_default();
let file_declares = record.declared_env.contains(key);
let map = env_override_map(&mut record, name)?;
let was_overridden = map.contains_key(key);
let tombstoned = map.get(key).is_some_and(serde_json::Value::is_null);
match value {
Some(value) => map.insert(
key.to_string(),
serde_json::Value::String(value.to_string()),
),
None if tombstoned => None,
None => map.remove(key),
};
if value.is_none() && was_in_config && (!was_overridden || file_declares) {
map.insert(key.to_string(), serde_json::Value::Null);
}
let emptied = map.is_empty();
if emptied {
record.fields.remove("env");
}
let overridden: Vec<String> = record.fields.keys().cloned().collect();
let changes = BTreeMap::from([(name.to_string(), Some(record))]);
overrides::update(&self.paths.overrides, &changes)
.map_err(|err| SupervisorError::Overrides(err.to_string()))?;
for id in self.ids_of_name(name) {
let Some(slot) = self.sheep.get_mut(&id) else {
continue;
};
slot.entry.pending = Some(parked.clone());
slot.entry.overridden.clone_from(&overridden);
}
Ok(Some(parked))
}
fn handle_set_sheep_env_batch(
&mut self,
name: &str,
entries: &BTreeMap<String, String>,
force: bool,
dry_run: bool,
) -> Result<Option<EnvBatch>, SupervisorError> {
let Some(id) = self.representative_id(name) else {
return Ok(None);
};
if self
.sheep
.get(&id)
.is_some_and(|slot| slot.entry.dog.is_some())
{
return Err(SupervisorError::IsADog(dog_config_refusal(name)));
}
let Some(mut intended) = self.intended_spec(id).map(|spec| spec.config().clone()) else {
return Ok(None);
};
let mut set = Vec::new();
let mut unchanged = Vec::new();
let mut collisions = Vec::new();
for (key, value) in entries {
match intended.env.get(key) {
Some(current) if current == value => unchanged.push(key.clone()),
Some(_) => {
collisions.push(key.clone());
if force {
set.push(key.clone());
}
}
None => set.push(key.clone()),
}
}
if !collisions.is_empty() && !force {
return Ok(Some(EnvBatch {
app: None,
set: Vec::new(),
unchanged,
collisions,
}));
}
for key in &set {
intended.env.insert(key.clone(), entries[key].clone());
}
let parked =
normalize(intended).map_err(|err| SupervisorError::InvalidEnv(err.to_string()))?;
if dry_run || set.is_empty() {
return Ok(Some(EnvBatch {
app: None,
set,
unchanged,
collisions,
}));
}
let mut record = overrides::get(&self.paths.overrides, name)
.map_err(|err| SupervisorError::Overrides(err.to_string()))?
.unwrap_or_default();
let map = env_override_map(&mut record, name)?;
for key in &set {
map.insert(key.clone(), serde_json::Value::String(entries[key].clone()));
}
let overridden: Vec<String> = record.fields.keys().cloned().collect();
let changes = BTreeMap::from([(name.to_string(), Some(record))]);
overrides::update(&self.paths.overrides, &changes)
.map_err(|err| SupervisorError::Overrides(err.to_string()))?;
for id in self.ids_of_name(name) {
let Some(slot) = self.sheep.get_mut(&id) else {
continue;
};
slot.entry.pending = Some(parked.clone());
slot.entry.overridden.clone_from(&overridden);
}
Ok(Some(EnvBatch {
app: Some(parked),
set,
unchanged,
collisions,
}))
}
fn handle_set_sheep_field(
&mut self,
name: &str,
key: &str,
value: &serde_json::Value,
) -> Result<Option<FieldSet>, SupervisorError> {
if key == "env" {
return Err(SupervisorError::InvalidField(
"env is set one key at a time; use `SetSheepEnv`".to_string(),
));
}
if apply_group(key) == ApplyGroup::Structural {
return Err(SupervisorError::InvalidField(format!(
"{key} is not a config write; `shep stock` moves an instance count, and a \
name change is a different sheep"
)));
}
let Some(id) = self.representative_id(name) else {
return Ok(None);
};
if self
.sheep
.get(&id)
.is_some_and(|slot| slot.entry.dog.is_some())
{
return Err(SupervisorError::IsADog(dog_config_refusal(name)));
}
let Some(slot) = self.sheep.get(&id) else {
return Ok(None);
};
let running = slot.entry.spec.config().clone();
let intended = slot
.entry
.pending
.as_ref()
.map_or_else(|| running.clone(), |parked| parked.config().clone());
let Ok(serde_json::Value::Object(mut object)) = serde_json::to_value(&intended) else {
return Err(SupervisorError::InvalidField(
"an app config must serialize as an object".to_string(),
));
};
if !object.contains_key(key) {
return Err(SupervisorError::InvalidField(format!(
"no config field named {key}"
)));
}
object.insert(key.to_string(), value.clone());
let edited: AppConfig = serde_json::from_value(serde_json::Value::Object(object))
.map_err(|err| SupervisorError::InvalidField(format!("{key}: {err}")))?;
let merged = normalize(edited)
.map_err(|err| SupervisorError::InvalidField(format!("{key}: {err}")))?;
let group = apply_group(key);
let reaches = matches!(group, ApplyGroup::Live | ApplyGroup::NextSpawn);
let reaching: Vec<String> = if reaches {
vec![key.to_string()]
} else {
Vec::new()
};
let next_spec = reached_spec(&running, merged.config(), &reaching, running.instances).ok();
let park_all = next_spec.is_none();
let mut record = overrides::get(&self.paths.overrides, name)
.map_err(|err| SupervisorError::Overrides(err.to_string()))?
.unwrap_or_default();
record.fields.insert(key.to_string(), value.clone());
let overridden: Vec<String> = record.fields.keys().cloned().collect();
let changes = BTreeMap::from([(name.to_string(), Some(record))]);
overrides::update(&self.paths.overrides, &changes)
.map_err(|err| SupervisorError::Overrides(err.to_string()))?;
let parked_wanted = park_all
|| !reaches
|| self.ids_of_name(name).iter().any(|id| {
self.sheep
.get(id)
.is_some_and(|s| s.entry.pending.is_some())
});
let parked = parked_wanted.then(|| merged.clone());
for id in self.ids_of_name(name) {
let Some(slot) = self.sheep.get_mut(&id) else {
continue;
};
if let Some(parked) = &parked {
let spawned = slot.entry.spec.config();
slot.entry.pending_reidentifies |=
spawned.user != parked.config().user || spawned.group != parked.config().group;
}
if let Some(next_spec) = next_spec.clone() {
slot.entry.spec = next_spec;
}
if let Some(parked) = parked.clone() {
slot.entry.pending = Some(parked);
}
slot.entry.overridden.clone_from(&overridden);
}
if !park_all && group == ApplyGroup::Live && EXTRAS_FIELDS.contains(&key) {
self.rearm_name(name);
}
let in_force =
!park_all && (group == ApplyGroup::Live || matches!(key, "autostart" | "depends_on"));
Ok(Some(FieldSet {
app: parked.unwrap_or_else(|| next_spec.unwrap_or(merged)),
pending: !in_force,
}))
}
fn apply_one(
&mut self,
incoming: &DeclaredApp,
reset: ResetDepth,
overrides: &AppOverrides,
changes: &mut BTreeMap<String, Option<AppOverrides>>,
) -> Applied {
let name = incoming.config.name.clone();
let refuse = |message: String| Applied {
name: name.clone(),
applied: Vec::new(),
pending: Vec::new(),
refused: Some(message),
app: None,
};
let ids = self.ids_of_name(&name);
let Some(slot) = self
.representative_id(&name)
.and_then(|id| self.sheep.get(&id))
else {
return refuse(format!(
"{name} is not registered; `shep start` it before a config can be applied to it"
));
};
if slot.entry.dog.is_some() {
return refuse(dog_config_refusal(&name));
}
let running = slot.entry.spec.config().clone();
let intended = slot
.entry
.pending
.as_ref()
.map_or_else(|| running.clone(), |parked| parked.config().clone());
let (merged, mut next_overrides) =
match merge_declared(&intended, incoming, overrides, reset) {
Ok(merged) => merged,
Err(message) => return refuse(message),
};
let merged = match normalize(merged) {
Ok(merged) => merged,
Err(err) => return refuse(err.to_string()),
};
let mut live = Vec::new();
let mut next_spawn = Vec::new();
let mut respawn = Vec::new();
let mut instances = false;
for field in intended.drifted_fields(merged.config()) {
match apply_group(&field) {
ApplyGroup::Live => live.push(field),
ApplyGroup::NextSpawn => next_spawn.push(field),
ApplyGroup::NeedsRespawn => respawn.push(field),
ApplyGroup::Structural => instances |= field == "instances",
_ => respawn.push(field),
}
}
let mut refusals = Vec::new();
let mut count = running.instances;
if !matches!(
reset,
ResetDepth::Policy | ResetDepth::All | ResetDepth::File
) && incoming.declared.contains("instances")
&& incoming.config.instances != running.instances
{
refusals.push(format!(
"instances: this load never reshapes a flock; no mode scales without also \
putting back every setting the file declares, and `--reset=file` is the \
narrowest that does, taking the file's count of {}",
incoming.config.instances
));
}
if instances {
if self.shutting_down {
refusals.push(
"instances: the daemon is shutting down and cannot reshape a flock".to_string(),
);
} else {
let (reply, mut answer) = oneshot::channel();
self.handle_scale(&name, merged.config().instances, reply);
match answer
.try_recv()
.expect("handle_scale answers before it returns")
{
Ok(scaled) => {
count = scaled.achieved();
if let Some(shortfall) = scaled.shortfall {
refusals.push(format!("instances: {shortfall}"));
}
}
Err(err) => return refuse(err.to_string()),
}
}
}
let reaching: Vec<String> = live.iter().chain(next_spawn.iter()).cloned().collect();
let next_spec = reached_spec(&running, merged.config(), &reaching, count).ok();
let park_all = next_spec.is_none();
let parked_wanted = park_all
|| !respawn.is_empty()
|| ids.iter().any(|id| {
self.sheep
.get(id)
.is_some_and(|slot| slot.entry.pending.is_some())
});
let parked = if parked_wanted {
with_count(&merged, count)
} else {
None
};
let parked_failed = parked_wanted && parked.is_none();
for id in self.ids_of_name(&name) {
let Some(slot) = self.sheep.get_mut(&id) else {
continue;
};
if let Some(parked) = &parked {
let running = slot.entry.spec.config();
slot.entry.pending_reidentifies |=
running.user != parked.config().user || running.group != parked.config().group;
}
if let Some(next_spec) = next_spec.clone() {
slot.entry.spec = next_spec;
}
if let Some(parked) = parked.clone() {
slot.entry.pending = Some(parked);
}
}
if !park_all
&& live
.iter()
.any(|field| EXTRAS_FIELDS.contains(&field.as_str()))
{
self.rearm_name(&name);
}
let (already_in_force, later): (Vec<String>, Vec<String>) = next_spawn
.into_iter()
.partition(|field| matches!(field.as_str(), "autostart" | "depends_on"));
let unparked: Vec<String> = if !parked_failed {
Vec::new()
} else if park_all {
live.iter()
.cloned()
.chain(already_in_force.iter().cloned())
.chain(later.iter().cloned())
.chain(respawn.iter().cloned())
.collect()
} else {
respawn.clone()
};
if parked_failed {
let what = if unparked.is_empty() {
"an earlier load's parked config could not be rebuilt, so a respawn will put \
its values back"
.to_string()
} else {
format!(
"{} could not be parked for a next spawn",
unparked.join(", ")
)
};
refusals.push(format!(
"{what}: the merged config does not hold at {count} instance(s)"
));
}
for field in &unparked {
match overrides.fields.get(field) {
Some(previous) => next_overrides
.fields
.insert(field.clone(), previous.clone()),
None => next_overrides.fields.remove(field),
};
if !overrides.declared.contains(field) {
next_overrides.declared.remove(field);
}
if field == "env" {
next_overrides
.declared_env
.clone_from(&overrides.declared_env);
}
}
let overridden_names: Vec<String> = if matches!(reset, ResetDepth::All) {
Vec::new()
} else {
next_overrides.fields.keys().cloned().collect()
};
for id in self.ids_of_name(&name) {
if let Some(slot) = self.sheep.get_mut(&id) {
slot.entry.overridden.clone_from(&overridden_names);
}
}
changes.insert(
name.clone(),
(!matches!(reset, ResetDepth::All)).then_some(next_overrides),
);
let (mut applied, mut pending): (Vec<String>, Vec<String>) = if park_all {
(
Vec::new(),
if parked_failed {
Vec::new()
} else {
live.iter()
.cloned()
.chain(already_in_force)
.chain(later)
.chain(respawn)
.collect()
},
)
} else {
(
live.iter().cloned().chain(already_in_force).collect(),
if parked_failed {
later
} else {
later.into_iter().chain(respawn).collect()
},
)
};
if instances && count != running.instances {
applied.push("instances".to_string());
}
applied.sort_unstable();
pending.sort_unstable();
let app = if count == merged.config().instances {
Some(merged)
} else {
with_count(&merged, count)
};
Applied {
name,
applied,
pending,
refused: (!refusals.is_empty()).then(|| refusals.join("; ")),
app,
}
}
fn rearm_name(&mut self, name: &str) {
let Some(extras) = self.extras.as_ref() else {
return;
};
let supervisor = SupervisorHandle {
tx: self.tx.clone(),
};
let mut armable: Vec<(u32, &ProcessEntry)> = self
.sheep
.iter()
.filter(|(_, slot)| {
slot.entry.spec.config().name == name
&& slot.entry.status == ProcStatus::Online
&& slot.entry.pid.is_some()
})
.map(|(id, slot)| (*id, &slot.entry))
.collect();
armable.sort_unstable_by_key(|(id, _)| *id);
let entries: Vec<&ProcessEntry> = armable.into_iter().map(|(_, entry)| entry).collect();
let specs = self.rearm_specs(&entries);
self.registry.rearm_name(
name,
&entries,
|entry| {
spec_prober(
specs
.get(&entry.id)
.expect("rearm_name: `rearm_specs` described every entry passed to it"),
)
},
extras,
&supervisor,
);
}
fn rearm_specs(&self, entries: &[&ProcessEntry]) -> HashMap<u32, SpawnSpec> {
let mut views: HashMap<&str, SecretView> = HashMap::new();
let mut specs = HashMap::with_capacity(entries.len());
for entry in entries {
let environment = entry
.spec
.config()
.environment
.as_deref()
.unwrap_or(&self.host_environment);
let secrets = views
.entry(environment)
.or_insert_with(|| self.secret_view(&entry.spec));
let credentials = match entry.credentials {
SpawnIdentity::Resolved(credentials) => credentials,
SpawnIdentity::Unresolved => None,
};
specs.insert(
entry.id,
describe(
&entry.spec,
entry.instance,
&self.paths,
credentials,
secrets,
),
);
}
specs
}
fn handle_reload(
&mut self,
selector: &ProcessSelector,
reply: oneshot::Sender<Result<Vec<ProcessInfo>, SupervisorError>>,
) {
let matched = self.matching_ids(selector);
if matched.is_empty() {
let _ = reply.send(Err(SupervisorError::NotFound));
return;
}
let in_flight = matched.iter().find_map(|id| {
let slot = self
.sheep
.get(id)
.expect("handle_reload: `matched` holds ids read off this map a moment ago");
let name = &slot.entry.spec.config().name;
self.reloads.contains_key(name).then(|| name.clone())
});
if let Some(name) = in_flight {
let _ = reply.send(Err(SupervisorError::ReloadInFlight(name)));
return;
}
let mut accepted: Vec<ProcessInfo> = matched
.iter()
.map(|id| {
let slot = self
.sheep
.get(id)
.expect("handle_reload: `matched` holds ids read off this map a moment ago");
to_info(&slot.entry, &self.smits)
})
.collect();
sort_flock(&mut accepted);
let mut queues: BTreeMap<String, Vec<(u32, u32)>> = BTreeMap::new();
for id in matched {
let slot = self
.sheep
.get(&id)
.expect("handle_reload: `matched` holds ids read off this map a moment ago");
if !reload_eligible(slot) {
continue;
}
let entry = &slot.entry;
queues
.entry(entry.spec.config().name.clone())
.or_default()
.push((entry.instance, id));
}
let _ = reply.send(Ok(accepted));
for (name, mut instances) in queues {
instances.sort_unstable();
let queue = instances.into_iter().map(|(_, id)| id).collect();
self.advance_reload(&name, queue);
}
}
fn advance_reload(&mut self, name: &str, mut queue: VecDeque<u32>) {
if self.shutting_down {
self.reloads.remove(name);
return;
}
while let Some(old_id) = queue.pop_front() {
let slot = self.sheep.get(&old_id);
let replaceable =
slot.is_some_and(|slot| reload_eligible(slot) && slot.manual.is_none());
if !replaceable {
tracing::debug!(
name,
old_id,
status = ?slot.map(|slot| slot.entry.status),
"reload skipped an instance: it is gone, no longer online, or its next \
exit is already claimed"
);
continue;
}
if self.reload_mode_of(old_id) == ReloadMode::Serial {
self.begin_serial_drain(name, old_id, queue);
return;
}
match self.spawn_replacement(old_id, ReloadMode::Overlap) {
Ok(new_id) => {
self.reloads.insert(
name.to_string(),
ReloadJob {
queue,
mode: ReloadMode::Overlap,
deadline: 0,
swap: ReloadSwap {
old_id,
new_id: Some(new_id),
phase: ReloadPhase::AwaitReady,
},
},
);
self.arm_reload_deadline(name, new_id);
return;
}
Err(error) => {
tracing::warn!(
name,
old_id,
error,
"reload abandoned: the replacement could not be spawned"
);
self.reloads.remove(name);
if let Some(slot) = self.sheep.get(&old_id) {
let info = to_info(&slot.entry, &self.smits);
self.emit(ProcessEventKind::ReloadAbandoned, info, true);
}
return;
}
}
}
self.reloads.remove(name);
}
fn reload_mode_of(&self, old_id: u32) -> ReloadMode {
let config = self
.intended_spec(old_id)
.expect("reload_mode_of: the instance was found replaceable a moment ago")
.config();
let source = ReadinessSource::of(config)
.expect("ResolvedApp already passed ProbeTarget::parse in normalize");
ReloadMode::of(config, &source)
}
fn begin_serial_drain(&mut self, name: &str, old_id: u32, queue: VecDeque<u32>) {
let drainee = self
.sheep
.get_mut(&old_id)
.expect("begin_serial_drain: the instance was found replaceable a moment ago");
drainee.entry.status = ProcStatus::Stopping;
drainee.entry.reload = ReloadState::Drainee { new_id: None };
let info = to_info(&drainee.entry, &self.smits);
self.reloads.insert(
name.to_string(),
ReloadJob {
queue,
mode: ReloadMode::Serial,
deadline: 0,
swap: ReloadSwap {
old_id,
new_id: None,
phase: ReloadPhase::DrainFirst,
},
},
);
self.arm_reload_deadline(name, old_id);
self.emit(ProcessEventKind::Reload, info, true);
self.claim_manual(
old_id,
PendingManual {
kind: ManualKind::Stop,
origin: CommandOrigin::Operator,
},
LadderCap::Drain,
);
}
fn spawn_replacement(&mut self, old_id: u32, mode: ReloadMode) -> Result<u32, String> {
let credentials = self
.intended_credentials(old_id)
.map_err(|err| err.to_string())?;
let app = self
.intended_spec(old_id)
.expect("spawn_replacement: the instance was found replaceable a moment ago")
.clone();
let drainee = &self.sheep[&old_id].entry;
let instance = drainee.instance;
let restarts = drainee.restarts;
let dog = drainee.dog.clone();
let last_exit = drainee.last_exit;
let overridden = drainee.overridden.clone();
let new_id = self.next_id;
self.next_id += 1;
let spec = assemble(
&app,
instance,
&self.paths,
credentials,
&self.secret_view(&app),
)
.map_err(|err| err.to_string())?;
let out_file = spec.out_file.clone();
let err_file = spec.err_file.clone();
let source = ReadinessSource::of(app.config())
.expect("ResolvedApp already passed ProbeTarget::parse in normalize");
let drainee = self
.sheep
.get_mut(&old_id)
.expect("spawn_replacement: the drainee was read a moment ago");
drainee.entry.status = ProcStatus::Stopping;
drainee.entry.reload = ReloadState::Drainee {
new_id: Some(new_id),
};
match self.runner.spawn(&spec) {
Ok((proc, io)) => {
let pid = proc.pid();
let entry = ProcessEntry {
id: new_id,
spec: app.clone(),
pending: None,
pending_reidentifies: false,
overridden,
instance,
status: ProcStatus::Starting,
pid: Some(pid),
restarts,
started_at: Some(tokio::time::Instant::now()),
budget: RestartBudget::default(),
reload: ReloadState::Replacement,
credentials: SpawnIdentity::Resolved(credentials),
out_file,
err_file,
dog,
last_exit,
};
let info = to_info(&entry, &self.smits);
let log_ctl = io.log_ctl.clone();
let to_child = io.to_child.clone();
let to_stdin = io.to_stdin.clone();
let handles = spawn_sheep_task::<R::Proc>(
new_id,
proc,
io,
app.clone(),
self.events.clone(),
self.tx.clone(),
);
let ready_tx = spawn_readiness_task(
new_id,
0,
true,
source,
app.config().listen_timeout.as_duration(),
spec_prober(&spec),
self.tx.clone(),
);
self.sheep.insert(
new_id,
SheepSlot {
ctl: Some(handles.ctl),
log_ctl: Some(log_ctl),
to_child: Some(to_child),
signals: Some(handles.signals),
to_stdin: Some(to_stdin),
ready_tx: Some(ready_tx),
..SheepSlot::new(entry)
},
);
if mode == ReloadMode::Overlap {
let drainee = to_info(&self.sheep[&old_id].entry, &self.smits);
self.emit(ProcessEventKind::Reload, drainee, true);
}
self.emit(ProcessEventKind::Start, info, true);
Ok(new_id)
}
Err(error) => {
if mode == ReloadMode::Overlap {
let drainee = self
.sheep
.get_mut(&old_id)
.expect("spawn_replacement: the drainee was marked a moment ago");
drainee.entry.status = restored_status(drainee);
drainee.entry.reload = ReloadState::None;
}
Err(error.to_string())
}
}
}
fn reload_ready_result(&mut self, new_id: u32, manually: bool, readiness: Readiness) {
let Some(name) = self.reload_of(new_id) else {
tracing::warn!(
id = new_id,
"a replacement resolved with no reload to belong to"
);
self.clear_reload(new_id);
let info = self.set_status(new_id, ProcStatus::Online);
self.went_online(new_id, info, manually);
return;
};
if readiness == Readiness::TimedOut {
let old_id = self.reloads[&name].swap.old_id;
if self.sheep.contains_key(&old_id) {
self.abort_reload(&name, "the replacement was not ready inside listen_timeout");
return;
}
tracing::warn!(
name,
new_id,
"reload abandoned: the replacement was not ready inside listen_timeout, with \
the instance it replaced already gone; it is left running and not online"
);
self.reloads.remove(&name);
self.clear_reload(new_id);
if let Some(slot) = self.sheep.get_mut(&new_id) {
slot.ready_failed = true;
let info = to_info(&slot.entry, &self.smits);
self.emit(ProcessEventKind::ReloadAbandoned, info, true);
}
return;
}
let info = self.set_status(new_id, ProcStatus::Online);
self.went_online(new_id, info, manually);
self.begin_drain(&name);
}
fn begin_drain(&mut self, name: &str) {
let Some(job) = self.reloads.get_mut(name) else {
return;
};
job.swap.phase = ReloadPhase::DrainOld;
let old_id = job.swap.old_id;
if !self.sheep.contains_key(&old_id) {
self.finish_swap(name);
return;
}
self.claim_manual(
old_id,
PendingManual {
kind: ManualKind::Stop,
origin: CommandOrigin::Operator,
},
LadderCap::Drain,
);
}
fn finish_swap(&mut self, name: &str) {
let Some(job) = self.reloads.remove(name) else {
return;
};
let new_id = job
.swap
.new_id
.expect("finish_swap: a swap with no replacement never reaches the drain");
let serving = self
.sheep
.get(&new_id)
.is_some_and(|slot| slot.entry.status == ProcStatus::Online);
if serving && let Some(source) = self.post_drain_probe(new_id, job.mode) {
self.reloads.insert(
name.to_string(),
ReloadJob {
swap: ReloadSwap {
phase: ReloadPhase::Verify,
..job.swap
},
..job
},
);
self.arm_reload_deadline(name, new_id);
self.spawn_verify_task(name, new_id, source);
return;
}
self.clear_reload(new_id);
if serving {
let info = to_info(&self.sheep[&new_id].entry, &self.smits);
self.emit(ProcessEventKind::Reloaded, info, true);
self.advance_reload(name, job.queue);
return;
}
tracing::warn!(
name,
new_id,
"reload abandoned: the replacement was no longer serving when the instance it \
replaced went"
);
if let Some(slot) = self.sheep.get(&new_id) {
let info = to_info(&slot.entry, &self.smits);
self.emit(ProcessEventKind::ReloadAbandoned, info, true);
}
}
fn post_drain_probe(&self, new_id: u32, mode: ReloadMode) -> Option<ReadinessSource> {
if mode != ReloadMode::Overlap {
return None;
}
let config = self.sheep.get(&new_id)?.entry.spec.config();
let source = ReadinessSource::of(config)
.expect("ResolvedApp already passed ProbeTarget::parse in normalize");
matches!(source, ReadinessSource::Probe(..)).then_some(source)
}
fn spawn_verify_task(&self, name: &str, new_id: u32, source: ReadinessSource) {
let slot = self
.sheep
.get(&new_id)
.expect("spawn_verify_task: the replacement was read a moment ago");
let deadline = slot.entry.spec.config().listen_timeout.as_duration();
let credentials = match slot.entry.credentials {
SpawnIdentity::Resolved(creds) => creds,
SpawnIdentity::Unresolved => None,
};
let described = describe(
&slot.entry.spec,
slot.entry.instance,
&self.paths,
credentials,
&self.secret_view(&slot.entry.spec),
);
let prober = spec_prober(&described);
let (_ready_tx, ready_rx) = oneshot::channel();
let tx = self.tx.clone();
let name = name.to_string();
tokio::spawn(async move {
let readiness = await_ready(&source, deadline, ready_rx, prober).await;
let _ = tx
.send(Msg::ReloadVerified {
name,
new_id,
readiness,
})
.await;
});
}
fn handle_reload_verified(&mut self, name: &str, new_id: u32, readiness: Readiness) {
let Some(job) = self.reloads.get(name) else {
return;
};
if job.swap.new_id != Some(new_id) || job.swap.phase != ReloadPhase::Verify {
return;
}
let job = self
.reloads
.remove(name)
.expect("handle_reload_verified: the job was read a moment ago");
self.clear_reload(new_id);
if readiness == Readiness::Ready {
let info = to_info(&self.sheep[&new_id].entry, &self.smits);
self.emit(ProcessEventKind::Reloaded, info, true);
self.advance_reload(name, job.queue);
return;
}
tracing::warn!(
name,
new_id,
"reload abandoned: the replacement did not answer its readiness probe once the \
instance it replaced was gone, so the probe that put it online was answered by \
that instance"
);
let info = self.set_status(new_id, ProcStatus::Starting);
if let Some(slot) = self.sheep.get_mut(&new_id) {
slot.ready_failed = true;
}
self.emit(ProcessEventKind::ReloadAbandoned, info, true);
}
fn abort_reload(&mut self, name: &str, reason: &str) {
let Some(job) = self.reloads.remove(name) else {
return;
};
debug_assert_eq!(
job.swap.phase,
ReloadPhase::AwaitReady,
"abort_reload: a committed swap has no old instance to go back to"
);
tracing::warn!(
name,
old_id = job.swap.old_id,
new_id = job.swap.new_id,
reason,
"reload abandoned"
);
let kept = self.sheep.get_mut(&job.swap.old_id).map(|drainee| {
drainee.entry.reload = ReloadState::None;
if drainee.ctl.is_some() && drainee.manual.is_none() {
drainee.entry.status = restored_status(drainee);
}
to_info(&drainee.entry, &self.smits)
});
if let Some(info) = kept {
self.emit(ProcessEventKind::ReloadAbandoned, info, true);
}
let Some(new_id) = job.swap.new_id else {
return;
};
let Some(replacement) = self.sheep.get_mut(&new_id) else {
return;
};
replacement.entry.reload = ReloadState::None;
if replacement.ctl.is_none() {
return;
}
replacement.pending_delete = true;
self.claim_manual(
new_id,
PendingManual {
kind: ManualKind::Delete,
origin: CommandOrigin::Operator,
},
LadderCap::Stop,
);
}
fn arm_reload_deadline(&mut self, name: &str, id: u32) {
let app = self
.sheep
.get(&id)
.expect("arm_reload_deadline: the swap's entry was read a moment ago")
.entry
.spec
.config();
let deadline = app.listen_timeout.as_duration()
+ app.graceful_timeout.as_duration()
+ RELOAD_DEADLINE_SLACK;
let stamp = self.next_deadline;
self.next_deadline += 1;
let Some(job) = self.reloads.get_mut(name) else {
debug_assert!(false, "arm_reload_deadline: no job to arm for");
return;
};
job.deadline = stamp;
let tx = self.tx.clone();
let name = name.to_string();
tokio::spawn(async move {
tokio::time::sleep(deadline).await;
let _ = tx.send(Msg::ReloadDeadline { name, stamp }).await;
});
}
fn handle_reload_deadline(&mut self, name: &str, stamp: u64) {
let Some(job) = self.reloads.get(name) else {
return;
};
if job.deadline != stamp {
return;
}
let old_id = job.swap.old_id;
match job.swap.phase {
ReloadPhase::DrainFirst => {
tracing::warn!(
name,
old_id,
"reload abandoned: the instance being drained passed the swap's deadline \
without exiting, so its replacement was never spawned"
);
self.reloads.remove(name);
self.clear_reload(old_id);
if let Some(slot) = self.sheep.get(&old_id) {
let info = to_info(&slot.entry, &self.smits);
self.emit(ProcessEventKind::ReloadAbandoned, info, true);
}
}
ReloadPhase::AwaitReady => {
self.abort_reload(
name,
"the swap passed its deadline with no readiness result",
);
}
ReloadPhase::DrainOld | ReloadPhase::Verify => {
let new_id = job
.swap
.new_id
.expect("a swap past AwaitReady has a replacement");
tracing::warn!(
name,
old_id,
new_id,
drainee_registered = self.sheep.contains_key(&old_id),
"reload abandoned: the swap passed its deadline, so the message that would \
have ended it is not coming"
);
self.reloads.remove(name);
self.clear_reload(new_id);
if let Some(slot) = self.sheep.get(&new_id) {
let info = to_info(&slot.entry, &self.smits);
self.emit(ProcessEventKind::ReloadAbandoned, info, true);
}
}
}
}
fn reap_drainee(&mut self, old_id: u32) -> bool {
if let Some(name) = self.serial_drain_of(old_id) {
if self.sheep[&old_id].pending_delete {
tracing::warn!(
name,
old_id,
"reload abandoned: the instance being drained was deleted, so no \
replacement was spawned"
);
self.reloads.remove(&name);
} else {
self.spawn_serial_replacement(&name, old_id);
}
}
let terminal = self.deregister_on_exit(old_id);
let Some(name) = self.reload_of(old_id) else {
return terminal;
};
match self.reloads[&name].swap.phase {
ReloadPhase::DrainOld | ReloadPhase::Verify => self.finish_swap(&name),
ReloadPhase::AwaitReady => {
self.reloads
.get_mut(&name)
.expect("reap_drainee: the phase was read a moment ago")
.swap
.phase = ReloadPhase::DrainOld;
}
ReloadPhase::DrainFirst => {
debug_assert!(false, "reap_drainee: DrainFirst outlived its own spawn");
}
}
terminal
}
fn serial_drain_of(&self, old_id: u32) -> Option<String> {
self.reloads
.iter()
.find(|(_, job)| job.swap.phase == ReloadPhase::DrainFirst && job.swap.old_id == old_id)
.map(|(name, _)| name.clone())
}
fn spawn_serial_replacement(&mut self, name: &str, old_id: u32) {
match self.spawn_replacement(old_id, ReloadMode::Serial) {
Ok(new_id) => {
let job = self
.reloads
.get_mut(name)
.expect("spawn_serial_replacement: the job was read a moment ago");
job.swap.new_id = Some(new_id);
job.swap.phase = ReloadPhase::AwaitReady;
self.arm_reload_deadline(name, new_id);
}
Err(error) => {
tracing::warn!(
name,
old_id,
error,
"reload abandoned: the instance was drained and its replacement could not \
be spawned, so the instance slot is empty"
);
self.reloads.remove(name);
if let Some(slot) = self.sheep.get(&old_id) {
let info = to_info(&slot.entry, &self.smits);
self.emit(ProcessEventKind::ReloadAbandoned, info, true);
}
}
}
}
fn deregister_on_exit(&mut self, id: u32) -> bool {
let mut removed = self
.sheep
.remove(&id)
.expect("deregister_on_exit: unknown id");
removed.entry.status = ProcStatus::Stopped;
removed.entry.reload = ReloadState::None;
let info = to_info(&removed.entry, &self.smits);
self.emit(ProcessEventKind::Delete, info.clone(), true);
self.disarm_extras(id, &info.name);
self.resolve_pending(id, info)
}
fn uncommitted_swap_of(&self, id: u32) -> Option<String> {
self.reloads
.iter()
.find(|(_, job)| {
job.swap.phase == ReloadPhase::AwaitReady
&& (job.swap.old_id == id || job.swap.new_id == Some(id))
})
.map(|(name, _)| name.clone())
}
fn in_an_uncommitted_swap(&self, id: u32) -> bool {
self.uncommitted_swap_of(id).is_some()
}
fn reload_of(&self, id: u32) -> Option<String> {
self.reloads
.iter()
.find(|(_, job)| job.swap.old_id == id || job.swap.new_id == Some(id))
.map(|(name, _)| name.clone())
}
fn clear_reload(&mut self, id: u32) {
if let Some(slot) = self.sheep.get_mut(&id) {
slot.entry.reload = ReloadState::None;
}
}
fn handle_reopen(
&self,
selector: &ProcessSelector,
reply: oneshot::Sender<Result<Vec<ProcessInfo>, SupervisorError>>,
) {
let mut matched: Vec<ProcessInfo> = Vec::new();
let mut paths: BTreeSet<PathBuf> = BTreeSet::new();
for id in self.matching_ids(selector) {
let slot = self
.sheep
.get(&id)
.expect("`matching_ids` answers with ids read off this map a moment ago");
paths.insert(slot.entry.out_file.clone());
paths.insert(slot.entry.err_file.clone());
matched.push(to_info(&slot.entry, &self.smits));
}
if matched.is_empty() {
let _ = reply.send(Err(SupervisorError::NotFound));
return;
}
let mut pumps: Vec<(ProcessInfo, mpsc::Sender<LogCtl>)> = self
.sheep
.values()
.filter(|slot| {
paths.contains(&slot.entry.out_file) || paths.contains(&slot.entry.err_file)
})
.filter_map(|slot| {
slot.log_ctl
.clone()
.map(|log_ctl| (to_info(&slot.entry, &self.smits), log_ctl))
})
.collect();
pumps.sort_unstable_by_key(|(info, _)| info.id);
sort_flock(&mut matched);
spawn_reopen_task(matched, pumps, reply);
}
#[cfg(unix)]
fn handle_handover_snapshot(
&self,
fds: DaemonFds,
reply: oneshot::Sender<Result<Snapshot, SupervisorError>>,
) {
let mut drafts: Vec<HandoverDraft> = self
.sheep
.values()
.map(|slot| HandoverDraft {
entry: slot.entry.clone(),
manual: slot.manual,
pending_delete: slot.pending_delete,
epoch: slot.epoch,
ready_failed: slot.ready_failed,
restart_due: slot.restart_due,
log_ctl: slot.log_ctl.clone(),
channel_open: slot.open_channel().is_some(),
})
.collect();
drafts.sort_unstable_by_key(|draft| draft.entry.id);
let mut reloads: Vec<CarriedReload> = self
.reloads
.iter()
.map(|(app, job)| CarriedReload {
app: app.clone(),
queue: job.queue.iter().copied().collect(),
mode: job.mode,
swap: job.swap,
})
.collect();
reloads.sort_unstable_by(|left, right| left.app.cmp(&right.app));
spawn_handover_task(
drafts,
fds,
Counters {
next_id: self.next_id,
next_deadline: self.next_deadline,
next_action_stamp: self.next_action_stamp,
},
reloads,
reply,
);
}
#[cfg(unix)]
fn handle_handover_fitness(&self, reply: oneshot::Sender<Result<Fitness, SupervisorError>>) {
let mut slots: Vec<&SheepSlot> = self.sheep.values().collect();
slots.sort_unstable_by_key(|slot| slot.entry.id);
let candidates: Vec<Candidate<'_>> = slots
.iter()
.map(|slot| Candidate {
entry: &slot.entry,
pump_unresponsive: false,
})
.collect();
let _ = reply.send(Ok(fitness(&candidates)));
}
fn handle_flush(
&self,
selector: &ProcessSelector,
reply: oneshot::Sender<Result<Vec<ProcessInfo>, SupervisorError>>,
) {
let mut matched: Vec<ProcessInfo> = Vec::new();
let mut paths: BTreeSet<PathBuf> = BTreeSet::new();
for id in self.matching_ids(selector) {
let slot = self
.sheep
.get(&id)
.expect("`matching_ids` answers with ids read off this map a moment ago");
paths.insert(slot.entry.out_file.clone());
paths.insert(slot.entry.err_file.clone());
matched.push(to_info(&slot.entry, &self.smits));
}
if matched.is_empty() {
let _ = reply.send(Err(SupervisorError::NotFound));
return;
}
let mut pumps: Vec<(u32, mpsc::Sender<LogCtl>)> = self
.sheep
.iter()
.filter(|(_, slot)| {
paths.contains(&slot.entry.out_file) || paths.contains(&slot.entry.err_file)
})
.filter_map(|(id, slot)| slot.log_ctl.clone().map(|log_ctl| (*id, log_ctl)))
.collect();
pumps.sort_unstable_by_key(|&(id, _)| id);
sort_flock(&mut matched);
let pumps = pumps.into_iter().map(|(_, log_ctl)| log_ctl).collect();
spawn_flush_task(matched, pumps, paths, reply);
}
fn begin_shutdown(&mut self, reply: oneshot::Sender<()>) -> bool {
self.shutting_down = true;
self.reloads.clear();
let online: HashSet<u32> = self
.sheep
.iter()
.filter(|(_, slot)| slot.ctl.is_some())
.map(|(&id, _)| id)
.collect();
if online.is_empty() {
let _ = reply.send(());
return true;
}
for &id in &online {
self.claim_manual(
id,
PendingManual {
kind: ManualKind::Stop,
origin: CommandOrigin::Operator,
},
LadderCap::Stop,
);
}
self.pending.push(PendingReply {
remaining: online,
results: Vec::new(),
reply: ReplyKind::Shutdown(reply),
});
false
}
fn handle_exited(&mut self, id: u32, outcome: ExitOutcome) -> bool {
let Some(slot) = self.sheep.get_mut(&id) else {
tracing::warn!(id, "Msg::Exited for an unregistered id");
return false;
};
slot.ctl = None;
slot.to_child = None;
slot.signals = None;
slot.to_stdin = None;
slot.actions.abandon_all();
slot.entry.pid = None;
slot.ready_failed = false;
slot.entry.last_exit = Some(outcome.into());
let manual = slot.manual.take();
let kind = manual.map(|pending| pending.kind);
let pending_delete = std::mem::take(&mut slot.pending_delete);
let reload = slot.entry.reload;
let started_at = slot.entry.started_at.take();
match reload {
ReloadState::Drainee { .. } => {
match self.uncommitted_swap_of(id) {
Some(name) if kind.is_some() => {
self.abort_reload(&name, "an operator's command reached the drainee first");
}
_ => return self.reap_drainee(id),
}
}
ReloadState::Replacement => {
self.clear_reload(id);
if let Some(name) = self.uncommitted_swap_of(id) {
self.abort_reload(&name, "the replacement exited before it was ready");
return self.deregister_on_exit(id);
}
if let Some(name) = self.reload_of(id) {
let old_id = self.reloads[&name].swap.old_id;
if !self.sheep.contains_key(&old_id) {
debug_assert_ne!(
self.sheep.get(&id).map(|slot| slot.entry.status),
Some(ProcStatus::Online),
"a swap committed by the drainee's death cannot have had a live \
replacement"
);
tracing::warn!(
name,
new_id = id,
"reload abandoned: the replacement exited before it was ready, with \
the instance it replaced already gone"
);
self.reloads.remove(&name);
if let Some(slot) = self.sheep.get(&id) {
let info = to_info(&slot.entry, &self.smits);
self.emit(ProcessEventKind::ReloadAbandoned, info, true);
}
}
}
}
ReloadState::None => {}
}
let Some(started_at) = started_at else {
tracing::warn!(
id,
"Msg::Exited for an entry with no started_at (duplicate?)"
);
if kind == Some(ManualKind::Delete) || pending_delete {
let mut removed = self.sheep.remove(&id).expect("checked above");
removed.entry.status = ProcStatus::Stopped;
let info = to_info(&removed.entry, &self.smits);
self.emit(ProcessEventKind::Delete, info.clone(), true);
self.disarm_extras(id, &info.name);
return self.resolve_pending(id, info);
}
let info = to_info(
&self.sheep.get(&id).expect("checked above").entry,
&self.smits,
);
return self.resolve_pending(id, info);
};
let uptime = tokio::time::Instant::now().saturating_duration_since(started_at);
if kind == Some(ManualKind::Restart) && !self.shutting_down && !pending_delete {
self.sheep
.get_mut(&id)
.expect("checked above")
.entry
.budget
.reset();
let manually = matches!(
manual,
Some(PendingManual {
origin: CommandOrigin::Operator,
..
})
);
let info = self.respawn(id, manually);
return self.resolve_pending(id, info);
}
let decision = {
let slot = self.sheep.get_mut(&id).expect("checked above");
decide_on_exit(
slot.entry.spec.config(),
&mut slot.entry.budget,
uptime,
outcome,
kind.is_some(),
)
};
let info = match decision {
Decision::Restart { delay } => {
let info = self.set_status(id, ProcStatus::WaitingRestart);
self.emit(ProcessEventKind::Exit, info.clone(), false);
let epoch = self.sheep.get(&id).expect("checked above").epoch;
self.sheep.get_mut(&id).expect("checked above").restart_due = SystemTime::now()
.checked_add(delay.unwrap_or(Duration::ZERO))
.filter(|due| due.duration_since(SystemTime::UNIX_EPOCH).is_ok());
self.schedule_restart(id, epoch, delay);
info
}
Decision::Errored => {
let info = self.set_status(id, ProcStatus::Errored);
self.emit(ProcessEventKind::Errored, info.clone(), kind.is_some());
self.disarm_extras(id, &info.name);
info
}
Decision::CleanStop if kind == Some(ManualKind::Delete) || pending_delete => {
let mut removed = self.sheep.remove(&id).expect("checked above");
removed.entry.status = ProcStatus::Stopped;
let info = to_info(&removed.entry, &self.smits);
self.emit(ProcessEventKind::Delete, info.clone(), true);
self.disarm_extras(id, &info.name);
info
}
Decision::CleanStop => {
let info = self.set_status(id, ProcStatus::Stopped);
self.disarm_extras(id, &info.name);
self.emit(
ProcessEventKind::Stop,
info.clone(),
kind == Some(ManualKind::Stop),
);
info
}
};
self.resolve_pending(id, info)
}
fn handle_extra_restart(
&mut self,
id: u32,
pid: u32,
epoch: Option<u64>,
observed: Option<MemSize>,
) {
if self.shutting_down {
tracing::debug!(id, pid, "extra restart dropped: engine is shutting down");
return;
}
let Some(slot) = self.sheep.get(&id) else {
tracing::debug!(id, pid, "extra restart dropped: no such sheep");
return;
};
if slot.entry.pid != Some(pid) {
tracing::debug!(
id,
pid,
current = slot.entry.pid,
"extra restart dropped: the reported pid is no longer this sheep's"
);
return;
}
if slot.entry.status != ProcStatus::Online {
tracing::debug!(
id,
pid,
status = %slot.entry.status,
"extra restart dropped: the sheep is no longer online"
);
return;
}
if let Some(epoch) = epoch {
let current = self.registry.liveness_epoch(id);
if current != epoch {
tracing::debug!(
id,
pid,
epoch,
current,
"extra restart dropped: the reporting probe has been replaced"
);
return;
}
}
if let Some(observed) = observed {
let ceiling = self
.sheep
.get(&id)
.and_then(|slot| slot.entry.spec.config().max_memory);
if ceiling.is_none_or(|limit| observed <= limit) {
tracing::debug!(
id,
pid,
observed = observed.bytes(),
"extra restart dropped: the ceiling it breached is no longer in force"
);
return;
}
}
let (reply, _dropped) = oneshot::channel();
self.begin_manual(
ProcessSelector::Id(id),
ManualKind::Restart,
CommandOrigin::Automatic,
ReplyKind::Info(reply),
);
}
fn handle_restart_due(&mut self, id: u32, epoch: u64) {
if self.shutting_down {
return;
}
let Some(slot) = self.sheep.get(&id) else {
return;
};
if slot.entry.status != ProcStatus::WaitingRestart {
return;
}
if slot.epoch != epoch {
return;
}
self.respawn(id, false);
}
fn handle_ready_signal(&mut self, id: u32) {
let Some(slot) = self.sheep.get_mut(&id) else {
return;
};
if let Some(tx) = slot.ready_tx.take() {
let _ = tx.send(());
}
}
fn handle_ready_result(&mut self, id: u32, epoch: u64, manually: bool, readiness: Readiness) {
if self.shutting_down {
return;
}
let Some(slot) = self.sheep.get(&id) else {
return;
};
if slot.epoch != epoch {
return;
}
if slot.entry.status != ProcStatus::Starting {
return;
}
if matches!(slot.entry.reload, ReloadState::Replacement) {
self.reload_ready_result(id, manually, readiness);
return;
}
if readiness == Readiness::TimedOut {
tracing::warn!(id, "readiness deadline elapsed; marking online anyway");
}
let info = self.set_status(id, ProcStatus::Online);
self.went_online(id, info, manually);
}
fn begin_action(
&mut self,
selector: &ProcessSelector,
action: String,
params: Option<String>,
reply: oneshot::Sender<Result<Vec<ActionReply>, SupervisorError>>,
) {
let matched = self.matching_ids(selector);
if matched.is_empty() {
let _ = reply.send(Err(SupervisorError::NotFound));
return;
}
let mut refused = Vec::new();
let mut waits = Vec::new();
for id in matched {
let slot = self
.sheep
.get(&id)
.expect("begin_action: `matched` holds ids read off this map a moment ago");
let config = slot.entry.spec.config();
let name = config.name.clone();
let action_timeout = config.action_timeout.as_duration();
if matches!(slot.entry.reload, ReloadState::Drainee { .. }) {
refused.push(ActionReply {
id,
name,
outcome: ActionOutcome::Skipped,
});
continue;
}
let Some(to_child) = slot.open_channel().cloned() else {
refused.push(ActionReply {
id,
name,
outcome: ActionOutcome::NoChannel,
});
continue;
};
let answer =
self.arm_action(id, to_child, action.clone(), params.clone(), action_timeout);
waits.push((id, name, answer));
}
if waits.is_empty() {
let skipped = refused
.iter()
.filter(|row| row.outcome == ActionOutcome::Skipped)
.count();
tracing::warn!(
action,
matched = refused.len(),
skipped,
"no matched sheep could take this action; nothing was delivered"
);
let _ = reply.send(Ok(refused));
return;
}
spawn_trigger_task(refused, waits, reply);
}
fn begin_signal(
&mut self,
selector: &ProcessSelector,
sig: OperatorSignal,
reply: oneshot::Sender<Result<Vec<SignalReply>, SupervisorError>>,
) {
let matched = self.matching_ids(selector);
if matched.is_empty() {
let _ = reply.send(Err(SupervisorError::NotFound));
return;
}
let mut settled = Vec::new();
let mut waits = Vec::new();
for id in matched {
let slot = self
.sheep
.get(&id)
.expect("begin_signal: `matched` holds ids read off this map a moment ago");
let name = slot.entry.spec.config().name.clone();
let Some(signals) = slot.signals.clone() else {
settled.push(SignalReply {
id,
name,
outcome: SignalOutcome::NotRunning,
});
continue;
};
let (done, answer) = oneshot::channel();
if signals.try_send(SignalRequest { sig, done }).is_err() {
settled.push(SignalReply {
id,
name,
outcome: SignalOutcome::NotRunning,
});
continue;
}
waits.push((id, name, answer));
}
spawn_signal_task(settled, waits, reply);
}
fn begin_send_line(
&mut self,
selector: &ProcessSelector,
line: String,
reply: oneshot::Sender<Result<Vec<LineReply>, SupervisorError>>,
) {
let matched = self.matching_ids(selector);
if matched.is_empty() {
let _ = reply.send(Err(SupervisorError::NotFound));
return;
}
let mut settled = Vec::new();
let mut waits = Vec::new();
for id in matched {
let slot = self
.sheep
.get(&id)
.expect("begin_send_line: `matched` holds ids read off this map a moment ago");
let name = slot.entry.spec.config().name.clone();
let Some(to_stdin) = slot.open_stdin().cloned() else {
settled.push(LineReply {
id,
name,
outcome: LineOutcome::NoStdin,
});
continue;
};
let (done, answer) = oneshot::channel();
match to_stdin.try_send(StdinWrite {
line: line.clone(),
done,
}) {
Ok(()) => waits.push((id, name, answer)),
Err(mpsc::error::TrySendError::Full(_)) => settled.push(LineReply {
id,
name,
outcome: LineOutcome::NotWritten {
reason: "the app is not reading its stdin (its queue was \
already full when this line arrived)"
.to_string(),
},
}),
Err(mpsc::error::TrySendError::Closed(_)) => settled.push(LineReply {
id,
name,
outcome: LineOutcome::NoStdin,
}),
}
}
spawn_send_line_task(settled, waits, reply);
}
fn arm_action(
&mut self,
id: u32,
to_child: mpsc::Sender<ShepherdMessage>,
action: String,
params: Option<String>,
timeout: Duration,
) -> oneshot::Receiver<ActionOutcome> {
let (reply, answer) = oneshot::channel();
let stamp = self.next_action_stamp;
self.next_action_stamp += 1;
let waiter = spawn_action_task(
id,
stamp,
ShepherdMessage::Action {
name: action.clone(),
params,
id: stamp,
},
to_child,
timeout,
self.tx.clone(),
);
self.sheep
.get_mut(&id)
.expect("arm_action: the slot was read a moment ago")
.actions
.arm(PendingAction {
stamp,
action,
waiter: Some(waiter),
reply,
});
answer
}
fn handle_action_reply(&mut self, id: u32, action: &str, body: String, stamp: Option<u64>) {
let Some(slot) = self.sheep.get_mut(&id) else {
return;
};
if let Some(waiter) = slot.actions.answer(action, stamp) {
let _ = waiter.send(body);
}
}
fn handle_action_result(&mut self, id: u32, stamp: u64, outcome: ActionOutcome) {
let Some(slot) = self.sheep.get_mut(&id) else {
return;
};
if let Some(reply) = slot.actions.resolve(stamp) {
let _ = reply.send(outcome);
}
}
fn schedule_restart(&self, id: u32, epoch: u64, delay: Option<Duration>) {
let tx = self.tx.clone();
tokio::spawn(async move {
if let Some(delay) = delay {
tokio::time::sleep(delay).await;
}
let _ = tx.send(Msg::RestartDue { id, epoch }).await;
});
}
fn resolve_pending(&mut self, id: u32, info: ProcessInfo) -> bool {
let mut shutdown_completed = false;
let mut i = 0;
while i < self.pending.len() {
if self.pending[i].remaining.remove(&id) {
self.pending[i].results.push(info.clone());
}
if self.pending[i].remaining.is_empty() {
let mut pending = self.pending.remove(i);
sort_flock(&mut pending.results);
if matches!(pending.reply, ReplyKind::Shutdown(_)) {
shutdown_completed = true;
}
send_reply(pending.reply, Ok(pending.results));
} else {
i += 1;
}
}
shutdown_completed
}
fn went_online(&mut self, id: u32, info: ProcessInfo, manually: bool) {
if let Some(slot) = self.sheep.get_mut(&id) {
slot.ready_failed = false;
}
self.emit(ProcessEventKind::Online, info, manually);
self.arm_extras(id);
}
fn arm_extras(&mut self, id: u32) {
let Some(extras) = self.extras.as_ref() else {
return;
};
let Some(slot) = self.sheep.get(&id) else {
return;
};
let supervisor = SupervisorHandle {
tx: self.tx.clone(),
};
let credentials = match slot.entry.credentials {
SpawnIdentity::Resolved(creds) => creds,
SpawnIdentity::Unresolved => None,
};
let described = describe(
&slot.entry.spec,
slot.entry.instance,
&self.paths,
credentials,
&self.secret_view(&slot.entry.spec),
);
self.registry
.arm(&slot.entry, spec_prober(&described), extras, &supervisor);
}
fn disarm_extras(&mut self, id: u32, name: &str) {
self.registry.disarm(id, name);
}
fn set_status(&mut self, id: u32, status: ProcStatus) -> ProcessInfo {
let slot = self.sheep.get_mut(&id).expect("set_status: unknown id");
slot.entry.status = status;
to_info(&slot.entry, &self.smits)
}
fn handle_set_smit(
&mut self,
conn: ConnId,
sheep: &str,
smit: Option<Smit>,
) -> Result<Vec<ProcessInfo>, SupervisorError> {
if !self
.sheep
.values()
.any(|slot| slot.entry.spec.config().name == sheep)
{
return Err(SupervisorError::NotFound);
}
match smit {
Some(smit) => {
self.smits
.insert(sheep.to_string(), (conn, smit.to_string()));
}
None => {
if self
.smits
.get(sheep)
.is_some_and(|(painter, _)| *painter == conn)
{
self.smits.remove(sheep);
}
}
}
Ok(self
.snapshot_all()
.into_iter()
.filter(|info| info.name == sheep)
.collect())
}
fn snapshot_all(&self) -> Vec<ProcessInfo> {
let mut listing: Vec<ProcessInfo> = self
.sheep
.values()
.map(|slot| to_info(&slot.entry, &self.smits))
.collect();
sort_flock(&mut listing);
listing
}
fn emit(&self, event: ProcessEventKind, info: ProcessInfo, manually: bool) {
let _ = self.events.send(SharedEvent::new(BusEvent::Process {
event,
info,
manually,
at_ms: crate::now_ms(),
}));
}
}
fn send_reply(reply: ReplyKind, outcome: Result<Vec<ProcessInfo>, SupervisorError>) {
match reply {
ReplyKind::Info(tx) => {
let _ = tx.send(outcome);
}
ReplyKind::Ids(tx) => {
let ids = outcome.map(|infos| infos.into_iter().map(|info| info.id).collect());
let _ = tx.send(ids);
}
ReplyKind::Shutdown(tx) => {
let _ = tx.send(());
}
}
}
fn to_info(entry: &ProcessEntry, smits: &Smits) -> ProcessInfo {
let uptime_ms = entry.started_at.map_or(0, |started_at| {
tokio::time::Instant::now()
.saturating_duration_since(started_at)
.as_millis() as u64
});
ProcessInfo::builder(entry.id, entry.spec.config().name.clone(), entry.status)
.pid(entry.pid)
.restarts(entry.restarts)
.uptime_ms(uptime_ms)
.fold(entry.spec.config().fold.clone())
.depends_on(entry.spec.config().depends_on.clone())
.out_file(Some(entry.out_file.to_string_lossy().into_owned()))
.err_file(Some(entry.err_file.to_string_lossy().into_owned()))
.cpu_percent(None)
.memory_bytes(None)
.dog(entry.dog.clone())
.last_exit(entry.last_exit)
.smit(
smits
.get(&entry.spec.config().name)
.map(|(_, smit)| smit.clone()),
)
.instance(Some(entry.instance))
.pending({
let fields = pending_fields(entry);
(!fields.is_empty()).then_some(fields)
})
.overridden((!entry.overridden.is_empty()).then(|| entry.overridden.clone()))
.max_memory(entry.spec.config().max_memory.map(MemSize::bytes))
.build()
}
impl From<ExitOutcome> for ExitInfo {
fn from(outcome: ExitOutcome) -> Self {
Self {
code: outcome.code,
signal: outcome.signal,
}
}
}
fn reload_eligible(slot: &SheepSlot) -> bool {
slot.entry.status == ProcStatus::Online || slot.ready_failed
}
fn restored_status(slot: &SheepSlot) -> ProcStatus {
if slot.ready_failed {
ProcStatus::Starting
} else {
ProcStatus::Online
}
}
fn spec_prober(spec: &SpawnSpec) -> Arc<dyn Prober> {
Arc::new(OsProber::new(spec.cwd.clone(), spec.env.clone()))
}
fn spawn_readiness_task(
id: u32,
epoch: u64,
manually: bool,
source: ReadinessSource,
deadline: Duration,
prober: Arc<dyn Prober>,
actor_tx: mpsc::Sender<Msg>,
) -> oneshot::Sender<()> {
let (ready_tx, ready_rx) = oneshot::channel();
tokio::spawn(async move {
let readiness = await_ready(&source, deadline, ready_rx, prober).await;
let _ = actor_tx
.send(Msg::ReadyResult {
id,
epoch,
manually,
readiness,
})
.await;
});
ready_tx
}
fn spawn_action_task(
id: u32,
stamp: u64,
message: ShepherdMessage,
to_child: mpsc::Sender<ShepherdMessage>,
deadline: Duration,
actor_tx: mpsc::Sender<Msg>,
) -> oneshot::Sender<String> {
let (reply_tx, reply_rx) = oneshot::channel();
tokio::spawn(async move {
let delivered = tokio::time::timeout(deadline, async move {
if to_child.send(message).await.is_err() {
return None;
}
reply_rx.await.ok()
})
.await;
let outcome = match delivered {
Ok(Some(body)) => ActionOutcome::Replied { body },
Ok(None) => ActionOutcome::NoChannel,
Err(_elapsed) => ActionOutcome::TimedOut,
};
let _ = actor_tx
.send(Msg::ActionResult { id, stamp, outcome })
.await;
});
reply_tx
}
fn spawn_trigger_task(
mut rows: Vec<ActionReply>,
waits: Vec<(u32, String, oneshot::Receiver<ActionOutcome>)>,
reply: oneshot::Sender<Result<Vec<ActionReply>, SupervisorError>>,
) {
tokio::spawn(async move {
for (id, name, answer) in waits {
let outcome = answer.await.unwrap_or(ActionOutcome::NoChannel);
rows.push(ActionReply { id, name, outcome });
}
rows.sort_unstable_by(|a, b| (a.name.as_str(), a.id).cmp(&(b.name.as_str(), b.id)));
let _ = reply.send(Ok(rows));
});
}
type SignalWait = (u32, String, oneshot::Receiver<Result<(), RunnerError>>);
fn spawn_signal_task(
mut rows: Vec<SignalReply>,
waits: Vec<SignalWait>,
reply: oneshot::Sender<Result<Vec<SignalReply>, SupervisorError>>,
) {
tokio::spawn(async move {
for (id, name, answer) in waits {
let outcome = match answer.await {
Ok(Ok(())) => SignalOutcome::Delivered,
Ok(Err(err)) => SignalOutcome::Failed {
reason: err.to_string(),
},
Err(_dropped) => SignalOutcome::NotRunning,
};
rows.push(SignalReply { id, name, outcome });
}
rows.sort_unstable_by(|a, b| (a.name.as_str(), a.id).cmp(&(b.name.as_str(), b.id)));
let _ = reply.send(Ok(rows));
});
}
type LineWait = (u32, String, oneshot::Receiver<Result<(), RunnerError>>);
fn spawn_send_line_task(
settled: Vec<LineReply>,
waits: Vec<LineWait>,
reply: oneshot::Sender<Result<Vec<LineReply>, SupervisorError>>,
) {
tokio::spawn(async move {
let mut rows = settled;
rows.extend(
futures_util::future::join_all(waits.into_iter().map(
|(id, name, answer)| async move {
let outcome = match tokio::time::timeout(STDIN_WRITE_TIMEOUT, answer).await {
Ok(Ok(Ok(()))) => LineOutcome::Sent,
Ok(Ok(Err(err))) => LineOutcome::NotWritten {
reason: err.to_string(),
},
Ok(Err(_recv)) => LineOutcome::NoStdin,
Err(_elapsed) => LineOutcome::NotWritten {
reason: format!(
"the app did not read its stdin within {}s; this line \
may still land if it drains",
STDIN_WRITE_TIMEOUT.as_secs()
),
},
};
LineReply { id, name, outcome }
},
))
.await,
);
rows.sort_unstable_by(|a, b| (a.name.as_str(), a.id).cmp(&(b.name.as_str(), b.id)));
let _ = reply.send(Ok(rows));
});
}
#[cfg(unix)]
type Snapshot = (Vec<OwnedCandidate>, Handover, ParkedPumps);
#[cfg(unix)]
#[derive(Debug, Default)]
pub(crate) struct ParkedPumps(Vec<mpsc::Sender<LogCtl>>);
#[cfg(unix)]
impl ParkedPumps {
pub(crate) async fn resume(&self) {
for pump in &self.0 {
let _ = pump.send(LogCtl::Resume).await;
}
}
}
#[cfg(unix)]
#[derive(Debug)]
struct HandoverDraft {
entry: ProcessEntry,
manual: Option<PendingManual>,
pending_delete: bool,
epoch: u64,
ready_failed: bool,
restart_due: Option<SystemTime>,
log_ctl: Option<mpsc::Sender<LogCtl>>,
channel_open: bool,
}
#[cfg(unix)]
fn spawn_handover_task(
drafts: Vec<HandoverDraft>,
fds: DaemonFds,
counters: Counters,
reloads: Vec<CarriedReload>,
reply: oneshot::Sender<Result<Snapshot, SupervisorError>>,
) {
tokio::spawn(async move {
let visited = futures_util::future::join_all(drafts.into_iter().map(|draft| async move {
let (mut fds, pump_unresponsive, parked_pump) = match &draft.log_ctl {
Some(log_ctl) => match report_fds(log_ctl).await {
PumpReport::Parked(fds) => (fds, false, Some(log_ctl.clone())),
PumpReport::Gone => (CarriedFds::none(), false, None),
PumpReport::Unresponsive => (CarriedFds::none(), true, None),
},
None => (CarriedFds::none(), false, None),
};
if !draft.channel_open {
fds.channel = None;
}
let carried = CarriedSheep::from_entry(
&draft.entry,
draft.epoch,
fds,
draft.pending_delete,
draft.manual,
draft.ready_failed,
draft.restart_due,
);
let candidate = OwnedCandidate {
entry: draft.entry,
pump_unresponsive,
};
(candidate, carried, parked_pump)
}))
.await;
let mut candidates = Vec::with_capacity(visited.len());
let mut carried = Vec::with_capacity(visited.len());
let mut parked = ParkedPumps::default();
for (candidate, sheep, parked_pump) in visited {
candidates.push(candidate);
carried.push(sheep);
if let Some(log_ctl) = parked_pump {
parked.0.push(log_ctl);
}
}
let blob = Handover::new(carried, fds, counters, reloads);
let _ = reply.send(Ok((candidates, blob, parked)));
});
}
#[cfg(unix)]
#[derive(Debug)]
enum PumpReport {
Parked(CarriedFds),
Gone,
Unresponsive,
}
#[cfg(unix)]
const REPORT_DEADLINE: Duration = Duration::from_secs(2);
#[cfg(unix)]
async fn report_fds(log_ctl: &mpsc::Sender<LogCtl>) -> PumpReport {
let (done, ack) = oneshot::channel();
let answer = async {
if log_ctl.send(LogCtl::ReportFds { done }).await.is_err() {
return PumpReport::Gone;
}
ack.await.map_or(PumpReport::Gone, PumpReport::Parked)
};
tokio::time::timeout(REPORT_DEADLINE, answer)
.await
.unwrap_or(PumpReport::Unresponsive)
}
fn spawn_reopen_task(
matched: Vec<ProcessInfo>,
pumps: Vec<(ProcessInfo, mpsc::Sender<LogCtl>)>,
reply: oneshot::Sender<Result<Vec<ProcessInfo>, SupervisorError>>,
) {
tokio::spawn(async move {
let mut failures = Vec::new();
for (info, log_ctl) in &pumps {
if let Err(error) = reopen_logs(log_ctl).await {
failures.push(format!("{} (id {}): {error}", info.name, info.id));
}
}
let _ = reply.send(if failures.is_empty() {
Ok(matched)
} else {
Err(SupervisorError::ReopenFailed(failures.join("; ")))
});
});
}
async fn reopen_logs(log_ctl: &mpsc::Sender<LogCtl>) -> Result<(), ReopenError> {
let (done, ack) = oneshot::channel();
if log_ctl.send(LogCtl::Reopen { done }).await.is_err() {
return Ok(());
}
ack.await.unwrap_or(Ok(()))
}
fn spawn_flush_task(
matched: Vec<ProcessInfo>,
pumps: Vec<mpsc::Sender<LogCtl>>,
paths: BTreeSet<PathBuf>,
reply: oneshot::Sender<Result<Vec<ProcessInfo>, SupervisorError>>,
) {
tokio::spawn(async move {
let mut failures = Vec::new();
for log_ctl in &pumps {
if let Err(error) = flush_logs(log_ctl).await {
failures.push(error.message);
}
}
for path in paths {
if let Err(error) = truncate_log(&path).await {
failures.push(error.message);
}
}
let _ = reply.send(if failures.is_empty() {
Ok(matched)
} else {
Err(SupervisorError::FlushFailed(failures.join("; ")))
});
});
}
async fn flush_logs(log_ctl: &mpsc::Sender<LogCtl>) -> Result<(), FlushError> {
let (done, ack) = oneshot::channel();
if log_ctl.send(LogCtl::Flush { done }).await.is_err() {
return Ok(());
}
ack.await.unwrap_or(Ok(()))
}
async fn truncate_log(path: &Path) -> Result<(), FlushError> {
let refused = |error: &dyn fmt::Display| FlushError {
message: format!("{}: {error}", path.display()),
};
if let Err(error) = check_log_ancestry(path) {
return Err(refused(&error));
}
let mut options = tokio::fs::OpenOptions::new();
options.write(true).truncate(true);
match open_log_path(&mut options, path).await {
Ok(_) => Ok(()),
Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(()),
Err(error) => Err(refused(&error)),
}
}
#[derive(Debug)]
struct SignalRequest {
sig: OperatorSignal,
done: oneshot::Sender<Result<(), RunnerError>>,
}
struct SheepHandles {
ctl: mpsc::Sender<SheepCtl>,
signals: mpsc::Sender<SignalRequest>,
}
fn spawn_sheep_task<P: RunningProcess>(
id: u32,
proc: P,
io: ProcIo,
app: ResolvedApp,
events: Bus,
actor_tx: mpsc::Sender<Msg>,
) -> SheepHandles {
let (ctl_tx, ctl_rx) = mpsc::channel(SHEEP_CTL_CAPACITY);
let (signal_tx, signal_rx) = mpsc::channel(SIGNAL_CAPACITY);
tokio::spawn(run_sheep(
id, proc, io, app, ctl_rx, signal_rx, events, actor_tx,
));
SheepHandles {
ctl: ctl_tx,
signals: signal_tx,
}
}
#[allow(clippy::too_many_arguments)]
async fn run_sheep<P: RunningProcess>(
id: u32,
mut proc: P,
io: ProcIo,
app: ResolvedApp,
mut ctl_rx: mpsc::Receiver<SheepCtl>,
mut signal_rx: mpsc::Receiver<SignalRequest>,
events: Bus,
actor_tx: mpsc::Sender<Msg>,
) {
let ProcIo {
mut logs,
mut from_child,
to_child,
log_ctl: _log_ctl,
to_stdin: _to_stdin,
} = io;
let mut ctl_open = true;
let mut logs_open = true;
let mut from_child_open = true;
let mut signals_open = true;
loop {
tokio::select! {
outcome = proc.wait() => {
let _ = actor_tx.send(Msg::Exited { id, outcome }).await;
break;
}
maybe_ctl = ctl_rx.recv(), if ctl_open => {
match maybe_ctl {
Some(SheepCtl::Kill { grace }) => {
let outcome =
kill_process(&mut proc, app.config(), Some(&to_child), grace).await;
let _ = actor_tx.send(Msg::Exited { id, outcome }).await;
break;
}
None => ctl_open = false,
}
}
maybe_signal = signal_rx.recv(), if signals_open => {
match maybe_signal {
Some(SignalRequest { sig, done }) => {
let _ = done.send(proc.signal_process(sig));
}
None => signals_open = false,
}
}
maybe_line = logs.recv(), if logs_open => {
match maybe_line {
Some(line) => {
events.publish_log(if line.err {
BusEvent::LogErr { id, line: line.line }
} else {
BusEvent::LogOut { id, line: line.line }
});
}
None => logs_open = false,
}
}
maybe_msg = from_child.recv(), if from_child_open => {
match maybe_msg {
Some(message) => {
let _ = events.send(SharedEvent::new(BusEvent::Channel {
id,
message: message.clone(),
}));
match message {
ChildMessage::Ready => {
let _ = actor_tx.send(Msg::Ready { id }).await;
}
ChildMessage::Metric { name, value } => {
tracing::debug!(
id,
name,
value,
"child metric forwarded to the bus as channel.metric"
);
}
ChildMessage::ActionReply {
action,
body,
id: stamp,
} => {
let _ = actor_tx
.send(Msg::ActionReply { id, action, body, stamp })
.await;
}
}
}
None => from_child_open = false,
}
}
}
}
}
#[cfg(test)]
mod tests {
use shep_core::config::{AppConfig, ProbeConfig, ProbeKind, normalize};
use shep_core::protocol::DogSource;
use shep_core::status::ProcStatus;
use shep_core::values::{MemSize, UpDuration};
use super::*;
use crate::cron::{DEFAULT_MAX_CRON_SLEEP, SystemClock};
use crate::extras::{ExtrasReports, spawn_extras_reporter};
use crate::fake::{ProcScript, ScriptedRunner};
use crate::limits::LimitEnforcer;
use crate::testing::capture_logs;
use crate::testing::{
Harness, RecordingEnforcer, ScriptedProber, SharedRunner, app_with, armed_entry, harness,
idle_stats, probe_config, test_paths,
};
#[cfg(unix)]
use crate::tokio_runner::TokioRunner;
use tokio::sync::watch;
use std::sync::atomic::{AtomicUsize, Ordering as AtomicOrdering};
fn drained_process_kinds(
rx: &mut tokio::sync::broadcast::Receiver<SharedEvent>,
) -> Vec<ProcessEventKind> {
let mut kinds = Vec::new();
while let Ok(BusEvent::Process { event, .. }) = rx.try_recv().map(|event| event.to_event())
{
kinds.push(event);
}
kinds
}
async fn await_event(
rx: &mut tokio::sync::broadcast::Receiver<SharedEvent>,
id: u32,
kind: ProcessEventKind,
) -> bool {
loop {
match rx.recv().await.map(|event| event.to_event()) {
Ok(BusEvent::Process {
event,
info,
manually,
..
}) if info.id == id && event == kind => {
return manually;
}
Ok(_) => continue,
Err(tokio::sync::broadcast::error::RecvError::Lagged(_)) => continue,
Err(e) => panic!("event stream closed before {kind:?} for id {id}: {e}"),
}
}
}
async fn assert_no_event_within(
rx: &mut tokio::sync::broadcast::Receiver<SharedEvent>,
id: u32,
kind: ProcessEventKind,
window: Duration,
) {
match tokio::time::timeout(window, await_event(rx, id, kind)).await {
Err(_elapsed) => {} Ok(_manually) => panic!("unexpected {kind:?} for id {id} within {window:?}"),
}
}
fn test_actor(
paths: ShepPaths,
scripts: Vec<ProcScript>,
sheep: HashMap<u32, SheepSlot>,
tx: mpsc::Sender<Msg>,
) -> Actor<ScriptedRunner> {
let (events, _events_rx) = crate::bus::test_bus(64);
let provider_secrets = Arc::new(ProviderSecrets::load(&paths.secrets_cache));
Actor {
runner: ScriptedRunner::new(scripts),
next_id: sheep.len() as u32,
paths,
events,
host_environment: DEFAULT_ENVIRONMENT.to_string(),
provider_secrets,
tx,
sheep,
next_deadline: 0,
next_action_stamp: 0,
pending: Vec::new(),
shutting_down: false,
extras: None,
registry: ExtrasRegistry::default(),
reloads: HashMap::new(),
smits: Smits::new(),
}
}
#[tokio::test(start_paused = true)]
async fn wait_ready_app_stays_starting_until_the_channel_signals() {
let (events, mut rx) = crate::bus::test_bus(64);
let runner = ScriptedRunner::new(vec![ProcScript::never_exits()]);
let dir = tempfile::tempdir().unwrap();
let handle = spawn_supervisor(runner, test_paths(&dir), events);
let mut app = AppConfig::minimal("web", "./srv");
app.wait_ready = true;
handle.start(vec![normalize(app).unwrap()]).await.unwrap();
assert_eq!(handle.list().await[0].status, ProcStatus::Starting);
assert_no_event_within(
&mut rx,
0,
ProcessEventKind::Online,
Duration::from_millis(500),
)
.await;
assert_eq!(handle.list().await[0].status, ProcStatus::Starting);
handle.tx.send(Msg::Ready { id: 0 }).await.unwrap();
tokio::time::timeout(
Duration::from_secs(1),
await_event(&mut rx, 0, ProcessEventKind::Online),
)
.await
.expect("Online once the channel signals");
assert_eq!(handle.list().await[0].status, ProcStatus::Online);
}
#[tokio::test]
async fn readiness_probe_app_stays_starting_until_the_probe_passes() {
let reserved = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
let addr = reserved.local_addr().unwrap();
drop(reserved);
let (events, mut rx) = crate::bus::test_bus(64);
let runner = ScriptedRunner::new(vec![ProcScript::never_exits()]);
let dir = tempfile::tempdir().unwrap();
let handle = spawn_supervisor(runner, test_paths(&dir), events);
let mut app = AppConfig::minimal("web", "./srv");
app.readiness_probe = Some(ProbeConfig {
interval: UpDuration::from_millis(50),
timeout: UpDuration::from_millis(200),
..probe_config(ProbeKind::Tcp, &addr.to_string())
});
handle.start(vec![normalize(app).unwrap()]).await.unwrap();
assert_eq!(handle.list().await[0].status, ProcStatus::Starting);
assert_no_event_within(
&mut rx,
0,
ProcessEventKind::Online,
Duration::from_millis(220),
)
.await;
assert_eq!(handle.list().await[0].status, ProcStatus::Starting);
let listener = tokio::net::TcpListener::bind(addr).await.unwrap();
let _accept = tokio::spawn(async move { while listener.accept().await.is_ok() {} });
tokio::time::timeout(
Duration::from_secs(2),
await_event(&mut rx, 0, ProcessEventKind::Online),
)
.await
.expect("Online once the probe starts passing");
assert_eq!(handle.list().await[0].status, ProcStatus::Online);
}
#[cfg(unix)]
#[tokio::test]
async fn an_exec_readiness_probe_sees_the_assembled_env_not_the_apps_own() {
let (events, mut rx) = crate::bus::test_bus(64);
let runner = ScriptedRunner::new(vec![ProcScript::never_exits()]);
let dir = tempfile::tempdir().unwrap();
let ready_file = dir.path().join("ready-0");
let handle = spawn_supervisor(runner, test_paths(&dir), events);
let mut app = AppConfig::minimal("web", "./srv");
app.readiness_probe = Some(ProbeConfig {
interval: UpDuration::from_millis(50),
timeout: UpDuration::from_millis(500),
..probe_config(
ProbeKind::Exec,
&format!(r#"test -f "{}/ready-$SHEP_INSTANCE""#, dir.path().display()),
)
});
app.listen_timeout = UpDuration::from_millis(60_000);
handle.start(vec![normalize(app).unwrap()]).await.unwrap();
assert_eq!(handle.list().await[0].status, ProcStatus::Starting);
assert_no_event_within(
&mut rx,
0,
ProcessEventKind::Online,
Duration::from_millis(220),
)
.await;
assert_eq!(handle.list().await[0].status, ProcStatus::Starting);
std::fs::write(&ready_file, b"").unwrap();
tokio::time::timeout(
Duration::from_secs(5),
await_event(&mut rx, 0, ProcessEventKind::Online),
)
.await
.expect("Online once the exec probe can resolve $SHEP_INSTANCE");
assert_eq!(handle.list().await[0].status, ProcStatus::Online);
}
#[tokio::test(start_paused = true)]
async fn a_gated_apps_online_carries_the_same_manually_flag_an_ungated_one_does() {
let (events, mut rx) = crate::bus::test_bus(64);
let runner = ScriptedRunner::new(vec![
ProcScript::never_exits(), ProcScript::never_exits(), ProcScript::never_exits(), ]);
let dir = tempfile::tempdir().unwrap();
let handle = spawn_supervisor(runner, test_paths(&dir), events);
let mut gated = AppConfig::minimal("gated", "./srv");
gated.wait_ready = true;
let ungated = AppConfig::minimal("ungated", "./srv");
handle
.start(vec![normalize(gated).unwrap(), normalize(ungated).unwrap()])
.await
.unwrap();
let ungated_manually = tokio::time::timeout(
Duration::from_secs(1),
await_event(&mut rx, 1, ProcessEventKind::Online),
)
.await
.expect("an ungated app is Online at spawn");
assert!(ungated_manually, "sanity: a Start is a manual event");
handle.tx.send(Msg::Ready { id: 0 }).await.unwrap();
let gated_manually = tokio::time::timeout(
Duration::from_secs(1),
await_event(&mut rx, 0, ProcessEventKind::Online),
)
.await
.expect("Online once the channel signals");
assert_eq!(
gated_manually, ungated_manually,
"the same `shep start` must report the same flag, gated or not"
);
let restarted = handle.restart(ProcessSelector::Id(0)).await.unwrap();
assert_eq!(restarted[0].status, ProcStatus::Starting);
handle.tx.send(Msg::Ready { id: 0 }).await.unwrap();
let restarted_manually = tokio::time::timeout(
Duration::from_secs(1),
await_event(&mut rx, 0, ProcessEventKind::Online),
)
.await
.expect("Online once the respawned sheep signals");
assert!(
restarted_manually,
"a manual Restart's Online must stay manual through the readiness gate"
);
}
#[tokio::test(start_paused = true)]
async fn an_operators_restart_is_reported_as_a_user_action() {
let (events, mut rx) = crate::bus::test_bus(64);
let runner = ScriptedRunner::new(vec![ProcScript::never_exits(); 2]);
let dir = tempfile::tempdir().unwrap();
let handle = spawn_supervisor(runner, test_paths(&dir), events);
handle
.start(vec![normalize(AppConfig::minimal("web", "./srv")).unwrap()])
.await
.unwrap();
let restarted = handle.restart(ProcessSelector::All).await.unwrap();
assert_eq!(restarted[0].restarts, 1, "the restart really respawned");
let manually = tokio::time::timeout(
Duration::from_secs(1),
await_event(&mut rx, 0, ProcessEventKind::Restart),
)
.await
.expect("the respawn `restart` performed");
assert!(
manually,
"a person typed `shep restart`; the bus must say a user action caused it"
);
}
#[tokio::test(start_paused = true)]
async fn a_gated_app_whose_deadline_elapses_goes_online_anyway() {
let (events, mut rx) = crate::bus::test_bus(64);
let runner = ScriptedRunner::new(vec![ProcScript::never_exits()]);
let dir = tempfile::tempdir().unwrap();
let handle = spawn_supervisor(runner, test_paths(&dir), events);
let mut app = AppConfig::minimal("web", "./srv");
app.wait_ready = true; handle.start(vec![normalize(app).unwrap()]).await.unwrap();
assert_eq!(handle.list().await[0].status, ProcStatus::Starting);
tokio::time::timeout(
Duration::from_secs(4), await_event(&mut rx, 0, ProcessEventKind::Online),
)
.await
.expect("Online once the readiness deadline elapses");
assert_eq!(handle.list().await[0].status, ProcStatus::Online);
}
#[tokio::test(start_paused = true)]
async fn a_gated_app_that_exits_while_starting_never_reaches_online_from_the_old_wait() {
let (events, mut rx) = crate::bus::test_bus(64);
let runner = ScriptedRunner::new(vec![
ProcScript::stable_then_exit(500, 1), ProcScript::never_exits(), ]);
let dir = tempfile::tempdir().unwrap();
let handle = spawn_supervisor(runner, test_paths(&dir), events);
let mut app = AppConfig::minimal("web", "./srv");
app.wait_ready = true; handle.start(vec![normalize(app).unwrap()]).await.unwrap();
assert_eq!(handle.list().await[0].status, ProcStatus::Starting);
tokio::time::timeout(
Duration::from_secs(1),
await_event(&mut rx, 0, ProcessEventKind::Restart),
)
.await
.expect("the automatic respawn after the unstable exit");
assert_eq!(handle.list().await[0].status, ProcStatus::Starting);
assert_eq!(handle.list().await[0].restarts, 1);
assert_no_event_within(
&mut rx,
0,
ProcessEventKind::Online,
Duration::from_millis(2_700),
)
.await;
assert_eq!(
handle.list().await[0].status,
ProcStatus::Starting,
"the OLD wait's stale TimedOut must not have marked the respawned process online"
);
tokio::time::timeout(
Duration::from_secs(2),
await_event(&mut rx, 0, ProcessEventKind::Online),
)
.await
.expect("the new process's own readiness deadline");
assert_eq!(handle.list().await[0].status, ProcStatus::Online);
}
#[tokio::test(start_paused = true)]
async fn a_gated_app_stopped_while_starting_ignores_the_old_wait() {
let (events, mut rx) = crate::bus::test_bus(64);
let runner = ScriptedRunner::new(vec![ProcScript::stable_then_exit(500, 1)]);
let dir = tempfile::tempdir().unwrap();
let handle = spawn_supervisor(runner, test_paths(&dir), events);
let mut app = AppConfig::minimal("web", "./srv");
app.wait_ready = true; app.autorestart = false; handle.start(vec![normalize(app).unwrap()]).await.unwrap();
assert_eq!(handle.list().await[0].status, ProcStatus::Starting);
tokio::time::timeout(
Duration::from_secs(1),
await_event(&mut rx, 0, ProcessEventKind::Stop),
)
.await
.expect("the natural exit at 500ms");
assert_eq!(handle.list().await[0].status, ProcStatus::Stopped);
assert_no_event_within(&mut rx, 0, ProcessEventKind::Online, Duration::from_secs(3)).await;
assert_eq!(handle.list().await[0].status, ProcStatus::Stopped);
}
#[tokio::test(start_paused = true)]
async fn a_gated_app_restarted_while_starting_ignores_the_old_wait() {
let (events, mut rx) = crate::bus::test_bus(64);
let runner =
ScriptedRunner::new(vec![ProcScript::never_exits(), ProcScript::never_exits()]);
let dir = tempfile::tempdir().unwrap();
let handle = spawn_supervisor(runner, test_paths(&dir), events);
let mut app = AppConfig::minimal("web", "./srv");
app.wait_ready = true; handle.start(vec![normalize(app).unwrap()]).await.unwrap();
assert_eq!(handle.list().await[0].status, ProcStatus::Starting);
tokio::time::sleep(Duration::from_millis(500)).await;
handle
.restart(ProcessSelector::Name("web".to_string()))
.await
.unwrap();
assert_eq!(
handle.list().await[0].status,
ProcStatus::Starting,
"the respawned process is gated too, so it must still be Starting"
);
assert_no_event_within(
&mut rx,
0,
ProcessEventKind::Online,
Duration::from_millis(2_700),
)
.await;
assert_eq!(
handle.list().await[0].status,
ProcStatus::Starting,
"the old wait's stale TimedOut must not have marked the new process online"
);
tokio::time::timeout(
Duration::from_secs(2),
await_event(&mut rx, 0, ProcessEventKind::Online),
)
.await
.expect("the new process's own readiness deadline");
assert_eq!(handle.list().await[0].status, ProcStatus::Online);
}
#[tokio::test(start_paused = true)]
async fn start_lists_online_instances() {
let (events, _rx) = crate::bus::test_bus(64);
let runner =
ScriptedRunner::new(vec![ProcScript::never_exits(), ProcScript::never_exits()]);
let dir = tempfile::tempdir().unwrap();
let handle = spawn_supervisor(runner, test_paths(&dir), events);
let mut app = AppConfig::minimal("web", "./srv");
app.instances = 2;
let infos = handle.start(vec![normalize(app).unwrap()]).await.unwrap();
assert_eq!(infos.len(), 2);
let list = handle.list().await;
assert_eq!(list.len(), 2);
assert!(list.iter().all(|i| i.status == ProcStatus::Online));
assert_eq!(list.iter().map(|i| i.id).collect::<Vec<_>>(), vec![0, 1]);
}
#[tokio::test(start_paused = true)]
async fn listed_log_paths_are_the_derived_defaults() {
let (events, _rx) = crate::bus::test_bus(64);
let runner = ScriptedRunner::new(vec![ProcScript::never_exits()]);
let dir = tempfile::tempdir().unwrap();
let paths = test_paths(&dir);
let logs = paths.logs.clone();
let handle = spawn_supervisor(runner, paths, events);
handle
.start(vec![normalize(AppConfig::minimal("web", "./srv")).unwrap()])
.await
.unwrap();
let list = handle.list().await;
assert_eq!(
list[0].out_file.as_deref(),
logs.join("web-0-out.log").to_str()
);
assert_eq!(
list[0].err_file.as_deref(),
logs.join("web-0-err.log").to_str()
);
}
#[tokio::test(start_paused = true)]
async fn listed_log_paths_honour_an_explicit_out_file() {
let (events, _rx) = crate::bus::test_bus(64);
let runner = ScriptedRunner::new(vec![ProcScript::never_exits()]);
let dir = tempfile::tempdir().unwrap();
let paths = test_paths(&dir);
let logs = paths.logs.clone();
let handle = spawn_supervisor(runner, paths, events);
let mut app = AppConfig::minimal("web", "./srv");
app.out_file = Some("/var/log/myapp.log".to_string());
handle.start(vec![normalize(app).unwrap()]).await.unwrap();
let list = handle.list().await;
assert_eq!(list[0].out_file.as_deref(), Some("/var/log/myapp.log"));
assert_eq!(
list[0].err_file.as_deref(),
logs.join("web-0-err.log").to_str(),
"err_file was not configured, so it must still be the default"
);
}
#[tokio::test(start_paused = true)]
async fn crash_loop_erroreds_after_budget_with_pinned_delays() {
let (events, mut rx) = crate::bus::test_bus(1024);
let runner = ScriptedRunner::new((0..16).map(|_| ProcScript::const_exit(1)).collect());
let dir = tempfile::tempdir().unwrap();
let handle = spawn_supervisor(runner, test_paths(&dir), events);
let mut app = AppConfig::minimal("crash", "./boom");
app.exp_backoff_restart_delay = Some("100".parse().unwrap());
handle.start(vec![normalize(app).unwrap()]).await.unwrap();
await_event(&mut rx, 0, ProcessEventKind::Errored).await;
let list = handle.list().await;
assert_eq!(list[0].status, ProcStatus::Errored);
assert_eq!(list[0].restarts, 15); assert_eq!(
list[0].last_exit,
Some(ExitInfo {
code: Some(1),
signal: None,
})
);
}
#[tokio::test(start_paused = true)]
async fn crash_loop_budget_check_fires_before_script_exhaustion_at_real_default() {
let (events, mut rx) = crate::bus::test_bus(1024);
let runner = ScriptedRunner::new((0..20).map(|_| ProcScript::const_exit(1)).collect());
let dir = tempfile::tempdir().unwrap();
let handle = spawn_supervisor(runner, test_paths(&dir), events);
let app = AppConfig::minimal("crash-default", "./boom"); handle.start(vec![normalize(app).unwrap()]).await.unwrap();
await_event(&mut rx, 0, ProcessEventKind::Errored).await;
let list = handle.list().await;
assert_eq!(list[0].status, ProcStatus::Errored);
assert_eq!(list[0].restarts, 15);
}
#[tokio::test(start_paused = true)]
async fn stable_run_resets_budget() {
let (events, mut rx) = crate::bus::test_bus(256);
let mut script = vec![
ProcScript::const_exit(1),
ProcScript::const_exit(1),
ProcScript::const_exit(1),
];
script.push(ProcScript::stable_then_exit(2000, 1)); script.extend((0..16).map(|_| ProcScript::const_exit(1)));
let runner = ScriptedRunner::new(script);
let dir = tempfile::tempdir().unwrap();
let handle = spawn_supervisor(runner, test_paths(&dir), events);
let app = AppConfig::minimal("flappy", "./f");
handle.start(vec![normalize(app).unwrap()]).await.unwrap();
await_event(&mut rx, 0, ProcessEventKind::Errored).await;
let list = handle.list().await;
assert_eq!(list[0].status, ProcStatus::Errored);
assert_eq!(list[0].restarts, 19);
}
#[tokio::test(start_paused = true)]
async fn manual_stop_prevents_restart() {
let (events, _rx) = crate::bus::test_bus(64);
let runner = ScriptedRunner::new(vec![ProcScript {
delay_ms: u64::MAX,
outcome: ExitOutcome {
code: None,
signal: None,
},
obeys_signal: true,
obeys_kill: true,
lamb_holds_the_pipe: false,
reads_stdin: true,
}]);
let dir = tempfile::tempdir().unwrap();
let handle = spawn_supervisor(runner, test_paths(&dir), events);
let app = AppConfig::minimal("svc", "./svc");
handle.start(vec![normalize(app).unwrap()]).await.unwrap();
let stopped = handle
.stop(ProcessSelector::Name("svc".to_string()))
.await
.unwrap();
assert_eq!(stopped[0].status, ProcStatus::Stopped); tokio::time::advance(std::time::Duration::from_secs(60)).await;
assert_eq!(handle.list().await[0].status, ProcStatus::Stopped);
}
#[tokio::test(start_paused = true)]
async fn stop_exit_codes_mean_clean_stop() {
let (events, mut rx) = crate::bus::test_bus(64);
let runner = ScriptedRunner::new(vec![ProcScript::const_exit(0)]);
let dir = tempfile::tempdir().unwrap();
let handle = spawn_supervisor(runner, test_paths(&dir), events);
let mut app = AppConfig::minimal("oneshot", "./job");
app.stop_exit_codes = vec![0];
handle.start(vec![normalize(app).unwrap()]).await.unwrap();
await_event(&mut rx, 0, ProcessEventKind::Stop).await;
assert_eq!(handle.list().await[0].status, ProcStatus::Stopped);
}
#[tokio::test(start_paused = true)]
async fn spawn_failure_surfaces_and_erroreds() {
let (events, _rx) = crate::bus::test_bus(64);
let runner = ScriptedRunner::new(vec![]); let dir = tempfile::tempdir().unwrap();
let handle = spawn_supervisor(runner, test_paths(&dir), events);
let app = AppConfig::minimal("ghost", "./missing");
let err = handle
.start(vec![normalize(app).unwrap()])
.await
.unwrap_err();
assert!(matches!(err, SupervisorError::SpawnFailed(_)));
assert_eq!(handle.list().await[0].status, ProcStatus::Errored);
}
#[tokio::test(start_paused = true)]
async fn an_unresolvable_user_fails_the_start() {
let dir = tempfile::tempdir().unwrap();
let (events, _rx) = crate::bus::test_bus(64);
let handle = spawn_supervisor(
ScriptedRunner::new(vec![ProcScript::never_exits()]),
test_paths(&dir),
events,
);
let mut app = AppConfig::minimal("svc", "./svc");
app.user = Some("definitely-not-a-real-shep-user".to_string());
let err = handle
.start(vec![normalize(app).unwrap()])
.await
.unwrap_err();
assert!(matches!(err, SupervisorError::CannotStart(_)), "{err:?}");
assert!(
handle.list().await.is_empty(),
"a refusal before the registering pass must leave nothing registered"
);
}
#[tokio::test(start_paused = true)]
async fn a_refusal_is_counted_once_per_app_not_once_per_failed_check() {
let dir = tempfile::tempdir().unwrap();
let (events, _rx) = crate::bus::test_bus(64);
let handle = spawn_supervisor(
ScriptedRunner::new(vec![ProcScript::never_exits()]).refusing(&["one", "two"]),
test_paths(&dir),
events,
);
let both_bad = |name: &str| {
let mut app = AppConfig::minimal(name, "./definitely-not-here");
app.cwd = Some(dir.path().display().to_string());
app.user = Some("definitely-not-a-real-shep-user".to_string());
normalize(app).unwrap()
};
let err = handle
.start(vec![both_bad("one"), both_bad("two")])
.await
.unwrap_err();
let SupervisorError::CannotStart(msg) = &err else {
panic!("expected CannotStart, got {err:?}");
};
assert!(
msg.contains("2 of 2 apps cannot start"),
"one refusal per app, never one per failed check: {msg}"
);
}
#[tokio::test(start_paused = true)]
async fn delete_and_selectors_route() {
let (events, _rx) = crate::bus::test_bus(64);
let runner =
ScriptedRunner::new(vec![ProcScript::never_exits(), ProcScript::never_exits()]);
let dir = tempfile::tempdir().unwrap();
let handle = spawn_supervisor(runner, test_paths(&dir), events);
let mut a = AppConfig::minimal("api", "./a");
a.fold = Some("backend".to_string());
let b = AppConfig::minimal("web", "./w");
handle
.start(vec![normalize(a).unwrap(), normalize(b).unwrap()])
.await
.unwrap();
let hits = handle
.stop(ProcessSelector::Fold("backend".to_string()))
.await
.unwrap();
assert_eq!(hits.len(), 1);
assert_eq!(hits[0].name, "api");
let deleted = handle
.delete(ProcessSelector::Name("web".to_string()))
.await
.unwrap();
assert_eq!(deleted, vec![1]);
assert_eq!(handle.list().await.len(), 1);
}
async fn wait_for_restarts_online(handle: &SupervisorHandle, restarts: u32) -> ProcessInfo {
let mut info = handle.list().await.remove(0);
for _ in 0..200 {
if info.restarts == restarts && info.status == ProcStatus::Online {
break;
}
tokio::task::yield_now().await;
info = handle.list().await.remove(0);
}
info
}
#[tokio::test(start_paused = true)]
async fn manual_restart_resets_budget_and_respawns() {
let (events, _rx) = crate::bus::test_bus(64);
let runner = ScriptedRunner::new(vec![
ProcScript::const_exit(1),
ProcScript::const_exit(1),
ProcScript::never_exits(),
ProcScript::const_exit(1),
ProcScript::never_exits(),
]);
let dir = tempfile::tempdir().unwrap();
let handle = spawn_supervisor(runner, test_paths(&dir), events);
let mut app = AppConfig::minimal("svc", "./svc");
app.max_restarts = 3;
app.exp_backoff_restart_delay = None;
handle.start(vec![normalize(app).unwrap()]).await.unwrap();
let info = wait_for_restarts_online(&handle, 2).await;
assert_eq!(
(info.status, info.restarts),
(ProcStatus::Online, 2),
"never reached the never_exits proc -- got {info:?}"
);
let restarted = handle
.restart(ProcessSelector::Name("svc".to_string()))
.await
.unwrap();
assert_eq!(restarted[0].status, ProcStatus::Online);
let mut settled = handle.list().await.remove(0);
for _ in 0..200 {
if settled.status == ProcStatus::Errored
|| (settled.status == ProcStatus::Online && settled.restarts == 4)
{
break;
}
tokio::task::yield_now().await;
settled = handle.list().await.remove(0);
}
assert_eq!(
(settled.status, settled.restarts),
(ProcStatus::Online, 4),
"an operator's restart left the two spent unstable exits on the \
books -- got {settled:?}"
);
}
#[tokio::test(start_paused = true)]
async fn an_automatic_restart_resets_the_budget_like_an_operators_does() {
let (events, _rx) = crate::bus::test_bus(64);
let runner = ScriptedRunner::new(vec![
ProcScript::const_exit(1),
ProcScript::const_exit(1),
ProcScript::never_exits(),
ProcScript::const_exit(1),
ProcScript::never_exits(),
]);
let dir = tempfile::tempdir().unwrap();
let handle = spawn_supervisor(runner, test_paths(&dir), events);
let mut app = AppConfig::minimal("svc", "./svc");
app.max_restarts = 3;
app.exp_backoff_restart_delay = None;
handle.start(vec![normalize(app).unwrap()]).await.unwrap();
let info = wait_for_restarts_online(&handle, 2).await;
assert_eq!(
(info.status, info.restarts),
(ProcStatus::Online, 2),
"never reached the never_exits proc -- got {info:?}"
);
handle
.restart_automatic(ProcessSelector::Name("svc".to_string()))
.await
.unwrap();
let mut settled = handle.list().await.remove(0);
for _ in 0..200 {
if settled.status == ProcStatus::Errored
|| (settled.status == ProcStatus::Online && settled.restarts == 4)
{
break;
}
tokio::task::yield_now().await;
settled = handle.list().await.remove(0);
}
assert_eq!(
(settled.status, settled.restarts),
(ProcStatus::Online, 4),
"an automatic restart left the two spent unstable exits on the \
books -- got {settled:?}"
);
}
#[tokio::test(start_paused = true)]
async fn restarting_a_stopped_sheep_resets_the_budget() {
let (events, _rx) = crate::bus::test_bus(64);
let runner = ScriptedRunner::new(vec![
ProcScript::const_exit(1),
ProcScript::const_exit(1),
ProcScript::never_exits(),
ProcScript::const_exit(1),
ProcScript::never_exits(),
]);
let dir = tempfile::tempdir().unwrap();
let handle = spawn_supervisor(runner, test_paths(&dir), events);
let mut app = AppConfig::minimal("svc", "./svc");
app.max_restarts = 3;
app.exp_backoff_restart_delay = None;
handle.start(vec![normalize(app).unwrap()]).await.unwrap();
let info = wait_for_restarts_online(&handle, 2).await;
assert_eq!(
(info.status, info.restarts),
(ProcStatus::Online, 2),
"never reached the never_exits proc -- got {info:?}"
);
let stopped = handle
.stop(ProcessSelector::Name("svc".to_string()))
.await
.unwrap();
assert_eq!(stopped[0].status, ProcStatus::Stopped);
let restarted = handle
.restart(ProcessSelector::Name("svc".to_string()))
.await
.unwrap();
assert_eq!(
(restarted[0].status, restarted[0].restarts),
(ProcStatus::Online, 3)
);
let mut settled = handle.list().await.remove(0);
for _ in 0..200 {
if settled.status == ProcStatus::Errored
|| (settled.status == ProcStatus::Online && settled.restarts == 4)
{
break;
}
tokio::task::yield_now().await;
settled = handle.list().await.remove(0);
}
assert_eq!(
(settled.status, settled.restarts),
(ProcStatus::Online, 4),
"restarting a stopped sheep left the two spent unstable exits on \
the books -- got {settled:?}"
);
}
#[tokio::test(start_paused = true)]
async fn shutdown_kills_all_and_stops_the_engine() {
let (events, _rx) = crate::bus::test_bus(64);
let runner =
ScriptedRunner::new(vec![ProcScript::never_exits(), ProcScript::never_exits()]);
let dir = tempfile::tempdir().unwrap();
let handle = spawn_supervisor(runner, test_paths(&dir), events);
let mut app = AppConfig::minimal("web", "./srv");
app.instances = 2;
handle.start(vec![normalize(app).unwrap()]).await.unwrap();
handle.shutdown().await; assert!(handle.list_checked().await.is_err());
}
async fn drain_kinds(
rx: &mut tokio::sync::broadcast::Receiver<SharedEvent>,
) -> Vec<(u32, ProcessEventKind)> {
let mut out = Vec::new();
while let Ok(ev) = rx.try_recv().map(|event| event.to_event()) {
if let BusEvent::Process { event, info, .. } = ev {
out.push((info.id, event));
}
}
out
}
#[tokio::test(start_paused = true)]
async fn shutdown_ignores_a_pending_restart_timer() {
let (events, mut rx) = crate::bus::test_bus(1024);
let runner = ScriptedRunner::new(vec![
ProcScript::const_exit(1), ProcScript::ignores_signals(), ProcScript::never_exits(), ]);
let dir = tempfile::tempdir().unwrap();
let handle = spawn_supervisor(runner, test_paths(&dir), events);
let mut crash = AppConfig::minimal("crash", "./boom");
crash.exp_backoff_restart_delay = Some("500".parse().unwrap());
let web = AppConfig::minimal("web", "./srv");
handle
.start(vec![normalize(crash).unwrap(), normalize(web).unwrap()])
.await
.unwrap();
await_event(&mut rx, 0, ProcessEventKind::Exit).await;
handle.shutdown().await;
let seen = drain_kinds(&mut rx).await;
let ghost = seen
.iter()
.any(|(id, k)| *id == 0 && *k == ProcessEventKind::Restart);
assert!(
!ghost,
"GHOST RESPAWN during shutdown: events after shutdown = {seen:?}"
);
}
#[tokio::test(start_paused = true)]
async fn shutdown_ignores_a_pending_readiness_wait() {
let (events, mut rx) = crate::bus::test_bus(1024);
let runner = ScriptedRunner::new(vec![ProcScript::ignores_signals()]);
let dir = tempfile::tempdir().unwrap();
let handle = spawn_supervisor(runner, test_paths(&dir), events);
let mut app = AppConfig::minimal("gated", "./g");
app.wait_ready = true; app.listen_timeout = UpDuration::from_millis(1_000);
handle.start(vec![normalize(app).unwrap()]).await.unwrap();
assert_eq!(
handle.list().await[0].status,
ProcStatus::Starting,
"the readiness wait has to be armed for its result to be droppable"
);
handle.shutdown().await;
let seen = drain_kinds(&mut rx).await;
assert!(
seen.contains(&(0, ProcessEventKind::Stop)),
"the kill ladder must have outlasted the readiness deadline: events = {seen:?}"
);
assert!(
!seen.contains(&(0, ProcessEventKind::Online)),
"a readiness wait resolved during shutdown marked a dying sheep online: \
events = {seen:?}"
);
}
#[tokio::test(start_paused = true)]
async fn late_start_racing_shutdown_never_orphans() {
let (events, mut rx) = crate::bus::test_bus(1024);
let runner = ScriptedRunner::new(vec![
ProcScript::ignores_signals(), ProcScript::never_exits(), ]);
let dir = tempfile::tempdir().unwrap();
let handle = spawn_supervisor(runner, test_paths(&dir), events);
let web = AppConfig::minimal("web", "./srv");
handle.start(vec![normalize(web).unwrap()]).await.unwrap();
let h2 = handle.clone();
let late = tokio::spawn(async move {
let app = AppConfig::minimal("late", "./l");
h2.start(vec![normalize(app).unwrap()]).await
});
handle.shutdown().await;
let outcome = late.await.unwrap();
let seen = drain_kinds(&mut rx).await;
match outcome {
Err(SupervisorError::EngineStopped) => {} Ok(infos) => {
let late_id = infos[0].id;
assert!(
seen.iter().any(|(id, k)| *id == late_id
&& matches!(k, ProcessEventKind::Stop | ProcessEventKind::Exit)),
"late Start raced ahead of shutdown but was never killed: events = {seen:?}"
);
}
Err(other) => panic!("unexpected error from a late Start during shutdown: {other:?}"),
}
}
#[tokio::test(start_paused = true)]
async fn stale_restart_timer_never_short_circuits_a_newer_backoff() {
let (events, mut rx) = crate::bus::test_bus(1024);
let runner = ScriptedRunner::new(vec![
ProcScript::const_exit(1), ProcScript::stable_then_exit(1500, 1), ProcScript::never_exits(), ProcScript::never_exits(),
]);
let dir = tempfile::tempdir().unwrap();
let handle = spawn_supervisor(runner, test_paths(&dir), events);
let mut app = AppConfig::minimal("crash", "./boom");
app.exp_backoff_restart_delay = Some("2000".parse().unwrap());
app.min_uptime = "5000".parse().unwrap(); handle.start(vec![normalize(app).unwrap()]).await.unwrap();
await_event(&mut rx, 0, ProcessEventKind::Exit).await;
let out = handle.restart(ProcessSelector::All).await.unwrap();
assert_eq!(
out[0].status,
ProcStatus::Online,
"manual restart respawned"
);
tokio::time::advance(Duration::from_millis(2500)).await;
for _ in 0..20 {
tokio::task::yield_now().await;
}
let info = handle.list().await.remove(0);
assert_eq!(
(info.status, info.restarts),
(ProcStatus::WaitingRestart, 1),
"at t=2500 the sheep should still be waiting on its 3500ms backoff; \
got {info:?} -- the stale 2000ms timer fired early"
);
}
#[tokio::test(start_paused = true)]
async fn overlapping_stop_and_restart_agree_on_one_outcome() {
let (events, _rx) = crate::bus::test_bus(1024);
let runner = ScriptedRunner::new(vec![
ProcScript::ignores_signals(), ProcScript::never_exits(),
]);
let dir = tempfile::tempdir().unwrap();
let handle = spawn_supervisor(runner, test_paths(&dir), events);
let app = AppConfig::minimal("svc", "./svc");
handle.start(vec![normalize(app).unwrap()]).await.unwrap();
let h2 = handle.clone();
let stopper = tokio::spawn(async move { h2.stop(ProcessSelector::All).await });
for _ in 0..10 {
tokio::task::yield_now().await;
}
let restarted = handle.restart(ProcessSelector::All).await.unwrap();
let stopped = stopper.await.unwrap().unwrap();
assert_eq!(
stopped[0].status,
ProcStatus::Stopped,
"stop() reported a non-stopped sheep"
);
assert_eq!(
restarted[0].status,
ProcStatus::Stopped,
"restart() lost the race to the earlier stop() but got a different \
answer than the stop() caller -- the two callers disagree about \
what happened to the same sheep"
);
}
#[tokio::test(start_paused = true)]
async fn actor_never_blocks_behind_a_busy_kill_ladder() {
let (events, mut rx) = crate::bus::test_bus(1024);
let runner = ScriptedRunner::new(vec![
ProcScript::ignores_signals(), ProcScript::stable_then_exit(800, 0), ]);
let dir = tempfile::tempdir().unwrap();
let handle = spawn_supervisor(runner, test_paths(&dir), events);
let a = AppConfig::minimal("a", "./a");
let mut b = AppConfig::minimal("b", "./b");
b.autorestart = false;
handle
.start(vec![normalize(a).unwrap(), normalize(b).unwrap()])
.await
.unwrap();
let t0 = tokio::time::Instant::now();
let mut tasks = Vec::new();
for _ in 0..8 {
let h = handle.clone();
tasks.push(tokio::spawn(async move {
h.stop(ProcessSelector::Name("a".to_string())).await
}));
}
await_event(&mut rx, 1, ProcessEventKind::Stop).await;
let seen_at = t0.elapsed();
for t in tasks {
let _ = t.await;
}
assert!(
seen_at < Duration::from_millis(1000),
"sheep b's own exit was only processed at {seen_at:?} -- the actor \
was parked inside ctl.send() for sheep a's kill ladder"
);
}
#[tokio::test(start_paused = true)]
async fn mailbox_flood_during_a_kill_never_deadlocks() {
let (events, mut rx) = crate::bus::test_bus(4096);
let runner = ScriptedRunner::new(vec![ProcScript::ignores_signals()]);
let dir = tempfile::tempdir().unwrap();
let handle = spawn_supervisor(runner, test_paths(&dir), events);
let a = AppConfig::minimal("a", "./a");
handle.start(vec![normalize(a).unwrap()]).await.unwrap();
for _ in 0..8 {
let h = handle.clone();
tokio::spawn(async move {
let _ = h.stop(ProcessSelector::All).await;
});
}
for _ in 0..40 {
tokio::task::yield_now().await;
}
for _ in 0..400 {
let h = handle.clone();
tokio::spawn(async move {
let _ = h.list_checked().await;
});
}
for _ in 0..200 {
tokio::task::yield_now().await;
}
let r = tokio::time::timeout(
Duration::from_secs(600),
await_event(&mut rx, 0, ProcessEventKind::Stop),
)
.await;
assert!(
r.is_ok(),
"DEADLOCK: actor parked in ctl.send() while the sheep task is \
parked in actor_tx.send() -- the daemon never recovers"
);
}
#[tokio::test(start_paused = true)]
async fn delete_racing_shutdown_still_deregisters_the_sheep() {
let (events, _rx) = crate::bus::test_bus(1024);
let runner = ScriptedRunner::new(vec![
ProcScript::ignores_signals(), ProcScript::ignores_signals(), ]);
let dir = tempfile::tempdir().unwrap();
let handle = spawn_supervisor(runner, test_paths(&dir), events);
let target = AppConfig::minimal("svc", "./svc");
let mut decoy = AppConfig::minimal("decoy", "./decoy");
decoy.kill_timeout = "600000".parse().unwrap(); let started = handle
.start(vec![normalize(target).unwrap(), normalize(decoy).unwrap()])
.await
.unwrap();
let id = started
.iter()
.find(|info| info.name == "svc")
.expect("target sheep registered")
.id;
let h2 = handle.clone();
let shutter = tokio::spawn(async move { h2.shutdown().await });
for _ in 0..10 {
tokio::task::yield_now().await; }
let deleted = handle.delete(ProcessSelector::Id(id)).await.unwrap();
assert_eq!(deleted, vec![id], "the caller was told this id was deleted");
assert!(
handle.list().await.iter().all(|info| info.id != id),
"a Delete that raced a Shutdown must still deregister the sheep, \
not just tell its caller it did"
);
drop(shutter);
}
#[tokio::test(start_paused = true)]
async fn delete_racing_restart_still_deregisters_the_sheep() {
let (events, _rx) = crate::bus::test_bus(1024);
let runner = ScriptedRunner::new(vec![
ProcScript::ignores_signals(), ProcScript::never_exits(),
]);
let dir = tempfile::tempdir().unwrap();
let handle = spawn_supervisor(runner, test_paths(&dir), events);
let app = AppConfig::minimal("svc", "./svc");
let started = handle.start(vec![normalize(app).unwrap()]).await.unwrap();
let id = started[0].id;
let h2 = handle.clone();
let restarter = tokio::spawn(async move { h2.restart(ProcessSelector::All).await });
for _ in 0..10 {
tokio::task::yield_now().await; }
let deleted = handle.delete(ProcessSelector::Id(id)).await.unwrap();
let restarted = restarter.await.unwrap().unwrap();
assert_eq!(deleted, vec![id], "the caller was told this id was deleted");
assert_eq!(restarted[0].id, id);
assert_eq!(
restarted[0].status,
ProcStatus::Stopped,
"restart() must not report a respawned process once a racing \
Delete has claimed this id -- got {restarted:?}"
);
assert!(
handle.list().await.iter().all(|info| info.id != id),
"a Delete that raced a Restart must still deregister the sheep, \
not respawn a brand-new live process while telling its caller \
the sheep was deleted"
);
}
#[tokio::test(start_paused = true)]
async fn an_operators_stop_beats_an_automatic_restart_mid_ladder() {
let (events, _rx) = crate::bus::test_bus(1024);
let runner = ScriptedRunner::new(vec![
ProcScript::ignores_signals(), ProcScript::never_exits(), ]);
let dir = tempfile::tempdir().unwrap();
let handle = spawn_supervisor(runner, test_paths(&dir), events);
let app = AppConfig::minimal("svc", "./svc");
handle.start(vec![normalize(app).unwrap()]).await.unwrap();
let running = handle.list().await.remove(0);
let pid = running.pid.expect("an online sheep has a pid");
handle.extra_restart(running.id, pid, None, None).await;
let stopped = handle.stop(ProcessSelector::All).await.unwrap();
assert_eq!(stopped.len(), 1);
assert_eq!(stopped[0].id, running.id);
assert_eq!(
(stopped[0].status, stopped[0].restarts),
(ProcStatus::Stopped, 0),
"an operator's stop was silently converted into the automatic \
restart it raced -- got {stopped:?}"
);
let listed = handle.list().await;
assert_eq!(
(listed[0].status, listed[0].pid),
(ProcStatus::Stopped, None),
"the sheep an operator stopped is running again -- got {listed:?}"
);
}
#[tokio::test(start_paused = true)]
async fn an_operators_delete_beats_an_automatic_restart_mid_ladder() {
let (events, _rx) = crate::bus::test_bus(1024);
let runner = ScriptedRunner::new(vec![
ProcScript::ignores_signals(),
ProcScript::never_exits(),
]);
let dir = tempfile::tempdir().unwrap();
let handle = spawn_supervisor(runner, test_paths(&dir), events);
let app = AppConfig::minimal("svc", "./svc");
handle.start(vec![normalize(app).unwrap()]).await.unwrap();
let running = handle.list().await.remove(0);
let pid = running.pid.expect("an online sheep has a pid");
handle.extra_restart(running.id, pid, None, None).await;
let deleted = handle
.delete(ProcessSelector::Id(running.id))
.await
.unwrap();
assert_eq!(deleted, vec![running.id]);
assert!(
handle.list().await.is_empty(),
"a delete that raced an automatic restart must still deregister \
the sheep, not leave one behind for the restart to bring back"
);
}
#[tokio::test(start_paused = true)]
async fn an_operators_stop_still_shows_its_signal_as_the_last_exit() {
let (events, _rx) = crate::bus::test_bus(64);
let runner = ScriptedRunner::new(vec![ProcScript::never_exits()]);
let dir = tempfile::tempdir().unwrap();
let handle = spawn_supervisor(runner, test_paths(&dir), events);
let app = AppConfig::minimal("svc", "./svc");
handle.start(vec![normalize(app).unwrap()]).await.unwrap();
let stopped = handle.stop(ProcessSelector::All).await.unwrap();
assert_eq!(stopped.len(), 1);
assert_eq!(stopped[0].status, ProcStatus::Stopped);
assert_eq!(
stopped[0].last_exit,
Some(ExitInfo {
code: None,
signal: Some(15),
}),
"an operator's own stop must still show up as a last exit: {stopped:?}"
);
}
#[tokio::test(start_paused = true)]
async fn a_respawn_that_fails_to_spawn_clears_the_previous_exit_code() {
let (events, _rx) = crate::bus::test_bus(64);
let runner = ScriptedRunner::new(vec![ProcScript::const_exit(1)]);
let dir = tempfile::tempdir().unwrap();
let handle = spawn_supervisor(runner, test_paths(&dir), events);
let app = AppConfig::minimal("svc", "./svc");
handle.start(vec![normalize(app).unwrap()]).await.unwrap();
tokio::time::sleep(Duration::from_secs(30)).await;
let listing = handle.list().await;
assert_eq!(listing.len(), 1);
assert_eq!(
listing[0].status,
ProcStatus::Errored,
"a respawn that cannot spawn is still terminal: {listing:?}"
);
assert_eq!(
listing[0].last_exit, None,
"nothing exited on the failed respawn, so the earlier code must \
not still be showing: {listing:?}"
);
}
fn actor_with_stopping_drainee(
dir: &tempfile::TempDir,
pid: u32,
epoch: u64,
) -> (Actor<ScriptedRunner>, mpsc::Receiver<SheepCtl>) {
let paths = test_paths(dir);
let app = normalize(AppConfig::minimal("web", "./srv")).unwrap();
let mut entry = armed_entry(0, 0, pid, app, &paths);
entry.status = ProcStatus::Stopping;
let (ctl_tx, ctl_rx) = mpsc::channel(1);
let slot = SheepSlot {
ctl: Some(ctl_tx),
epoch,
..SheepSlot::new(entry)
};
let mut sheep = HashMap::new();
sheep.insert(0, slot);
let (tx, _rx) = mpsc::channel(16);
let actor = test_actor(paths, Vec::new(), sheep, tx);
(actor, ctl_rx)
}
fn actor_with_a_drainee_holding_a_signal_mailbox(
dir: &tempfile::TempDir,
) -> (Actor<ScriptedRunner>, mpsc::Receiver<SignalRequest>) {
let (mut actor, _mailbox) = actor_with_one_online_sheep(dir, vec![]);
let slot = actor.sheep.get_mut(&0).expect("the fixture registers id 0");
slot.entry.status = ProcStatus::Stopping;
slot.entry.reload = ReloadState::Drainee { new_id: Some(1) };
let (signals, signal_rx) = mpsc::channel(16);
slot.signals = Some(signals);
(actor, signal_rx)
}
#[test]
fn a_stopping_sheep_rejects_an_extra_restart() {
let dir = tempfile::tempdir().unwrap();
let (mut actor, mut ctl_rx) = actor_with_stopping_drainee(&dir, 4242, 0);
actor.handle_extra_restart(0, 4242, None, None);
let slot = actor.sheep.get(&0).expect("the sheep stays registered");
assert_eq!(
slot.entry.status,
ProcStatus::Stopping,
"a rejected extra restart must never touch status"
);
assert!(
slot.manual.is_none(),
"a Stopping sheep must never claim the manual marker off an extra restart"
);
assert!(
ctl_rx.try_recv().is_err(),
"a Stopping sheep must never receive a second Kill"
);
}
#[tokio::test]
async fn a_stale_liveness_failure_from_a_replaced_probe_does_not_restart() {
let dir = tempfile::tempdir().unwrap();
let paths = test_paths(&dir);
let mut app = AppConfig::minimal("web", "./srv");
app.liveness_probe = Some(ProbeConfig {
failure_threshold: 1,
..probe_config(ProbeKind::Tcp, "localhost:5432")
});
let app = normalize(app).unwrap();
let pid = 1111;
let entry = armed_entry(0, 0, pid, app, &paths);
let (ctl_tx, mut ctl_rx) = mpsc::channel(1);
let mut sheep = HashMap::new();
sheep.insert(
0,
SheepSlot {
ctl: Some(ctl_tx),
..SheepSlot::new(entry)
},
);
let (tx, _mailbox) = mpsc::channel(16);
let mut actor = test_actor(paths, Vec::new(), sheep, tx);
let entry = actor
.sheep
.get(&0)
.expect("the fixture registers id 0")
.entry
.clone();
let supervisor = SupervisorHandle {
tx: actor.tx.clone(),
};
let (breach_tx, _breaches) = mpsc::channel(1);
let (live_tx, _liveness) = mpsc::channel(1);
let extras = Extras {
clock: Arc::new(SystemClock),
enforcer: Arc::new(RecordingEnforcer::default()),
max_cron_sleep: DEFAULT_MAX_CRON_SLEEP,
reports: ExtrasReports {
breaches: breach_tx,
liveness: live_tx,
},
stats: idle_stats(),
};
let prober: Arc<dyn Prober> = Arc::new(ScriptedProber::new(vec![]));
actor
.registry
.arm(&entry, Arc::clone(&prober), &extras, &supervisor);
let stale_epoch = actor.registry.liveness_epoch(0);
actor.registry.arm(&entry, prober, &extras, &supervisor);
assert_ne!(
actor.registry.liveness_epoch(0),
stale_epoch,
"a re-arm must advance the epoch"
);
actor.handle_extra_restart(0, pid, Some(stale_epoch), None);
let slot = actor.sheep.get(&0).expect("the sheep stays registered");
assert!(
slot.manual.is_none(),
"a stale liveness failure from a replaced probe must never claim the manual marker"
);
assert!(
ctl_rx.try_recv().is_err(),
"a stale liveness failure from a replaced probe must never queue a Kill"
);
assert_eq!(
slot.entry.restarts, 0,
"a dropped report must never bump the restart count"
);
assert_eq!(
slot.entry.pid,
Some(pid),
"a dropped report must never touch the running pid"
);
}
#[test]
fn a_stopping_sheep_rejects_a_restart_due() {
let dir = tempfile::tempdir().unwrap();
let (mut actor, _ctl_rx) = actor_with_stopping_drainee(&dir, 4242, 7);
actor.handle_restart_due(0, 7);
let slot = actor.sheep.get(&0).expect("the sheep stays registered");
assert_eq!(
slot.entry.status,
ProcStatus::Stopping,
"a rejected restart-due must never touch status"
);
assert_eq!(
slot.entry.restarts, 0,
"a rejected restart-due must never respawn"
);
}
fn probed_app(name: &str) -> AppConfig {
let mut app = AppConfig::minimal(name, "./srv");
app.readiness_probe = Some(probe_config(ProbeKind::Tcp, "127.0.0.1:9"));
app
}
#[test]
fn reload_mode_serialises_exactly_the_apps_whose_probe_can_lie() {
let cases = [
(false, false, false, ReloadMode::Overlap),
(false, true, false, ReloadMode::Overlap),
(true, false, false, ReloadMode::Serial),
(true, true, false, ReloadMode::Overlap),
(true, false, true, ReloadMode::Overlap),
];
for (probe, wait_ready, reuse_port, want) in cases {
let mut app = if probe {
probed_app("web")
} else {
AppConfig::minimal("web", "./srv")
};
app.wait_ready = wait_ready;
app.reuse_port = reuse_port;
let source = ReadinessSource::of(&app).expect("the target parses");
assert_eq!(
ReloadMode::of(&app, &source),
want,
"probe={probe} wait_ready={wait_ready} reuse_port={reuse_port}"
);
}
}
#[tokio::test(start_paused = true)]
async fn a_probed_reload_drains_before_it_spawns_anything() {
let dir = tempfile::tempdir().unwrap();
let (mut actor, _mailbox) = actor_with_one_online_sheep_of(&dir, probed_app("web"), vec![]);
let (ctl_tx, mut ctl_rx) = mpsc::channel(SHEEP_CTL_CAPACITY);
actor.sheep.get_mut(&0).expect("the fixture's sheep").ctl = Some(ctl_tx);
actor.advance_reload("web", VecDeque::from([0]));
let job = &actor.reloads["web"];
assert_eq!(job.mode, ReloadMode::Serial);
assert_eq!(job.swap.phase, ReloadPhase::DrainFirst);
assert_eq!(
job.swap.new_id, None,
"a serial reload has no replacement until the drain is over"
);
assert_eq!(
actor.sheep.len(),
1,
"nothing was spawned into the slot the drain is emptying"
);
let drainee = &actor.sheep[&0];
assert_eq!(drainee.entry.status, ProcStatus::Stopping);
assert_eq!(
drainee.entry.reload,
ReloadState::Drainee { new_id: None },
"the marker is what keeps this exit off the ordinary respawn path"
);
assert!(
drainee.manual.is_some(),
"the drain owns the drainee's next exit"
);
assert!(
ctl_rx.try_recv().is_ok(),
"the drain asked the instance to go"
);
}
#[tokio::test(start_paused = true)]
async fn reuse_port_buys_back_the_overlap_a_probe_would_otherwise_cost() {
let dir = tempfile::tempdir().unwrap();
let mut app = probed_app("web");
app.reuse_port = true;
let (mut actor, _mailbox) =
actor_with_one_online_sheep_of(&dir, app, vec![ProcScript::never_exits()]);
actor.advance_reload("web", VecDeque::from([0]));
let job = &actor.reloads["web"];
assert_eq!(job.mode, ReloadMode::Overlap);
assert_eq!(job.swap.phase, ReloadPhase::AwaitReady);
let new_id = job
.swap
.new_id
.expect("an overlapping reload spawns at once");
assert_eq!(
actor.sheep[&new_id].entry.instance, actor.sheep[&0].entry.instance,
"the replacement takes the same instance slot"
);
assert_eq!(
actor.sheep[&0].entry.status,
ProcStatus::Stopping,
"the instance being replaced is still there, marked and serving"
);
}
#[tokio::test(start_paused = true)]
async fn a_serial_reload_spawns_into_the_slot_the_drain_emptied() {
let dir = tempfile::tempdir().unwrap();
let (mut actor, _mailbox) = actor_with_one_online_sheep_of(
&dir,
probed_app("web"),
vec![ProcScript::never_exits()],
);
let (ctl_tx, _ctl_rx) = mpsc::channel(SHEEP_CTL_CAPACITY);
actor.sheep.get_mut(&0).expect("the fixture's sheep").ctl = Some(ctl_tx);
let instance = actor.sheep[&0].entry.instance;
let restarts = actor.sheep[&0].entry.restarts;
actor.advance_reload("web", VecDeque::from([0]));
actor.handle_exited(
0,
ExitOutcome {
code: Some(0),
signal: None,
},
);
assert!(
!actor.sheep.contains_key(&0),
"the drained instance is deregistered, not left as a second row"
);
let new_id = actor.reloads["web"]
.swap
.new_id
.expect("the reap is where a serial reload spawns");
let replacement = &actor.sheep[&new_id].entry;
assert_eq!(replacement.instance, instance);
assert_eq!(replacement.restarts, restarts);
assert_eq!(replacement.status, ProcStatus::Starting);
assert_eq!(replacement.reload, ReloadState::Replacement);
assert_eq!(
actor.reloads["web"].swap.phase,
ReloadPhase::DrainOld,
"with the drainee gone there is nothing left to abandon back to"
);
}
#[tokio::test(start_paused = true)]
async fn a_reload_can_still_replace_the_instance_a_failed_reload_parked() {
let dir = tempfile::tempdir().unwrap();
let (mut actor, _mailbox) = actor_with_one_online_sheep_of(&dir, probed_app("web"), vec![]);
let (ctl_tx, _ctl_rx) = mpsc::channel(SHEEP_CTL_CAPACITY);
let slot = actor.sheep.get_mut(&0).expect("the fixture's sheep");
slot.ctl = Some(ctl_tx);
slot.entry.status = ProcStatus::Starting;
slot.ready_failed = true;
let (reply, _rx) = oneshot::channel();
actor.handle_reload(&ProcessSelector::Name("web".to_string()), reply);
assert!(
actor.reloads.contains_key("web"),
"a parked instance is still replaceable, or nothing can roll it back"
);
assert_eq!(actor.reloads["web"].swap.old_id, 0);
}
#[tokio::test(start_paused = true)]
async fn an_ordinarily_starting_sheep_is_still_skipped_by_a_reload() {
let dir = tempfile::tempdir().unwrap();
let (mut actor, _mailbox) = actor_with_one_online_sheep_of(&dir, probed_app("web"), vec![]);
let (ctl_tx, _ctl_rx) = mpsc::channel(SHEEP_CTL_CAPACITY);
let slot = actor.sheep.get_mut(&0).expect("the fixture's sheep");
slot.ctl = Some(ctl_tx);
slot.entry.status = ProcStatus::Starting;
let (reply, _rx) = oneshot::channel();
actor.handle_reload(&ProcessSelector::Name("web".to_string()), reply);
assert!(
actor.reloads.is_empty(),
"a sheep still waiting on its own readiness is left to finish"
);
assert_eq!(actor.sheep[&0].entry.status, ProcStatus::Starting);
}
#[tokio::test(start_paused = true)]
async fn undoing_a_swap_never_promotes_a_parked_drainee_to_online() {
let dir = tempfile::tempdir().unwrap();
let mut app = probed_app("web");
app.reuse_port = true;
let (mut actor, _mailbox) = actor_with_one_online_sheep_of(&dir, app, vec![]);
let slot = actor.sheep.get_mut(&0).expect("the fixture's sheep");
slot.entry.status = ProcStatus::Starting;
slot.ready_failed = true;
actor
.spawn_replacement(0, ReloadMode::Overlap)
.expect_err("a runner with no scripts left cannot spawn");
let slot = actor.sheep.get(&0).expect("the drainee stays registered");
assert_eq!(
slot.entry.status,
ProcStatus::Starting,
"an instance that never proved itself is not restored to online"
);
assert_eq!(slot.entry.reload, ReloadState::None);
assert!(
slot.ready_failed,
"and it stays replaceable, or the next attempt cannot reach it"
);
}
#[tokio::test(start_paused = true)]
async fn abandoning_a_swap_never_promotes_a_parked_drainee_to_online() {
let dir = tempfile::tempdir().unwrap();
let mut app = probed_app("web");
app.reuse_port = true;
let (mut actor, _mailbox) =
actor_with_one_online_sheep_of(&dir, app, vec![ProcScript::never_exits()]);
let (ctl_tx, _ctl_rx) = mpsc::channel(SHEEP_CTL_CAPACITY);
let slot = actor.sheep.get_mut(&0).expect("the fixture's sheep");
slot.ctl = Some(ctl_tx);
slot.entry.status = ProcStatus::Starting;
slot.ready_failed = true;
let (reply, _rx) = oneshot::channel();
actor.handle_reload(&ProcessSelector::Name("web".to_string()), reply);
assert!(
actor.reloads.contains_key("web"),
"the parked instance is the one being replaced"
);
actor.abort_reload("web", "the replacement was not ready inside listen_timeout");
assert_eq!(
actor.sheep[&0].entry.status,
ProcStatus::Starting,
"an instance that never proved itself is not restored to online"
);
assert!(actor.sheep[&0].ready_failed);
}
#[test]
fn only_an_overlapping_probe_is_re_asked_after_the_drain() {
let dir = tempfile::tempdir().unwrap();
let mut probed = probed_app("web");
probed.reuse_port = true;
let mut channel = AppConfig::minimal("web", "./srv");
channel.wait_ready = true;
for (app, mode, want) in [
(probed.clone(), ReloadMode::Overlap, true),
(probed, ReloadMode::Serial, false),
(channel, ReloadMode::Overlap, false),
(
AppConfig::minimal("web", "./srv"),
ReloadMode::Overlap,
false,
),
] {
let (actor, _mailbox) = actor_with_one_online_sheep_of(&dir, app, vec![]);
assert_eq!(
actor.post_drain_probe(0, mode).is_some(),
want,
"mode={mode:?}"
);
}
}
fn actor_awaiting_a_verdict(
dir: &tempfile::TempDir,
) -> (Actor<ScriptedRunner>, mpsc::Receiver<Msg>) {
let mut app = probed_app("web");
app.reuse_port = true;
let (mut actor, mailbox) = actor_with_one_online_sheep_of(dir, app, vec![]);
actor
.sheep
.get_mut(&0)
.expect("the fixture's sheep")
.entry
.reload = ReloadState::Replacement;
actor.reloads.insert(
"web".to_string(),
ReloadJob {
queue: VecDeque::new(),
mode: ReloadMode::Overlap,
deadline: 0,
swap: ReloadSwap {
old_id: 99,
new_id: Some(0),
phase: ReloadPhase::Verify,
},
},
);
(actor, mailbox)
}
#[tokio::test(start_paused = true)]
async fn a_replacement_that_fails_the_second_probe_is_demoted_and_abandoned() {
let dir = tempfile::tempdir().unwrap();
let (mut actor, _mailbox) = actor_awaiting_a_verdict(&dir);
let mut rx = actor.events.subscribe();
actor.handle_reload_verified("web", 0, Readiness::TimedOut);
assert!(actor.reloads.is_empty(), "the reload is over either way");
assert!(
actor.sheep.contains_key(&0),
"the only instance the app has left is not killed as well"
);
assert_eq!(
actor.sheep[&0].entry.status,
ProcStatus::Starting,
"an instance that never answered alone is not online"
);
assert_eq!(
drained_process_kinds(&mut rx),
vec![ProcessEventKind::ReloadAbandoned],
"an abandonment, and never the `Reloaded` that claims a success"
);
}
#[tokio::test(start_paused = true)]
async fn a_replacement_that_answers_alone_finishes_the_reload() {
let dir = tempfile::tempdir().unwrap();
let (mut actor, _mailbox) = actor_awaiting_a_verdict(&dir);
let mut rx = actor.events.subscribe();
actor.handle_reload_verified("web", 0, Readiness::Ready);
assert!(actor.reloads.is_empty());
assert_eq!(actor.sheep[&0].entry.status, ProcStatus::Online);
assert_eq!(actor.sheep[&0].entry.reload, ReloadState::None);
assert_eq!(
drained_process_kinds(&mut rx),
vec![ProcessEventKind::Reloaded]
);
}
#[tokio::test(start_paused = true)]
async fn a_re_armed_watchdog_retires_the_one_it_replaced() {
let dir = tempfile::tempdir().unwrap();
let (mut actor, _mailbox) = actor_awaiting_a_verdict(&dir);
actor.arm_reload_deadline("web", 0);
let first = actor.reloads["web"].deadline;
actor.arm_reload_deadline("web", 0);
let second = actor.reloads["web"].deadline;
assert_ne!(first, second, "each arming takes a stamp of its own");
actor.handle_reload_deadline("web", first);
assert!(
actor.reloads.contains_key("web"),
"the retired watchdog must not end a swap the live one is still watching"
);
actor.handle_reload_deadline("web", second);
assert!(actor.reloads.is_empty(), "the live one still ends it");
}
#[tokio::test(start_paused = true)]
async fn a_delete_during_a_serial_drain_spawns_no_replacement() {
let dir = tempfile::tempdir().unwrap();
let (mut actor, _mailbox) = actor_with_one_online_sheep_of(&dir, probed_app("web"), vec![]);
let (ctl_tx, _ctl_rx) = mpsc::channel(SHEEP_CTL_CAPACITY);
actor.sheep.get_mut(&0).expect("the fixture's sheep").ctl = Some(ctl_tx);
actor.advance_reload("web", VecDeque::from([0]));
assert_eq!(actor.reloads["web"].swap.phase, ReloadPhase::DrainFirst);
actor.sheep.get_mut(&0).expect("the drainee").pending_delete = true;
actor.handle_exited(
0,
ExitOutcome {
code: Some(0),
signal: None,
},
);
assert!(actor.sheep.is_empty(), "the delete emptied the flock");
assert!(actor.reloads.is_empty(), "and the reload ended with it");
}
#[tokio::test(start_paused = true)]
async fn a_parked_instances_verdict_does_not_outlive_its_process() {
let dir = tempfile::tempdir().unwrap();
let mut app = probed_app("web");
app.autorestart = false;
let (mut actor, _mailbox) = actor_with_one_online_sheep_of(&dir, app, vec![]);
let (ctl_tx, _ctl_rx) = mpsc::channel(SHEEP_CTL_CAPACITY);
let slot = actor.sheep.get_mut(&0).expect("the fixture's sheep");
slot.ctl = Some(ctl_tx);
slot.entry.status = ProcStatus::Starting;
slot.ready_failed = true;
actor.handle_exited(
0,
ExitOutcome {
code: Some(0),
signal: None,
},
);
let slot = actor.sheep.get(&0).expect("a clean stop keeps the row");
assert_ne!(slot.entry.status, ProcStatus::Online);
assert!(!slot.ready_failed);
assert!(
!reload_eligible(slot),
"a row with no process behind it is not something a reload may replace"
);
}
const SWAP_WINDOW: Duration = Duration::from_secs(30);
async fn expect_event(
rx: &mut tokio::sync::broadcast::Receiver<SharedEvent>,
id: u32,
kind: ProcessEventKind,
) {
assert!(
tokio::time::timeout(SWAP_WINDOW, await_event(rx, id, kind))
.await
.is_ok(),
"no {kind:?} for id {id} within {SWAP_WINDOW:?}"
);
}
async fn started(
dir: &tempfile::TempDir,
app: AppConfig,
scripts: Vec<ProcScript>,
) -> (
SupervisorHandle,
Arc<ScriptedRunner>,
tokio::sync::broadcast::Receiver<SharedEvent>,
) {
let (events, rx) = crate::bus::test_bus(256);
let runner = Arc::new(ScriptedRunner::new(scripts));
let handle = spawn_supervisor(SharedRunner(Arc::clone(&runner)), test_paths(dir), events);
handle.start(vec![normalize(app).unwrap()]).await.unwrap();
(handle, runner, rx)
}
fn actor_with_one_online_sheep(
dir: &tempfile::TempDir,
scripts: Vec<ProcScript>,
) -> (Actor<ScriptedRunner>, mpsc::Receiver<Msg>) {
actor_with_one_online_sheep_of(dir, AppConfig::minimal("web", "./srv"), scripts)
}
fn actor_with_one_online_sheep_of(
dir: &tempfile::TempDir,
app: AppConfig,
scripts: Vec<ProcScript>,
) -> (Actor<ScriptedRunner>, mpsc::Receiver<Msg>) {
let paths = test_paths(dir);
let app = normalize(app).unwrap();
let mut sheep = HashMap::new();
sheep.insert(0, SheepSlot::new(armed_entry(0, 0, 1111, app, &paths)));
let (tx, rx) = mpsc::channel(MAILBOX_CAPACITY);
let actor = test_actor(paths, scripts, sheep, tx);
(actor, rx)
}
const SHEEP_ID: u32 = 0;
const DOG_ID: u32 = 1;
fn actor_with_a_sheep_and_a_dog(
dir: &tempfile::TempDir,
) -> (Actor<ScriptedRunner>, mpsc::Receiver<Msg>) {
let paths = test_paths(dir);
let mut sheep = HashMap::new();
for (id, name, dog) in [
(SHEEP_ID, "web", None),
(DOG_ID, "bark", Some(DogSource::BuiltIn)),
] {
let app = app_with(name, |config| config.fold = Some("svc".to_string()));
let mut entry = armed_entry(id, 0, 1111 + id, app, &paths);
entry.dog = dog;
sheep.insert(id, SheepSlot::new(entry));
}
let (tx, rx) = mpsc::channel(MAILBOX_CAPACITY);
let actor = test_actor(paths, Vec::new(), sheep, tx);
(actor, rx)
}
#[test]
fn a_wildcard_passes_a_dog_by_and_its_own_name_still_reaches_it() {
let dir = tempfile::tempdir().unwrap();
let (actor, _mailbox) = actor_with_a_sheep_and_a_dog(&dir);
assert_eq!(
actor.matching_ids(&ProcessSelector::All),
vec![SHEEP_ID],
"`all` is the flock, not the kennel"
);
assert_eq!(
actor.matching_ids(&ProcessSelector::parse("/^(web|bark)$/").unwrap()),
vec![SHEEP_ID],
"a sweep that spells both names out is still a sweep"
);
assert_eq!(
actor.matching_ids(&ProcessSelector::Fold("svc".into())),
vec![SHEEP_ID],
"a dog shares its fold with the flock and is still not swept by it"
);
assert_eq!(
actor.matching_ids(&ProcessSelector::Name("bark".into())),
vec![DOG_ID]
);
assert_eq!(
actor.matching_ids(&ProcessSelector::Id(DOG_ID)),
vec![DOG_ID]
);
}
#[test]
fn a_listing_reports_where_a_dog_came_from() {
let dir = tempfile::tempdir().unwrap();
let (actor, _mailbox) = actor_with_a_sheep_and_a_dog(&dir);
assert_eq!(
to_info(&actor.sheep[&DOG_ID].entry, &actor.smits).dog,
Some(DogSource::BuiltIn)
);
assert_eq!(
to_info(&actor.sheep[&SHEEP_ID].entry, &actor.smits).dog,
None
);
}
async fn start_app(h: &Harness, app: AppConfig) -> Vec<ProcessInfo> {
h.ctx
.supervisor
.start(vec![normalize(app).unwrap()])
.await
.unwrap()
}
async fn instance_slots_of(h: &Harness, name: &str) -> Vec<u32> {
let mut slots: Vec<u32> = h
.ctx
.supervisor
.list()
.await
.iter()
.filter(|info| info.name == name)
.map(|info| {
let out = info
.out_file
.as_deref()
.expect("a listed sheep has a log path");
let stem = std::path::Path::new(out)
.file_name()
.and_then(std::ffi::OsStr::to_str)
.and_then(|file| file.strip_suffix("-out.log"))
.and_then(|stem| stem.strip_prefix(&format!("{name}-")))
.expect("a derived log path is `<name>-<instance>-out.log`");
stem.parse().expect("the instance slot is a number")
})
.collect();
slots.sort_unstable();
slots
}
async fn settle_to(h: &Harness, name: &str, count: usize) {
let settled = tokio::time::timeout(SWAP_WINDOW, async {
loop {
let live = h
.ctx
.supervisor
.list()
.await
.iter()
.filter(|info| info.name == name)
.count();
if live == count {
return;
}
tokio::task::yield_now().await;
}
})
.await;
assert!(settled.is_ok(), "{name} never settled to {count} instances");
}
fn actor_with_a_scaled_app(
dir: &tempfile::TempDir,
instances: u32,
scripts: Vec<ProcScript>,
) -> Actor<ScriptedRunner> {
let paths = test_paths(dir);
let app = normalize(AppConfig {
instances,
..AppConfig::minimal("web", "./srv")
})
.unwrap();
let mut sheep = HashMap::new();
for instance in 0..instances {
sheep.insert(
instance,
SheepSlot::new(armed_entry(
instance,
instance,
1111 + instance,
app.clone(),
&paths,
)),
);
}
let (tx, _rx) = mpsc::channel(MAILBOX_CAPACITY);
test_actor(paths, scripts, sheep, tx)
}
fn stored_instance_counts(actor: &Actor<ScriptedRunner>) -> Vec<u32> {
let mut ids: Vec<u32> = actor.sheep.keys().copied().collect();
ids.sort_unstable();
ids.iter()
.map(|id| actor.sheep[id].entry.spec.config().instances)
.collect()
}
fn reload_job_for(name: &str) -> ReloadJob {
let _ = name;
ReloadJob {
queue: VecDeque::new(),
mode: ReloadMode::Overlap,
deadline: 0,
swap: ReloadSwap {
old_id: 0,
new_id: Some(1),
phase: ReloadPhase::AwaitReady,
},
}
}
#[tokio::test(start_paused = true)]
async fn scaling_up_fills_the_lowest_free_slots() {
let h = harness(vec![ProcScript::never_exits(); 4]);
start_app(
&h,
AppConfig {
instances: 2,
..AppConfig::minimal("web", "./srv")
},
)
.await;
let scaled = h.ctx.supervisor.scale("web", 4).await.unwrap();
assert_eq!(scaled.instances.len(), 4);
assert_eq!(instance_slots_of(&h, "web").await, vec![0, 1, 2, 3]);
}
#[tokio::test(start_paused = true)]
async fn scaling_down_removes_the_highest_slots_so_a_round_trip_returns() {
let h = harness(vec![ProcScript::never_exits(); 4]);
start_app(
&h,
AppConfig {
instances: 2,
..AppConfig::minimal("web", "./srv")
},
)
.await;
h.ctx.supervisor.scale("web", 4).await.unwrap();
let scaled = h.ctx.supervisor.scale("web", 2).await.unwrap();
assert_eq!(scaled.instances.len(), 2);
settle_to(&h, "web", 2).await;
assert_eq!(instance_slots_of(&h, "web").await, vec![0, 1]);
}
#[tokio::test(start_paused = true)]
async fn scaling_to_the_current_count_is_a_no_op() {
let h = harness(vec![ProcScript::never_exits(); 2]);
start_app(
&h,
AppConfig {
instances: 2,
..AppConfig::minimal("web", "./srv")
},
)
.await;
let before = h.ctx.supervisor.list().await;
let scaled = h.ctx.supervisor.scale("web", 2).await.unwrap();
assert_eq!(scaled.instances.len(), 2);
let after = h.ctx.supervisor.list().await;
assert_eq!(
after.iter().map(|i| i.id).collect::<Vec<_>>(),
before.iter().map(|i| i.id).collect::<Vec<_>>(),
"a no-op scale replaced processes"
);
}
#[tokio::test(start_paused = true)]
async fn scaling_to_zero_is_refused_and_names_delete() {
let h = harness(vec![ProcScript::never_exits()]);
start_app(&h, AppConfig::minimal("web", "./srv")).await;
let err = h.ctx.supervisor.scale("web", 0).await.unwrap_err();
let SupervisorError::InvalidScale(message) = err else {
panic!("expected InvalidScale, got {err:?}");
};
assert!(message.contains("delete"), "{message}");
}
#[tokio::test(start_paused = true)]
async fn scaling_an_unregistered_app_is_not_found() {
let h = harness(vec![]);
assert_eq!(
h.ctx.supervisor.scale("ghost", 2).await.unwrap_err(),
SupervisorError::NotFound
);
}
#[tokio::test(start_paused = true)]
async fn a_dog_cannot_be_scaled() {
let dir = tempfile::tempdir().unwrap();
let (mut actor, _mailbox) = actor_with_a_sheep_and_a_dog(&dir);
let (reply, answer) = oneshot::channel();
actor.handle_command(Command::Scale {
name: "bark".to_string(),
count: 2,
reply,
});
let SupervisorError::InvalidScale(message) = answer.await.unwrap().unwrap_err() else {
panic!("expected InvalidScale");
};
assert!(message.contains("dog"), "{message}");
}
#[tokio::test(start_paused = true)]
async fn an_app_mid_reload_refuses_a_scale() {
let dir = tempfile::tempdir().unwrap();
let mut actor = actor_with_a_scaled_app(&dir, 2, vec![]);
actor
.reloads
.insert("web".to_string(), reload_job_for("web"));
let (reply, answer) = oneshot::channel();
actor.handle_command(Command::Scale {
name: "web".to_string(),
count: 4,
reply,
});
assert_eq!(
answer.await.unwrap().unwrap_err(),
SupervisorError::ReloadInFlight("web".to_string())
);
}
#[tokio::test(start_paused = true)]
async fn a_scale_is_refused_while_an_earlier_ones_departures_are_still_leaving() {
let h = harness(vec![
ProcScript::never_exits(),
ProcScript::never_reports_its_exit(),
ProcScript::never_reports_its_exit(),
ProcScript::never_reports_its_exit(),
]);
start_app(
&h,
AppConfig {
instances: 4,
..AppConfig::minimal("web", "./srv")
},
)
.await;
let down = h.ctx.supervisor.scale("web", 1).await.unwrap();
assert_eq!(down.instances.len(), 1);
let err = h.ctx.supervisor.scale("web", 4).await.unwrap_err();
let SupervisorError::InvalidScale(message) = err else {
panic!("expected InvalidScale, got {err:?}");
};
assert!(
message.contains("3 instance(s) still shutting down"),
"the refusal has to say how much of the flock is still moving: {message}"
);
assert!(
message.contains("shep flock"),
"the refusal has to name what to wait for: {message}"
);
}
#[tokio::test(start_paused = true)]
async fn a_scale_is_accepted_again_once_the_departures_have_left() {
let h = harness(vec![ProcScript::never_exits(); 7]);
start_app(
&h,
AppConfig {
instances: 4,
..AppConfig::minimal("web", "./srv")
},
)
.await;
h.ctx.supervisor.scale("web", 1).await.unwrap();
settle_to(&h, "web", 1).await;
let up = h.ctx.supervisor.scale("web", 4).await.unwrap();
assert_eq!(up.instances.len(), 4);
assert_eq!(up.shortfall, None);
assert_eq!(instance_slots_of(&h, "web").await, vec![0, 1, 2, 3]);
}
#[tokio::test(start_paused = true)]
async fn a_scale_updates_the_stored_instance_count_on_every_slot() {
let dir = tempfile::tempdir().unwrap();
let mut actor = actor_with_a_scaled_app(&dir, 2, vec![ProcScript::never_exits(); 2]);
let (reply, answer) = oneshot::channel();
actor.handle_command(Command::Scale {
name: "web".to_string(),
count: 4,
reply,
});
let scaled = answer.await.unwrap().unwrap();
assert_eq!(scaled.app.config().instances, 4);
assert_eq!(stored_instance_counts(&actor), vec![4, 4, 4, 4]);
}
#[tokio::test(start_paused = true)]
async fn a_partial_scale_up_stores_the_count_it_achieved() {
let dir = tempfile::tempdir().unwrap();
let mut actor = actor_with_a_scaled_app(&dir, 1, vec![ProcScript::never_exits()]);
let (reply, answer) = oneshot::channel();
actor.handle_command(Command::Scale {
name: "web".to_string(),
count: 3,
reply,
});
let scaled = answer.await.unwrap().expect(
"a partial scale-up is a partial success: an `Err` here would take \
the achieved config with it and leave the roll pre-scale",
);
assert_eq!(scaled.requested, 3);
assert_eq!(scaled.achieved(), 2);
assert!(
scaled.shortfall.is_some(),
"the shortfall has to survive the reply, or nothing downstream can \
tell the operator they got two of three"
);
assert_eq!(scaled.app.config().instances, 2);
assert_eq!(
stored_instance_counts(&actor),
vec![2, 2, 2],
"the flock achieved two, so every registered slot — including the \
errored attempt — must say two"
);
}
#[tokio::test]
async fn a_listing_groups_an_apps_instances_under_its_name() {
let h = harness(vec![ProcScript::never_exits(); 4]);
start_app(
&h,
AppConfig {
instances: 2,
..AppConfig::minimal("zebra", "./z")
},
)
.await;
start_app(
&h,
AppConfig {
instances: 2,
..AppConfig::minimal("alpha", "./a")
},
)
.await;
let listed = h.ctx.supervisor.list().await;
let names: Vec<&str> = listed.iter().map(|i| i.name.as_str()).collect();
assert_eq!(names, ["alpha", "alpha", "zebra", "zebra"]);
let ids: Vec<u32> = listed.iter().map(|i| i.id).collect();
assert_ne!(
ids,
{
let mut sorted = ids.clone();
sorted.sort_unstable();
sorted
},
"the fixture must make id order and name order disagree, or it proves nothing"
);
}
fn dog_app(name: &str) -> ResolvedApp {
normalize(AppConfig::minimal(name, "/nonexistent/shep")).unwrap()
}
fn dog_row(listed: &[ProcessInfo], id: u32) -> ProcessInfo {
listed
.iter()
.find(|info| info.id == id)
.unwrap_or_else(|| panic!("id {id} left the flock: {listed:?}"))
.clone()
}
#[tokio::test(start_paused = true)]
async fn a_dog_that_restarts_is_still_a_dog() {
let dir = tempfile::tempdir().unwrap();
let (events, mut rx) = crate::bus::test_bus(64);
let runner =
ScriptedRunner::new(vec![ProcScript::const_exit(1), ProcScript::never_exits()]);
let handle = spawn_supervisor(runner, test_paths(&dir), events);
let dog = handle
.start_dog(dog_app("bark"), DogSource::BuiltIn)
.await
.unwrap();
assert_eq!(dog.dog, Some(DogSource::BuiltIn));
expect_event(&mut rx, dog.id, ProcessEventKind::Restart).await;
let after = dog_row(&handle.list().await, dog.id);
assert_eq!(
after.restarts, 1,
"the ordinary restart path, not a dog one"
);
assert_eq!(after.dog, Some(DogSource::BuiltIn));
}
#[tokio::test(start_paused = true)]
async fn a_dog_that_cannot_be_spawned_is_still_a_dog() {
let dir = tempfile::tempdir().unwrap();
let (events, _rx) = crate::bus::test_bus(64);
let handle = spawn_supervisor(ScriptedRunner::new(Vec::new()), test_paths(&dir), events);
let failed = handle
.start_dog(dog_app("bark"), DogSource::BuiltIn)
.await
.expect_err("a spawn with no script behind it cannot succeed");
assert!(matches!(failed, SupervisorError::SpawnFailed(_)));
let listed = handle.list().await;
let errored = dog_row(&listed, 0);
assert_eq!(errored.status, ProcStatus::Errored);
assert_eq!(errored.dog, Some(DogSource::BuiltIn));
}
#[tokio::test(start_paused = true)]
async fn an_app_in_the_gate_set_holds_at_starting_without_a_probe() {
let dir = tempfile::tempdir().unwrap();
let (mut actor, _mailbox) =
actor_with_one_online_sheep(&dir, vec![ProcScript::never_exits()]);
let (reply, rx) = oneshot::channel();
let mut app = AppConfig::minimal("db", "./db");
app.listen_timeout = UpDuration::from_millis(50);
actor.handle_command(Command::Start {
apps: vec![normalize(app).unwrap()],
policy: BatchPolicy::AllOrNothing,
gate: BTreeSet::from(["db".to_string()]),
reply,
});
let started = rx.await.unwrap().unwrap();
assert_eq!(started[0].status, ProcStatus::Starting);
}
#[tokio::test(start_paused = true)]
async fn an_app_outside_the_gate_set_is_online_at_spawn() {
let dir = tempfile::tempdir().unwrap();
let (mut actor, _mailbox) =
actor_with_one_online_sheep(&dir, vec![ProcScript::never_exits()]);
let (reply, rx) = oneshot::channel();
actor.handle_command(Command::Start {
apps: vec![normalize(AppConfig::minimal("db", "./db")).unwrap()],
policy: BatchPolicy::AllOrNothing,
gate: BTreeSet::new(),
reply,
});
let started = rx.await.unwrap().unwrap();
assert_eq!(started[0].status, ProcStatus::Online);
}
#[cfg(unix)]
#[tokio::test(start_paused = true)]
async fn a_credential_failure_never_shifts_another_apps_identity() {
let dir = tempfile::tempdir().unwrap();
let (mut actor, _mailbox) =
actor_with_one_online_sheep(&dir, vec![ProcScript::never_exits(); 2]);
let own_uid = nix::unistd::geteuid();
let own_user = nix::unistd::User::from_uid(own_uid)
.expect("the passwd database is readable")
.expect("this process's own uid has a passwd entry")
.name;
let mut bad = AppConfig::minimal("a-bad", "./a");
bad.user = Some("definitely-not-a-real-shep-user".to_string());
let mut mine = AppConfig::minimal("b-mine", "./b");
mine.user = Some(own_user);
let plain = AppConfig::minimal("c-plain", "./c");
let (reply, answer) = oneshot::channel();
actor.handle_command(Command::Start {
apps: vec![
normalize(bad).unwrap(),
normalize(mine).unwrap(),
normalize(plain).unwrap(),
],
policy: BatchPolicy::PerApp,
gate: BTreeSet::new(),
reply,
});
let err = answer
.await
.expect("the actor answers every Start")
.expect_err("one app has no resolvable user");
assert!(
err.to_string().contains("a-bad"),
"the failure must name the app whose user could not be resolved: {err}"
);
let identity = |name: &str| {
actor
.sheep
.values()
.find(|slot| slot.entry.spec.config().name == name)
.map(|slot| slot.entry.credentials)
};
assert_eq!(
identity("a-bad"),
Some(SpawnIdentity::Unresolved),
"an app with no resolvable user is registered `Errored` so it is \
visible, and must hold NO identity: `Unresolved` is what stops a \
later restart reusing one, and it is also the proof this app did \
not inherit b-mine's"
);
assert_eq!(
identity("b-mine"),
Some(SpawnIdentity::Resolved(Some(Credentials {
uid: own_uid.as_raw(),
gid: None
}))),
"the app that asked for a user must run as that user"
);
assert_eq!(
identity("c-plain"),
Some(SpawnIdentity::Resolved(None)),
"the app that asked for no user must be registered, and must run \
as the daemon rather than as somebody else's uid. `Resolved(None)` \
rather than a bare `None`: this app was ASKED, and answered \
nobody, which is not the same fact as never having been asked"
);
}
#[tokio::test(start_paused = true)]
async fn an_all_or_nothing_start_stops_at_the_first_failed_spawn() {
let dir = tempfile::tempdir().unwrap();
let (events, _rx) = crate::bus::test_bus(64);
let handle = spawn_supervisor(
ScriptedRunner::new(vec![ProcScript::never_exits()]).failing_to_spawn(&["first"]),
test_paths(&dir),
events,
);
let err = handle
.start(vec![
normalize(AppConfig::minimal("first", "./first")).unwrap(),
normalize(AppConfig::minimal("second", "./second")).unwrap(),
])
.await
.expect_err("the first app cannot spawn");
assert!(matches!(err, SupervisorError::SpawnFailed(_)), "{err:?}");
let listed = handle.list().await;
assert_eq!(
listed
.iter()
.map(|i| (i.name.as_str(), i.status))
.collect::<Vec<_>>(),
vec![("first", ProcStatus::Errored)],
"an operator's `shep start` must not go on past a failure: the \
second app is never reached: {listed:?}"
);
handle.shutdown().await;
}
#[tokio::test(start_paused = true)]
async fn a_dog_that_cannot_spawn_is_registered_errored_rather_than_refused() {
let dir = tempfile::tempdir().unwrap();
let (events, _rx) = crate::bus::test_bus(64);
let runner = ScriptedRunner::new(Vec::new()).refusing(&["bark"]);
let handle = spawn_supervisor(runner, test_paths(&dir), events);
let err = handle
.start_dog(dog_app("bark"), DogSource::BuiltIn)
.await
.expect_err("nothing can spawn against an empty script pool");
assert!(
matches!(err, SupervisorError::SpawnFailed(_)),
"a dog must reach its spawn and fail there, not be refused \
before it registers: {err:?}"
);
let listed = handle.list().await;
assert_eq!(
listed
.iter()
.map(|i| (i.name.as_str(), i.status, i.dog.clone()))
.collect::<Vec<_>>(),
vec![("bark", ProcStatus::Errored, Some(DogSource::BuiltIn))],
"the dogs table must still show the dog, and show it broken"
);
handle.shutdown().await;
}
#[tokio::test(start_paused = true)]
async fn a_reloaded_dog_is_still_a_dog() {
let dir = tempfile::tempdir().unwrap();
let (events, mut rx) = crate::bus::test_bus(256);
let runner = ScriptedRunner::new(vec![ProcScript::never_exits(); 3]);
let handle = spawn_supervisor(runner, test_paths(&dir), events);
let dog = handle
.start_dog(dog_app("bark"), DogSource::BuiltIn)
.await
.unwrap();
handle
.reload(ProcessSelector::Name("bark".to_string()))
.await
.expect("a reload that names the dog is accepted");
let replacement = dog.id + 1;
expect_event(&mut rx, replacement, ProcessEventKind::Reloaded).await;
let listed = handle.list().await;
assert_eq!(
listed.len(),
1,
"the swap is over, not in flight: {listed:?}"
);
assert_eq!(
dog_row(&listed, replacement).dog,
Some(DogSource::BuiltIn),
"the half that arrived is the same dog the half that left was"
);
}
#[tokio::test(start_paused = true)]
async fn a_dog_is_refused_once_a_shutdown_has_begun() {
let dir = tempfile::tempdir().unwrap();
let (mut actor, _mailbox) =
actor_with_one_online_sheep(&dir, vec![ProcScript::never_exits()]);
actor.shutting_down = true;
let (reply, rx) = oneshot::channel();
actor.handle_command(Command::StartDog {
app: Box::new(dog_app("bark")),
source: DogSource::BuiltIn,
reply,
});
assert_eq!(rx.await, Ok(Err(SupervisorError::EngineStopped)));
assert_eq!(actor.sheep.len(), 1, "nothing new was registered");
}
#[tokio::test(start_paused = true)]
async fn enabling_a_dog_twice_starts_one_process() {
let dir = tempfile::tempdir().unwrap();
let (events, _rx) = crate::bus::test_bus(64);
let runner = ScriptedRunner::new(vec![ProcScript::never_exits(); 2]);
let handle = spawn_supervisor(runner, test_paths(&dir), events);
let first = handle
.start_dog(dog_app("bark"), DogSource::BuiltIn)
.await
.unwrap();
let second = handle
.start_dog(dog_app("bark"), DogSource::BuiltIn)
.await
.unwrap();
assert_eq!(first.id, second.id);
assert_eq!(second.pid, first.pid, "the same process, not a fresh one");
let listed = handle.list().await;
assert_eq!(listed.iter().filter(|i| i.name == "bark").count(), 1);
}
struct RefusesOneSpawn {
inner: ScriptedRunner,
refuse: usize,
attempts: Arc<AtomicUsize>,
}
impl fmt::Debug for RefusesOneSpawn {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("RefusesOneSpawn").finish_non_exhaustive()
}
}
impl ProcessRunner for RefusesOneSpawn {
type Proc = crate::fake::FakeProc;
fn spawn(
&self,
spec: &SpawnSpec,
) -> Result<(Self::Proc, ProcIo), crate::runner::RunnerError> {
let nth = self.attempts.fetch_add(1, AtomicOrdering::SeqCst);
if nth == self.refuse {
return Err(crate::runner::RunnerError::SpawnFailed(
"refused by the fixture".to_string(),
));
}
self.inner.spawn(spec)
}
}
#[tokio::test(start_paused = true)]
async fn a_swap_puts_each_half_of_a_reload_on_the_entry_that_owns_it() {
let dir = tempfile::tempdir().unwrap();
let (mut actor, _mailbox) =
actor_with_one_online_sheep(&dir, vec![ProcScript::never_exits()]);
let new_id = actor
.spawn_replacement(0, ReloadMode::Overlap)
.expect("the fixture's one script covers this spawn");
assert_ne!(new_id, 0, "a replacement never reuses the drainee's id");
let drainee = &actor.sheep[&0].entry;
let replacement = &actor.sheep[&new_id].entry;
assert_eq!(drainee.status, ProcStatus::Stopping);
assert_eq!(
drainee.reload,
ReloadState::Drainee {
new_id: Some(new_id)
}
);
assert_ne!(
replacement.status,
ProcStatus::Stopping,
"`Stopping` belongs to the instance going away, not the one arriving"
);
assert_eq!(replacement.status, ProcStatus::Starting);
assert_eq!(replacement.reload, ReloadState::Replacement);
assert_eq!(
replacement.instance, drainee.instance,
"a replacement takes the drainee's instance slot, or an app deriving \
its port from it binds a different one and nothing overlaps"
);
}
#[tokio::test(start_paused = true)]
async fn a_reload_is_refused_once_a_shutdown_has_begun() {
let dir = tempfile::tempdir().unwrap();
let (mut actor, _mailbox) =
actor_with_one_online_sheep(&dir, vec![ProcScript::never_exits()]);
actor.shutting_down = true;
let (reply, rx) = oneshot::channel();
actor.handle_command(Command::Reload {
selector: ProcessSelector::All,
reply,
});
assert_eq!(rx.await, Ok(Err(SupervisorError::EngineStopped)));
actor.advance_reload("web", VecDeque::from([0]));
assert_eq!(actor.sheep.len(), 1, "nothing new was registered");
assert_eq!(actor.sheep[&0].entry.status, ProcStatus::Online);
assert_eq!(actor.sheep[&0].entry.reload, ReloadState::None);
assert!(actor.reloads.is_empty(), "no job was started");
}
#[tokio::test(start_paused = true)]
async fn a_reload_gives_the_replacement_a_new_id_in_the_drainees_slot() {
let dir = tempfile::tempdir().unwrap();
let (handle, runner, mut rx) = started(
&dir,
AppConfig::minimal("web", "./srv"),
vec![ProcScript::never_exits(), ProcScript::never_exits()],
)
.await;
let before = handle.list().await;
let accepted = handle
.reload(ProcessSelector::Name("web".to_string()))
.await
.expect("the reload is accepted");
assert_eq!(
accepted.iter().map(|info| info.id).collect::<Vec<_>>(),
vec![0],
"the answer is the flock as it stood when the reload was accepted"
);
expect_event(&mut rx, 1, ProcessEventKind::Online).await;
expect_event(&mut rx, 0, ProcessEventKind::Delete).await;
let after = handle.list().await;
assert_eq!(after.len(), 1, "the drainee's registration goes with it");
assert_eq!(after[0].id, 1);
assert_eq!(after[0].status, ProcStatus::Online);
assert_eq!(after[0].out_file, before[0].out_file);
assert_eq!(after[0].err_file, before[0].err_file);
assert_eq!(
runner.kill_counts().len(),
2,
"one original and one replacement, and nothing else"
);
}
#[tokio::test(start_paused = true)]
async fn a_reload_stops_counting_the_drainee_as_running_before_its_replacement_starts() {
let dir = tempfile::tempdir().unwrap();
let (handle, _runner, mut rx) = started(
&dir,
AppConfig::minimal("web", "./srv"),
vec![ProcScript::never_exits(), ProcScript::never_exits()],
)
.await;
handle
.reload(ProcessSelector::Name("web".to_string()))
.await
.expect("the reload is accepted");
expect_event(&mut rx, 1, ProcessEventKind::Start).await;
let mid = handle.list().await;
assert_eq!(mid.len(), 2, "both entries are registered mid-swap");
assert_eq!(mid[0].status, ProcStatus::Stopping, "the drainee");
assert_eq!(mid[1].status, ProcStatus::Starting, "its replacement");
let running = mid
.iter()
.filter(|info| {
matches!(
info.status,
ProcStatus::Online | ProcStatus::Starting | ProcStatus::WaitingRestart
)
})
.count();
assert_eq!(
running, 1,
"a one-instance app must never count as two running instances"
);
}
#[tokio::test(start_paused = true)]
async fn a_reload_of_an_app_with_no_readiness_signal_completes_at_its_deadline() {
let dir = tempfile::tempdir().unwrap();
let mut app = AppConfig::minimal("web", "./srv");
app.listen_timeout = UpDuration::from_millis(2_500);
let (handle, _runner, mut rx) = started(
&dir,
app,
vec![ProcScript::never_exits(), ProcScript::never_exits()],
)
.await;
let start = tokio::time::Instant::now();
handle
.reload(ProcessSelector::Name("web".to_string()))
.await
.expect("the reload is accepted");
expect_event(&mut rx, 1, ProcessEventKind::Online).await;
assert_eq!(
tokio::time::Instant::now() - start,
Duration::from_millis(2_500)
);
expect_event(&mut rx, 0, ProcessEventKind::Delete).await;
let after = handle.list().await;
assert_eq!(after.len(), 1);
assert_eq!(after[0].id, 1);
assert_eq!(after[0].status, ProcStatus::Online);
}
#[tokio::test(start_paused = true)]
async fn a_reload_commits_the_moment_the_replacement_signals_ready() {
let dir = tempfile::tempdir().unwrap();
let mut app = AppConfig::minimal("web", "./srv");
app.wait_ready = true;
let (handle, _runner, mut rx) = started(
&dir,
app,
vec![ProcScript::never_exits(), ProcScript::never_exits()],
)
.await;
handle.tx.send(Msg::Ready { id: 0 }).await.unwrap();
expect_event(&mut rx, 0, ProcessEventKind::Online).await;
let start = tokio::time::Instant::now();
handle
.reload(ProcessSelector::Name("web".to_string()))
.await
.expect("the reload is accepted");
expect_event(&mut rx, 1, ProcessEventKind::Start).await;
handle.tx.send(Msg::Ready { id: 1 }).await.unwrap();
expect_event(&mut rx, 1, ProcessEventKind::Online).await;
assert!(
tokio::time::Instant::now() - start < Duration::from_millis(3_000),
"the swap committed at the signal, not at `listen_timeout`"
);
expect_event(&mut rx, 0, ProcessEventKind::Delete).await;
assert_eq!(handle.list().await[0].id, 1);
}
#[tokio::test(start_paused = true)]
async fn a_replacement_that_never_becomes_ready_leaves_the_old_instance_serving() {
let dir = tempfile::tempdir().unwrap();
let mut app = AppConfig::minimal("web", "./srv");
app.wait_ready = true; let (handle, runner, mut rx) = started(
&dir,
app,
vec![ProcScript::never_exits(), ProcScript::never_exits()],
)
.await;
handle.tx.send(Msg::Ready { id: 0 }).await.unwrap();
expect_event(&mut rx, 0, ProcessEventKind::Online).await;
handle
.reload(ProcessSelector::Name("web".to_string()))
.await
.expect("the reload is accepted");
expect_event(&mut rx, 1, ProcessEventKind::Delete).await;
let after = handle.list().await;
assert_eq!(after.len(), 1, "the abandoned replacement is deregistered");
assert_eq!(after[0].id, 0);
assert_eq!(
after[0].status,
ProcStatus::Online,
"the instance that can serve keeps serving"
);
assert_eq!(
runner.signals(1),
vec![15],
"the replacement went through the stop ladder rather than being dropped"
);
assert_eq!(runner.kill_counts(), vec![0, 0], "neither needed SIGKILL");
}
#[tokio::test(start_paused = true)]
async fn a_reload_drains_the_old_instance_under_graceful_timeout() {
let dir = tempfile::tempdir().unwrap();
let (handle, runner, mut rx) = started(
&dir,
AppConfig::minimal("web", "./srv"),
vec![ProcScript::ignores_signals(), ProcScript::never_exits()],
)
.await;
let start = tokio::time::Instant::now();
handle
.reload(ProcessSelector::Name("web".to_string()))
.await
.expect("the reload is accepted");
expect_event(&mut rx, 1, ProcessEventKind::Online).await;
expect_event(&mut rx, 0, ProcessEventKind::Delete).await;
assert_eq!(
tokio::time::Instant::now() - start,
Duration::from_millis(11_000)
);
assert_eq!(
runner.kill_counts(),
vec![1, 0],
"only the defiant drainee reached the SIGKILL rung"
);
}
#[tokio::test(start_paused = true)]
async fn a_reload_replaces_a_clustered_apps_instances_one_at_a_time() {
let dir = tempfile::tempdir().unwrap();
let mut app = AppConfig::minimal("web", "./srv");
app.instances = 2;
let (handle, runner, mut rx) = started(&dir, app, vec![ProcScript::never_exits(); 4]).await;
handle
.reload(ProcessSelector::Name("web".to_string()))
.await
.expect("the reload is accepted");
expect_event(&mut rx, 2, ProcessEventKind::Start).await;
assert_eq!(
runner.kill_counts().len(),
3,
"only the first instance's replacement exists yet"
);
assert_eq!(
handle
.list()
.await
.iter()
.filter(|info| info.status == ProcStatus::Stopping)
.count(),
1,
"one instance is being replaced, and the other is untouched"
);
expect_event(&mut rx, 0, ProcessEventKind::Delete).await;
expect_event(&mut rx, 3, ProcessEventKind::Start).await;
assert_eq!(runner.kill_counts().len(), 4);
expect_event(&mut rx, 1, ProcessEventKind::Delete).await;
let after = handle.list().await;
assert_eq!(
after.iter().map(|info| info.id).collect::<Vec<_>>(),
vec![2, 3]
);
assert!(after.iter().all(|info| info.status == ProcStatus::Online));
}
#[tokio::test(start_paused = true)]
async fn a_replacement_that_cannot_be_spawned_leaves_every_instance_alone() {
let dir = tempfile::tempdir().unwrap();
let mut app = AppConfig::minimal("web", "./srv");
app.instances = 2;
let attempts = Arc::new(AtomicUsize::new(0));
let runner = RefusesOneSpawn {
inner: ScriptedRunner::new(vec![ProcScript::never_exits(); 4]),
refuse: 2,
attempts: Arc::clone(&attempts),
};
let (events, mut rx) = crate::bus::test_bus(256);
let handle = spawn_supervisor(runner, test_paths(&dir), events);
handle.start(vec![normalize(app).unwrap()]).await.unwrap();
handle
.reload(ProcessSelector::Name("web".to_string()))
.await
.expect("the reload is accepted before anything is spawned");
assert_no_event_within(&mut rx, 3, ProcessEventKind::Start, Duration::from_secs(10)).await;
let after = handle.list().await;
assert_eq!(
after.iter().map(|info| info.id).collect::<Vec<_>>(),
vec![0, 1]
);
assert!(
after.iter().all(|info| info.status == ProcStatus::Online),
"both old instances keep serving: {after:?}"
);
assert_eq!(
attempts.load(AtomicOrdering::SeqCst),
3,
"two originals and one refused replacement, and no second attempt"
);
}
#[tokio::test(start_paused = true)]
async fn a_second_reload_of_an_app_already_reloading_is_refused() {
let dir = tempfile::tempdir().unwrap();
let mut app = AppConfig::minimal("web", "./srv");
app.instances = 2;
let (handle, runner, mut rx) = started(&dir, app, vec![ProcScript::never_exits(); 4]).await;
handle
.reload(ProcessSelector::Name("web".to_string()))
.await
.expect("the first reload is accepted");
expect_event(&mut rx, 2, ProcessEventKind::Start).await;
let refused = handle.reload(ProcessSelector::All).await;
assert_eq!(
refused,
Err(SupervisorError::ReloadInFlight("web".to_string()))
);
assert_eq!(
runner.kill_counts().len(),
3,
"no second replacement was spawned"
);
assert_eq!(
handle.list().await.len(),
3,
"one drainee, its replacement, and the instance untouched so far"
);
}
#[tokio::test(start_paused = true)]
async fn a_reload_of_a_sheep_that_is_not_online_is_a_no_op_success() {
let dir = tempfile::tempdir().unwrap();
let (handle, runner, _rx) = started(
&dir,
AppConfig::minimal("web", "./srv"),
vec![ProcScript::never_exits()],
)
.await;
handle.stop(ProcessSelector::All).await.unwrap();
let reloaded = handle
.reload(ProcessSelector::All)
.await
.expect("a reload that has nothing to replace still succeeds");
assert_eq!(reloaded.len(), 1);
assert_eq!(reloaded[0].status, ProcStatus::Stopped);
assert_eq!(handle.list().await.len(), 1, "nothing was registered");
assert_eq!(runner.kill_counts().len(), 1, "nothing was spawned");
}
#[tokio::test(start_paused = true)]
async fn a_reload_skips_an_instance_whose_kill_ladder_is_already_running() {
let dir = tempfile::tempdir().unwrap();
let (handle, runner, mut rx) = started(
&dir,
AppConfig::minimal("web", "./srv"),
vec![ProcScript::ignores_signals(), ProcScript::never_exits()],
)
.await;
let pid = handle.list().await[0].pid.expect("a live sheep has a pid");
handle.extra_restart(0, pid, None, None).await;
let reloaded = handle
.reload(ProcessSelector::Name("web".to_string()))
.await
.expect("a reload with nothing left to replace still succeeds");
assert_eq!(reloaded.len(), 1, "the reload still answers for the match");
assert_eq!(
handle.list().await.len(),
1,
"no replacement was registered against an instance on its way out"
);
assert_eq!(runner.kill_counts().len(), 1, "nothing was spawned");
expect_event(&mut rx, 0, ProcessEventKind::Restart).await;
let after = handle.list().await;
assert_eq!(after.len(), 1);
assert_eq!(after[0].id, 0);
assert_eq!(
runner.kill_counts().len(),
2,
"the original and its respawn"
);
}
#[tokio::test(start_paused = true)]
async fn a_report_raised_against_a_drainee_never_takes_it_off_the_reload() {
let dir = tempfile::tempdir().unwrap();
let (handle, runner, mut rx) = started(
&dir,
AppConfig::minimal("web", "./srv"),
vec![ProcScript::never_exits(); 3],
)
.await;
let pid = handle.list().await[0].pid.expect("a live sheep has a pid");
handle
.reload(ProcessSelector::Name("web".to_string()))
.await
.expect("the reload is accepted");
expect_event(&mut rx, 1, ProcessEventKind::Start).await;
handle.extra_restart(0, pid, None, None).await;
assert_no_event_within(
&mut rx,
0,
ProcessEventKind::Restart,
Duration::from_millis(1_000),
)
.await;
assert_eq!(handle.list().await[0].status, ProcStatus::Stopping);
expect_event(&mut rx, 1, ProcessEventKind::Online).await;
expect_event(&mut rx, 0, ProcessEventKind::Delete).await;
assert_eq!(
runner.kill_counts().len(),
2,
"the report never caused a spawn"
);
}
#[tokio::test(start_paused = true)]
async fn a_drainee_whose_liveness_probe_fails_is_reaped_rather_than_restarted() {
let dir = tempfile::tempdir().unwrap();
let reserved = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
let addr = reserved.local_addr().unwrap();
drop(reserved);
let mut app = AppConfig::minimal("web", "./srv");
app.wait_ready = true;
app.listen_timeout = UpDuration::from_millis(60_000);
app.liveness_probe = Some(ProbeConfig {
interval: UpDuration::from_millis(1_000),
timeout: UpDuration::from_millis(500),
failure_threshold: 1,
..probe_config(ProbeKind::Tcp, &addr.to_string())
});
let (events, mut rx) = crate::bus::test_bus(256);
let runner = Arc::new(ScriptedRunner::new(vec![ProcScript::never_exits(); 4]));
let (breaches_tx, breaches_rx) = mpsc::channel(8);
let (liveness_tx, liveness_rx) = mpsc::channel(8);
let handle =
SupervisorBuilder::new(SharedRunner(Arc::clone(&runner)), test_paths(&dir), events)
.extras(Extras {
clock: Arc::new(SystemClock),
enforcer: Arc::new(RecordingEnforcer::default()),
max_cron_sleep: DEFAULT_MAX_CRON_SLEEP,
reports: ExtrasReports {
breaches: breaches_tx,
liveness: liveness_tx,
},
stats: idle_stats(),
})
.spawn();
let _reporter = spawn_extras_reporter(breaches_rx, liveness_rx, handle.clone());
handle.start(vec![normalize(app).unwrap()]).await.unwrap();
handle.tx.send(Msg::Ready { id: 0 }).await.unwrap();
expect_event(&mut rx, 0, ProcessEventKind::Online).await;
handle
.reload(ProcessSelector::Name("web".to_string()))
.await
.expect("the reload is accepted");
expect_event(&mut rx, 1, ProcessEventKind::Start).await;
assert_no_event_within(
&mut rx,
0,
ProcessEventKind::Restart,
Duration::from_secs(5),
)
.await;
assert_eq!(handle.list().await[0].status, ProcStatus::Stopping);
assert_eq!(
runner.kill_counts().len(),
2,
"the failing probe never caused a spawn"
);
handle.tx.send(Msg::Ready { id: 1 }).await.unwrap();
expect_event(&mut rx, 1, ProcessEventKind::Online).await;
expect_event(&mut rx, 0, ProcessEventKind::Delete).await;
let after = handle.list().await;
assert_eq!(after.len(), 1, "the drainee left no registration behind");
assert_eq!(after[0].id, 1);
expect_event(&mut rx, 1, ProcessEventKind::Restart).await;
}
#[tokio::test(start_paused = true)]
async fn a_drainee_that_exits_on_its_own_is_never_restarted() {
let dir = tempfile::tempdir().unwrap();
let (handle, runner, mut rx) = started(
&dir,
AppConfig::minimal("web", "./srv"),
vec![
ProcScript::stable_then_exit(1_000, 1),
ProcScript::never_exits(),
],
)
.await;
handle
.reload(ProcessSelector::Name("web".to_string()))
.await
.expect("the reload is accepted");
expect_event(&mut rx, 1, ProcessEventKind::Start).await;
expect_event(&mut rx, 0, ProcessEventKind::Delete).await;
expect_event(&mut rx, 1, ProcessEventKind::Online).await;
let after = handle.list().await;
assert_eq!(after.len(), 1, "the drainee left no registration behind");
assert_eq!(after[0].id, 1);
assert_eq!(after[0].status, ProcStatus::Online);
assert_eq!(runner.kill_counts().len(), 2, "the drainee never respawned");
}
#[tokio::test(start_paused = true)]
async fn an_operators_stop_mid_reload_leaves_the_app_stopped_and_registered() {
let dir = tempfile::tempdir().unwrap();
let (handle, runner, mut rx) = started(
&dir,
AppConfig::minimal("web", "./srv"),
vec![ProcScript::never_exits(), ProcScript::ignores_signals()],
)
.await;
handle
.reload(ProcessSelector::Name("web".to_string()))
.await
.expect("the reload is accepted");
expect_event(&mut rx, 1, ProcessEventKind::Start).await;
let stopped = handle
.stop(ProcessSelector::Name("web".to_string()))
.await
.expect("the stop reaches both entries");
assert_eq!(stopped.len(), 2, "a stop answers for every id it matched");
let after = handle.list().await;
assert_eq!(
after.iter().map(|info| info.id).collect::<Vec<_>>(),
vec![0],
"the instance stays registered; only the abandoned replacement goes"
);
assert_eq!(after[0].status, ProcStatus::Stopped);
assert_eq!(runner.kill_counts().len(), 2, "nothing was spawned again");
}
#[tokio::test(start_paused = true)]
async fn a_replacement_is_kept_but_not_online_when_the_deadline_elapses_with_no_drainee() {
let dir = tempfile::tempdir().unwrap();
let mut app = AppConfig::minimal("web", "./srv");
app.wait_ready = true; let (handle, runner, mut rx) = started(
&dir,
app,
vec![
ProcScript::stable_then_exit(1_000, 1),
ProcScript::never_exits(),
],
)
.await;
handle.tx.send(Msg::Ready { id: 0 }).await.unwrap();
expect_event(&mut rx, 0, ProcessEventKind::Online).await;
handle
.reload(ProcessSelector::Name("web".to_string()))
.await
.expect("the reload is accepted");
expect_event(&mut rx, 0, ProcessEventKind::Delete).await;
expect_event(&mut rx, 1, ProcessEventKind::ReloadAbandoned).await;
let after = handle.list().await;
assert_eq!(after.len(), 1, "the app still has its instance");
assert_eq!(after[0].id, 1);
assert_eq!(
after[0].status,
ProcStatus::Starting,
"a replacement that never answered is kept, and never called online"
);
assert_eq!(runner.kill_counts(), vec![0, 0], "neither was SIGKILLed");
}
#[tokio::test(start_paused = true)]
async fn an_abandoned_reload_never_reports_a_dying_drainee_as_online() {
let dir = tempfile::tempdir().unwrap();
let mut app = AppConfig::minimal("web", "./srv");
app.wait_ready = true;
let (handle, _runner, mut rx) = started(
&dir,
app,
vec![ProcScript::ignores_signals(), ProcScript::never_exits()],
)
.await;
handle.tx.send(Msg::Ready { id: 0 }).await.unwrap();
expect_event(&mut rx, 0, ProcessEventKind::Online).await;
handle
.reload(ProcessSelector::Name("web".to_string()))
.await
.expect("the reload is accepted");
expect_event(&mut rx, 1, ProcessEventKind::Start).await;
let stopper = {
let handle = handle.clone();
tokio::spawn(async move { handle.stop(ProcessSelector::Name("web".to_string())).await })
};
expect_event(&mut rx, 1, ProcessEventKind::Delete).await;
let mid = handle.list().await;
assert_eq!(mid.len(), 1, "only the abandoned replacement has gone");
assert_eq!(mid[0].id, 0);
assert_eq!(
mid[0].status,
ProcStatus::Stopping,
"a drainee an operator already claimed is not back to serving"
);
let stopped = stopper.await.unwrap().expect("the stop is answered");
assert_eq!(stopped.len(), 2, "a stop answers for every id it matched");
assert_eq!(handle.list().await[0].status, ProcStatus::Stopped);
}
#[tokio::test(start_paused = true)]
async fn an_automatic_restart_never_lands_on_either_half_of_a_swap() {
let dir = tempfile::tempdir().unwrap();
let mut app = AppConfig::minimal("web", "./srv");
app.wait_ready = true;
let (handle, runner, mut rx) = started(
&dir,
app,
vec![ProcScript::never_exits(), ProcScript::never_exits()],
)
.await;
handle.tx.send(Msg::Ready { id: 0 }).await.unwrap();
expect_event(&mut rx, 0, ProcessEventKind::Online).await;
handle
.reload(ProcessSelector::Name("web".to_string()))
.await
.expect("the reload is accepted");
expect_event(&mut rx, 1, ProcessEventKind::Start).await;
let restarted = handle
.restart_automatic(ProcessSelector::Name("web".to_string()))
.await
.expect("the selector matches both halves of the swap");
assert!(
restarted.is_empty(),
"neither half of a swap is an automatic restart's to take"
);
handle.tx.send(Msg::Ready { id: 1 }).await.unwrap();
expect_event(&mut rx, 1, ProcessEventKind::Online).await;
expect_event(&mut rx, 0, ProcessEventKind::Delete).await;
let after = handle.list().await;
assert_eq!(after.len(), 1);
assert_eq!(after[0].id, 1, "the replacement, not a restarted drainee");
assert_eq!(after[0].status, ProcStatus::Online);
assert_eq!(after[0].restarts, 0, "nothing counted a restart");
assert_eq!(
runner.kill_counts().len(),
2,
"one original and one replacement, and nothing else"
);
}
#[tokio::test(start_paused = true)]
async fn a_reload_ends_when_its_replacement_goes_with_the_drainee_already_gone() {
let dir = tempfile::tempdir().unwrap();
let mut app = AppConfig::minimal("web", "./srv");
app.wait_ready = true; let (handle, _runner, mut rx) = started(
&dir,
app,
vec![
ProcScript::stable_then_exit(1_000, 1),
ProcScript::never_exits(),
],
)
.await;
handle.tx.send(Msg::Ready { id: 0 }).await.unwrap();
expect_event(&mut rx, 0, ProcessEventKind::Online).await;
handle
.reload(ProcessSelector::Name("web".to_string()))
.await
.expect("the reload is accepted");
expect_event(&mut rx, 0, ProcessEventKind::Delete).await;
handle
.stop(ProcessSelector::Name("web".to_string()))
.await
.expect("the stop reaches the replacement");
let seen = events_through(&mut rx, 1, ProcessEventKind::Stop).await;
assert!(
at(&seen, 1, ProcessEventKind::ReloadAbandoned) < at(&seen, 1, ProcessEventKind::Stop),
"the reload gives up before the exit that ended it is reported, so a \
subscriber reads the two in the order they happened: {seen:?}"
);
let again = handle
.reload(ProcessSelector::Name("web".to_string()))
.await
.map(|infos| infos.iter().map(|info| info.id).collect::<Vec<_>>());
assert_eq!(
again,
Ok(vec![1]),
"the reload is over, so the app is reloadable again"
);
}
#[tokio::test(start_paused = true)]
async fn a_reload_carries_the_drainees_restart_count_to_its_replacement() {
let dir = tempfile::tempdir().unwrap();
let (handle, _runner, mut rx) = started(
&dir,
AppConfig::minimal("web", "./srv"),
vec![ProcScript::never_exits(); 3],
)
.await;
handle
.restart(ProcessSelector::Name("web".to_string()))
.await
.unwrap();
assert_eq!(handle.list().await[0].restarts, 1);
handle
.reload(ProcessSelector::Name("web".to_string()))
.await
.expect("the reload is accepted");
expect_event(&mut rx, 1, ProcessEventKind::Online).await;
expect_event(&mut rx, 0, ProcessEventKind::Delete).await;
let after = handle.list().await;
assert_eq!(after[0].id, 1);
assert_eq!(after[0].restarts, 1);
}
#[tokio::test(start_paused = true)]
async fn a_reload_carries_the_drainees_last_exit_to_its_replacement() {
let dir = tempfile::tempdir().unwrap();
let (handle, _runner, mut rx) = started(
&dir,
AppConfig::minimal("web", "./srv"),
vec![ProcScript::never_exits(); 3],
)
.await;
handle
.restart(ProcessSelector::Name("web".to_string()))
.await
.unwrap();
let restarted = handle.list().await;
let last_exit = restarted[0].last_exit;
assert!(
last_exit.is_some(),
"a manual restart is itself an exit, so this must not be None: {restarted:?}"
);
handle
.reload(ProcessSelector::Name("web".to_string()))
.await
.expect("the reload is accepted");
expect_event(&mut rx, 1, ProcessEventKind::Online).await;
expect_event(&mut rx, 0, ProcessEventKind::Delete).await;
let after = handle.list().await;
assert_eq!(after[0].id, 1);
assert_eq!(
after[0].last_exit, last_exit,
"a reload is not an exit -- the replacement must inherit the drainee's \
last_exit rather than reset it to None"
);
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
struct Seen {
id: u32,
kind: ProcessEventKind,
status: ProcStatus,
manually: bool,
}
async fn events_through(
rx: &mut tokio::sync::broadcast::Receiver<SharedEvent>,
id: u32,
kind: ProcessEventKind,
) -> Vec<Seen> {
let collect = async {
let mut seen = Vec::new();
loop {
match rx.recv().await.map(|event| event.to_event()) {
Ok(BusEvent::Process {
event,
info,
manually,
..
}) => {
seen.push(Seen {
id: info.id,
kind: event,
status: info.status,
manually,
});
if info.id == id && event == kind {
return seen;
}
}
Ok(_) => continue,
Err(tokio::sync::broadcast::error::RecvError::Lagged(n)) => {
panic!("the event stream lagged by {n}; no ordering claim survives that")
}
Err(e) => panic!("event stream closed before {kind:?} for id {id}: {e}"),
}
}
};
tokio::time::timeout(SWAP_WINDOW, collect)
.await
.unwrap_or_else(|_| panic!("no {kind:?} for id {id} within {SWAP_WINDOW:?}"))
}
fn at(seen: &[Seen], id: u32, kind: ProcessEventKind) -> usize {
seen.iter()
.position(|e| e.id == id && e.kind == kind)
.unwrap_or_else(|| panic!("no {kind:?} for id {id} in {seen:?}"))
}
#[tokio::test(start_paused = true)]
async fn a_completed_swap_reports_itself_on_the_bus() {
let dir = tempfile::tempdir().unwrap();
let (handle, _runner, mut rx) = started(
&dir,
AppConfig::minimal("web", "./srv"),
vec![ProcScript::never_exits(); 3],
)
.await;
handle
.reload(ProcessSelector::Name("web".to_string()))
.await
.expect("the reload is accepted");
let seen = events_through(&mut rx, 1, ProcessEventKind::Reloaded).await;
assert!(
at(&seen, 0, ProcessEventKind::Reload) < at(&seen, 1, ProcessEventKind::Start),
"the instance being replaced is named before its replacement starts: {seen:?}"
);
assert_eq!(
seen[at(&seen, 0, ProcessEventKind::Reload)],
Seen {
id: 0,
kind: ProcessEventKind::Reload,
status: ProcStatus::Stopping,
manually: true,
}
);
assert!(
at(&seen, 0, ProcessEventKind::Delete) < at(&seen, 1, ProcessEventKind::Reloaded),
"a swap is not over until the instance it replaced is gone: {seen:?}"
);
assert_eq!(
seen[at(&seen, 1, ProcessEventKind::Reloaded)],
Seen {
id: 1,
kind: ProcessEventKind::Reloaded,
status: ProcStatus::Online,
manually: true,
}
);
}
#[tokio::test(start_paused = true)]
async fn a_swap_is_not_announced_for_a_replacement_that_is_no_longer_serving() {
let dir = tempfile::tempdir().unwrap();
let mut app = AppConfig::minimal("web", "./srv");
app.autorestart = false; let (handle, _runner, mut rx) = started(
&dir,
app,
vec![
ProcScript::ignores_signals(),
ProcScript::stable_then_exit(5_000, 1),
],
)
.await;
handle
.reload(ProcessSelector::Name("web".to_string()))
.await
.expect("the reload is accepted");
let seen = events_through(&mut rx, 1, ProcessEventKind::ReloadAbandoned).await;
assert_eq!(
seen[at(&seen, 1, ProcessEventKind::ReloadAbandoned)],
Seen {
id: 1,
kind: ProcessEventKind::ReloadAbandoned,
status: ProcStatus::Stopped,
manually: true,
},
"a replacement that died inside the drain window is what the \
abandonment names, carrying the status it actually reached"
);
assert!(
!seen.iter().any(|e| e.kind == ProcessEventKind::Reloaded),
"no swap succeeded, so nothing may say one did: {seen:?}"
);
}
#[tokio::test(start_paused = true)]
async fn a_swap_whose_drainee_never_reports_its_exit_gives_up_on_its_own() {
let dir = tempfile::tempdir().unwrap();
let (handle, runner, mut rx) = started(
&dir,
AppConfig::minimal("web", "./srv"),
vec![
ProcScript::never_reports_its_exit(),
ProcScript::never_exits(),
ProcScript::never_exits(),
],
)
.await;
handle
.reload(ProcessSelector::Name("web".to_string()))
.await
.expect("the reload is accepted");
let seen = events_through(&mut rx, 1, ProcessEventKind::ReloadAbandoned).await;
assert_eq!(
seen[at(&seen, 1, ProcessEventKind::ReloadAbandoned)],
Seen {
id: 1,
kind: ProcessEventKind::ReloadAbandoned,
status: ProcStatus::Online,
manually: true,
},
"the replacement took the slot over and is what is left holding it"
);
assert_eq!(
runner.kill_counts(),
vec![1, 0],
"the drain did reach `SIGKILL`; what never came back was the exit"
);
let again = handle
.reload(ProcessSelector::Name("web".to_string()))
.await
.map(|infos| infos.iter().map(|info| info.id).collect::<Vec<_>>());
assert_eq!(
again,
Ok(vec![0, 1]),
"a wedged instance must not refuse the app's next reload"
);
}
#[tokio::test(start_paused = true)]
async fn a_deadline_from_a_finished_swap_never_ends_the_one_that_followed_it() {
let dir = tempfile::tempdir().unwrap();
let mut app = AppConfig::minimal("web", "./srv");
app.instances = 2;
let (handle, _runner, mut rx) = started(
&dir,
app,
vec![
ProcScript::ignores_signals(),
ProcScript::ignores_signals(),
ProcScript::never_exits(),
ProcScript::never_exits(),
],
)
.await;
handle
.reload(ProcessSelector::Name("web".to_string()))
.await
.expect("the reload is accepted");
expect_event(&mut rx, 2, ProcessEventKind::Reloaded).await;
expect_event(&mut rx, 3, ProcessEventKind::Reloaded).await;
let after = handle.list().await;
assert_eq!(
after.iter().map(|info| info.id).collect::<Vec<_>>(),
vec![2, 3],
"both instances were replaced: {after:?}"
);
assert!(after.iter().all(|info| info.status == ProcStatus::Online));
}
#[tokio::test(start_paused = true)]
async fn a_deadline_before_the_commit_puts_the_instance_being_replaced_back() {
let dir = tempfile::tempdir().unwrap();
let (mut actor, _mailbox) =
actor_with_one_online_sheep(&dir, vec![ProcScript::never_exits()]);
let (ctl_tx, _ctl_rx) = mpsc::channel(SHEEP_CTL_CAPACITY);
actor.sheep.get_mut(&0).expect("the fixture's sheep").ctl = Some(ctl_tx);
let new_id = actor
.spawn_replacement(0, ReloadMode::Overlap)
.expect("the fixture's one script covers this spawn");
actor.reloads.insert(
"web".to_string(),
ReloadJob {
queue: VecDeque::new(),
mode: ReloadMode::Overlap,
deadline: 0,
swap: ReloadSwap {
old_id: 0,
new_id: Some(new_id),
phase: ReloadPhase::AwaitReady,
},
},
);
actor.handle_reload_deadline("web", actor.reloads["web"].deadline);
assert!(
actor.reloads.is_empty(),
"the job is gone, so the app is reloadable again"
);
let drainee = &actor.sheep[&0];
assert_eq!(
drainee.entry.status,
ProcStatus::Online,
"nothing was ever killed, so the instance being replaced goes back to serving"
);
assert_eq!(drainee.entry.reload, ReloadState::None);
assert_eq!(
actor.sheep[&new_id].manual.map(|pending| pending.kind),
Some(ManualKind::Delete),
"the replacement that never proved itself is taken back down"
);
}
#[tokio::test(start_paused = true)]
async fn an_abandoned_reload_says_so_on_the_bus() {
let dir = tempfile::tempdir().unwrap();
let mut app = AppConfig::minimal("web", "./srv");
app.wait_ready = true; let (handle, _runner, mut rx) =
started(&dir, app, vec![ProcScript::never_exits(); 2]).await;
handle.tx.send(Msg::Ready { id: 0 }).await.unwrap();
expect_event(&mut rx, 0, ProcessEventKind::Online).await;
handle
.reload(ProcessSelector::Name("web".to_string()))
.await
.expect("the reload is accepted");
let seen = events_through(&mut rx, 0, ProcessEventKind::ReloadAbandoned).await;
assert_eq!(
seen[at(&seen, 0, ProcessEventKind::ReloadAbandoned)],
Seen {
id: 0,
kind: ProcessEventKind::ReloadAbandoned,
status: ProcStatus::Online,
manually: true,
},
"the abandoned reload's own instance is still the one serving"
);
}
#[tokio::test(start_paused = true)]
async fn a_reload_whose_replacement_cannot_spawn_says_so_on_the_bus() {
let dir = tempfile::tempdir().unwrap();
let (handle, _runner, mut rx) = started(
&dir,
AppConfig::minimal("web", "./srv"),
vec![ProcScript::never_exits()],
)
.await;
handle
.reload(ProcessSelector::Name("web".to_string()))
.await
.expect("the reload is accepted before anything is spawned");
let seen = events_through(&mut rx, 0, ProcessEventKind::ReloadAbandoned).await;
assert_eq!(
seen[at(&seen, 0, ProcessEventKind::ReloadAbandoned)],
Seen {
id: 0,
kind: ProcessEventKind::ReloadAbandoned,
status: ProcStatus::Online,
manually: true,
},
"a failed spawn leaves the instance it was replacing serving"
);
let after = handle.list().await;
assert_eq!(after.len(), 1, "no replacement was registered: {after:?}");
assert_eq!(after[0].id, 0);
}
#[tokio::test(start_paused = true)]
async fn a_live_sheep_keeps_holding_its_log_control_sender() {
let runner = ScriptedRunner::new(vec![ProcScript::never_exits()]);
let (proc, io) = runner.spawn(&log_ctl_spec()).unwrap();
assert!(runner.log_ctl_live(0), "sanity: the fake starts it live");
let (events, _rx) = crate::bus::test_bus(64);
let (_ctl_tx, ctl_rx) = mpsc::channel(8);
let (_signal_tx, signal_rx) = mpsc::channel(8);
let (actor_tx, _actor_rx) = mpsc::channel(8);
let app = normalize(AppConfig::minimal("svc", "./svc")).unwrap();
tokio::spawn(run_sheep(
7, proc, io, app, ctl_rx, signal_rx, events, actor_tx,
));
for _ in 0..100 {
tokio::task::yield_now().await;
}
assert!(
runner.log_ctl_live(0),
"run_sheep dropped ProcIo::log_ctl while its sheep was still \
running: against the real runner that closes the read ends of \
the child's stdout and stderr"
);
}
#[tokio::test(start_paused = true)]
async fn a_pump_is_reaped_when_its_sheep_ends_even_with_a_lamb_on_the_pipe() {
let (events, mut rx) = crate::bus::test_bus(64);
let runner = Arc::new(ScriptedRunner::new(vec![
ProcScript::stable_then_exit(1_000, 0).with_a_lamb_holding_the_pipe(),
]));
let dir = tempfile::tempdir().unwrap();
let handle = spawn_supervisor(SharedRunner(Arc::clone(&runner)), test_paths(&dir), events);
let mut app = AppConfig::minimal("web", "./srv");
app.autorestart = false;
handle.start(vec![normalize(app).unwrap()]).await.unwrap();
assert!(
runner.log_ctl_live(0),
"sanity: a running sheep has a live pump"
);
await_event(&mut rx, 0, ProcessEventKind::Stop).await;
assert_eq!(
handle.list().await.len(),
1,
"sanity: the sheep is still registered, so its slot still holds a \
clone of the control sender"
);
let reaped = tokio::time::timeout(Duration::from_secs(5), async {
while runner.log_ctl_live(0) {
tokio::time::sleep(Duration::from_millis(10)).await;
}
})
.await;
assert!(
reaped.is_ok(),
"the pump outlived its sheep task, holding both log files and \
both pipe read ends open"
);
}
#[tokio::test(start_paused = true)]
async fn a_writer_task_is_reaped_when_its_sheep_exits() {
let dir = tempfile::tempdir().unwrap();
let mut app = AppConfig::minimal("web", "./srv");
app.autorestart = false;
app.channel = true;
let (handle, runner, mut rx) =
started(&dir, app, vec![ProcScript::stable_then_exit(1_000, 0)]).await;
let mut io = runner.io_handles(0);
assert_eq!(
io.to_child_rx.try_recv(),
Err(mpsc::error::TryRecvError::Empty),
"sanity: a running sheep's channel is open and quiet, so the \
`None` below is a close and not a channel that never opened"
);
await_event(&mut rx, 0, ProcessEventKind::Stop).await;
assert_eq!(
handle.list().await.len(),
1,
"sanity: the sheep is still registered, so nothing but the \
clearing can have let go of the slot's clone"
);
let reaped = tokio::time::timeout(Duration::from_secs(5), io.to_child_rx.recv()).await;
assert_eq!(
reaped.ok(),
Some(None),
"the writer task outlived its sheep, parked on `recv()` and \
holding the daemon's end of the shepherd channel"
);
}
#[tokio::test(start_paused = true)]
async fn a_failed_spawn_names_the_sheep_and_the_path_it_tried() {
let dir = tempfile::tempdir().unwrap();
let (mut actor, _mailbox) = actor_with_one_online_sheep(&dir, Vec::new());
let (reply, answer) = oneshot::channel();
actor.handle_command(Command::Start {
apps: vec![normalize(AppConfig::minimal("api", "./api")).unwrap()],
policy: BatchPolicy::AllOrNothing,
gate: BTreeSet::new(),
reply,
});
let err = answer
.await
.expect("the actor answers every Start")
.expect_err("an empty script pool cannot spawn");
assert_eq!(
err.to_string(),
"spawn failed: api: process spawn failed: script exhausted; tried `./api`"
);
}
#[tokio::test(start_paused = true)]
async fn config_drift_names_an_edited_sheeps_fields_and_changes_nothing() {
let dir = tempfile::tempdir().unwrap();
let (actor, _mailbox) = actor_with_one_online_sheep(&dir, vec![ProcScript::never_exits()]);
let mut edited = AppConfig::minimal("web", "./srv");
edited.cwd = Some("/srv/new".to_string());
edited
.env
.insert("DATABASE_URL".to_string(), "postgres://hunter2".to_string());
let unknown = AppConfig::minimal("api", "./api");
let drift = actor.config_drift(&[normalize(edited).unwrap(), normalize(unknown).unwrap()]);
assert_eq!(
drift,
vec![SheepDrift::new(
"web",
vec!["cwd".to_string(), "env".to_string()]
)]
);
assert!(
!format!("{drift:?}").contains("hunter2"),
"a value must never travel with the field name that changed: {drift:?}"
);
assert_eq!(
actor.sheep[&0].entry.spec.config().cwd,
None,
"asking which fields differ must not apply them"
);
}
#[tokio::test(start_paused = true)]
async fn every_spawn_leaves_the_daemons_end_of_the_channel_on_the_slot() {
let dir = tempfile::tempdir().unwrap();
let (mut actor, _mailbox) = actor_with_one_online_sheep(
&dir,
vec![
ProcScript::never_exits(),
ProcScript::never_exits(),
ProcScript::never_exits(),
],
);
let mut app = AppConfig::minimal("api", "./api");
app.autorestart = false;
let (reply, _answer) = oneshot::channel();
actor.handle_command(Command::Start {
apps: vec![normalize(app).unwrap()],
policy: BatchPolicy::AllOrNothing,
gate: BTreeSet::new(),
reply,
});
assert!(
actor.sheep[&1].to_child.is_some(),
"a fresh spawn's slot holds the daemon's end of its shepherd channel"
);
actor.handle_exited(
1,
ExitOutcome {
code: Some(0),
signal: None,
},
);
assert!(
actor.sheep[&1].to_child.is_none(),
"the clone goes with the process it was cloned for"
);
actor.respawn(1, false);
assert!(
actor.sheep[&1].to_child.is_some(),
"a respawn hands the slot the new process's channel, not the dead \
one's"
);
actor.advance_reload("web", VecDeque::from([0]));
let new_id = actor.reloads["web"]
.swap
.new_id
.expect("an overlapping reload spawns its replacement at once");
assert!(
actor.sheep[&new_id].to_child.is_some(),
"a reload's replacement holds the daemon's end of its own channel"
);
}
const ACTION_TIMEOUT: Duration = Duration::from_secs(20);
const ACTION_WINDOW: Duration = Duration::from_secs(120);
fn actor_with_an_open_channel(
dir: &tempfile::TempDir,
) -> (
Actor<ScriptedRunner>,
mpsc::Receiver<Msg>,
mpsc::Receiver<ShepherdMessage>,
) {
let (mut actor, mailbox) = actor_with_one_online_sheep(dir, vec![]);
let (to_child, child_rx) = mpsc::channel(16);
actor
.sheep
.get_mut(&0)
.expect("the fixture registers id 0")
.to_child = Some(to_child);
(actor, mailbox, child_rx)
}
fn trigger_action(
actor: &mut Actor<ScriptedRunner>,
action: &str,
) -> oneshot::Receiver<ActionOutcome> {
let to_child = actor.sheep[&0]
.to_child
.clone()
.expect("the fixture's sheep holds the daemon's end of a channel");
actor.arm_action(0, to_child, action.to_string(), None, ACTION_TIMEOUT)
}
async fn settle_action(
actor: &mut Actor<ScriptedRunner>,
mailbox: &mut mpsc::Receiver<Msg>,
) -> ActionOutcome {
let msg = tokio::time::timeout(ACTION_WINDOW, mailbox.recv())
.await
.expect("an action wait reported nothing within the window")
.expect("the actor's mailbox closed");
match msg {
Msg::ActionResult { id, stamp, outcome } => {
actor.handle_action_result(id, stamp, outcome.clone());
outcome
}
other => panic!("expected an action result, got {other:?}"),
}
}
async fn sent_action(child_rx: &mut mpsc::Receiver<ShepherdMessage>) -> ShepherdMessage {
tokio::time::timeout(ACTION_WINDOW, child_rx.recv())
.await
.expect("nothing reached the child's end of the channel")
.expect("the child's end of the channel closed")
}
#[tokio::test(start_paused = true)]
async fn a_triggered_action_answers_with_the_apps_reply() {
let dir = tempfile::tempdir().unwrap();
let mut app = AppConfig::minimal("web", "./srv");
app.channel = true;
let (handle, runner, _events) = started(&dir, app, vec![ProcScript::never_exits()]).await;
let mut io = runner.io_handles(0);
let triggered = tokio::spawn(async move {
handle
.trigger(
ProcessSelector::Name("web".to_string()),
"gc".to_string(),
Some("--full".to_string()),
)
.await
});
assert_eq!(
sent_action(&mut io.to_child_rx).await,
ShepherdMessage::Action {
name: "gc".to_string(),
params: Some("--full".to_string()),
id: 0,
},
"the action reaches the child's end of the channel as it was asked for"
);
io.from_child_tx
.send(ChildMessage::ActionReply {
action: "gc".to_string(),
body: "swept 3".to_string(),
id: Some(0),
})
.await
.unwrap();
assert_eq!(
triggered.await.unwrap(),
Ok(vec![ActionReply {
id: 0,
name: "web".to_string(),
outcome: ActionOutcome::Replied {
body: "swept 3".to_string()
},
}]),
"the app's reply body is what the caller is answered with"
);
}
#[tokio::test(start_paused = true)]
async fn a_ready_on_the_channel_reaches_both_the_bus_and_the_readiness_wait() {
let dir = tempfile::tempdir().unwrap();
let mut app = AppConfig::minimal("web", "./srv");
app.channel = true;
app.wait_ready = true;
let (handle, runner, mut events) =
started(&dir, app, vec![ProcScript::never_exits()]).await;
let io = runner.io_handles(0);
io.from_child_tx.send(ChildMessage::Ready).await.unwrap();
let seen = tokio::time::timeout(ACTION_WINDOW, async {
loop {
if let BusEvent::Channel { id, message } = events.recv().await.unwrap().to_event() {
break (id, message);
}
}
})
.await
.expect("no channel event within the window");
assert_eq!(seen, (0, ChildMessage::Ready));
let listed = handle.list().await;
assert_eq!(listed[0].status, ProcStatus::Online);
}
#[tokio::test(start_paused = true)]
async fn a_metric_on_the_channel_reaches_the_bus_with_its_name_and_value() {
let dir = tempfile::tempdir().unwrap();
let mut app = AppConfig::minimal("web", "./srv");
app.channel = true;
let (_handle, runner, mut events) =
started(&dir, app, vec![ProcScript::never_exits()]).await;
let io = runner.io_handles(0);
io.from_child_tx
.send(ChildMessage::Metric {
name: "rps".to_string(),
value: 42.0,
})
.await
.unwrap();
let seen = tokio::time::timeout(ACTION_WINDOW, async {
loop {
if let BusEvent::Channel { id, message } = events.recv().await.unwrap().to_event() {
break (id, message);
}
}
})
.await
.expect("no channel event within the window");
assert_eq!(
seen,
(
0,
ChildMessage::Metric {
name: "rps".to_string(),
value: 42.0,
}
)
);
}
#[tokio::test(start_paused = true)]
async fn an_action_reply_no_trigger_is_waiting_for_still_reaches_the_bus() {
let dir = tempfile::tempdir().unwrap();
let mut app = AppConfig::minimal("web", "./srv");
app.channel = true;
let (_handle, runner, mut events) =
started(&dir, app, vec![ProcScript::never_exits()]).await;
let io = runner.io_handles(0);
io.from_child_tx
.send(ChildMessage::ActionReply {
action: "gc".to_string(),
body: "unprompted".to_string(),
id: None,
})
.await
.unwrap();
let seen = tokio::time::timeout(ACTION_WINDOW, async {
loop {
if let BusEvent::Channel { message, .. } = events.recv().await.unwrap().to_event() {
break message;
}
}
})
.await
.expect("no channel event within the window");
let ChildMessage::ActionReply { body, .. } = seen else {
panic!("expected an action reply, got {seen:?}");
};
assert_eq!(body, "unprompted");
}
#[tokio::test(start_paused = true)]
async fn a_triggered_action_times_out_when_the_app_never_answers() {
let dir = tempfile::tempdir().unwrap();
let mut app = AppConfig::minimal("api", "./api");
app.channel = true;
let (handle, runner, _events) = started(&dir, app, vec![ProcScript::never_exits()]).await;
let mut io = runner.io_handles(0);
assert_eq!(
handle
.trigger(
ProcessSelector::Name("api".to_string()),
"stats".to_string(),
None,
)
.await,
Ok(vec![ActionReply {
id: 0,
name: "api".to_string(),
outcome: ActionOutcome::TimedOut,
}]),
"an app that says nothing is reported as saying nothing, not waited on forever"
);
assert_eq!(
sent_action(&mut io.to_child_rx).await,
ShepherdMessage::Action {
name: "stats".to_string(),
params: None,
id: 0,
},
"the timeout is an app that did not answer, not an action that was never sent"
);
}
#[test]
fn a_stamped_reply_wakes_its_own_wait_even_with_a_debt_outstanding() {
let mut waits = ActionWaits::default();
let (t1_reply, _t1_out) = oneshot::channel();
let (t1_waiter, _t1_body) = oneshot::channel();
waits.arm(PendingAction {
stamp: 1,
action: "gc".to_string(),
waiter: Some(t1_waiter),
reply: t1_reply,
});
assert!(waits.resolve(1).is_some(), "T1 must have been live");
let (t2_reply, _t2_out) = oneshot::channel();
let (t2_waiter, t2_body) = oneshot::channel();
waits.arm(PendingAction {
stamp: 2,
action: "gc".to_string(),
waiter: Some(t2_waiter),
reply: t2_reply,
});
let woken = waits
.answer("gc", Some(2))
.expect("a reply stamped with the live wait's own stamp must reach it");
woken.send("collected".to_string()).unwrap();
assert_eq!(t2_body.blocking_recv().unwrap(), "collected");
}
#[test]
fn an_unstamped_reply_still_settles_the_oldest_debt_first() {
let mut waits = ActionWaits::default();
let (t1_reply, _t1_out) = oneshot::channel();
let (t1_waiter, _t1_body) = oneshot::channel();
waits.arm(PendingAction {
stamp: 1,
action: "gc".to_string(),
waiter: Some(t1_waiter),
reply: t1_reply,
});
waits.resolve(1);
let (t2_reply, _t2_out) = oneshot::channel();
let (t2_waiter, _t2_body) = oneshot::channel();
waits.arm(PendingAction {
stamp: 2,
action: "gc".to_string(),
waiter: Some(t2_waiter),
reply: t2_reply,
});
assert!(
waits.answer("gc", None).is_none(),
"an unstamped reply pays the debt, exactly as it did before stamping"
);
assert!(
waits.answer("gc", None).is_some(),
"and the next one reaches the live wait, exactly as it did before"
);
}
#[test]
fn a_stamped_reply_for_a_dead_wait_does_not_reach_a_live_one() {
let mut waits = ActionWaits::default();
let (t1_reply, _t1_out) = oneshot::channel();
let (t1_waiter, _t1_body) = oneshot::channel();
waits.arm(PendingAction {
stamp: 1,
action: "gc".to_string(),
waiter: Some(t1_waiter),
reply: t1_reply,
});
waits.resolve(1);
let (t2_reply, _t2_out) = oneshot::channel();
let (t2_waiter, _t2_body) = oneshot::channel();
waits.arm(PendingAction {
stamp: 2,
action: "gc".to_string(),
waiter: Some(t2_waiter),
reply: t2_reply,
});
assert!(
waits.answer("gc", Some(1)).is_none(),
"T1's own late reply belongs to T1's debt, not to T2"
);
assert!(
waits.answer("gc", Some(2)).is_some(),
"and T2 is still waiting for its own"
);
}
#[tokio::test(start_paused = true)]
async fn a_reply_owed_by_a_timed_out_action_never_answers_the_next_one() {
let dir = tempfile::tempdir().unwrap();
let (mut actor, mut mailbox, mut child_rx) = actor_with_an_open_channel(&dir);
let first = trigger_action(&mut actor, "gc");
sent_action(&mut child_rx).await;
assert_eq!(
settle_action(&mut actor, &mut mailbox).await,
ActionOutcome::TimedOut
);
assert_eq!(first.await.unwrap(), ActionOutcome::TimedOut);
let second = trigger_action(&mut actor, "gc");
sent_action(&mut child_rx).await;
actor.handle_action_reply(0, "gc", "swept 3".to_string(), None);
assert_eq!(
settle_action(&mut actor, &mut mailbox).await,
ActionOutcome::TimedOut,
"a reply the first `gc` was owed was handed to the second one"
);
assert_eq!(second.await.unwrap(), ActionOutcome::TimedOut);
let third = trigger_action(&mut actor, "gc");
sent_action(&mut child_rx).await;
actor.handle_action_reply(0, "gc", "swept 7".to_string(), None);
actor.handle_action_reply(0, "gc", "swept 11".to_string(), None);
assert_eq!(
settle_action(&mut actor, &mut mailbox).await,
ActionOutcome::Replied {
body: "swept 11".to_string()
},
"the debts outlived the replies that settled them"
);
assert_eq!(
third.await.unwrap(),
ActionOutcome::Replied {
body: "swept 11".to_string()
}
);
}
#[tokio::test(start_paused = true)]
async fn a_second_reply_to_an_answered_action_is_dropped() {
let dir = tempfile::tempdir().unwrap();
let (mut actor, mut mailbox, mut child_rx) = actor_with_an_open_channel(&dir);
let answered = trigger_action(&mut actor, "gc");
sent_action(&mut child_rx).await;
actor.handle_action_reply(0, "gc", "swept 3".to_string(), None);
assert_eq!(
settle_action(&mut actor, &mut mailbox).await,
ActionOutcome::Replied {
body: "swept 3".to_string()
}
);
assert_eq!(
answered.await.unwrap(),
ActionOutcome::Replied {
body: "swept 3".to_string()
}
);
actor.handle_action_reply(0, "gc", "spent".to_string(), None);
let next = trigger_action(&mut actor, "gc");
sent_action(&mut child_rx).await;
assert_eq!(
settle_action(&mut actor, &mut mailbox).await,
ActionOutcome::TimedOut,
"a spare reply was kept and used to answer a wait that came after it"
);
assert_eq!(next.await.unwrap(), ActionOutcome::TimedOut);
}
#[tokio::test(start_paused = true)]
async fn two_waits_for_one_action_are_answered_in_the_order_they_were_asked() {
let dir = tempfile::tempdir().unwrap();
let (mut actor, mut mailbox, mut child_rx) = actor_with_an_open_channel(&dir);
let first = trigger_action(&mut actor, "gc");
sent_action(&mut child_rx).await;
let second = trigger_action(&mut actor, "gc");
sent_action(&mut child_rx).await;
actor.handle_action_reply(0, "gc", "swept 3".to_string(), None);
actor.handle_action_reply(0, "gc", "swept 7".to_string(), None);
settle_action(&mut actor, &mut mailbox).await;
settle_action(&mut actor, &mut mailbox).await;
assert_eq!(
first.await.unwrap(),
ActionOutcome::Replied {
body: "swept 3".to_string()
},
"the earlier trigger was answered with the later reply"
);
assert_eq!(
second.await.unwrap(),
ActionOutcome::Replied {
body: "swept 7".to_string()
},
"the second reply was dropped and its wait left waiting"
);
}
#[tokio::test(start_paused = true)]
async fn an_action_that_cannot_even_be_delivered_still_ends() {
let dir = tempfile::tempdir().unwrap();
let (mut actor, mut mailbox) = actor_with_one_online_sheep(&dir, vec![]);
let (to_child, _wedged) = mpsc::channel(1);
to_child.try_send(ShepherdMessage::Shutdown).unwrap();
actor
.sheep
.get_mut(&0)
.expect("the fixture registers id 0")
.to_child = Some(to_child);
let answer = trigger_action(&mut actor, "gc");
assert_eq!(
settle_action(&mut actor, &mut mailbox).await,
ActionOutcome::TimedOut,
"an action that never got onto the channel left its wait parked"
);
assert_eq!(answer.await.unwrap(), ActionOutcome::TimedOut);
}
fn register_sheep(
actor: &mut Actor<ScriptedRunner>,
dir: &tempfile::TempDir,
name: &str,
to_child: Option<mpsc::Sender<ShepherdMessage>>,
) -> u32 {
let id = actor.next_id;
actor.next_id += 1;
let paths = test_paths(dir);
let app = normalize(AppConfig::minimal(name, "./srv")).unwrap();
actor.sheep.insert(
id,
SheepSlot {
to_child,
..SheepSlot::new(armed_entry(id, 0, 2000 + id, app, &paths))
},
);
id
}
fn trigger_flock(
actor: &mut Actor<ScriptedRunner>,
selector: ProcessSelector,
action: &str,
) -> oneshot::Receiver<Result<Vec<ActionReply>, SupervisorError>> {
let (reply, answer) = oneshot::channel();
actor.handle_command(Command::Trigger {
selector,
action: action.to_string(),
params: None,
reply,
});
answer
}
async fn triggered(
answer: oneshot::Receiver<Result<Vec<ActionReply>, SupervisorError>>,
) -> Result<Vec<ActionReply>, SupervisorError> {
tokio::time::timeout(ACTION_WINDOW, answer)
.await
.expect("a trigger reported nothing within the window")
.expect("the trigger's reply channel was dropped")
}
fn row(id: u32, name: &str, outcome: ActionOutcome) -> ActionReply {
ActionReply {
id,
name: name.to_string(),
outcome,
}
}
#[tokio::test(start_paused = true)]
async fn a_trigger_answers_every_sheep_it_matched_before_it_answers_at_all() {
let dir = tempfile::tempdir().unwrap();
let (mut actor, mut mailbox, mut child_rx) = actor_with_an_open_channel(&dir);
register_sheep(&mut actor, &dir, "zeus", None);
let (silent_tx, mut silent_rx) = mpsc::channel(16);
register_sheep(&mut actor, &dir, "worker", Some(silent_tx));
let mut answer = trigger_flock(&mut actor, ProcessSelector::All, "gc");
sent_action(&mut child_rx).await;
sent_action(&mut silent_rx).await;
actor.handle_action_reply(0, "gc", "swept 3".to_string(), None);
assert_eq!(
settle_action(&mut actor, &mut mailbox).await,
ActionOutcome::Replied {
body: "swept 3".to_string()
}
);
tokio::task::yield_now().await;
assert!(
matches!(answer.try_recv(), Err(oneshot::error::TryRecvError::Empty)),
"a trigger answered while a sheep it matched was still being waited on"
);
assert_eq!(
settle_action(&mut actor, &mut mailbox).await,
ActionOutcome::TimedOut
);
assert_eq!(
triggered(answer).await,
Ok(vec![
row(
0,
"web",
ActionOutcome::Replied {
body: "swept 3".to_string()
}
),
row(2, "worker", ActionOutcome::TimedOut),
row(1, "zeus", ActionOutcome::NoChannel),
])
);
}
#[tokio::test(start_paused = true)]
async fn a_sheep_with_no_channel_is_refused_in_its_own_row() {
let dir = tempfile::tempdir().unwrap();
let (mut actor, mut mailbox, mut child_rx) = actor_with_an_open_channel(&dir);
register_sheep(&mut actor, &dir, "api", None);
let answer = trigger_flock(&mut actor, ProcessSelector::All, "gc");
sent_action(&mut child_rx).await;
actor.handle_action_reply(0, "gc", "swept 3".to_string(), None);
settle_action(&mut actor, &mut mailbox).await;
assert_eq!(
triggered(answer).await,
Ok(vec![
row(1, "api", ActionOutcome::NoChannel),
row(
0,
"web",
ActionOutcome::Replied {
body: "swept 3".to_string()
}
),
])
);
assert!(
matches!(mailbox.try_recv(), Err(mpsc::error::TryRecvError::Empty)),
"a sheep with no channel armed a wait anyway"
);
}
#[tokio::test(start_paused = true)]
async fn a_sheep_whose_channel_has_no_far_end_is_refused_too() {
let dir = tempfile::tempdir().unwrap();
let (mut actor, mut mailbox) = actor_with_one_online_sheep(&dir, vec![]);
let (to_child, receiver) = mpsc::channel(16);
actor
.sheep
.get_mut(&0)
.expect("the fixture registers id 0")
.to_child = Some(to_child);
drop(receiver);
assert_eq!(
triggered(trigger_flock(&mut actor, ProcessSelector::All, "gc")).await,
Ok(vec![row(0, "web", ActionOutcome::NoChannel)])
);
assert!(
matches!(mailbox.try_recv(), Err(mpsc::error::TryRecvError::Empty)),
"a sheep whose channel has no far end armed a wait anyway"
);
}
#[tokio::test(start_paused = true)]
async fn a_trigger_no_sheep_can_take_is_a_success_that_says_so() {
let dir = tempfile::tempdir().unwrap();
let (mut actor, mut mailbox) = actor_with_one_online_sheep(&dir, vec![]);
register_sheep(&mut actor, &dir, "api", None);
assert_eq!(
triggered(trigger_flock(&mut actor, ProcessSelector::All, "gc")).await,
Ok(vec![
row(0, "web", ActionOutcome::NoChannel),
row(1, "api", ActionOutcome::NoChannel),
])
);
assert!(
matches!(mailbox.try_recv(), Err(mpsc::error::TryRecvError::Empty)),
"a flock with nothing to deliver to armed a wait anyway"
);
}
#[tokio::test(start_paused = true)]
async fn a_reload_drainee_is_skipped_and_its_replacement_answers() {
let dir = tempfile::tempdir().unwrap();
let (mut actor, mut mailbox, mut child_rx) = actor_with_an_open_channel(&dir);
let (replacement_tx, mut replacement_rx) = mpsc::channel(16);
let new_id = register_sheep(&mut actor, &dir, "web", Some(replacement_tx));
let drainee = actor.sheep.get_mut(&0).expect("the fixture registers id 0");
drainee.entry.status = ProcStatus::Stopping;
drainee.entry.reload = ReloadState::Drainee {
new_id: Some(new_id),
};
actor
.sheep
.get_mut(&new_id)
.expect("the replacement was just registered")
.entry
.reload = ReloadState::Replacement;
let answer = trigger_flock(&mut actor, ProcessSelector::Name("web".to_string()), "gc");
sent_action(&mut replacement_rx).await;
actor.handle_action_reply(new_id, "gc", "swept 3".to_string(), None);
settle_action(&mut actor, &mut mailbox).await;
assert_eq!(
triggered(answer).await,
Ok(vec![
row(0, "web", ActionOutcome::Skipped),
row(
new_id,
"web",
ActionOutcome::Replied {
body: "swept 3".to_string()
}
),
])
);
assert!(
matches!(child_rx.try_recv(), Err(mpsc::error::TryRecvError::Empty)),
"an action was delivered to a process that is on its way out"
);
}
#[tokio::test(start_paused = true)]
async fn a_trigger_matching_no_sheep_is_an_error() {
let dir = tempfile::tempdir().unwrap();
let (mut actor, _mailbox) = actor_with_one_online_sheep(&dir, vec![]);
assert_eq!(
triggered(trigger_flock(
&mut actor,
ProcessSelector::Name("ghost".to_string()),
"gc"
))
.await,
Err(SupervisorError::NotFound)
);
}
#[tokio::test(start_paused = true)]
async fn a_sheep_exiting_answers_every_action_waiting_on_it() {
let dir = tempfile::tempdir().unwrap();
let (mut actor, mut mailbox, mut child_rx) = actor_with_an_open_channel(&dir);
let timed_out = trigger_action(&mut actor, "gc");
sent_action(&mut child_rx).await;
settle_action(&mut actor, &mut mailbox).await;
assert_eq!(timed_out.await.unwrap(), ActionOutcome::TimedOut);
let waiting = trigger_action(&mut actor, "stats");
sent_action(&mut child_rx).await;
actor.handle_exited(
0,
ExitOutcome {
code: Some(0),
signal: None,
},
);
let answered = tokio::time::timeout(ACTION_WINDOW, waiting)
.await
.expect("a wait outlived the process it was waiting on")
.unwrap();
assert_eq!(
answered,
ActionOutcome::NoChannel,
"a wait outlived the process it was waiting on"
);
assert!(
actor.sheep[&0].actions.abandoned.is_empty(),
"a debt owed by a process that has exited outlived it, and would \
have swallowed a reply from whatever runs under this id next"
);
}
struct SilentPumpRunner {
inner: ScriptedRunner,
seen: watch::Sender<u32>,
}
impl SilentPumpRunner {
fn new(scripts: Vec<ProcScript>) -> (Self, watch::Receiver<u32>) {
let (seen, requests) = watch::channel(0);
(
Self {
inner: ScriptedRunner::new(scripts),
seen,
},
requests,
)
}
}
impl fmt::Debug for SilentPumpRunner {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("SilentPumpRunner").finish_non_exhaustive()
}
}
impl ProcessRunner for SilentPumpRunner {
type Proc = crate::fake::FakeProc;
fn spawn(
&self,
spec: &SpawnSpec,
) -> Result<(Self::Proc, ProcIo), crate::runner::RunnerError> {
let (proc, mut io) = self.inner.spawn(spec)?;
let (tx, mut rx) = mpsc::channel(SHEEP_CTL_CAPACITY);
io.log_ctl = tx;
let seen = self.seen.clone();
tokio::spawn(async move {
let mut held = Vec::new();
while let Some(request) = rx.recv().await {
held.push(request);
seen.send_modify(|count| *count += 1);
}
});
Ok((proc, io))
}
}
#[tokio::test(start_paused = true)]
async fn the_actor_keeps_answering_while_a_reopen_waits_on_a_silent_pump() {
let (events, _rx) = crate::bus::test_bus(64);
let (runner, mut requests) = SilentPumpRunner::new(vec![ProcScript::never_exits()]);
let dir = tempfile::tempdir().unwrap();
let handle = spawn_supervisor(runner, test_paths(&dir), events);
handle
.start(vec![normalize(AppConfig::minimal("web", "./srv")).unwrap()])
.await
.unwrap();
let reopening = tokio::spawn({
let handle = handle.clone();
async move { handle.reopen(ProcessSelector::All).await }
});
tokio::time::timeout(Duration::from_secs(5), requests.wait_for(|seen| *seen == 1))
.await
.expect("the reopen must reach the pump")
.expect("the runner outlives this wait, so its sender cannot have closed");
let listed = tokio::time::timeout(Duration::from_secs(5), handle.list())
.await
.expect("the actor must keep answering while a reopen is outstanding");
assert_eq!(listed.len(), 1);
assert!(
!reopening.is_finished(),
"sanity: nothing can acknowledge this reopen, so `list` answering \
above is not just the reopen having finished first"
);
reopening.abort();
}
#[tokio::test(start_paused = true)]
async fn a_reopen_reaches_every_matched_sheep_and_no_others() {
let (events, _rx) = crate::bus::test_bus(64);
let runner = Arc::new(ScriptedRunner::new(vec![
ProcScript::never_exits(),
ProcScript::never_exits(),
ProcScript::never_exits(),
]));
let dir = tempfile::tempdir().unwrap();
let handle = spawn_supervisor(SharedRunner(Arc::clone(&runner)), test_paths(&dir), events);
let mut web = AppConfig::minimal("web", "./srv");
web.instances = 2;
handle
.start(vec![
normalize(web).unwrap(),
normalize(AppConfig::minimal("api", "./api")).unwrap(),
])
.await
.unwrap();
let reopened = handle
.reopen(ProcessSelector::Name("web".to_string()))
.await
.unwrap();
assert_eq!(
reopened.iter().map(|info| info.id).collect::<Vec<_>>(),
vec![0, 1],
"the reply must carry both `web` instances, id-sorted, and no `api`"
);
assert_eq!(runner.reopens(0), 1, "web's first instance");
assert_eq!(runner.reopens(1), 1, "web's second instance");
assert_eq!(runner.reopens(2), 0, "api was never named");
}
#[tokio::test(start_paused = true)]
async fn a_reopen_after_a_restart_reaches_the_pump_the_restart_spawned() {
let (events, _rx) = crate::bus::test_bus(64);
let runner = Arc::new(ScriptedRunner::new(vec![
ProcScript::never_exits(),
ProcScript::never_exits(),
]));
let dir = tempfile::tempdir().unwrap();
let handle = spawn_supervisor(SharedRunner(Arc::clone(&runner)), test_paths(&dir), events);
handle
.start(vec![normalize(AppConfig::minimal("web", "./srv")).unwrap()])
.await
.unwrap();
let restarted = handle
.restart(ProcessSelector::Name("web".to_string()))
.await
.unwrap();
assert_eq!(
(restarted[0].restarts, restarted[0].status),
(1, ProcStatus::Online),
"the sheep must be back up on a second process before the reopen"
);
let reopened =
tokio::time::timeout(Duration::from_secs(5), handle.reopen(ProcessSelector::All))
.await
.expect("a live pump must answer rather than leave the reopen waiting")
.expect("a running sheep's reopen must succeed");
assert_eq!(reopened.len(), 1);
assert_eq!(
runner.reopens(1),
1,
"the reopen must reach the pump the restart spawned"
);
assert_eq!(
runner.reopens(0),
0,
"the pre-restart pump belongs to a process that is gone; a reopen \
sent there reaches nothing and is reported as a success"
);
}
#[tokio::test(start_paused = true)]
async fn a_reopen_matching_nothing_is_not_found() {
let (events, _rx) = crate::bus::test_bus(64);
let runner = ScriptedRunner::new(vec![]);
let dir = tempfile::tempdir().unwrap();
let handle = spawn_supervisor(runner, test_paths(&dir), events);
assert_eq!(
handle.reopen(ProcessSelector::All).await,
Err(SupervisorError::NotFound)
);
}
#[tokio::test(start_paused = true)]
async fn a_stopped_sheep_is_a_no_op_success() {
let (events, _rx) = crate::bus::test_bus(64);
let runner = ScriptedRunner::new(vec![ProcScript::never_exits()]);
let dir = tempfile::tempdir().unwrap();
let handle = spawn_supervisor(runner, test_paths(&dir), events);
let mut app = AppConfig::minimal("web", "./srv");
app.autorestart = false;
handle.start(vec![normalize(app).unwrap()]).await.unwrap();
handle
.stop(ProcessSelector::Name("web".to_string()))
.await
.unwrap();
assert_eq!(handle.list().await[0].status, ProcStatus::Stopped);
let reopened =
tokio::time::timeout(Duration::from_secs(5), handle.reopen(ProcessSelector::All))
.await
.expect("a reopen aimed at a stopped sheep must not wait for an acknowledgement")
.expect("a stopped sheep has nothing to reopen, which is a success");
assert_eq!(reopened.len(), 1);
assert_eq!(reopened[0].status, ProcStatus::Stopped);
}
#[tokio::test(start_paused = true)]
async fn a_reopen_whose_pump_is_already_gone_returns_at_once() {
let (tx, rx) = mpsc::channel(1);
drop(rx);
let outcome = tokio::time::timeout(Duration::from_secs(5), reopen_logs(&tx))
.await
.expect("a failed send must end the reopen, not leave it waiting");
assert_eq!(
outcome,
Ok(()),
"a pump that was never reached reopened nothing, which is a no-op \
success rather than a reopen that failed"
);
}
#[tokio::test(start_paused = true)]
async fn a_pump_that_ends_mid_request_still_ends_the_reopen() {
let (tx, mut rx) = mpsc::channel(1);
tokio::spawn(async move {
let request = rx.recv().await.expect("the reopen must reach the pump");
drop(request); });
let outcome = tokio::time::timeout(Duration::from_secs(5), reopen_logs(&tx))
.await
.expect("a dropped acknowledgement must end the reopen, not leave it waiting");
assert_eq!(
outcome,
Ok(()),
"a pump that ended mid-request reopened nothing, which is the same \
no-op success a failed send is"
);
}
#[tokio::test(start_paused = true)]
async fn a_pump_that_ends_mid_request_still_ends_the_flush() {
let (tx, mut rx) = mpsc::channel(1);
tokio::spawn(async move {
let request = rx.recv().await.expect("the flush must reach the pump");
drop(request); });
let outcome = tokio::time::timeout(Duration::from_secs(5), flush_logs(&tx))
.await
.expect("a dropped acknowledgement must end the flush, not leave it waiting");
assert_eq!(
outcome,
Ok(()),
"a pump that ended mid-request owes no bytes, which is the same \
no-op success a failed send is"
);
}
const PUMP_REFUSAL: &str = "/gone/web-out.log: No such file or directory";
const REFUSING_SHEEP: &str = "web";
#[derive(Debug)]
struct FailingPumpRunner {
inner: Arc<ScriptedRunner>,
}
impl FailingPumpRunner {
fn new(inner: Arc<ScriptedRunner>) -> Self {
Self { inner }
}
}
impl ProcessRunner for FailingPumpRunner {
type Proc = crate::fake::FakeProc;
fn spawn(
&self,
spec: &SpawnSpec,
) -> Result<(Self::Proc, ProcIo), crate::runner::RunnerError> {
let (proc, mut io) = self.inner.spawn(spec)?;
if spec.name != REFUSING_SHEEP {
return Ok((proc, io));
}
let (tx, mut rx) = mpsc::channel(SHEEP_CTL_CAPACITY);
io.log_ctl = tx;
tokio::spawn(async move {
while let Some(ctl) = rx.recv().await {
match ctl {
LogCtl::Reopen { done } => {
let _ = done.send(Err(ReopenError {
message: PUMP_REFUSAL.to_string(),
}));
}
LogCtl::Flush { done } => {
let _ = done.send(Err(FlushError {
message: PUMP_REFUSAL.to_string(),
}));
}
#[cfg(unix)]
LogCtl::ReportFds { done } => {
let _ = done.send(CarriedFds::none());
}
#[cfg(unix)]
LogCtl::Resume => {}
}
}
});
Ok((proc, io))
}
}
#[tokio::test(start_paused = true)]
async fn a_pump_that_could_not_reopen_fails_the_request_and_names_its_sheep() {
let (events, _rx) = crate::bus::test_bus(64);
let scripted = Arc::new(ScriptedRunner::new(vec![
ProcScript::never_exits(),
ProcScript::never_exits(),
]));
let dir = tempfile::tempdir().unwrap();
let handle = spawn_supervisor(
FailingPumpRunner::new(Arc::clone(&scripted)),
test_paths(&dir),
events,
);
handle
.start(vec![
normalize(AppConfig::minimal("web", "./srv")).unwrap(),
normalize(AppConfig::minimal("api", "./api")).unwrap(),
])
.await
.unwrap();
let error =
tokio::time::timeout(Duration::from_secs(5), handle.reopen(ProcessSelector::All))
.await
.expect("a pump that answers must not leave the reopen waiting")
.expect_err("a reopen a pump could not carry out must not answer Ok");
assert_eq!(
error,
SupervisorError::ReopenFailed(format!("web (id 0): could not reopen {PUMP_REFUSAL}")),
"the failure must carry the sheep and the path, and only the sheep that failed"
);
assert_eq!(
scripted.reopens(1),
1,
"the healthy sheep must still have been reopened"
);
}
#[tokio::test(start_paused = true)]
async fn a_signal_reaches_the_sheeps_own_process_and_not_its_group() {
let dir = tempfile::tempdir().unwrap();
let (handle, runner, _events) = started(
&dir,
AppConfig::minimal("web", "./srv"),
vec![ProcScript::never_exits()],
)
.await;
let rows = handle
.signal(ProcessSelector::Id(0), OperatorSignal::Hup)
.await
.unwrap();
assert_eq!(
rows,
vec![SignalReply {
id: 0,
name: "web".to_string(),
outcome: SignalOutcome::Delivered,
}]
);
assert_eq!(runner.process_signals(0), vec![OperatorSignal::Hup]);
assert!(
runner.signals(0).is_empty(),
"shep signal must not reach the process group"
);
}
#[tokio::test(start_paused = true)]
async fn a_stopped_sheep_answers_not_running_rather_than_delivered() {
let dir = tempfile::tempdir().unwrap();
let (handle, _runner, _events) = started(
&dir,
AppConfig::minimal("web", "./srv"),
vec![ProcScript::never_exits()],
)
.await;
handle.stop(ProcessSelector::Id(0)).await.unwrap();
let rows = handle
.signal(ProcessSelector::Id(0), OperatorSignal::Hup)
.await
.unwrap();
assert_eq!(rows.len(), 1);
assert_eq!(rows[0].outcome, SignalOutcome::NotRunning);
}
#[tokio::test(start_paused = true)]
async fn a_selector_that_matches_nothing_is_not_found() {
let h = harness(vec![]);
let err = h
.ctx
.supervisor
.signal(
ProcessSelector::Name("ghost".to_string()),
OperatorSignal::Hup,
)
.await
.unwrap_err();
assert_eq!(err, SupervisorError::NotFound);
}
#[tokio::test(start_paused = true)]
async fn a_reload_drainee_is_signalled_like_any_other_live_sheep() {
let dir = tempfile::tempdir().unwrap();
let (mut actor, mut signal_rx) = actor_with_a_drainee_holding_a_signal_mailbox(&dir);
let (reply, answer) = oneshot::channel();
actor.handle_command(Command::Signal {
selector: ProcessSelector::Id(0),
sig: OperatorSignal::Hup,
reply,
});
let request = tokio::time::timeout(ACTION_WINDOW, signal_rx.recv())
.await
.expect("no signal reached the drainee's mailbox within the window")
.expect("the drainee's signal mailbox closed");
assert_eq!(request.sig, OperatorSignal::Hup);
let _ = request.done.send(Ok(()));
let rows = tokio::time::timeout(ACTION_WINDOW, answer)
.await
.expect("the signal reported nothing within the window")
.expect("the signal's reply channel was dropped")
.unwrap();
assert_eq!(rows[0].outcome, SignalOutcome::Delivered);
}
#[tokio::test(start_paused = true)]
async fn a_line_reaches_a_sheep_that_asked_for_stdin() {
let dir = tempfile::tempdir().unwrap();
let mut app = AppConfig::minimal("repl", "./repl");
app.stdin = true;
let (handle, runner, _events) = started(&dir, app, vec![ProcScript::never_exits()]).await;
let rows = handle
.send_line(ProcessSelector::Id(0), "reload-config".to_string())
.await
.unwrap();
assert_eq!(
rows,
vec![LineReply {
id: 0,
name: "repl".to_string(),
outcome: LineOutcome::Sent,
}]
);
assert_eq!(runner.stdin_lines(0), vec!["reload-config".to_string()]);
}
#[tokio::test(start_paused = true)]
async fn a_sheep_without_stdin_answers_no_stdin() {
let dir = tempfile::tempdir().unwrap();
let (handle, _runner, _events) = started(
&dir,
AppConfig::minimal("web", "./srv"),
vec![ProcScript::never_exits()],
)
.await;
let rows = handle
.send_line(ProcessSelector::Id(0), "hello".to_string())
.await
.unwrap();
assert_eq!(rows[0].outcome, LineOutcome::NoStdin);
}
#[tokio::test(start_paused = true)]
async fn a_mixed_flock_reports_per_sheep_rather_than_failing() {
let dir = tempfile::tempdir().unwrap();
let mut piped = AppConfig::minimal("repl", "./repl");
piped.stdin = true;
let (handle, runner, _events) = started(
&dir,
piped,
vec![ProcScript::never_exits(), ProcScript::never_exits()],
)
.await;
handle
.start(vec![normalize(AppConfig::minimal("web", "./srv")).unwrap()])
.await
.unwrap();
let rows = handle
.send_line(ProcessSelector::All, "hello".to_string())
.await
.unwrap();
let outcome = |id| rows.iter().find(|r| r.id == id).unwrap().outcome.clone();
assert_eq!(outcome(0), LineOutcome::Sent);
assert_eq!(outcome(1), LineOutcome::NoStdin);
assert_eq!(runner.stdin_lines(1), Vec::<String>::new());
assert!(rows.windows(2).all(|w| w[0].id < w[1].id));
}
#[tokio::test(start_paused = true)]
async fn a_write_that_never_lands_times_out_and_says_so() {
let dir = tempfile::tempdir().unwrap();
let mut app = AppConfig::minimal("stuck", "./stuck");
app.stdin = true;
let (handle, _runner, _events) =
started(&dir, app, vec![ProcScript::never_reads_its_stdin()]).await;
let rows = tokio::time::timeout(
STDIN_WRITE_TIMEOUT * 4,
handle.send_line(ProcessSelector::Id(0), "hello".to_string()),
)
.await
.expect("send_line did not honour its own bound")
.unwrap();
let LineOutcome::NotWritten { reason } = rows[0].outcome.clone() else {
panic!("expected NotWritten, got {:?}", rows[0].outcome);
};
assert!(reason.contains("read"), "{reason}");
}
#[tokio::test(start_paused = true)]
async fn a_flock_of_wedged_sheep_is_bounded_once_and_not_once_each() {
let dir = tempfile::tempdir().unwrap();
let mut app = AppConfig::minimal("stuck", "./stuck");
app.stdin = true;
app.instances = 3;
let (handle, _runner, _events) =
started(&dir, app, vec![ProcScript::never_reads_its_stdin(); 3]).await;
let started_at = tokio::time::Instant::now();
let rows = handle
.send_line(ProcessSelector::All, "hello".to_string())
.await
.unwrap();
let elapsed = started_at.elapsed();
assert_eq!(rows.len(), 3);
assert!(
rows.iter()
.all(|row| matches!(row.outcome, LineOutcome::NotWritten { .. })),
"{rows:?}"
);
assert!(
elapsed < STDIN_WRITE_TIMEOUT * 2,
"three wedged sheep cost {elapsed:?}; the bound is per-CALL, not per-sheep"
);
}
#[tokio::test(start_paused = true)]
async fn a_selector_that_matches_nothing_is_not_found_for_send_line() {
let h = harness(vec![]);
assert_eq!(
h.ctx
.supervisor
.send_line(ProcessSelector::Name("ghost".to_string()), "x".to_string())
.await
.unwrap_err(),
SupervisorError::NotFound
);
}
#[tokio::test(start_paused = true)]
async fn a_flush_reaches_every_matched_sheep_and_no_others() {
let (events, _rx) = crate::bus::test_bus(64);
let runner = Arc::new(ScriptedRunner::new(vec![
ProcScript::never_exits(),
ProcScript::never_exits(),
ProcScript::never_exits(),
]));
let dir = tempfile::tempdir().unwrap();
let handle = spawn_supervisor(SharedRunner(Arc::clone(&runner)), test_paths(&dir), events);
let mut web = AppConfig::minimal("web", "./srv");
web.instances = 2;
handle
.start(vec![
normalize(web).unwrap(),
normalize(AppConfig::minimal("api", "./api")).unwrap(),
])
.await
.unwrap();
let flushed = handle
.flush(ProcessSelector::Name("web".to_string()))
.await
.unwrap();
assert_eq!(
flushed.iter().map(|info| info.id).collect::<Vec<_>>(),
vec![0, 1],
"the reply must carry both `web` instances, id-sorted, and no `api`"
);
assert_eq!(runner.flushes(0), 1, "web's first instance");
assert_eq!(runner.flushes(1), 1, "web's second instance");
assert_eq!(runner.flushes(2), 0, "api was never named");
assert_eq!(
runner.reopens(0),
0,
"a flush must push `LogCtl::Flush`, never `LogCtl::Reopen` — a \
flush wired to the neighbouring variant would swap the flock's \
handles and empty nothing"
);
}
#[tokio::test(start_paused = true)]
async fn the_actor_keeps_answering_while_a_flush_waits_on_a_silent_pump() {
let (events, _rx) = crate::bus::test_bus(64);
let (runner, mut requests) = SilentPumpRunner::new(vec![ProcScript::never_exits()]);
let dir = tempfile::tempdir().unwrap();
let handle = spawn_supervisor(runner, test_paths(&dir), events);
handle
.start(vec![normalize(AppConfig::minimal("web", "./srv")).unwrap()])
.await
.unwrap();
let flushing = tokio::spawn({
let handle = handle.clone();
async move { handle.flush(ProcessSelector::All).await }
});
tokio::time::timeout(Duration::from_secs(5), requests.wait_for(|seen| *seen == 1))
.await
.expect("the flush must reach the pump")
.expect("the runner outlives this wait, so its sender cannot have closed");
let listed = tokio::time::timeout(Duration::from_secs(5), handle.list())
.await
.expect("the actor must keep answering while a flush is outstanding");
assert_eq!(listed.len(), 1);
assert!(
!flushing.is_finished(),
"sanity: nothing can acknowledge this flush, so `list` answering \
above is not just the flush having finished first"
);
flushing.abort();
}
const LATE_LINE: &str = "landed-while-the-flush-was-being-answered\n";
const LATE_WRITING_SHEEP: &str = "latecomer";
struct LateWritingPumpRunner {
inner: ScriptedRunner,
}
impl LateWritingPumpRunner {
fn new(inner: ScriptedRunner) -> Self {
Self { inner }
}
}
impl fmt::Debug for LateWritingPumpRunner {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("LateWritingPumpRunner")
.finish_non_exhaustive()
}
}
impl ProcessRunner for LateWritingPumpRunner {
type Proc = crate::fake::FakeProc;
fn spawn(
&self,
spec: &SpawnSpec,
) -> Result<(Self::Proc, ProcIo), crate::runner::RunnerError> {
let (proc, mut io) = self.inner.spawn(spec)?;
if spec.name != LATE_WRITING_SHEEP {
return Ok((proc, io));
}
let (tx, mut rx) = mpsc::channel(SHEEP_CTL_CAPACITY);
io.log_ctl = tx;
let out_file = spec.out_file.clone();
tokio::spawn(async move {
while let Some(ctl) = rx.recv().await {
match ctl {
LogCtl::Flush { done } => {
if let Some(parent) = out_file.parent() {
std::fs::create_dir_all(parent).unwrap();
}
let mut file = std::fs::OpenOptions::new()
.create(true)
.append(true)
.open(&out_file)
.unwrap();
std::io::Write::write_all(&mut file, LATE_LINE.as_bytes()).unwrap();
let _ = done.send(Ok(()));
}
LogCtl::Reopen { done } => {
let _ = done.send(Ok(()));
}
#[cfg(unix)]
LogCtl::ReportFds { done } => {
let _ = done.send(CarriedFds::none());
}
#[cfg(unix)]
LogCtl::Resume => {}
}
}
});
Ok((proc, io))
}
}
#[tokio::test(start_paused = true)]
async fn a_flush_truncates_only_after_its_pump_has_answered() {
let (events, _rx) = crate::bus::test_bus(64);
let dir = tempfile::tempdir().unwrap();
let handle = spawn_supervisor(
LateWritingPumpRunner::new(ScriptedRunner::new(vec![ProcScript::never_exits()])),
test_paths(&dir),
events,
);
handle
.start(vec![
normalize(AppConfig::minimal(LATE_WRITING_SHEEP, "./srv")).unwrap(),
])
.await
.unwrap();
let out_file = PathBuf::from(
handle.list().await[0]
.out_file
.clone()
.expect("the daemon reports its own resolved log paths"),
);
handle.flush(ProcessSelector::All).await.unwrap();
assert!(
out_file.exists(),
"the pump never wrote, so this flush never reached one: {}",
out_file.display()
);
assert_eq!(
std::fs::read_to_string(&out_file).unwrap(),
"",
"a line the pump landed as it answered the flush must not survive \
the truncate that follows it"
);
}
#[tokio::test(start_paused = true)]
async fn a_stopped_sheeps_log_file_is_truncated_too() {
let (events, _rx) = crate::bus::test_bus(64);
let runner = ScriptedRunner::new(vec![ProcScript::never_exits()]);
let dir = tempfile::tempdir().unwrap();
let handle = spawn_supervisor(runner, test_paths(&dir), events);
let mut app = AppConfig::minimal("web", "./srv");
app.autorestart = false;
handle.start(vec![normalize(app).unwrap()]).await.unwrap();
handle
.stop(ProcessSelector::Name("web".to_string()))
.await
.unwrap();
let listed = handle.list().await;
assert_eq!(listed[0].status, ProcStatus::Stopped);
let out_file = PathBuf::from(listed[0].out_file.clone().unwrap());
std::fs::create_dir_all(out_file.parent().unwrap()).unwrap();
std::fs::write(&out_file, "what the sheep logged before it stopped\n").unwrap();
let flushed =
tokio::time::timeout(Duration::from_secs(5), handle.flush(ProcessSelector::All))
.await
.expect("a flush aimed at a stopped sheep must not wait for an acknowledgement")
.expect("a stopped sheep has no pump to flush, which is not a failure");
assert_eq!(flushed.len(), 1);
assert_eq!(flushed[0].status, ProcStatus::Stopped);
assert_eq!(
std::fs::read_to_string(&out_file).unwrap(),
"",
"a stopped sheep's log is still readable, so it is still emptied"
);
}
#[tokio::test(start_paused = true)]
async fn instances_sharing_one_log_path_answer_one_row_each() {
let (events, _rx) = crate::bus::test_bus(64);
let runner =
ScriptedRunner::new(vec![ProcScript::never_exits(), ProcScript::never_exits()]);
let dir = tempfile::tempdir().unwrap();
let handle = spawn_supervisor(runner, test_paths(&dir), events);
let mut web = AppConfig::minimal("web", "./srv");
web.instances = 2;
web.merge_logs = true;
handle.start(vec![normalize(web).unwrap()]).await.unwrap();
let listed = handle.list().await;
assert_eq!(
listed[0].out_file, listed[1].out_file,
"fixture check: `merge_logs` must really point both instances at \
one path, or this case proves nothing"
);
let shared = PathBuf::from(listed[0].out_file.clone().unwrap());
std::fs::create_dir_all(shared.parent().unwrap()).unwrap();
std::fs::write(&shared, "both instances wrote here\n").unwrap();
let flushed = handle.flush(ProcessSelector::All).await.unwrap();
assert_eq!(
flushed.iter().map(|info| info.id).collect::<Vec<_>>(),
vec![0, 1],
"one row per sheep, not per file emptied"
);
assert_eq!(std::fs::read_to_string(&shared).unwrap(), "");
}
#[tokio::test(start_paused = true)]
async fn a_sibling_sharing_a_path_is_flushed_even_when_the_selector_skips_it() {
let (events, _rx) = crate::bus::test_bus(64);
let dir = tempfile::tempdir().unwrap();
let handle = spawn_supervisor(
LateWritingPumpRunner::new(ScriptedRunner::new(vec![
ProcScript::never_exits(),
ProcScript::never_exits(),
])),
test_paths(&dir),
events,
);
let shared = dir.path().join("shared-out.log");
let mut named = AppConfig::minimal("web", "./srv");
named.out_file = Some(shared.display().to_string());
let mut sibling = AppConfig::minimal(LATE_WRITING_SHEEP, "./api");
sibling.out_file = Some(shared.display().to_string());
handle
.start(vec![normalize(named).unwrap(), normalize(sibling).unwrap()])
.await
.unwrap();
let listed = handle.list().await;
assert_eq!(
listed[0].out_file, listed[1].out_file,
"fixture check: both apps must really resolve to one path, or \
this case proves nothing"
);
let flushed = handle
.flush(ProcessSelector::Name("web".to_string()))
.await
.unwrap();
assert_eq!(
flushed.iter().map(|info| info.id).collect::<Vec<_>>(),
vec![0],
"the reply answers the selector: the sibling's file was emptied \
too, but the operator never named the sibling and it is not a \
row here"
);
assert!(
shared.exists(),
"the sibling's pump writes as it answers a flush, so a path that \
is not there means it was never asked: {}",
shared.display()
);
assert_eq!(
std::fs::read_to_string(&shared).unwrap(),
"",
"a line the unmatched sibling landed as it answered must not \
survive the truncate of the path it shares"
);
}
#[tokio::test(start_paused = true)]
async fn a_pump_that_could_not_flush_fails_the_request() {
let (events, _rx) = crate::bus::test_bus(64);
let scripted = Arc::new(ScriptedRunner::new(vec![
ProcScript::never_exits(),
ProcScript::never_exits(),
]));
let dir = tempfile::tempdir().unwrap();
let handle = spawn_supervisor(
FailingPumpRunner::new(Arc::clone(&scripted)),
test_paths(&dir),
events,
);
handle
.start(vec![
normalize(AppConfig::minimal(REFUSING_SHEEP, "./srv")).unwrap(),
normalize(AppConfig::minimal("api", "./api")).unwrap(),
])
.await
.unwrap();
let error =
tokio::time::timeout(Duration::from_secs(5), handle.flush(ProcessSelector::All))
.await
.expect("a pump that answers must not leave the flush waiting")
.expect_err("a flush a pump could not carry out must not answer Ok");
assert_eq!(
error,
SupervisorError::FlushFailed(PUMP_REFUSAL.to_string()),
"the failure must carry the path, and only the path that failed"
);
assert_eq!(
scripted.flushes(1),
1,
"the healthy sheep must still have been flushed"
);
}
#[tokio::test(start_paused = true)]
async fn a_flush_matching_nothing_is_not_found() {
let (events, _rx) = crate::bus::test_bus(64);
let dir = tempfile::tempdir().unwrap();
let handle = spawn_supervisor(ScriptedRunner::new(vec![]), test_paths(&dir), events);
assert_eq!(
handle.flush(ProcessSelector::All).await,
Err(SupervisorError::NotFound)
);
}
#[tokio::test]
async fn truncating_a_path_that_is_not_there_creates_nothing_and_succeeds() {
let dir = tempfile::tempdir().unwrap();
let missing = dir.path().join("never-started-out.log");
assert_eq!(truncate_log(&missing).await, Ok(()));
assert!(
!missing.exists(),
"a flush must not create the log file it did not find"
);
}
#[tokio::test]
async fn truncating_a_path_that_is_a_directory_reports_the_failure() {
let dir = tempfile::tempdir().unwrap();
let blocked = dir.path().join("web-out.log");
std::fs::create_dir(&blocked).unwrap();
let error = truncate_log(&blocked)
.await
.expect_err("a path that could not be truncated must not answer Ok");
assert!(
error
.message
.starts_with(&format!("{}: ", blocked.display())),
"the failure must name the path it could not empty: {error}"
);
}
#[cfg(unix)]
#[tokio::test]
async fn truncating_a_symlinked_log_path_refuses_and_leaves_its_target_alone() {
let dir = tempfile::tempdir().unwrap();
let target = dir.path().join("precious.txt");
let link = dir.path().join("web-out.log");
std::fs::write(&target, b"do not empty me").unwrap();
std::os::unix::fs::symlink(&target, &link).unwrap();
let error = truncate_log(&link)
.await
.expect_err("a symlinked log path must not be truncated through");
assert_eq!(
std::fs::read(&target).unwrap(),
b"do not empty me",
"the symlink's target must still hold every byte it did"
);
assert!(
std::fs::symlink_metadata(&link)
.unwrap()
.file_type()
.is_symlink(),
"the refusal must leave the symlink itself in place, not replace it"
);
assert_eq!(
error.message,
format!("{}: {}", link.display(), crate::runner::SYMLINK_REFUSED),
"the failure must name the path and say the word symlink: {error}"
);
}
#[tokio::test(start_paused = true)]
async fn a_flush_naming_a_replacement_still_drains_the_drainee_sharing_its_path() {
let dir = tempfile::tempdir().unwrap();
let (handle, runner, mut rx) = started(
&dir,
AppConfig::minimal("web", "./srv"),
vec![ProcScript::never_exits(), ProcScript::never_exits()],
)
.await;
handle
.reload(ProcessSelector::Name("web".to_string()))
.await
.expect("the reload is accepted");
expect_event(&mut rx, 1, ProcessEventKind::Start).await;
let mid = handle.list().await;
assert_eq!(
mid.len(),
2,
"fixture check: both halves of the swap must be registered, or \
there is no shared path to widen to"
);
assert_eq!(
mid[0].out_file, mid[1].out_file,
"fixture check: one instance slot must really give both entries \
one out path, or this case proves nothing"
);
assert_eq!(
mid[0].err_file, mid[1].err_file,
"fixture check: and one err path"
);
let flushed = handle.flush(ProcessSelector::Id(1)).await.unwrap();
assert_eq!(
flushed.iter().map(|info| info.id).collect::<Vec<_>>(),
vec![1],
"the reply answers the selector: the drainee's pump was drained \
too, but the operator named only the replacement"
);
assert_eq!(
runner.flushes(0),
1,
"the drainee's pump is what this case exists for — it is still \
holding the file the truncate is about to empty"
);
assert_eq!(
runner.flushes(1),
1,
"the replacement's pump, which the selector did name"
);
}
#[tokio::test(start_paused = true)]
async fn a_reopen_naming_a_replacement_still_reaches_the_drainee_sharing_its_path() {
let dir = tempfile::tempdir().unwrap();
let (handle, runner, mut rx) = started(
&dir,
AppConfig::minimal("web", "./srv"),
vec![ProcScript::never_exits(), ProcScript::never_exits()],
)
.await;
handle
.reload(ProcessSelector::Name("web".to_string()))
.await
.expect("the reload is accepted");
expect_event(&mut rx, 1, ProcessEventKind::Start).await;
let mid = handle.list().await;
assert_eq!(
mid.len(),
2,
"fixture check: both halves of the swap must be registered"
);
assert_eq!(
mid[0].out_file, mid[1].out_file,
"fixture check: one instance slot must really give both entries \
one out path, or this case proves nothing"
);
assert_eq!(
mid[0].err_file, mid[1].err_file,
"fixture check: and one err path"
);
let reopened = handle.reopen(ProcessSelector::Id(1)).await.unwrap();
assert_eq!(
reopened.iter().map(|info| info.id).collect::<Vec<_>>(),
vec![1],
"the reply answers the selector, the same way `flush`'s does"
);
assert_eq!(
runner.reopens(0),
1,
"the drainee's pump is what this case exists for — unasked, it \
keeps the renamed inode open and goes on filling the archive"
);
assert_eq!(
runner.reopens(1),
1,
"the replacement's pump, which the selector did name"
);
assert_eq!(
runner.flushes(0),
0,
"a reopen must push `LogCtl::Reopen`, never `LogCtl::Flush` — the \
neighbouring variant would land the drainee's owed bytes and \
leave it on the renamed inode regardless"
);
}
fn log_ctl_spec() -> SpawnSpec {
SpawnSpec {
name: "svc".to_string(),
program: "./svc".to_string(),
args: Vec::new(),
cwd: None,
env: std::collections::BTreeMap::new(),
out_file: std::path::PathBuf::from("out.log"),
err_file: std::path::PathBuf::from("err.log"),
channel: false,
stdin: false,
credentials: None,
}
}
fn actor_with_an_empty_flock(
dir: &tempfile::TempDir,
scripts: Vec<ProcScript>,
) -> Actor<ScriptedRunner> {
let (tx, _rx) = mpsc::channel(MAILBOX_CAPACITY);
let paths = test_paths(dir);
test_actor(paths, scripts, HashMap::new(), tx)
}
#[cfg(unix)]
fn own_user_name() -> String {
nix::unistd::User::from_uid(nix::unistd::geteuid())
.unwrap()
.expect("this process has a passwd entry")
.name
}
const NO_SUCH_USER: &str = "definitely-not-a-real-shep-user";
#[cfg(unix)]
#[tokio::test(start_paused = true)]
async fn a_restored_app_restarts_under_its_configured_user() {
let dir = tempfile::tempdir().unwrap();
let mut actor = actor_with_an_empty_flock(&dir, vec![ProcScript::never_exits()]);
let app = app_with("web", |app| app.user = Some(own_user_name()));
let info = actor.register_at_rest(&app);
assert_eq!(
actor.sheep[&info.id].entry.credentials,
SpawnIdentity::Unresolved,
"a membership record is not a `Start`: nothing has looked this app's user up yet"
);
actor.respawn(info.id, true);
let wanted = Credentials {
uid: nix::unistd::geteuid().as_raw(),
gid: None,
};
assert_eq!(
actor.runner.spawned_as(0),
Some(wanted),
"the restarted child must carry the identity its `user` resolves to; `None` here is \
the child running as the shepherd, which is the downgrade"
);
assert_eq!(
actor.sheep[&info.id].entry.credentials,
SpawnIdentity::Resolved(Some(wanted)),
"resolved once at the first spawn and stored, so no later restart looks it up again"
);
}
#[tokio::test(start_paused = true)]
async fn a_restored_app_whose_user_cannot_be_resolved_is_refused() {
let dir = tempfile::tempdir().unwrap();
let mut actor = actor_with_an_empty_flock(&dir, vec![ProcScript::never_exits()]);
let app = app_with("web", |app| app.user = Some(NO_SUCH_USER.to_string()));
let info = actor.register_at_rest(&app);
let after = actor.respawn(info.id, true);
assert_eq!(
actor.runner.spawn_count(),
0,
"nothing may be spawned for an app whose identity could not be resolved"
);
assert_eq!(after.status, ProcStatus::Errored);
assert_eq!(
actor.sheep[&info.id].entry.credentials,
SpawnIdentity::Unresolved,
"a failed resolution settles nothing: the next restart must ask again"
);
}
#[cfg(unix)]
#[tokio::test(start_paused = true)]
async fn a_restart_reuses_a_resolved_identity_rather_than_looking_it_up_again() {
let dir = tempfile::tempdir().unwrap();
let (mut actor, _rx) = actor_with_one_online_sheep(&dir, vec![ProcScript::never_exits()]);
let pinned = Credentials {
uid: 4242,
gid: Some(4243),
};
let entry = &mut actor
.sheep
.get_mut(&0)
.expect("the fixture registers id 0")
.entry;
entry.spec = app_with("web", |app| app.user = Some(own_user_name()));
entry.credentials = SpawnIdentity::Resolved(Some(pinned));
actor.respawn(0, true);
assert_eq!(
actor.runner.spawned_as(0),
Some(pinned),
"a running app's identity must survive its restart untouched"
);
}
#[cfg(unix)]
#[tokio::test(start_paused = true)]
async fn a_start_refused_over_credentials_leaves_an_errored_row() {
let dir = tempfile::tempdir().unwrap();
let mut actor = actor_with_an_empty_flock(&dir, vec![ProcScript::never_exits()]);
let app = app_with("web", |app| app.user = Some(NO_SUCH_USER.to_string()));
let err = actor
.do_start(vec![app], None, BatchPolicy::PerApp, &BTreeSet::new())
.expect_err("an unresolvable user must refuse the start");
assert!(
err.to_string().contains(NO_SUCH_USER),
"the refusal must name the user it could not resolve: {err}"
);
assert_eq!(actor.runner.spawn_count(), 0);
let slot = actor
.sheep
.values()
.next()
.expect("the refusal must leave a row rather than vanishing");
assert_eq!(slot.entry.status, ProcStatus::Errored);
assert_eq!(
slot.entry.credentials,
SpawnIdentity::Unresolved,
"the row must not claim a settled identity: a `restart` would reuse it and bring the \
sheep up as the shepherd, which is the bug the row was added to make visible"
);
let id = slot.entry.id;
let after = actor.respawn(id, true);
assert_eq!(after.status, ProcStatus::Errored);
assert_eq!(actor.runner.spawn_count(), 0);
}
#[cfg(unix)]
#[tokio::test(start_paused = true)]
async fn an_all_or_nothing_start_refused_over_credentials_registers_nothing() {
let dir = tempfile::tempdir().unwrap();
let mut actor = actor_with_an_empty_flock(&dir, vec![ProcScript::never_exits()]);
let app = app_with("web", |app| app.user = Some(NO_SUCH_USER.to_string()));
let err = actor
.do_start(vec![app], None, BatchPolicy::AllOrNothing, &BTreeSet::new())
.expect_err("an unresolvable user must refuse the start");
assert!(
err.to_string().contains(NO_SUCH_USER),
"the refusal must name the user it could not resolve: {err}"
);
assert!(
actor.sheep.is_empty(),
"`AllOrNothing` registers nothing at all, this app included"
);
assert_eq!(actor.runner.spawn_count(), 0);
}
#[cfg(unix)]
#[tokio::test(start_paused = true)]
async fn scaling_an_app_whose_user_will_not_resolve_leaves_the_flock_alone() {
let dir = tempfile::tempdir().unwrap();
let mut actor = actor_with_an_empty_flock(&dir, vec![ProcScript::never_exits(); 4]);
let app = app_with("web", |app| app.user = Some(NO_SUCH_USER.to_string()));
let registered = actor.register_at_rest(&app);
let (reply, answer) = oneshot::channel();
actor.handle_scale("web", 3, reply);
let err = answer
.await
.expect("handle_scale answers every call")
.expect_err("an unresolvable user must refuse the scale");
let SupervisorError::CannotStart(message) = &err else {
panic!("expected CannotStart, got {err:?}");
};
assert!(
message.contains("web") && message.contains(NO_SUCH_USER),
"the refusal must name the app and the user it could not resolve: {message}"
);
assert_eq!(actor.runner.spawn_count(), 0);
assert_eq!(
actor.sheep.len(),
1,
"no instance may be registered by a scale that could not resolve an identity"
);
assert_eq!(
actor.sheep[®istered.id].entry.spec.config().instances,
1,
"the stored count must not be written back for a scale that did not happen"
);
assert_eq!(
actor.sheep[®istered.id].entry.credentials,
SpawnIdentity::Unresolved,
"a failed resolution settles nothing: the next attempt must ask again"
);
}
#[cfg(unix)]
#[tokio::test(start_paused = true)]
async fn a_second_restore_does_not_re_announce_an_errored_row() {
let dir = tempfile::tempdir().unwrap();
let mut actor = actor_with_an_empty_flock(&dir, vec![ProcScript::never_exits(); 2]);
let app = app_with("web", |app| app.user = Some(NO_SUCH_USER.to_string()));
let mut events = actor.events.subscribe();
for attempt in 1..=2 {
assert!(
actor
.do_start(
vec![app.clone()],
None,
BatchPolicy::PerApp,
&BTreeSet::new()
)
.is_err(),
"restore {attempt} must refuse the app"
);
}
assert_eq!(
actor.sheep.len(),
1,
"the second restore must find the row, not add a second one"
);
let mut errored = 0;
while let Ok(event) = events.try_recv().map(|event| event.to_event()) {
if let BusEvent::Process {
event: ProcessEventKind::Errored,
info,
..
} = event
&& info.name == "web"
{
errored += 1;
}
}
assert_eq!(
errored, 1,
"one Errored event, for the restore that actually registered the row: a second is a \
transition that did not happen"
);
}
#[cfg(unix)]
#[tokio::test(start_paused = true)]
async fn scaling_up_the_row_a_refused_start_left_meets_the_same_refusal() {
let dir = tempfile::tempdir().unwrap();
let mut actor = actor_with_an_empty_flock(&dir, vec![ProcScript::never_exits(); 4]);
let app = app_with("web", |app| app.user = Some(NO_SUCH_USER.to_string()));
actor
.do_start(vec![app], None, BatchPolicy::PerApp, &BTreeSet::new())
.expect_err("an unresolvable user must refuse the start");
let id = actor
.sheep
.values()
.next()
.expect("the refused start leaves a row")
.entry
.id;
assert_eq!(actor.sheep[&id].entry.status, ProcStatus::Errored);
let (reply, answer) = oneshot::channel();
actor.handle_scale("web", 3, reply);
let err = answer
.await
.expect("handle_scale answers every call")
.expect_err("the row's user still will not resolve");
assert!(
matches!(err, SupervisorError::CannotStart(_)),
"expected CannotStart, got {err:?}"
);
assert_eq!(actor.runner.spawn_count(), 0);
assert_eq!(
actor.sheep.len(),
1,
"no instance may be registered by a scale that could not resolve an identity"
);
}
#[cfg(unix)]
#[tokio::test(start_paused = true)]
async fn a_no_op_scale_asks_no_passwd_question() {
let dir = tempfile::tempdir().unwrap();
let mut actor = actor_with_an_empty_flock(&dir, vec![ProcScript::never_exits(); 2]);
let app = app_with("web", |app| app.user = Some(own_user_name()));
let registered = actor.register_at_rest(&app);
let (reply, answer) = oneshot::channel();
actor.handle_scale("web", 1, reply);
answer
.await
.expect("handle_scale answers every call")
.expect("scaling to the count an app already has is a no-op");
assert_eq!(
actor.sheep[®istered.id].entry.credentials,
SpawnIdentity::Unresolved,
"a scale that spawns nothing must not resolve an identity: nothing would use it, \
and asking is what lets the answer refuse the call"
);
assert_eq!(actor.runner.spawn_count(), 0);
}
#[cfg(unix)]
#[tokio::test(start_paused = true)]
async fn a_no_op_scale_survives_a_user_that_will_not_resolve() {
let dir = tempfile::tempdir().unwrap();
let mut actor = actor_with_an_empty_flock(&dir, vec![ProcScript::never_exits(); 2]);
let app = app_with("web", |app| app.user = Some(NO_SUCH_USER.to_string()));
let registered = actor.register_at_rest(&app);
let (reply, answer) = oneshot::channel();
actor.handle_scale("web", 1, reply);
let scaled = answer
.await
.expect("handle_scale answers every call")
.expect("a no-op scale must not be refused over an identity it never needed");
assert_eq!(scaled.instances.len(), 1);
assert_eq!(actor.runner.spawn_count(), 0);
assert_eq!(
actor.sheep[®istered.id].entry.status,
ProcStatus::Stopped,
"a no-op must leave the sheep exactly as it found it"
);
}
#[cfg(unix)]
#[test]
fn a_restart_that_starts_nothing_says_why_in_the_log() {
let dir = tempfile::tempdir().unwrap();
let rt = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.unwrap();
let refused = capture_logs(|| {
let mut actor = actor_with_an_empty_flock(&dir, Vec::new());
let info = actor.register_at_rest(&app_with("web", |_| {}));
rt.block_on(async { actor.respawn(info.id, true) });
});
assert!(
refused.contains("script exhausted"),
"the runner's own reason must reach the log: {refused}"
);
let unresolved = capture_logs(|| {
let mut actor = actor_with_an_empty_flock(&dir, vec![ProcScript::never_exits()]);
let app = app_with("web", |app| app.user = Some(NO_SUCH_USER.to_string()));
let info = actor.register_at_rest(&app);
rt.block_on(async { actor.respawn(info.id, true) });
});
assert!(
unresolved.contains(NO_SUCH_USER),
"the unresolvable user must reach the log by name: {unresolved}"
);
}
#[derive(Debug, Clone, Copy)]
enum Step {
List,
StopAll,
RestartAll,
DeleteFirst,
StartOne,
Report,
StaleReport,
}
fn step_strategy() -> impl proptest::strategy::Strategy<Value = Step> {
proptest::prop_oneof![
proptest::strategy::Just(Step::List),
proptest::strategy::Just(Step::StopAll),
proptest::strategy::Just(Step::RestartAll),
proptest::strategy::Just(Step::DeleteFirst),
proptest::strategy::Just(Step::StartOne),
proptest::strategy::Just(Step::Report),
proptest::strategy::Just(Step::StaleReport),
]
}
#[derive(Debug, Clone, Copy)]
enum Gate {
Ungated,
Channel(u64),
}
fn gate_strategy() -> impl proptest::strategy::Strategy<Value = Gate> {
use proptest::strategy::Strategy as _; proptest::prop_oneof![
2 => proptest::strategy::Just(Gate::Ungated),
1 => (1u64..4_000u64).prop_map(Gate::Channel),
]
}
fn script_strategy() -> impl proptest::strategy::Strategy<Value = ProcScript> {
proptest::prop_oneof![
6 => proptest::strategy::Just(ProcScript::never_exits()),
2 => proptest::strategy::Just(ProcScript::const_exit(1)),
1 => proptest::strategy::Just(ProcScript::stable_then_exit(2_000, 0)),
1 => proptest::strategy::Just(ProcScript::ignores_signals()),
]
}
const SCRIPT_POOL: usize = 512;
const QUIET_WINDOW: Duration = Duration::from_secs(60);
const EVENT_BUDGET: usize = 3 * SCRIPT_POOL;
proptest::proptest! {
#![proptest_config(crate::testing::proptest_config(128))]
#[test]
fn supervisor_upholds_its_invariants_under_any_interleaving(
steps in proptest::collection::vec(step_strategy(), 1..10),
gates in proptest::collection::vec(gate_strategy(), 1..10),
scripts in proptest::collection::vec(script_strategy(), SCRIPT_POOL..SCRIPT_POOL + 1),
) {
let runtime = tokio::runtime::Builder::new_current_thread()
.enable_all()
.start_paused(true)
.build()
.unwrap();
let dir = tempfile::tempdir().unwrap();
runtime.block_on(async move {
let (events, mut rx) = crate::bus::test_bus(8192);
let handle = spawn_supervisor(
ScriptedRunner::new(scripts),
test_paths(&dir),
events,
);
let mut started = 0u32;
let mut highest_restarts = std::collections::HashMap::<u32, u32>::new();
for step in steps {
match step {
Step::StartOne => {
let gate = gates[started as usize % gates.len()];
started += 1;
let mut app = AppConfig::minimal(&format!("sheep-{started}"), "./s");
if let Gate::Channel(ms) = gate {
app.wait_ready = true;
app.listen_timeout = UpDuration::from_millis(ms);
}
let _ = handle.start(vec![normalize(app).unwrap()]).await;
}
Step::StopAll => {
if let Ok(stopped) = handle.stop(ProcessSelector::All).await {
for info in stopped {
proptest::prop_assert_eq!(info.status, ProcStatus::Stopped);
}
}
}
Step::RestartAll => {
let _ = handle.restart(ProcessSelector::All).await;
}
Step::DeleteFirst => {
if let Some(first) = handle.list().await.first() {
let id = first.id;
if let Ok(deleted) = handle.delete(ProcessSelector::Id(id)).await {
proptest::prop_assert_eq!(deleted, vec![id]);
}
proptest::prop_assert!(
handle.list().await.iter().all(|i| i.id != id)
);
}
}
Step::Report => {
if let Some(first) = handle.list().await.first()
&& let Some(pid) = first.pid
{
handle.extra_restart(first.id, pid, None, None).await;
}
}
Step::StaleReport => {
if let Some(first) = handle.list().await.first() {
let stale = first.pid.unwrap_or(0).wrapping_add(1);
handle.extra_restart(first.id, stale, None, None).await;
}
}
Step::List => {}
}
let listed = handle.list().await;
let ids: Vec<u32> = listed.iter().map(|i| i.id).collect();
let mut sorted = ids.clone();
sorted.sort_unstable();
sorted.dedup();
proptest::prop_assert_eq!(&ids, &sorted);
for info in &listed {
let seen = highest_restarts.entry(info.id).or_default();
proptest::prop_assert!(info.restarts >= *seen);
*seen = info.restarts;
proptest::prop_assert!(matches!(
info.status,
ProcStatus::Starting | ProcStatus::Online | ProcStatus::Stopping
| ProcStatus::Stopped | ProcStatus::Errored | ProcStatus::WaitingRestart
));
}
}
let mut observed = Vec::new();
loop {
match tokio::time::timeout(QUIET_WINDOW, rx.recv()).await {
Ok(Ok(event)) => {
observed.push(event.to_event());
proptest::prop_assert!(
observed.len() <= EVENT_BUDGET,
"the flock never reached steady state: {} transitions after \
the last command",
observed.len()
);
}
Ok(Err(tokio::sync::broadcast::error::RecvError::Lagged(skipped))) => {
return Err(proptest::test_runner::TestCaseError::fail(format!(
"event stream lagged by {skipped}: the invariants below cannot \
be read off a stream with holes in it"
)));
}
Ok(Err(tokio::sync::broadcast::error::RecvError::Closed)) => break,
Err(_elapsed) => break, }
}
let mut live = std::collections::HashSet::<u32>::new();
let mut event_restarts = std::collections::HashMap::<u32, u32>::new();
for event in observed {
let BusEvent::Process { event, info, .. } = event else {
continue;
};
match event {
ProcessEventKind::Start => {
proptest::prop_assert!(
live.insert(info.id),
"two live spawns for id {}",
info.id
);
}
ProcessEventKind::Restart => {
live.insert(info.id);
}
ProcessEventKind::Online => {
proptest::prop_assert!(
live.contains(&info.id),
"id {} was marked online with no live process: a readiness \
wait resolved onto a sheep that had already gone terminal",
info.id
);
}
ProcessEventKind::Exit
| ProcessEventKind::Stop
| ProcessEventKind::Errored
| ProcessEventKind::Delete => {
live.remove(&info.id);
}
_ => {}
}
let seen = event_restarts.entry(info.id).or_default();
proptest::prop_assert!(
info.restarts >= *seen,
"restart count for id {} went backwards: {} after {}",
info.id,
info.restarts,
*seen
);
*seen = info.restarts;
}
Ok::<(), proptest::test_runner::TestCaseError>(())
})?;
}
}
#[cfg(unix)]
fn daemon_fds(dir: &tempfile::TempDir) -> (DaemonFds, [std::fs::File; 2]) {
use std::os::fd::AsRawFd as _;
let listener = std::fs::File::create(dir.path().join("listener.stand-in")).unwrap();
let pidfile = std::fs::File::create(dir.path().join("pidfile.stand-in")).unwrap();
let fds = DaemonFds {
listener: listener.as_raw_fd(),
pidfile: pidfile.as_raw_fd(),
};
(fds, [listener, pidfile])
}
#[cfg(unix)]
fn is_open(fd: std::os::fd::RawFd) -> bool {
nix::fcntl::fcntl(fd, nix::fcntl::FcntlArg::F_GETFD).is_ok()
}
#[cfg(unix)]
#[tokio::test(start_paused = true)]
async fn a_blob_from_a_live_flock_names_open_descriptors_per_sheep() {
let (events, _rx) = crate::bus::test_bus(64);
let runner =
ScriptedRunner::new(vec![ProcScript::never_exits(), ProcScript::never_exits()]);
let dir = tempfile::tempdir().unwrap();
let (fds, _held) = daemon_fds(&dir);
let handle = spawn_supervisor(runner, test_paths(&dir), events);
let mut talkative = AppConfig::minimal("api", "./srv");
talkative.channel = true;
handle
.start(vec![
normalize(AppConfig::minimal("web", "./srv")).unwrap(),
normalize(talkative).unwrap(),
])
.await
.unwrap();
let (candidates, blob, _parked) = handle.handover_snapshot(fds).await.unwrap();
assert_eq!(candidates.len(), 2, "both sheep must reach the gate");
assert_eq!(blob.sheep().len(), 2);
for sheep in blob.sheep() {
for fd in sheep.fds().all().into_iter().flatten() {
assert!(is_open(fd), "blob names a closed descriptor: {fd}");
}
}
}
#[cfg(unix)]
#[tokio::test(start_paused = true)]
async fn a_snapshot_names_no_channel_for_a_sheep_that_has_none() {
let (events, _rx) = crate::bus::test_bus(64);
let runner =
ScriptedRunner::new(vec![ProcScript::never_exits(), ProcScript::never_exits()]);
let dir = tempfile::tempdir().unwrap();
let (fds, _held) = daemon_fds(&dir);
let handle = spawn_supervisor(runner, test_paths(&dir), events);
let mut talkative = AppConfig::minimal("api", "./srv");
talkative.channel = true;
handle
.start(vec![
normalize(AppConfig::minimal("web", "./srv")).unwrap(),
normalize(talkative).unwrap(),
])
.await
.unwrap();
let (_candidates, blob, _parked) = handle.handover_snapshot(fds).await.unwrap();
let named: Vec<(&str, bool)> = blob
.sheep()
.iter()
.map(|sheep| (sheep.name(), sheep.fds().channel.is_some()))
.collect();
assert!(
named.contains(&("web", false)),
"a sheep with no channel must carry no channel descriptor: {named:?}"
);
assert!(
named.contains(&("api", true)),
"a sheep with a live channel must carry its descriptor: {named:?}"
);
}
#[cfg(unix)]
#[tokio::test(start_paused = true)]
async fn a_pump_that_never_reports_refuses_the_snapshot_instead_of_hanging() {
let (events, _rx) = crate::bus::test_bus(64);
let runner = ScriptedRunner::new(vec![ProcScript::never_exits(); 2])
.with_a_pump_that_never_reports(&["wedged"]);
let dir = tempfile::tempdir().unwrap();
let (fds, _held) = daemon_fds(&dir);
let handle = spawn_supervisor(runner, test_paths(&dir), events);
handle
.start(vec![
normalize(AppConfig::minimal("answering", "./srv")).unwrap(),
normalize(AppConfig::minimal("wedged", "./srv")).unwrap(),
])
.await
.unwrap();
let snapshot =
tokio::time::timeout(Duration::from_secs(3600), handle.handover_snapshot(fds))
.await
.expect("a snapshot over a wedged pump must answer rather than hang")
.unwrap();
let (candidates, _blob, _parked) = snapshot;
let borrowed: Vec<crate::handover::Candidate<'_>> = candidates
.iter()
.map(crate::handover::OwnedCandidate::as_candidate)
.collect();
assert_eq!(
crate::handover::fitness(&borrowed),
crate::handover::Fitness::Refused(crate::handover::RefusedReason::PumpUnresponsive {
sheep: "wedged".to_string(),
}),
"the gate must refuse, and must name the sheep whose pump went quiet"
);
for candidate in &candidates {
assert_eq!(
candidate.pump_unresponsive,
candidate.entry.spec.config().name == "wedged",
"exactly the wedged sheep is unresponsive, not its neighbour"
);
}
}
#[cfg(unix)]
#[tokio::test(start_paused = true)]
async fn a_pump_that_missed_the_deadline_is_not_in_the_parked_set() {
let (events, _rx) = crate::bus::test_bus(64);
let runner = Arc::new(
ScriptedRunner::new(vec![ProcScript::never_exits(); 2])
.with_a_pump_that_never_reports(&["wedged"]),
);
let dir = tempfile::tempdir().unwrap();
let (fds, _held) = daemon_fds(&dir);
let handle = spawn_supervisor(SharedRunner(Arc::clone(&runner)), test_paths(&dir), events);
handle
.start(vec![
normalize(AppConfig::minimal("answering", "./srv")).unwrap(),
normalize(AppConfig::minimal("wedged", "./srv")).unwrap(),
])
.await
.unwrap();
let (_candidates, _blob, parked) =
tokio::time::timeout(Duration::from_secs(3600), handle.handover_snapshot(fds))
.await
.expect("a snapshot over a wedged pump must answer rather than hang")
.unwrap();
parked.resume().await;
let answering = runner.spawn_index_of("answering").expect("started above");
let wedged = runner.spawn_index_of("wedged").expect("started above");
let delivered = async {
while runner.resumes(answering) == 0 {
tokio::time::sleep(Duration::from_millis(10)).await;
}
};
tokio::time::timeout(Duration::from_secs(10), delivered)
.await
.expect("the pump that answered was parked, and is owed a resume");
assert_eq!(
runner.resumes(wedged),
0,
"a pump that never answered never parked, so nothing may resume it"
);
}
#[cfg(unix)]
#[tokio::test(start_paused = true)]
async fn a_sweep_of_six_wedged_pumps_costs_one_deadline_not_six() {
const WEDGED: usize = 6;
let names: Vec<String> = (0..WEDGED).map(|i| format!("wedged{i}")).collect();
let name_refs: Vec<&str> = names.iter().map(String::as_str).collect();
let (events, _rx) = crate::bus::test_bus(64);
let runner = ScriptedRunner::new(vec![ProcScript::never_exits(); WEDGED])
.with_a_pump_that_never_reports(&name_refs);
let dir = tempfile::tempdir().unwrap();
let (fds, _held) = daemon_fds(&dir);
let handle = spawn_supervisor(runner, test_paths(&dir), events);
handle
.start(
names
.iter()
.map(|name| normalize(AppConfig::minimal(name, "./srv")).unwrap())
.collect(),
)
.await
.unwrap();
let started_at = tokio::time::Instant::now();
let (candidates, blob, _parked) =
tokio::time::timeout(Duration::from_secs(3600), handle.handover_snapshot(fds))
.await
.expect("a snapshot over six wedged pumps must answer rather than hang")
.unwrap();
let elapsed = started_at.elapsed();
assert!(
elapsed < REPORT_DEADLINE * 2,
"six wedged pumps swept concurrently should cost about one \
REPORT_DEADLINE, not six; took {elapsed:?}"
);
assert_eq!(candidates.len(), WEDGED);
for candidate in &candidates {
assert!(
candidate.pump_unresponsive,
"{} must still be reported unresponsive; racing the reports \
must not blur which pump answered",
candidate.entry.spec.config().name
);
}
assert!(
candidates.windows(2).all(|w| w[0].entry.id < w[1].entry.id),
"candidates must stay in id order across a concurrent sweep"
);
assert!(
blob.sheep().windows(2).all(|w| w[0].id() < w[1].id()),
"the blob must stay in id order too: it is a file an operator may read"
);
}
#[cfg(unix)]
#[tokio::test(start_paused = true)]
async fn the_snapshot_carries_the_actors_counters_and_slot_state() {
let dir = tempfile::tempdir().unwrap();
let (fds, _held) = daemon_fds(&dir);
let (mut actor, _ctl_rx) = actor_with_stopping_drainee(&dir, 4242, 7);
actor.sheep.get_mut(&0).unwrap().manual = Some(PendingManual {
kind: ManualKind::Stop,
origin: CommandOrigin::Operator,
});
actor.sheep.get_mut(&0).unwrap().ready_failed = true;
let (reply, rx) = oneshot::channel();
actor.handle_handover_snapshot(fds, reply);
let (candidates, blob, _parked) = rx.await.unwrap().unwrap();
assert!(
!candidates.is_empty(),
"the flock must still reach the gate at all"
);
assert_eq!(
blob.sheep()[0].manual(),
Some(PendingManual {
kind: ManualKind::Stop,
origin: CommandOrigin::Operator,
}),
"a manual stop must reach the successor, kind and origin both"
);
assert_eq!(
blob.sheep()[0].pending_delete(),
Some(false),
"nothing asked for this sheep to be deleted"
);
assert_eq!(
blob.sheep()[0].ready_failed(),
Some(true),
"an earlier reload's failed verdict must reach the successor, or the rollback that \
follows it has nothing left to replace"
);
assert!(
blob.next_id() > 0,
"a successor that reissues a live id collides"
);
assert_eq!(blob.sheep()[0].epoch(), 7, "a stale timer must stay stale");
assert_eq!(
blob.sheep()[0].fds(),
CarriedFds::none(),
"a sheep with no pump has no descriptors to carry"
);
}
#[cfg(unix)]
#[derive(Debug, Default)]
struct AdoptingRunner;
#[cfg(unix)]
const STAND_IN_SPAWN_PID: u32 = 7000;
#[cfg(unix)]
#[derive(Debug)]
struct StandInProc {
pid: u32,
}
#[cfg(unix)]
impl RunningProcess for StandInProc {
fn pid(&self) -> u32 {
self.pid
}
async fn wait(&mut self) -> ExitOutcome {
core::future::pending().await
}
fn signal(&mut self, _sig: crate::runner::StopSignal) -> Result<(), RunnerError> {
Ok(())
}
fn kill_tree(&mut self) -> Result<(), RunnerError> {
Ok(())
}
}
#[cfg(unix)]
fn stand_in_io() -> ProcIo {
let (_logs_tx, logs) = mpsc::channel(1);
let (_from_child_tx, from_child) = mpsc::channel(1);
let (to_child, to_child_rx) = mpsc::channel(1);
drop(to_child_rx);
let (log_ctl, log_ctl_rx) = mpsc::channel(1);
drop(log_ctl_rx);
let (to_stdin, to_stdin_rx) = mpsc::channel(1);
drop(to_stdin_rx);
ProcIo {
logs,
from_child,
to_child,
log_ctl,
to_stdin,
}
}
#[cfg(unix)]
impl ProcessRunner for AdoptingRunner {
type Proc = StandInProc;
fn spawn(&self, _spec: &SpawnSpec) -> Result<(Self::Proc, ProcIo), RunnerError> {
Ok((
StandInProc {
pid: STAND_IN_SPAWN_PID,
},
stand_in_io(),
))
}
fn adopt(
&self,
spec: crate::runner::AdoptSpec,
) -> Result<(Self::Proc, ProcIo), RunnerError> {
Ok((StandInProc { pid: spec.pid }, stand_in_io()))
}
}
#[cfg(unix)]
#[tokio::test(start_paused = true)]
async fn an_adopted_sheep_that_was_still_starting_reaches_online() {
let dir = tempfile::tempdir().unwrap();
let (events, _rx) = crate::bus::test_bus(64);
let sup = SupervisorBuilder::new(AdoptingRunner, test_paths(&dir), events)
.spawn_adopted(
vec![without_handles(carried("web", 7, Some(4242), |entry| {
let mut app = AppConfig::minimal("web", "./srv");
app.autorestart = false;
app.wait_ready = true;
entry.spec = normalize(app).unwrap();
entry.status = ProcStatus::Starting;
}))],
counters(9),
Vec::new(),
)
.expect("a carried flock installs");
assert_eq!(
sup.list().await[0].status,
ProcStatus::Starting,
"the blob said this sheep was still starting, so the successor must agree"
);
let goes_online = async {
while sup.list().await[0].status != ProcStatus::Online {
tokio::time::sleep(Duration::from_millis(50)).await;
}
};
tokio::time::timeout(Duration::from_secs(30), goes_online)
.await
.expect("an adopted sheep left `Starting` must still have a readiness wait over it");
}
#[cfg(unix)]
fn carried(
name: &str,
id: u32,
pid: Option<u32>,
mutate: impl FnOnce(&mut ProcessEntry),
) -> CarriedSheep {
carried_marked(name, id, pid, false, None, false, mutate)
}
#[cfg(unix)]
fn carried_ready_failed(
name: &str,
id: u32,
pid: Option<u32>,
mutate: impl FnOnce(&mut ProcessEntry),
) -> CarriedSheep {
carried_marked(name, id, pid, false, None, true, mutate)
}
#[cfg(unix)]
fn carried_marked(
name: &str,
id: u32,
pid: Option<u32>,
pending_delete: bool,
manual: Option<PendingManual>,
ready_failed: bool,
mutate: impl FnOnce(&mut ProcessEntry),
) -> CarriedSheep {
let mut app = AppConfig::minimal(name, "./srv");
app.autorestart = false;
let mut entry = ProcessEntry {
id,
spec: normalize(app).unwrap(),
pending: None,
pending_reidentifies: false,
overridden: Vec::new(),
instance: 0,
status: ProcStatus::Online,
pid,
restarts: 0,
started_at: None,
budget: RestartBudget::default(),
reload: ReloadState::None,
credentials: SpawnIdentity::Resolved(None),
out_file: PathBuf::new(),
err_file: PathBuf::new(),
dog: None,
last_exit: None,
};
mutate(&mut entry);
CarriedSheep::from_entry(
&entry,
0,
CarriedFds::none(),
pending_delete,
manual,
ready_failed,
None,
)
}
#[cfg(unix)]
fn carried_owed_a_restart(
name: &str,
id: u32,
delay: shep_core::values::UpDuration,
due: Option<SystemTime>,
) -> CarriedSheep {
let mut app = AppConfig::minimal(name, "./srv");
app.restart_delay = Some(delay);
let entry = ProcessEntry {
id,
spec: normalize(app).unwrap(),
pending: None,
pending_reidentifies: false,
overridden: Vec::new(),
instance: 0,
status: ProcStatus::WaitingRestart,
pid: None,
restarts: 1,
started_at: None,
budget: RestartBudget::default(),
reload: ReloadState::None,
credentials: SpawnIdentity::Resolved(None),
out_file: PathBuf::new(),
err_file: PathBuf::new(),
dog: None,
last_exit: None,
};
CarriedSheep::from_entry(&entry, 0, CarriedFds::none(), false, None, false, due)
}
#[cfg(unix)]
fn without_handles(carried: CarriedSheep) -> crate::handover::adopt::AdoptedSheep {
crate::handover::adopt::AdoptedSheep {
carried,
out_pipe: None,
err_pipe: None,
out_log: None,
err_log: None,
stdin_pipe: None,
channel: None,
}
}
#[cfg(unix)]
const fn counters(next_id: u32) -> Counters {
Counters {
next_id,
next_deadline: 0,
next_action_stamp: 0,
}
}
#[cfg(unix)]
fn by_id(mut info: Vec<ProcessInfo>) -> Vec<ProcessInfo> {
info.sort_unstable_by_key(|row| row.id);
info
}
#[cfg(unix)]
#[tokio::test(start_paused = true)]
async fn an_adopted_flock_keeps_its_pids_and_ids() {
let dir = tempfile::tempdir().unwrap();
let (events, _rx) = crate::bus::test_bus(64);
let sup = SupervisorBuilder::new(AdoptingRunner, test_paths(&dir), events)
.spawn_adopted(
vec![
without_handles(carried("web", 7, Some(4242), |_| {})),
without_handles(carried("api", 8, Some(4243), |_| {})),
],
counters(9),
Vec::new(),
)
.expect("a carried flock installs");
let info = by_id(sup.list().await);
assert_eq!(
info.iter().map(|row| row.id).collect::<Vec<_>>(),
vec![7, 8],
"the successor reissued ids instead of carrying them"
);
assert_eq!(
info.iter().map(|row| row.pid).collect::<Vec<_>>(),
vec![Some(4242), Some(4243)],
"a sheep whose pid moved was respawned"
);
}
#[cfg(unix)]
#[tokio::test(start_paused = true)]
async fn an_adopted_dog_keeps_its_marker_and_stays_out_of_a_wildcard() {
let dir = tempfile::tempdir().unwrap();
let (events, _rx) = crate::bus::test_bus(64);
let sup = SupervisorBuilder::new(AdoptingRunner, test_paths(&dir), events)
.spawn_adopted(
vec![without_handles(carried(
"log-rotate",
7,
Some(4242),
|entry| {
entry.dog = Some(DogSource::Adopted {
path: "/opt/bin/shep-log-rotate".to_string(),
});
},
))],
counters(9),
Vec::new(),
)
.expect("a carried flock installs");
let info = sup.list().await;
assert_eq!(
info[0].dog,
Some(DogSource::Adopted {
path: "/opt/bin/shep-log-rotate".to_string(),
}),
"a dog adopted as an ordinary sheep is one `shep dogs` has lost"
);
let swept = tokio::time::timeout(Duration::from_secs(5), sup.stop(ProcessSelector::All))
.await
.expect("a wildcard over a flock of nothing but dogs must answer rather than hang");
assert!(
matches!(swept, Err(SupervisorError::NotFound)),
"`all` is the flock, not the kennel: {swept:?}"
);
}
#[cfg(unix)]
#[tokio::test(start_paused = true)]
async fn an_adopted_sheep_keeps_its_counters_and_last_exit() {
let dir = tempfile::tempdir().unwrap();
let (events, _rx) = crate::bus::test_bus(64);
let sup = SupervisorBuilder::new(AdoptingRunner, test_paths(&dir), events)
.spawn_adopted(
vec![without_handles(carried("web", 7, Some(4242), |entry| {
entry.restarts = 4;
entry.last_exit = Some(ExitInfo {
code: Some(2),
signal: None,
});
}))],
counters(9),
Vec::new(),
)
.expect("a carried flock installs");
let info = sup.list().await;
assert_eq!(info[0].restarts, 4, "the restart count was reset");
assert_eq!(
info[0].last_exit,
Some(ExitInfo {
code: Some(2),
signal: None
}),
"the last exit was lost"
);
}
#[cfg(unix)]
#[tokio::test]
#[expect(
clippy::zombie_processes,
reason = "the adopted flock's reaper collects this status; a Child::wait would take it first"
)]
async fn an_adopted_sheeps_exit_flows_through_the_ordinary_path() {
let dir = tempfile::tempdir().unwrap();
let (events, _rx) = crate::bus::test_bus(64);
let child = std::process::Command::new("/bin/sh")
.args(["-c", "sleep 30"])
.spawn()
.expect("a test host can spawn a shell");
let pid = child.id();
let sup = SupervisorBuilder::new(TokioRunner::new(), test_paths(&dir), events)
.spawn_adopted(
vec![without_handles(carried("web", 7, Some(pid), |_| {}))],
counters(9),
Vec::new(),
)
.expect("a carried flock installs");
nix::sys::signal::kill(
nix::unistd::Pid::from_raw(i32::try_from(pid).unwrap()),
nix::sys::signal::Signal::SIGKILL,
)
.expect("the adopted child is signalable");
let info = tokio::time::timeout(Duration::from_secs(20), async {
loop {
let info = sup.list().await;
if info[0].status == ProcStatus::Stopped {
return info;
}
tokio::time::sleep(Duration::from_millis(20)).await;
}
})
.await
.expect("an adopted sheep's exit reaches the actor");
assert_eq!(
info[0].last_exit,
Some(ExitInfo {
code: None,
signal: Some(9)
}),
"the exit must be recorded, not lost"
);
}
#[cfg(unix)]
#[tokio::test]
#[expect(
clippy::zombie_processes,
reason = "the adopted flock's reaper collects this status; a Child::wait would take it first"
)]
async fn a_carried_pending_delete_deregisters_on_the_next_exit() {
let dir = tempfile::tempdir().unwrap();
let (events, _rx) = crate::bus::test_bus(64);
let child = std::process::Command::new("/bin/sh")
.args(["-c", "sleep 30"])
.spawn()
.expect("a test host can spawn a shell");
let pid = child.id();
let sup = SupervisorBuilder::new(TokioRunner::new(), test_paths(&dir), events)
.spawn_adopted(
vec![without_handles(carried_marked(
"web",
7,
Some(pid),
true,
None,
false,
|_| {},
))],
counters(9),
Vec::new(),
)
.expect("a carried flock installs");
nix::sys::signal::kill(
nix::unistd::Pid::from_raw(i32::try_from(pid).unwrap()),
nix::sys::signal::Signal::SIGKILL,
)
.expect("the adopted child is signalable");
let info = tokio::time::timeout(Duration::from_secs(20), async {
loop {
let info = sup.list().await;
if info.is_empty() {
return info;
}
tokio::time::sleep(Duration::from_millis(20)).await;
}
})
.await
.expect(
"a carried pending delete must deregister the sheep on its next exit, not merely \
stop it",
);
assert!(
info.is_empty(),
"the deleted sheep must be gone, not just stopped"
);
}
#[cfg(unix)]
fn adoptable_child(script: &str) -> u32 {
use std::os::unix::process::CommandExt as _;
std::process::Command::new("/bin/sh")
.args(["-c", script])
.process_group(0)
.stdin(std::process::Stdio::null())
.stdout(std::process::Stdio::null())
.stderr(std::process::Stdio::null())
.spawn()
.expect("a test host can spawn a shell")
.id()
}
#[cfg(unix)]
fn respawnable(mutate: impl FnOnce(&mut AppConfig)) -> impl FnOnce(&mut ProcessEntry) {
move |entry: &mut ProcessEntry| {
let mut app = AppConfig::minimal("web", "/bin/sh");
app.args = vec!["-c".to_owned(), "sleep 30".to_owned()];
mutate(&mut app);
entry.spec = normalize(app).unwrap();
}
}
#[cfg(unix)]
async fn flock_until(
sup: &SupervisorHandle,
done: impl Fn(&[ProcessInfo]) -> bool,
what: &str,
) -> Vec<ProcessInfo> {
tokio::time::timeout(Duration::from_secs(20), async {
loop {
let info = sup.list().await;
if done(&info) {
return info;
}
tokio::time::sleep(Duration::from_millis(20)).await;
}
})
.await
.unwrap_or_else(|_| panic!("{what}"))
}
#[cfg(unix)]
#[tokio::test]
async fn a_carried_manual_stop_stops_the_sheep_instead_of_respawning_it() {
let dir = tempfile::tempdir().unwrap();
let (events, _rx) = crate::bus::test_bus(64);
let pid = adoptable_child("sleep 30");
let sup = SupervisorBuilder::new(TokioRunner::new(), test_paths(&dir), events)
.spawn_adopted(
vec![without_handles(carried_marked(
"web",
7,
Some(pid),
false,
Some(PendingManual {
kind: ManualKind::Stop,
origin: CommandOrigin::Operator,
}),
false,
respawnable(|app| app.autorestart = true),
))],
counters(9),
Vec::new(),
)
.expect("a carried flock installs");
let info = flock_until(
&sup,
|info| info[0].status == ProcStatus::Stopped,
"a carried manual stop must still stop the sheep after the exec",
)
.await;
assert_eq!(
info[0].restarts, 0,
"a stop must not come back as a respawn"
);
assert_eq!(
info[0].last_exit,
Some(ExitInfo {
code: None,
signal: Some(15)
}),
"the re-armed ladder starts at its polite rung, not at SIGKILL"
);
assert_eq!(info[0].pid, None, "a stopped sheep holds no pid");
}
#[cfg(unix)]
#[tokio::test]
async fn a_carried_manual_stop_still_escalates_to_sigkill() {
let dir = tempfile::tempdir().unwrap();
let (events, _rx) = crate::bus::test_bus(64);
let armed = dir.path().join("trap-armed");
let pid = adoptable_child(&format!(
"trap '' TERM; : > {}; i=0; while [ $i -lt 60 ]; do sleep 1; i=$((i+1)); done",
armed.display()
));
tokio::time::timeout(Duration::from_secs(20), async {
while !armed.exists() {
tokio::time::sleep(Duration::from_millis(10)).await;
}
})
.await
.expect("the defiant child must arm its trap before anything signals it");
let sup = SupervisorBuilder::new(TokioRunner::new(), test_paths(&dir), events)
.spawn_adopted(
vec![without_handles(carried_marked(
"web",
7,
Some(pid),
false,
Some(PendingManual {
kind: ManualKind::Stop,
origin: CommandOrigin::Operator,
}),
false,
respawnable(|app| {
app.autorestart = false;
app.kill_timeout = "1000".parse().unwrap();
}),
))],
counters(9),
Vec::new(),
)
.expect("a carried flock installs");
let info = flock_until(
&sup,
|info| info[0].status == ProcStatus::Stopped,
"a sheep carried mid-ladder must still die on its own, without a hand-sent SIGKILL",
)
.await;
assert_eq!(
info[0].last_exit,
Some(ExitInfo {
code: None,
signal: Some(9)
}),
"the ladder must escalate: this child ignores SIGTERM, so anything else means it \
was never killed"
);
}
#[cfg(unix)]
#[tokio::test]
async fn a_carried_manual_restart_respawns_and_keeps_its_origin() {
let dir = tempfile::tempdir().unwrap();
let (events, mut rx) = crate::bus::test_bus(64);
let pid = adoptable_child("sleep 30");
let sup = SupervisorBuilder::new(TokioRunner::new(), test_paths(&dir), events)
.spawn_adopted(
vec![without_handles(carried_marked(
"web",
7,
Some(pid),
false,
Some(PendingManual {
kind: ManualKind::Restart,
origin: CommandOrigin::Automatic,
}),
false,
respawnable(|app| app.autorestart = false),
))],
counters(9),
Vec::new(),
)
.expect("a carried flock installs");
let manually = tokio::time::timeout(
Duration::from_secs(20),
await_event(&mut rx, 7, ProcessEventKind::Restart),
)
.await
.expect("a carried manual restart must respawn the sheep after the exec");
assert!(
!manually,
"an automatic restart carried across a handover must not be broadcast as a user \
action"
);
let info = flock_until(
&sup,
|info| info[0].restarts == 1,
"the respawn must be counted against the sheep that was carried",
)
.await;
assert_ne!(
info[0].pid,
Some(pid),
"a restart is a new process, not the adopted one"
);
assert_eq!(
info[0].status,
ProcStatus::Online,
"an ungated app is Online the moment it respawns"
);
sup.shutdown().await;
}
#[cfg(unix)]
#[tokio::test(start_paused = true)]
async fn the_successor_does_not_reissue_a_live_id() {
let dir = tempfile::tempdir().unwrap();
let (events, _rx) = crate::bus::test_bus(64);
let sup = SupervisorBuilder::new(AdoptingRunner, test_paths(&dir), events)
.spawn_adopted(
vec![without_handles(carried("web", 7, Some(4242), |_| {}))],
counters(9),
Vec::new(),
)
.expect("a carried flock installs");
let fresh = sup
.start(vec![normalize(AppConfig::minimal("api", "./srv")).unwrap()])
.await
.expect("a fresh app starts under a successor");
assert!(fresh[0].id >= 9, "reissued a live id: {}", fresh[0].id);
}
#[cfg(unix)]
#[tokio::test(start_paused = true)]
async fn an_adopted_sheep_owed_a_restart_still_gets_one() {
let dir = tempfile::tempdir().unwrap();
let (events, _rx) = crate::bus::test_bus(64);
let sup = SupervisorBuilder::new(AdoptingRunner, test_paths(&dir), events)
.spawn_adopted(
vec![without_handles(carried("web", 7, None, |entry| {
entry.status = ProcStatus::WaitingRestart;
}))],
counters(9),
Vec::new(),
)
.expect("a carried flock installs");
tokio::time::sleep(Duration::from_secs(5)).await;
let info = sup.list().await;
assert_eq!(
info[0].status,
ProcStatus::Online,
"a sheep owed a respawn was left waiting for a timer that died with the exec: {info:?}"
);
assert_eq!(
info[0].pid,
Some(STAND_IN_SPAWN_PID),
"the restart must be a real respawn, not a status edit: {info:?}"
);
}
#[cfg(unix)]
#[tokio::test(start_paused = true)]
async fn an_adopted_sheep_waits_out_only_what_is_left_of_its_delay() {
let hour = "1h".parse().unwrap();
let anchored_dir = tempfile::tempdir().unwrap();
let (anchored_events, _anchored_rx) = crate::bus::test_bus(64);
let anchored =
SupervisorBuilder::new(AdoptingRunner, test_paths(&anchored_dir), anchored_events)
.spawn_adopted(
vec![without_handles(carried_owed_a_restart(
"web",
7,
hour,
Some(SystemTime::now() + Duration::from_secs(2)),
))],
counters(9),
Vec::new(),
)
.expect("a carried flock installs");
let control_dir = tempfile::tempdir().unwrap();
let (control_events, _control_rx) = crate::bus::test_bus(64);
let control =
SupervisorBuilder::new(AdoptingRunner, test_paths(&control_dir), control_events)
.spawn_adopted(
vec![without_handles(carried_owed_a_restart(
"web", 7, hour, None,
))],
counters(9),
Vec::new(),
)
.expect("a carried flock installs");
tokio::time::sleep(Duration::from_secs(10)).await;
let anchored = anchored.list().await;
assert_eq!(
anchored[0].status,
ProcStatus::Online,
"a sheep two seconds from its own due time must not be made to wait another hour by \
the handover: {anchored:?}"
);
assert_eq!(
anchored[0].pid,
Some(STAND_IN_SPAWN_PID),
"the restart must be a real respawn, not a status edit: {anchored:?}"
);
let control = control.list().await;
assert_eq!(
control[0].status,
ProcStatus::WaitingRestart,
"with no carried deadline the whole delay is re-armed, so an hourly app must still \
be waiting ten seconds in — a control that respawned would mean the case above \
proved nothing: {control:?}"
);
}
#[cfg(unix)]
#[tokio::test(start_paused = true)]
async fn a_snapshot_of_a_waiting_sheep_carries_the_moment_its_respawn_falls_due() {
let (events, _rx) = crate::bus::test_bus(64);
let runner = ScriptedRunner::new(vec![ProcScript::const_exit(1)]);
let dir = tempfile::tempdir().unwrap();
let (fds, _held) = daemon_fds(&dir);
let handle = spawn_supervisor(runner, test_paths(&dir), events);
let mut app = AppConfig::minimal("web", "./srv");
app.restart_delay = Some("1h".parse().unwrap());
handle.start(vec![normalize(app).unwrap()]).await.unwrap();
let waits = async {
while handle.list().await[0].status != ProcStatus::WaitingRestart {
tokio::time::sleep(Duration::from_millis(10)).await;
}
};
tokio::time::timeout(Duration::from_secs(30), waits)
.await
.expect("an app that exits immediately with autorestart on must reach WaitingRestart");
let (_candidates, blob, _parked) = handle.handover_snapshot(fds).await.unwrap();
let due = blob.sheep()[0]
.restart_due()
.expect("a sheep owed a respawn must carry the moment it is owed at");
let left = due
.duration_since(SystemTime::now())
.expect("a deadline an hour out has not passed");
assert!(
left > Duration::from_secs(3595) && left <= Duration::from_secs(3600),
"the carried moment must be this sheep's own exit plus its own delay: {left:?}"
);
}
#[cfg(unix)]
#[tokio::test(start_paused = true)]
async fn a_second_handover_inside_one_delay_still_names_the_original_moment() {
let dir = tempfile::tempdir().unwrap();
let (fds, _held) = daemon_fds(&dir);
let (events, _rx) = crate::bus::test_bus(64);
let due = SystemTime::now() + Duration::from_secs(3000);
let handle = SupervisorBuilder::new(AdoptingRunner, test_paths(&dir), events)
.spawn_adopted(
vec![without_handles(carried_owed_a_restart(
"web",
7,
"1h".parse().unwrap(),
Some(due),
))],
counters(9),
Vec::new(),
)
.expect("a carried flock installs");
let (_candidates, blob, _parked) = handle.handover_snapshot(fds).await.unwrap();
let carried = blob.sheep()[0]
.restart_due()
.expect("a successor must hand its own successor the moment it was given");
let drift = carried
.duration_since(due)
.or_else(|_| due.duration_since(carried))
.unwrap();
assert!(
drift < Duration::from_secs(1),
"the second hop must name the same moment as the first, not a fresh one: {drift:?} \
of drift"
);
}
#[cfg(unix)]
#[tokio::test]
async fn a_snapshot_taken_mid_swap_carries_the_job_and_the_markers() {
let (events, _rx) = crate::bus::test_bus(64);
let runner = ScriptedRunner::new(vec![ProcScript::never_reports_its_exit(); 2]);
let dir = tempfile::tempdir().unwrap();
let (fds, _held) = daemon_fds(&dir);
let handle = spawn_supervisor(runner, test_paths(&dir), events);
let mut app = AppConfig::minimal("web", "./srv");
app.listen_timeout = UpDuration::from_millis(200);
app.readiness_probe = Some(probe_config(ProbeKind::Tcp, "127.0.0.1:9"));
handle.start(vec![normalize(app).unwrap()]).await.unwrap();
let online = loop {
let info = handle.list().await;
if info[0].status == ProcStatus::Online {
break info;
}
tokio::time::sleep(Duration::from_millis(20)).await;
};
let old_id = online[0].id;
handle
.reload(ProcessSelector::All)
.await
.expect("an online app reloads");
let (_candidates, blob, _parked) = handle.handover_snapshot(fds).await.unwrap();
assert_eq!(
blob.reloads(),
&[CarriedReload {
app: "web".to_owned(),
queue: Vec::new(),
mode: ReloadMode::Serial,
swap: ReloadSwap {
old_id,
new_id: None,
phase: ReloadPhase::DrainFirst,
},
}],
"the job the successor continues has to be in the blob, whole"
);
assert_eq!(
blob.sheep()[0].reload(),
Some(ReloadState::Drainee { new_id: None }),
"and the marker that routes this instance's exit with it"
);
}
#[cfg(unix)]
fn carried_in_swap(
name: &str,
id: u32,
pid: Option<u32>,
role: ReloadState,
status: ProcStatus,
manual: Option<PendingManual>,
mutate: impl FnOnce(&mut ProcessEntry),
) -> CarriedSheep {
carried_marked(name, id, pid, false, manual, false, move |entry| {
mutate(entry);
entry.reload = role;
entry.status = status;
})
}
#[cfg(unix)]
fn carried_job(
app: &str,
mode: ReloadMode,
old_id: u32,
new_id: Option<u32>,
phase: ReloadPhase,
) -> CarriedReload {
CarriedReload {
app: app.to_owned(),
queue: Vec::new(),
mode,
swap: ReloadSwap {
old_id,
new_id,
phase,
},
}
}
#[cfg(unix)]
#[tokio::test(start_paused = true)]
async fn a_carried_swap_that_cannot_finish_is_still_abandoned_on_time() {
let dir = tempfile::tempdir().unwrap();
let (events, mut rx) = crate::bus::test_bus(64);
let sup = SupervisorBuilder::new(AdoptingRunner, test_paths(&dir), events)
.spawn_adopted(
vec![
without_handles(carried_in_swap(
"web",
7,
Some(4242),
ReloadState::Drainee { new_id: Some(8) },
ProcStatus::Stopping,
None,
|_| {},
)),
without_handles(carried_in_swap(
"web",
8,
Some(4243),
ReloadState::Replacement,
ProcStatus::Online,
None,
|_| {},
)),
],
counters(9),
vec![carried_job(
"web",
ReloadMode::Overlap,
7,
Some(8),
ReloadPhase::DrainOld,
)],
)
.expect("a carried flock installs");
let refused = sup.reload(ProcessSelector::All).await;
assert!(
matches!(refused, Err(SupervisorError::ReloadInFlight(ref name)) if name == "web"),
"the carried job must be in the map, or this case proves nothing: {refused:?}"
);
tokio::time::timeout(
Duration::from_secs(3600),
await_event(&mut rx, 8, ProcessEventKind::ReloadAbandoned),
)
.await
.expect("a carried swap must still be bounded by a watchdog this image armed");
sup.reload(ProcessSelector::All)
.await
.expect("once the watchdog has ended the job, the app must be reloadable again");
}
#[cfg(unix)]
#[tokio::test(start_paused = true)]
async fn a_carried_reload_naming_no_registered_instance_is_dropped() {
let dir = tempfile::tempdir().unwrap();
let (events, _rx) = crate::bus::test_bus(64);
let sup = SupervisorBuilder::new(AdoptingRunner, test_paths(&dir), events)
.spawn_adopted(
vec![without_handles(carried("web", 7, Some(4242), |_| {}))],
counters(9),
vec![carried_job(
"web",
ReloadMode::Overlap,
98,
Some(99),
ReloadPhase::DrainOld,
)],
)
.expect("a carried flock installs");
sup.reload(ProcessSelector::All)
.await
.expect("a job naming nothing must be dropped, not left refusing every later reload");
}
#[cfg(unix)]
#[tokio::test]
async fn a_carried_swap_in_verify_is_asked_again_rather_than_abandoned() {
let probe_target = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
let addr = probe_target.local_addr().unwrap();
let dir = tempfile::tempdir().unwrap();
let (events, mut rx) = crate::bus::test_bus(64);
let _sup = SupervisorBuilder::new(AdoptingRunner, test_paths(&dir), events)
.spawn_adopted(
vec![without_handles(carried_in_swap(
"web",
8,
Some(4243),
ReloadState::Replacement,
ProcStatus::Online,
None,
move |entry| {
let mut app = AppConfig::minimal("web", "./srv");
app.autorestart = false;
app.reuse_port = true;
app.readiness_probe = Some(probe_config(ProbeKind::Tcp, &addr.to_string()));
entry.spec = normalize(app).unwrap();
},
))],
counters(9),
vec![carried_job(
"web",
ReloadMode::Overlap,
7,
Some(8),
ReloadPhase::Verify,
)],
)
.expect("a carried flock installs");
tokio::time::timeout(
Duration::from_secs(5),
await_event(&mut rx, 8, ProcessEventKind::Reloaded),
)
.await
.expect("a carried swap in Verify must be probed again, not left to its watchdog");
}
#[cfg(unix)]
#[tokio::test]
async fn a_carried_serial_drain_still_spawns_its_replacement() {
let dir = tempfile::tempdir().unwrap();
let (events, _rx) = crate::bus::test_bus(64);
let pid = adoptable_child("sleep 30");
let sup = SupervisorBuilder::new(TokioRunner::new(), test_paths(&dir), events)
.spawn_adopted(
vec![without_handles(carried_in_swap(
"web",
7,
Some(pid),
ReloadState::Drainee { new_id: None },
ProcStatus::Stopping,
Some(PendingManual {
kind: ManualKind::Stop,
origin: CommandOrigin::Operator,
}),
swappable,
))],
counters(9),
vec![carried_job(
"web",
ReloadMode::Serial,
7,
None,
ReloadPhase::DrainFirst,
)],
)
.expect("a carried flock installs");
let info = flock_until(
&sup,
|info| info.len() == 1 && info[0].id != 7,
"a carried serial drain must spawn its replacement once the instance it drained goes",
)
.await;
assert_eq!(
info[0].id, 9,
"the replacement takes the next carried id, not a reissued one"
);
assert_eq!(
info[0].restarts, 0,
"a reload is not a restart, so the count carries across the swap unchanged"
);
assert_ne!(info[0].pid, Some(pid), "the replacement is a new process");
sup.shutdown().await;
}
#[cfg(unix)]
#[tokio::test]
async fn a_carried_replacement_awaiting_readiness_commits_its_swap() {
let dir = tempfile::tempdir().unwrap();
let (events, mut rx) = crate::bus::test_bus(64);
let drainee = adoptable_child("sleep 30");
let replacement = adoptable_child("sleep 30");
let sup = SupervisorBuilder::new(TokioRunner::new(), test_paths(&dir), events)
.spawn_adopted(
vec![
without_handles(carried_in_swap(
"web",
7,
Some(drainee),
ReloadState::Drainee { new_id: Some(8) },
ProcStatus::Stopping,
None,
swappable,
)),
without_handles(carried_in_swap(
"web",
8,
Some(replacement),
ReloadState::Replacement,
ProcStatus::Starting,
None,
swappable,
)),
],
counters(9),
vec![carried_job(
"web",
ReloadMode::Overlap,
7,
Some(8),
ReloadPhase::AwaitReady,
)],
)
.expect("a carried flock installs");
let manually = tokio::time::timeout(
Duration::from_secs(20),
await_event(&mut rx, 8, ProcessEventKind::Online),
)
.await
.expect("a carried replacement must still resolve its readiness after the exec");
assert!(
manually,
"a replacement's Online is an operator's doing; reporting otherwise broadcasts a \
deploy as the daemon's own"
);
let info = flock_until(
&sup,
|info| info.len() == 1,
"committing the swap must drain the instance it replaced",
)
.await;
assert_eq!(info[0].id, 8, "the replacement is what is left: {info:?}");
assert_eq!(info[0].pid, Some(replacement));
sup.shutdown().await;
}
#[cfg(unix)]
#[tokio::test]
async fn a_carried_drainee_still_finishes_its_swap() {
let dir = tempfile::tempdir().unwrap();
let (events, mut rx) = crate::bus::test_bus(64);
let drainee = adoptable_child("sleep 30");
let replacement = adoptable_child("sleep 30");
let sup = SupervisorBuilder::new(TokioRunner::new(), test_paths(&dir), events)
.spawn_adopted(
vec![
without_handles(carried_in_swap(
"web",
7,
Some(drainee),
ReloadState::Drainee { new_id: Some(8) },
ProcStatus::Stopping,
Some(PendingManual {
kind: ManualKind::Stop,
origin: CommandOrigin::Operator,
}),
|entry| swappable_with(entry, |app| app.autorestart = true),
)),
without_handles(carried_in_swap(
"web",
8,
Some(replacement),
ReloadState::Replacement,
ProcStatus::Online,
None,
|entry| swappable_with(entry, |app| app.autorestart = true),
)),
],
counters(9),
vec![carried_job(
"web",
ReloadMode::Overlap,
7,
Some(8),
ReloadPhase::DrainOld,
)],
)
.expect("a carried flock installs");
tokio::time::timeout(
Duration::from_secs(20),
await_event(&mut rx, 8, ProcessEventKind::Reloaded),
)
.await
.expect("a carried drainee's exit must finish the swap it was half of");
let info = sup.list().await;
assert_eq!(
info.len(),
1,
"the drainee is deregistered by the swap, never respawned: {info:?}"
);
assert_eq!(info[0].id, 8);
assert_eq!(info[0].pid, Some(replacement));
sup.shutdown().await;
}
#[cfg(unix)]
#[tokio::test]
async fn a_carried_drainee_is_capped_by_graceful_timeout_not_kill_timeout() {
let dir = tempfile::tempdir().unwrap();
let (events, _rx) = crate::bus::test_bus(64);
let drainee = adoptable_child("trap '' TERM; sleep 300");
let replacement = adoptable_child("sleep 30");
let capped = |entry: &mut ProcessEntry| {
swappable_with(entry, |app| {
app.graceful_timeout = UpDuration::from_millis(250);
app.kill_timeout = UpDuration::from_millis(300_000);
});
};
let sup = SupervisorBuilder::new(TokioRunner::new(), test_paths(&dir), events)
.spawn_adopted(
vec![
without_handles(carried_in_swap(
"web",
7,
Some(drainee),
ReloadState::Drainee { new_id: Some(8) },
ProcStatus::Stopping,
Some(PendingManual {
kind: ManualKind::Stop,
origin: CommandOrigin::Operator,
}),
capped,
)),
without_handles(carried_in_swap(
"web",
8,
Some(replacement),
ReloadState::Replacement,
ProcStatus::Online,
None,
capped,
)),
],
counters(9),
vec![carried_job(
"web",
ReloadMode::Overlap,
7,
Some(8),
ReloadPhase::DrainOld,
)],
)
.expect("a carried flock installs");
flock_until(
&sup,
|info| info.len() == 1,
"a carried drainee must escalate at graceful_timeout, not at kill_timeout",
)
.await;
sup.shutdown().await;
}
#[cfg(unix)]
#[tokio::test]
async fn a_carried_ready_failed_instance_is_still_replaceable() {
let dir = tempfile::tempdir().unwrap();
let (events, _rx) = crate::bus::test_bus(64);
let pid = adoptable_child("sleep 30");
let sup = SupervisorBuilder::new(TokioRunner::new(), test_paths(&dir), events)
.spawn_adopted(
vec![without_handles(carried_ready_failed(
"web",
7,
Some(pid),
|entry| {
swappable(entry);
entry.status = ProcStatus::Starting;
},
))],
counters(9),
Vec::new(),
)
.expect("a carried flock installs");
sup.reload(ProcessSelector::All)
.await
.expect("a registered app is one a reload can name");
let info = flock_until(
&sup,
|info| info.len() == 1 && info[0].id != 7,
"a carried `ready_failed` instance must still be replaceable by the reload that \
rolls its release back",
)
.await;
assert_eq!(
info[0].id, 9,
"the replacement takes the next carried id, not a reissued one"
);
assert_ne!(info[0].pid, Some(pid), "the replacement is a new process");
sup.shutdown().await;
}
#[cfg(unix)]
#[tokio::test(start_paused = true)]
async fn a_carried_ready_failed_instance_gets_no_fresh_readiness_wait() {
let dir = tempfile::tempdir().unwrap();
let (events, _rx) = crate::bus::test_bus(64);
let sup = SupervisorBuilder::new(AdoptingRunner, test_paths(&dir), events)
.spawn_adopted(
vec![without_handles(carried_ready_failed(
"web",
7,
Some(4242),
|entry| {
let mut app = AppConfig::minimal("web", "./srv");
app.autorestart = false;
app.wait_ready = true;
entry.spec = normalize(app).unwrap();
entry.status = ProcStatus::Starting;
},
))],
counters(9),
Vec::new(),
)
.expect("a carried flock installs");
tokio::time::sleep(Duration::from_secs(120)).await;
assert_eq!(
sup.list().await[0].status,
ProcStatus::Starting,
"an instance whose readiness already failed must not be handed a second verdict by \
the successor that adopted it"
);
}
#[cfg(unix)]
fn swappable(entry: &mut ProcessEntry) {
swappable_with(entry, |_| {});
}
#[cfg(unix)]
fn swappable_with(entry: &mut ProcessEntry, mutate: impl FnOnce(&mut AppConfig)) {
let mut app = AppConfig::minimal("web", "/bin/sh");
app.args = vec!["-c".to_owned(), "sleep 30".to_owned()];
app.autorestart = false;
app.listen_timeout = UpDuration::from_millis(200);
mutate(&mut app);
entry.spec = normalize(app).unwrap();
}
const APPLY_FIRST_PID: u32 = 7100;
fn actor_over(
dir: &tempfile::TempDir,
apps: &[ResolvedApp],
) -> (Actor<ScriptedRunner>, Arc<RecordingEnforcer>) {
let paths = test_paths(dir);
let mut sheep = HashMap::new();
let mut next_id = 0;
for app in apps {
for instance in 0..app.config().instances {
let id = next_id;
next_id += 1;
sheep.insert(
id,
SheepSlot::new(armed_entry(
id,
instance,
APPLY_FIRST_PID + id,
app.clone(),
&paths,
)),
);
}
}
let enforcer = Arc::new(RecordingEnforcer::default());
let (breach_tx, _breaches) = mpsc::channel(1);
let (live_tx, _liveness) = mpsc::channel(1);
let extras = Extras {
clock: Arc::new(SystemClock),
enforcer: Arc::clone(&enforcer) as Arc<dyn LimitEnforcer>,
max_cron_sleep: DEFAULT_MAX_CRON_SLEEP,
reports: ExtrasReports {
breaches: breach_tx,
liveness: live_tx,
},
stats: idle_stats(),
};
let (tx, _rx) = mpsc::channel(MAILBOX_CAPACITY);
let scripts = vec![ProcScript::never_exits(); 4];
let actor = Actor {
extras: Some(extras),
..test_actor(paths, scripts, sheep, tx)
};
(actor, enforcer)
}
fn declared_app(config: AppConfig, keys: &[&str]) -> DeclaredApp {
let declared: BTreeSet<String> = keys.iter().map(|key| (*key).to_string()).collect();
let declared_env = if declared.contains("env") {
config.env.keys().cloned().collect()
} else {
BTreeSet::new()
};
DeclaredApp {
config,
declared,
declared_env,
}
}
fn established(
keys: &[&str],
fields: Vec<(&str, serde_json::Value)>,
) -> shep_core::overrides::AppOverrides {
shep_core::overrides::AppOverrides {
fields: fields
.into_iter()
.map(|(key, value)| (key.to_string(), value))
.collect(),
declared: keys.iter().map(|key| (*key).to_string()).collect(),
declared_env: BTreeSet::new(),
}
}
async fn apply_config(
actor: &mut Actor<ScriptedRunner>,
apps: Vec<DeclaredApp>,
reset: ResetDepth,
) -> Vec<Applied> {
let (reply, answer) = oneshot::channel();
actor.handle_command(Command::ApplyConfig { apps, reset, reply });
answer
.await
.expect("the actor answers an apply before it returns")
.expect("the fixture flock is registered")
}
#[tokio::test(start_paused = true)]
async fn a_file_load_does_not_overwrite_an_established_key() {
let dir = tempfile::tempdir().unwrap();
let (mut actor, _enforcer) =
actor_over(&dir, &[app_with("web", |app| app.max_restarts = 3)]);
shep_core::overrides::put(
&actor.paths.overrides,
"web",
&established(
&["name", "script", "max_restarts"],
vec![("max_restarts", serde_json::json!(3))],
),
)
.unwrap();
let mut file = AppConfig::minimal("web", "./srv");
file.max_restarts = 99;
let reply = apply_config(
&mut actor,
vec![declared_app(file, &["name", "script", "max_restarts"])],
ResetDepth::None,
)
.await;
assert_eq!(
actor.sheep[&0].entry.spec.config().max_restarts,
3,
"the file overwrote a key the operator had set"
);
assert!(
reply[0].applied.is_empty(),
"nothing applied, so nothing may be reported as applied: {reply:?}"
);
}
#[tokio::test(start_paused = true)]
async fn a_file_load_appends_a_key_nobody_had_established() {
let dir = tempfile::tempdir().unwrap();
let (mut actor, _enforcer) = actor_over(&dir, &[app_with("web", |_| {})]);
shep_core::overrides::put(
&actor.paths.overrides,
"web",
&established(&["name", "script"], Vec::new()),
)
.unwrap();
let mut file = AppConfig::minimal("web", "./srv");
file.max_memory = Some(MemSize::from_bytes(512 << 20));
let reply = apply_config(
&mut actor,
vec![declared_app(file, &["name", "script", "max_memory"])],
ResetDepth::None,
)
.await;
assert_eq!(
actor.sheep[&0].entry.spec.config().max_memory,
Some(MemSize::from_bytes(512 << 20)),
"a key nobody had established must be appended"
);
assert_eq!(reply[0].applied, vec!["max_memory".to_string()]);
}
#[tokio::test(start_paused = true)]
async fn a_live_field_lands_on_the_stored_spec() {
let dir = tempfile::tempdir().unwrap();
let (mut actor, _enforcer) = actor_over(&dir, &[app_with("web", |_| {})]);
let mut file = AppConfig::minimal("web", "./srv");
file.max_restarts = 42;
let reply = apply_config(
&mut actor,
vec![declared_app(file, &["name", "script", "max_restarts"])],
ResetDepth::None,
)
.await;
assert_eq!(actor.sheep[&0].entry.spec.config().max_restarts, 42);
assert_eq!(reply[0].applied, vec!["max_restarts".to_string()]);
assert!(reply[0].pending.is_empty(), "{reply:?}");
}
#[tokio::test(start_paused = true)]
async fn a_loaded_depends_on_is_in_force_and_never_reports_as_pending() {
let dir = tempfile::tempdir().unwrap();
let (mut actor, _enforcer) = actor_over(&dir, &[app_with("web", |_| {})]);
let mut file = AppConfig::minimal("web", "./srv");
file.depends_on = vec!["db".to_string()];
let reply = apply_config(
&mut actor,
vec![declared_app(file, &["name", "script", "depends_on"])],
ResetDepth::None,
)
.await;
assert_eq!(
actor.sheep[&0].entry.spec.config().depends_on,
vec!["db".to_string()],
"the edge has to be on the stored spec for the walk to read it"
);
assert_eq!(reply[0].applied, vec!["depends_on".to_string()]);
assert!(
reply[0].pending.is_empty(),
"an edge already being read is not pending: {reply:?}"
);
assert!(
actor.sheep[&0].entry.pending.is_none(),
"nothing may be parked for a respawn to promote"
);
}
#[tokio::test(start_paused = true)]
async fn a_needs_respawn_field_parks_as_pending_and_leaves_the_child_alone() {
let dir = tempfile::tempdir().unwrap();
let (mut actor, _enforcer) = actor_over(&dir, &[app_with("web", |_| {})]);
let mut file = AppConfig::minimal("web", "./srv");
file.env = BTreeMap::from([("MODE".to_string(), "blue".to_string())]);
let reply = apply_config(
&mut actor,
vec![declared_app(file, &["name", "script", "env"])],
ResetDepth::None,
)
.await;
let entry = &actor.sheep[&0].entry;
assert_eq!(
entry.pid,
Some(APPLY_FIRST_PID),
"the running child must not have been replaced"
);
assert!(
entry.spec.config().env.is_empty(),
"the running child's own config must keep describing what it was spawned from"
);
assert_eq!(
entry
.pending
.as_ref()
.expect("a NeedsRespawn change parks as pending")
.config()
.env
.get("MODE")
.map(String::as_str),
Some("blue")
);
assert_eq!(reply[0].pending, vec!["env".to_string()]);
assert!(reply[0].applied.is_empty(), "{reply:?}");
assert_eq!(
reply[0]
.app
.as_ref()
.expect("an applied app is recorded")
.config()
.env
.get("MODE")
.map(String::as_str),
Some("blue"),
"a reboot spawns everything afresh, so what it comes up on is the \
full merge and not what the running child is on"
);
}
#[tokio::test(start_paused = true)]
async fn an_unnormalizable_merge_refuses_one_app_and_applies_the_others() {
let dir = tempfile::tempdir().unwrap();
let (mut actor, _enforcer) =
actor_over(&dir, &[app_with("web", |_| {}), app_with("worker", |_| {})]);
let mut broken = AppConfig::minimal("web", "./srv");
broken.instances = 2;
broken.out_file = Some("/tmp/web.log".to_string());
let mut worker = AppConfig::minimal("worker", "./srv");
worker.max_restarts = 7;
let reply = apply_config(
&mut actor,
vec![
declared_app(broken, &["name", "script", "instances", "out_file"]),
declared_app(worker, &["name", "script", "max_restarts"]),
],
ResetDepth::Policy,
)
.await;
assert!(
reply[0].refused.is_some(),
"an unnormalizable merge must refuse: {reply:?}"
);
assert_eq!(
actor.sheep[&0].entry.spec.config().instances,
1,
"a refused app's stored config must be untouched"
);
assert!(actor.sheep[&0].entry.spec.config().out_file.is_none());
assert_eq!(actor.sheep[&1].entry.spec.config().max_restarts, 7);
assert_eq!(reply[1].applied, vec!["max_restarts".to_string()]);
}
#[tokio::test(start_paused = true)]
async fn an_app_absent_from_the_file_is_left_running() {
let dir = tempfile::tempdir().unwrap();
let (mut actor, _enforcer) =
actor_over(&dir, &[app_with("web", |_| {}), app_with("worker", |_| {})]);
let mut file = AppConfig::minimal("web", "./srv");
file.max_restarts = 5;
let reply = apply_config(
&mut actor,
vec![declared_app(file, &["name", "script", "max_restarts"])],
ResetDepth::None,
)
.await;
let worker = &actor.sheep[&1].entry;
assert_eq!(worker.spec.config().name, "worker");
assert_eq!(worker.status, ProcStatus::Online);
assert_eq!(worker.pid, Some(APPLY_FIRST_PID + 1));
assert!(
reply.iter().all(|applied| applied.name != "worker"),
"a load must not claim to have touched an app the file never named: {reply:?}"
);
}
#[tokio::test(start_paused = true)]
async fn a_load_during_a_reload_reads_the_replacement_and_not_the_drainee() {
let dir = tempfile::tempdir().unwrap();
let (mut actor, _enforcer) = actor_over(
&dir,
&[app_with("web", |app| {
app.instances = 2;
app.cwd = Some("/srv/new".to_string());
})],
);
{
let slot = actor
.sheep
.get_mut(&0)
.expect("the fixture registered two slots");
slot.entry.status = ProcStatus::Stopping;
slot.entry.reload = ReloadState::Drainee { new_id: Some(1) };
slot.entry.spec = app_with("web", |app| {
app.instances = 2;
app.cwd = Some("/srv/old".to_string());
});
}
let mut file = AppConfig::minimal("web", "./srv");
file.max_restarts = 7;
let reply = apply_config(
&mut actor,
vec![declared_app(file, &["name", "script", "max_restarts"])],
ResetDepth::None,
)
.await;
assert_eq!(
actor.sheep[&1].entry.spec.config().cwd.as_deref(),
Some("/srv/new"),
"the replacement's spec must keep describing what the replacement \
was spawned from: {reply:?}"
);
assert_eq!(actor.sheep[&1].entry.spec.config().max_restarts, 7);
assert_eq!(reply[0].applied, vec!["max_restarts".to_string()]);
assert!(reply[0].pending.is_empty(), "{reply:?}");
}
#[tokio::test(start_paused = true)]
async fn a_file_naming_a_dog_is_refused_rather_than_merged() {
let dir = tempfile::tempdir().unwrap();
let (mut actor, _enforcer) = actor_over(&dir, &[app_with("metrics", |_| {})]);
actor
.sheep
.get_mut(&0)
.expect("the fixture registered one slot")
.entry
.dog = Some(DogSource::BuiltIn);
let mut file = AppConfig::minimal("metrics", "/opt/evil");
file.max_restarts = 42;
let reply = apply_config(
&mut actor,
vec![declared_app(file, &["name", "script", "max_restarts"])],
ResetDepth::None,
)
.await;
assert_eq!(
reply[0].refused.as_deref(),
Some(
"metrics is a dog, and a dog's config comes from `shep adopt` rather than \
from a Flockfile"
),
"{reply:?}"
);
let entry = &actor.sheep[&0].entry;
assert_eq!(entry.spec.config().script, "./srv");
assert_eq!(
entry.spec.config().max_restarts,
AppConfig::default().max_restarts
);
assert!(entry.pending.is_none(), "a refused app parks nothing");
assert!(
shep_core::overrides::get(&actor.paths.overrides, "metrics")
.unwrap()
.is_none(),
"a refused app establishes nothing"
);
}
#[tokio::test(start_paused = true)]
async fn a_reset_against_an_unchanged_file_keeps_a_defaulted_cwd() {
let dir = tempfile::tempdir().unwrap();
let (mut actor, _enforcer) = actor_over(
&dir,
&[app_with("web", |app| {
app.cwd = Some("/srv/web".to_string());
app.fold = Some("edge".to_string());
})],
);
let mut file = AppConfig::minimal("web", "./srv");
file.cwd = Some("/srv/web".to_string());
file.fold = Some("edge".to_string());
let reply = apply_config(
&mut actor,
vec![declared_app(file, &["name", "script"])],
ResetDepth::Policy,
)
.await;
assert!(
actor.sheep[&0].entry.pending.is_none(),
"an unchanged file has nothing to park: {reply:?}"
);
assert!(reply[0].pending.is_empty(), "{reply:?}");
let config = actor.sheep[&0].entry.spec.config().clone();
assert_eq!(config.cwd.as_deref(), Some("/srv/web"));
assert_eq!(config.fold.as_deref(), Some("edge"));
}
#[tokio::test(start_paused = true)]
async fn reset_restores_every_setting_and_only_reset_all_takes_env() {
let stored = |name: &str| {
app_with(name, |app| {
app.max_restarts = 3;
app.env = BTreeMap::from([("OPERATOR".to_string(), "1".to_string())]);
app.min_uptime = UpDuration::from_millis(9000);
})
};
let record = || {
established(
&["name", "script", "max_restarts"],
vec![
("max_restarts", serde_json::json!(3)),
("env", serde_json::json!({ "OPERATOR": "1" })),
("min_uptime", serde_json::json!("9000ms")),
],
)
};
let file = || {
let mut file = AppConfig::minimal("web", "./srv");
file.max_restarts = 10;
declared_app(file, &["name", "script", "max_restarts"])
};
let settings_dir = tempfile::tempdir().unwrap();
let (mut actor, _enforcer) = actor_over(&settings_dir, &[stored("web")]);
shep_core::overrides::put(&actor.paths.overrides, "web", &record()).unwrap();
apply_config(&mut actor, vec![file()], ResetDepth::Policy).await;
let settings = actor.sheep[&0].entry.spec.config().clone();
let settings_pending = actor.sheep[&0].entry.pending.clone();
assert_eq!(
settings.max_restarts, 10,
"--reset=policy puts a declared setting back to the file's"
);
assert_eq!(
settings.min_uptime,
AppConfig::default().min_uptime,
"--reset=policy puts a field the file never declared back to the \
file's own value, which for an undeclared key is the compiled \
default"
);
assert_eq!(
settings_pending
.as_ref()
.map_or(&settings.env, |app| &app.config().env)
.get("OPERATOR")
.map(String::as_str),
Some("1"),
"--reset=policy keeps env"
);
let all_dir = tempfile::tempdir().unwrap();
let (mut actor, _enforcer) = actor_over(&all_dir, &[stored("web")]);
shep_core::overrides::put(&actor.paths.overrides, "web", &record()).unwrap();
apply_config(&mut actor, vec![file()], ResetDepth::All).await;
let all = actor.sheep[&0].entry.spec.config().clone();
let all_pending = actor.sheep[&0].entry.pending.clone();
assert_eq!(all.max_restarts, 10);
assert_eq!(
all.min_uptime,
AppConfig::default().min_uptime,
"--reset=all drops a field the operator added"
);
assert!(
all_pending
.as_ref()
.expect("dropping an env key needs a respawn")
.config()
.env
.is_empty(),
"--reset=all drops an env key the operator added"
);
assert!(
shep_core::overrides::get(&actor.paths.overrides, "web")
.unwrap()
.is_none(),
"--reset=all removes the override record"
);
}
fn reset_grid() -> (AppConfig, DeclaredApp, shep_core::overrides::AppOverrides) {
let mut stored = AppConfig::minimal("web", "./srv");
stored.max_restarts = 3;
stored.max_memory = Some(MemSize::from_bytes(2 << 30));
stored.env = BTreeMap::from([("DB".to_string(), "operator".to_string())]);
let mut template = AppConfig::minimal("web", "./srv");
template.max_restarts = 10;
template.env = BTreeMap::from([("DB".to_string(), "template".to_string())]);
let incoming = declared_app(template, &["name", "script", "max_restarts", "env"]);
let mut record = established(
&["name", "script", "max_restarts", "env"],
vec![
("max_restarts", serde_json::json!(3)),
("max_memory", serde_json::json!("2G")),
("env", serde_json::json!({ "DB": "operator" })),
],
);
record.declared_env = BTreeSet::from(["DB".to_string()]);
(stored, incoming, record)
}
fn merged_over_grid(reset: ResetDepth) -> AppConfig {
let (stored, incoming, record) = reset_grid();
merge_declared(&stored, &incoming, &record, reset)
.expect("the grid fixture travels through serde")
.0
}
fn record_over_grid(reset: ResetDepth) -> shep_core::overrides::AppOverrides {
let (stored, incoming, record) = reset_grid();
merge_declared(&stored, &incoming, &record, reset)
.expect("the grid fixture travels through serde")
.1
}
#[test]
fn file_puts_back_what_the_template_declares_and_leaves_the_rest() {
let merged = merged_over_grid(ResetDepth::File);
assert_eq!(merged.max_restarts, 10, "a declared key goes back");
assert_eq!(
merged.max_memory,
Some(MemSize::from_bytes(2 << 30)),
"a key the template never declares is not the template's to reset"
);
assert_eq!(
merged.env.get("DB").map(String::as_str),
Some("operator"),
"`file` keeps env"
);
}
#[test]
fn policy_puts_back_every_setting_declared_or_not_and_keeps_env() {
let merged = merged_over_grid(ResetDepth::Policy);
assert_eq!(merged.max_restarts, 10, "a declared key goes back");
assert_eq!(
merged.max_memory,
AppConfig::default().max_memory,
"`policy` resets a key the template is silent about, which for an \
undeclared key means the value a fresh start off that template gives it"
);
assert_eq!(
merged.env.get("DB").map(String::as_str),
Some("operator"),
"`policy` keeps env"
);
}
#[test]
fn env_resets_env_and_touches_no_setting_at_all() {
let merged = merged_over_grid(ResetDepth::Env);
assert_eq!(
merged.max_restarts, 3,
"a declared policy field is not `env`'s to reset"
);
assert_eq!(
merged.max_memory,
Some(MemSize::from_bytes(2 << 30)),
"and neither is one the template is silent on"
);
assert_eq!(
merged.env.get("DB").map(String::as_str),
Some("template"),
"`env` puts env back to the template"
);
}
#[test]
fn all_puts_back_every_setting_and_env_with_it() {
let merged = merged_over_grid(ResetDepth::All);
assert_eq!(merged.max_restarts, 10, "a declared key goes back");
assert_eq!(
merged.max_memory,
AppConfig::default().max_memory,
"`all` resets a key the template is silent about"
);
assert_eq!(
merged.env.get("DB").map(String::as_str),
Some("template"),
"`all` puts env back to the template"
);
}
#[test]
fn file_spends_only_the_override_it_put_back() {
let record = record_over_grid(ResetDepth::File);
let mut held: Vec<&String> = record.fields.keys().collect();
held.sort();
assert_eq!(
held,
vec!["env", "max_memory"],
"`file` keeps the undeclared override and env, and spends the rest"
);
}
#[test]
fn policy_spends_every_setting_override_and_keeps_env() {
let record = record_over_grid(ResetDepth::Policy);
let mut held: Vec<&String> = record.fields.keys().collect();
held.sort();
assert_eq!(
held,
vec!["env"],
"`policy` spends both setting overrides and keeps env"
);
}
#[test]
fn env_spends_only_the_env_override() {
let record = record_over_grid(ResetDepth::Env);
let mut held: Vec<&String> = record.fields.keys().collect();
held.sort();
assert_eq!(
held,
vec!["max_memory", "max_restarts"],
"`env` spends env and keeps every setting override"
);
}
#[test]
fn all_spends_every_override() {
let record = record_over_grid(ResetDepth::All);
assert!(
record.fields.is_empty(),
"`all` holds nothing back: {:?}",
record.fields.keys().collect::<Vec<_>>()
);
}
#[tokio::test(start_paused = true)]
async fn an_env_reset_keeps_the_override_record() {
let dir = tempfile::tempdir().unwrap();
let (mut actor, _enforcer) = actor_over(
&dir,
&[app_with("web", |app| {
app.max_memory = Some(MemSize::from_bytes(2 << 30));
app.env = BTreeMap::from([("DB".to_string(), "operator".to_string())]);
})],
);
let mut record = established(
&["name", "script", "env"],
vec![
("max_memory", serde_json::json!("2G")),
("env", serde_json::json!({ "DB": "operator" })),
],
);
record.declared_env = BTreeSet::from(["DB".to_string()]);
shep_core::overrides::put(&actor.paths.overrides, "web", &record).unwrap();
let mut file = AppConfig::minimal("web", "./srv");
file.env = BTreeMap::from([("DB".to_string(), "template".to_string())]);
let file = declared_app(file, &["name", "script", "env"]);
let reply = apply_config(&mut actor, vec![file], ResetDepth::Env).await;
let written = shep_core::overrides::get(&actor.paths.overrides, "web")
.unwrap()
.unwrap_or_else(|| panic!("`env` keeps the record: {reply:?}"));
assert!(
written.fields.contains_key("max_memory"),
"the undeclared override must survive an env reset: {reply:?}"
);
assert_eq!(
actor.sheep[&0].entry.overridden,
vec!["max_memory".to_string()],
"and `shep flock`'s CFG column must still say so"
);
}
#[test]
fn an_env_reset_still_appends_a_key_nobody_established() {
let (stored, _, record) = reset_grid();
let mut template = AppConfig::minimal("web", "./srv");
template.max_restarts = 10;
template.min_uptime = UpDuration::from_millis(9000);
template.env = BTreeMap::from([("DB".to_string(), "template".to_string())]);
let incoming = declared_app(
template,
&["name", "script", "max_restarts", "min_uptime", "env"],
);
let (merged, _) = merge_declared(&stored, &incoming, &record, ResetDepth::Env)
.expect("the grid fixture travels through serde");
assert_eq!(
merged.min_uptime,
UpDuration::from_millis(9000),
"a declared key nobody established is appended under `env` too"
);
assert_eq!(
merged.max_restarts, 3,
"and an established one is still not overwritten"
);
}
#[tokio::test(start_paused = true)]
async fn an_env_reset_never_reshapes_a_flock() {
let dir = tempfile::tempdir().unwrap();
let (mut actor, _enforcer) = actor_over(&dir, &[app_with("web", |app| app.instances = 4)]);
let mut file = AppConfig::minimal("web", "./srv");
file.instances = 2;
let file = declared_app(file, &["name", "script", "instances"]);
let reply = apply_config(&mut actor, vec![file], ResetDepth::Env).await;
assert_eq!(
actor.ids_of_name("web").len(),
4,
"`env` scaled a flock: {reply:?}"
);
assert_eq!(
reply[0].refused.as_deref(),
Some(
"instances: this load never reshapes a flock; no mode scales without also \
putting back every setting the file declares, and `--reset=file` is the \
narrowest that does, taking the file's count of 2"
),
"an operator whose count did not move must be told why, and what the \
mode they are pointed at would cost them: {reply:?}"
);
}
#[tokio::test(start_paused = true)]
async fn a_file_reset_does_not_scale_an_app_the_template_says_nothing_about() {
let stocked = || app_with("web", |app| app.instances = 4);
let silent_dir = tempfile::tempdir().unwrap();
let (mut actor, _enforcer) = actor_over(&silent_dir, &[stocked()]);
let silent = declared_app(AppConfig::minimal("web", "./srv"), &["name", "script"]);
let reply = apply_config(&mut actor, vec![silent], ResetDepth::File).await;
assert_eq!(
actor.ids_of_name("web").len(),
4,
"`file` scaled against a file with no `instances` line: {reply:?}"
);
assert!(
!reply[0].applied.contains(&"instances".to_string()),
"{reply:?}"
);
let declaring_dir = tempfile::tempdir().unwrap();
let (mut actor, _enforcer) = actor_over(&declaring_dir, &[stocked()]);
let mut declaring = AppConfig::minimal("web", "./srv");
declaring.instances = 2;
let declaring = declared_app(declaring, &["name", "script", "instances"]);
let reply = apply_config(&mut actor, vec![declaring], ResetDepth::File).await;
assert_eq!(
actor.ids_of_name("web").len(),
2,
"`file` must apply a count the template does declare: {reply:?}"
);
assert!(
reply[0].applied.contains(&"instances".to_string()),
"{reply:?}"
);
}
#[tokio::test(start_paused = true)]
async fn a_settings_reset_does_not_establish_an_env_key_it_never_merged() {
let dir = tempfile::tempdir().unwrap();
let (mut actor, _enforcer) = actor_over(&dir, &[app_with("web", |_| {})]);
shep_core::overrides::put(
&actor.paths.overrides,
"web",
&established(&["name", "script"], Vec::new()),
)
.unwrap();
let file = || {
let mut file = AppConfig::minimal("web", "./srv");
file.env = BTreeMap::from([("NEW_KEY".to_string(), "1".to_string())]);
declared_app(file, &["name", "script", "env"])
};
let reset = apply_config(&mut actor, vec![file()], ResetDepth::Policy).await;
assert!(
actor.sheep[&0].entry.spec.config().env.is_empty(),
"a `--reset=policy` merges no env at all: {reset:?}"
);
assert!(
shep_core::overrides::get(&actor.paths.overrides, "web")
.unwrap()
.expect("the load records what it established")
.declared_env
.is_empty(),
"a key that never merged is not established"
);
let plain = apply_config(&mut actor, vec![file()], ResetDepth::None).await;
assert_eq!(
actor.sheep[&0]
.entry
.pending
.as_ref()
.expect("an env change parks for the next spawn")
.config()
.env
.get("NEW_KEY")
.map(String::as_str),
Some("1"),
"{plain:?}"
);
}
#[tokio::test(start_paused = true)]
async fn a_changed_extras_field_rearms_the_name() {
let dir = tempfile::tempdir().unwrap();
let (mut actor, enforcer) = actor_over(
&dir,
&[app_with("web", |app| {
app.max_memory = Some(MemSize::from_bytes(100 << 20));
})],
);
let mut file = AppConfig::minimal("web", "./srv");
file.max_memory = Some(MemSize::from_bytes(512 << 20));
apply_config(
&mut actor,
vec![declared_app(file, &["name", "script", "max_memory"])],
ResetDepth::None,
)
.await;
assert_eq!(
enforcer
.arms()
.last()
.expect("a changed ceiling re-arms the name")
.limit,
MemSize::from_bytes(512 << 20),
"the registry was re-armed with the old ceiling"
);
}
#[tokio::test(start_paused = true)]
async fn a_breach_measured_under_a_since_raised_ceiling_does_not_restart() {
let dir = tempfile::tempdir().unwrap();
let (mut actor, _enforcer) = actor_over(
&dir,
&[app_with("web", |app| {
app.max_memory = Some(MemSize::from_bytes(100 << 20));
})],
);
let mut file = AppConfig::minimal("web", "./srv");
file.max_memory = Some(MemSize::from_bytes(512 << 20));
apply_config(
&mut actor,
vec![declared_app(file, &["name", "script", "max_memory"])],
ResetDepth::None,
)
.await;
actor.handle_extra_restart(
0,
APPLY_FIRST_PID,
None,
Some(MemSize::from_bytes(200 << 20)),
);
let slot = &actor.sheep[&0];
assert!(
slot.manual.is_none(),
"a breach against a ceiling the operator has raised must never claim the manual marker"
);
assert_eq!(slot.entry.pid, Some(APPLY_FIRST_PID));
}
#[tokio::test(start_paused = true)]
async fn a_second_load_of_the_same_file_keeps_the_first_loads_parked_config() {
let dir = tempfile::tempdir().unwrap();
let (mut actor, _enforcer) = actor_over(&dir, &[app_with("web", |_| {})]);
let file = || {
let mut file = AppConfig::minimal("web", "./srv");
file.env = BTreeMap::from([("MODE".to_string(), "blue".to_string())]);
declared_app(file, &["name", "script", "env"])
};
let first = apply_config(&mut actor, vec![file()], ResetDepth::None).await;
assert_eq!(first[0].pending, vec!["env".to_string()]);
let second = apply_config(&mut actor, vec![file()], ResetDepth::None).await;
assert_eq!(
actor.sheep[&0]
.entry
.pending
.as_ref()
.expect("the parked config survives a second load")
.config()
.env
.get("MODE")
.map(String::as_str),
Some("blue"),
"the second load erased what the first parked"
);
assert_eq!(
second[0]
.app
.as_ref()
.expect("an applied app is recorded")
.config()
.env
.get("MODE")
.map(String::as_str),
Some("blue"),
"the recorded app lost it too, so a reboot would come up without it"
);
assert!(
second[0].pending.is_empty(),
"the second load changes nothing that was not already coming: {second:?}"
);
}
#[tokio::test(start_paused = true)]
async fn a_scale_down_removing_a_non_running_instance_does_not_panic() {
let dir = tempfile::tempdir().unwrap();
let (mut actor, _enforcer) = actor_over(&dir, &[app_with("web", |app| app.instances = 2)]);
let stopped = actor.sheep.get_mut(&1).expect("the fixture registers two");
stopped.entry.status = ProcStatus::Stopped;
stopped.entry.pid = None;
let mut file = AppConfig::minimal("web", "./srv");
file.instances = 1;
let reply = apply_config(
&mut actor,
vec![declared_app(file, &["name", "script", "instances"])],
ResetDepth::Policy,
)
.await;
assert_eq!(actor.ids_of_name("web"), vec![0]);
assert_eq!(reply[0].applied, vec!["instances".to_string()]);
assert_eq!(actor.sheep[&0].entry.spec.config().instances, 1);
}
#[tokio::test(start_paused = true)]
async fn a_load_that_scales_and_cannot_reach_the_running_spec_reports_what_landed() {
let root = tempfile::tempdir().unwrap();
let dir = tempfile::tempdir().unwrap();
let (mut actor, _enforcer) = actor_over(&dir, &[app_with("web", |_| {})]);
let mut file = AppConfig::minimal("web", "./srv");
file.instances = 2;
file.watch = true;
file.cwd = Some(root.path().display().to_string());
let reply = apply_config(
&mut actor,
vec![declared_app(
file,
&["name", "script", "instances", "watch", "cwd"],
)],
ResetDepth::Policy,
)
.await;
assert_eq!(
actor.ids_of_name("web").len(),
2,
"the scale really did happen"
);
assert!(
reply[0].refused.is_none(),
"a merge that normalizes is not an invalid file: {reply:?}"
);
assert!(
reply[0].app.is_some(),
"the muster roll must not be left on the pre-load config: {reply:?}"
);
assert_eq!(reply[0].applied, vec!["instances".to_string()]);
assert!(
reply[0].pending.contains(&"watch".to_string())
&& reply[0].pending.contains(&"cwd".to_string()),
"a change no running instance can take must park, not vanish: {reply:?}"
);
assert!(
actor.sheep[&0].entry.pending.is_some(),
"and it must be parked on the entry, not only reported"
);
}
#[tokio::test(start_paused = true)]
async fn a_load_does_not_arm_a_stopped_instance() {
let dir = tempfile::tempdir().unwrap();
let (mut actor, _enforcer) = actor_over(
&dir,
&[app_with("web", |app| {
app.cron_restart = Some("0 * * * *".to_string());
})],
);
let stopped = actor.sheep.get_mut(&0).expect("the fixture registers one");
stopped.entry.status = ProcStatus::Stopped;
stopped.entry.pid = None;
let mut file = AppConfig::minimal("web", "./srv");
file.cron_restart = Some("*/5 * * * *".to_string());
apply_config(
&mut actor,
vec![declared_app(file, &["name", "script", "cron_restart"])],
ResetDepth::None,
)
.await;
assert_eq!(
actor.registry.group_members("web"),
None,
"a load armed a schedule that will start a sheep the operator stopped"
);
}
#[tokio::test(start_paused = true)]
async fn a_plain_load_never_scales_and_a_reset_does() {
let file = || {
let mut file = AppConfig::minimal("web", "./srv");
file.instances = 1;
declared_app(file, &["name", "script", "instances"])
};
let plain_dir = tempfile::tempdir().unwrap();
let (mut actor, _enforcer) =
actor_over(&plain_dir, &[app_with("web", |app| app.instances = 2)]);
let reply = apply_config(&mut actor, vec![file()], ResetDepth::None).await;
assert_eq!(
actor.ids_of_name("web").len(),
2,
"a plain load deleted an instance"
);
assert!(!reply[0].applied.contains(&"instances".to_string()));
assert_eq!(
reply[0].refused.as_deref(),
Some(
"instances: this load never reshapes a flock; no mode scales without also \
putting back every setting the file declares, and `--reset=file` is the \
narrowest that does, taking the file's count of 1"
),
"the refusal must name a mode an operator can actually type, \
not the bare flag `shep start --reset` now refuses on its \
own, and it must name what following the advice costs: {reply:?}"
);
assert_eq!(
reply[0]
.app
.as_ref()
.expect("an applied app is recorded")
.config()
.instances,
2,
"the recorded count must be the one really running"
);
let reset_dir = tempfile::tempdir().unwrap();
let (mut actor, _enforcer) =
actor_over(&reset_dir, &[app_with("web", |app| app.instances = 2)]);
let reply = apply_config(&mut actor, vec![file()], ResetDepth::Policy).await;
assert_eq!(actor.ids_of_name("web"), vec![0], "--reset must take it");
assert_eq!(reply[0].applied, vec!["instances".to_string()]);
}
#[tokio::test(start_paused = true)]
async fn every_extras_field_triggers_a_rearm() {
let root = tempfile::tempdir().unwrap();
let cwd = root.path().display().to_string();
let ceiling = MemSize::from_bytes(100 << 20);
let base = |cwd: &str| {
let cwd = cwd.to_string();
move |app: &mut AppConfig| {
app.max_memory = Some(ceiling);
app.cwd = Some(cwd.clone());
app.watch = true;
app.ignore_watch = vec!["target/**".to_string()];
app.watch_delay = Some(UpDuration::from_millis(1000));
app.watch_options = vec!["src/**".to_string()];
app.cron_restart = Some("0 * * * *".to_string());
app.cron_timezone = Some("UTC".to_string());
app.liveness_probe = Some(ProbeConfig {
failure_threshold: 3,
interval: UpDuration::from_millis(600_000),
timeout: UpDuration::from_millis(1000),
..probe_config(ProbeKind::Tcp, "127.0.0.1:1")
});
}
};
type Edit = (&'static str, fn(&mut AppConfig));
let edits: Vec<Edit> = vec![
("max_memory", |app| {
app.max_memory = Some(MemSize::from_bytes(512 << 20));
}),
("watch", |app| app.watch = false),
("ignore_watch", |app| {
app.ignore_watch = vec!["dist/**".to_string()];
}),
("watch_delay", |app| {
app.watch_delay = Some(UpDuration::from_millis(2500));
}),
("watch_options", |app| {
app.watch_options = vec!["lib/**".to_string()];
}),
("cron_restart", |app| {
app.cron_restart = Some("*/5 * * * *".to_string());
}),
("cron_timezone", |app| {
app.cron_timezone = Some("Europe/Berlin".to_string());
}),
("liveness_probe", |app| app.liveness_probe = None),
];
for (field, edit) in edits {
let dir = tempfile::tempdir().unwrap();
let (mut actor, enforcer) = actor_over(&dir, &[app_with("web", base(&cwd))]);
let mut file = AppConfig::minimal("web", "./srv");
base(&cwd)(&mut file);
edit(&mut file);
let reply = apply_config(
&mut actor,
vec![declared_app(
file,
&[
"name",
"script",
"cwd",
"max_memory",
"watch",
"ignore_watch",
"watch_delay",
"watch_options",
"cron_restart",
"cron_timezone",
"liveness_probe",
],
)],
ResetDepth::Policy,
)
.await;
assert!(
reply[0].applied.contains(&field.to_string()),
"{field} did not apply at all: {reply:?}"
);
assert!(
!enforcer.arms().is_empty(),
"changing {field} left the armed worker on the old value"
);
}
}
#[tokio::test(start_paused = true)]
async fn a_load_tears_down_a_group_whose_instances_are_all_down() {
let dir = tempfile::tempdir().unwrap();
let (mut actor, _enforcer) = actor_over(
&dir,
&[app_with("web", |app| {
app.cron_restart = Some("0 * * * *".to_string());
})],
);
actor.arm_extras(0);
assert_eq!(
actor.registry.group_members("web"),
Some(vec![0]),
"the fixture must really be armed, or this pins nothing"
);
let waiting = actor.sheep.get_mut(&0).expect("the fixture registers one");
waiting.entry.status = ProcStatus::WaitingRestart;
waiting.entry.pid = None;
let mut file = AppConfig::minimal("web", "./srv");
file.cron_restart = Some("*/5 * * * *".to_string());
let reply = apply_config(
&mut actor,
vec![declared_app(file, &["name", "script", "cron_restart"])],
ResetDepth::None,
)
.await;
assert_eq!(reply[0].applied, vec!["cron_restart".to_string()]);
assert_eq!(
actor.registry.group_members("web"),
None,
"a worker built from the replaced config survived a load that reported the \
field as applied"
);
}
#[tokio::test(start_paused = true)]
async fn a_change_that_cannot_be_parked_is_not_reported_as_parked() {
let dir = tempfile::tempdir().unwrap();
let (mut actor, _enforcer) = actor_over(&dir, &[app_with("web", |app| app.instances = 2)]);
let earlier = app_with("web", |app| {
app.instances = 1;
app.env = BTreeMap::from([("MODE".to_string(), "blue".to_string())]);
});
for id in [0, 1] {
actor
.sheep
.get_mut(&id)
.expect("the fixture registers two")
.entry
.pending = Some(earlier.clone());
}
let mut file = AppConfig::minimal("web", "./srv");
file.out_file = Some("/tmp/web.log".to_string());
let reply = apply_config(
&mut actor,
vec![declared_app(file, &["name", "script", "out_file"])],
ResetDepth::None,
)
.await;
assert!(
!reply[0].pending.contains(&"out_file".to_string()),
"a field that went nowhere must not be reported as coming: {reply:?}"
);
assert!(
reply[0]
.refused
.as_deref()
.is_some_and(|why| why.contains("out_file")),
"and the operator must be told which field it was: {reply:?}"
);
assert_eq!(
actor.sheep[&0]
.entry
.pending
.as_ref()
.expect("the earlier load's parked config survives")
.config()
.env
.get("MODE")
.map(String::as_str),
Some("blue"),
"the earlier load's parked config must not be cleared"
);
assert!(
reply[0].app.is_none(),
"there is no config that holds at the count really running, so nothing may be \
recorded: {reply:?}"
);
}
#[tokio::test(start_paused = true)]
async fn a_plain_load_whose_merge_cannot_normalize_refuses_and_touches_nothing() {
let dir = tempfile::tempdir().unwrap();
let (mut actor, _enforcer) = actor_over(&dir, &[app_with("web", |app| app.instances = 2)]);
let mut file = AppConfig::minimal("web", "./srv");
file.out_file = Some("/tmp/web.log".to_string());
let reply = apply_config(
&mut actor,
vec![declared_app(file, &["name", "script", "out_file"])],
ResetDepth::None,
)
.await;
assert!(reply[0].refused.is_some(), "{reply:?}");
assert!(reply[0].applied.is_empty() && reply[0].pending.is_empty());
assert!(reply[0].app.is_none());
assert!(actor.sheep[&0].entry.spec.config().out_file.is_none());
assert!(actor.sheep[&0].entry.pending.is_none());
assert_eq!(actor.ids_of_name("web").len(), 2);
}
#[tokio::test(start_paused = true)]
async fn a_parked_config_that_cannot_be_rebuilt_is_reported_even_with_no_field_to_name() {
let dir = tempfile::tempdir().unwrap();
let (mut actor, _enforcer) = actor_over(&dir, &[app_with("web", |app| app.instances = 2)]);
let earlier = app_with("web", |app| {
app.instances = 1;
app.max_restarts = 10;
app.out_file = Some("/tmp/web.log".to_string());
});
for id in [0, 1] {
actor
.sheep
.get_mut(&id)
.expect("the fixture registers two")
.entry
.pending = Some(earlier.clone());
}
let mut file = AppConfig::minimal("web", "./srv");
file.max_restarts = 99;
let reply = apply_config(
&mut actor,
vec![declared_app(file, &["name", "script", "max_restarts"])],
ResetDepth::None,
)
.await;
assert_eq!(actor.sheep[&0].entry.spec.config().max_restarts, 99);
assert!(
reply[0]
.refused
.as_deref()
.is_some_and(|why| why.contains("could not be rebuilt")),
"a respawn is going to put max_restarts back to 10 and nobody was told: {reply:?}"
);
assert_eq!(
actor.sheep[&0]
.entry
.pending
.as_ref()
.expect("the earlier parked config is still there")
.config()
.max_restarts,
10,
"and that is what makes the refusal true"
);
}
#[tokio::test(start_paused = true)]
async fn a_refused_key_is_not_established_so_the_same_file_still_tries() {
let dir = tempfile::tempdir().unwrap();
let (mut actor, _enforcer) = actor_over(&dir, &[app_with("web", |app| app.instances = 2)]);
let earlier = app_with("web", |app| app.instances = 1);
for id in [0, 1] {
actor
.sheep
.get_mut(&id)
.expect("the fixture registers two")
.entry
.pending = Some(earlier.clone());
}
let file = || {
let mut file = AppConfig::minimal("web", "./srv");
file.out_file = Some("/tmp/web.log".to_string());
declared_app(file, &["name", "script", "out_file"])
};
let names_it = |why: &str| why.contains("out_file");
let first = apply_config(&mut actor, vec![file()], ResetDepth::None).await;
assert!(
first[0].refused.as_deref().is_some_and(names_it),
"{first:?}"
);
let second = apply_config(&mut actor, vec![file()], ResetDepth::None).await;
assert!(
second[0].refused.as_deref().is_some_and(names_it),
"a retry of the same file must meet the same refusal, not silence: {second:?}"
);
assert!(
!shep_core::overrides::get(&actor.paths.overrides, "web")
.unwrap()
.expect("the load recorded what it established")
.declared
.contains("out_file"),
"a key that went nowhere was established by nobody"
);
}
#[tokio::test(start_paused = true)]
async fn a_key_a_load_took_is_established_against_the_next_load() {
let dir = tempfile::tempdir().unwrap();
let (mut actor, _enforcer) = actor_over(&dir, &[app_with("web", |_| {})]);
let with_budget = |budget: u32| {
let mut file = AppConfig::minimal("web", "./srv");
file.max_restarts = budget;
declared_app(file, &["name", "script", "max_restarts"])
};
let first = apply_config(&mut actor, vec![with_budget(99)], ResetDepth::None).await;
assert_eq!(first[0].applied, vec!["max_restarts".to_string()]);
assert_eq!(actor.sheep[&0].entry.spec.config().max_restarts, 99);
let dropped = apply_config(
&mut actor,
vec![declared_app(
AppConfig::minimal("web", "./srv"),
&["name", "script"],
)],
ResetDepth::None,
)
.await;
assert!(dropped[0].applied.is_empty(), "{dropped:?}");
let third = apply_config(&mut actor, vec![with_budget(5)], ResetDepth::None).await;
assert_eq!(
actor.sheep[&0].entry.spec.config().max_restarts,
99,
"a file overwrote a key an earlier load had established"
);
assert!(third[0].applied.is_empty(), "{third:?}");
assert!(
shep_core::overrides::get(&actor.paths.overrides, "web")
.unwrap()
.expect("a load records what it established")
.declared
.contains("max_restarts"),
"and the record is what makes that true"
);
}
#[tokio::test(start_paused = true)]
async fn a_refused_env_change_gives_the_operators_override_back() {
let dir = tempfile::tempdir().unwrap();
let (mut actor, _enforcer) = actor_over(&dir, &[app_with("web", |app| app.instances = 2)]);
let earlier = app_with("web", |app| {
app.instances = 1;
app.out_file = Some("/tmp/web.log".to_string());
});
for id in [0, 1] {
actor
.sheep
.get_mut(&id)
.expect("the fixture registers two")
.entry
.pending = Some(earlier.clone());
}
shep_core::overrides::put(
&actor.paths.overrides,
"web",
&established(
&["name", "script"],
vec![("env", serde_json::json!({ "OPERATOR": "1" }))],
),
)
.unwrap();
let mut file = AppConfig::minimal("web", "./srv");
file.env = BTreeMap::from([
("OPERATOR".to_string(), "2".to_string()),
("MODE".to_string(), "blue".to_string()),
]);
let reply = apply_config(
&mut actor,
vec![declared_app(file, &["name", "script", "env"])],
ResetDepth::None,
)
.await;
assert!(
reply[0]
.refused
.as_deref()
.is_some_and(|why| why.contains("env")),
"the fixture must really refuse the env change: {reply:?}"
);
let record = shep_core::overrides::get(&actor.paths.overrides, "web")
.unwrap()
.expect("a load records what it established");
assert_eq!(
record
.fields
.get("env")
.and_then(|env| env.get("OPERATOR"))
.and_then(serde_json::Value::as_str),
Some("1"),
"a load that changed nothing spent the operator's override"
);
assert!(
record.declared_env.is_empty(),
"and it established env keys that never landed: {:?}",
record.declared_env
);
}
#[tokio::test(start_paused = true)]
async fn reload_carries_the_overridden_cache_to_its_replacement() {
let dir = tempfile::tempdir().unwrap();
let (mut actor, _enforcer) = actor_over(&dir, &[app_with("web", |_| {})]);
actor.sheep.get_mut(&0).unwrap().entry.overridden = vec!["max_restarts".to_string()];
actor.advance_reload("web", VecDeque::from([0]));
let new_id = actor.reloads["web"]
.swap
.new_id
.expect("an overlapping reload spawns its replacement at once");
assert_eq!(
actor.sheep[&new_id].entry.overridden,
vec!["max_restarts".to_string()],
"the replacement must carry the drainee's own overridden cache, not start blank"
);
}
#[tokio::test(start_paused = true)]
async fn reload_promotes_pending_config() {
let dir = tempfile::tempdir().unwrap();
let (mut actor, _enforcer) = actor_over(&dir, &[app_with("web", |_| {})]);
let mut file = AppConfig::minimal("web", "./srv");
file.env = BTreeMap::from([("MODE".to_string(), "blue".to_string())]);
apply_config(
&mut actor,
vec![declared_app(file, &["name", "script", "env"])],
ResetDepth::None,
)
.await;
assert!(
actor.sheep[&0].entry.pending.is_some(),
"the fixture must really park the change, or this case proves nothing"
);
actor.advance_reload("web", VecDeque::from([0]));
let new_id = actor.reloads["web"]
.swap
.new_id
.expect("an overlapping reload spawns its replacement at once");
assert_eq!(
actor.sheep[&new_id]
.entry
.spec
.config()
.env
.get("MODE")
.map(String::as_str),
Some("blue"),
"the replacement must come up on the config the load parked"
);
assert!(
actor.sheep[&new_id].entry.pending.is_none(),
"and it is owed nothing further, having been built from what was owed"
);
assert!(
actor.sheep[&0].entry.pending.is_some(),
"the drainee keeps its copy until it is deregistered: this swap can still be \
abandoned, and the child it would go back to serving has not got the change"
);
assert_ne!(
actor.sheep[&new_id].entry.pid,
Some(APPLY_FIRST_PID),
"a promotion is only reachable through a process that actually replaced the old one"
);
}
#[tokio::test(start_paused = true)]
async fn restart_promotes_pending_config() {
let dir = tempfile::tempdir().unwrap();
let (mut actor, _enforcer) = actor_over(&dir, &[app_with("web", |_| {})]);
let mut file = AppConfig::minimal("web", "./srv");
file.env = BTreeMap::from([("MODE".to_string(), "blue".to_string())]);
apply_config(
&mut actor,
vec![declared_app(file, &["name", "script", "env"])],
ResetDepth::None,
)
.await;
assert!(
actor.sheep[&0].entry.pending.is_some(),
"the fixture must really park the change, or this case proves nothing"
);
let (reply, _answer) = oneshot::channel();
actor.begin_manual(
ProcessSelector::Name("web".to_string()),
ManualKind::Restart,
CommandOrigin::Operator,
ReplyKind::Info(reply),
);
actor.handle_exited(
0,
ExitOutcome {
code: Some(0),
signal: None,
},
);
let entry = &actor.sheep[&0].entry;
assert_eq!(
entry.spec.config().env.get("MODE").map(String::as_str),
Some("blue"),
"the restarted child must come up on the config the load parked"
);
assert!(
entry.pending.is_none(),
"a promoted config is owed no longer, so the slot must be empty"
);
assert_ne!(
entry.pid,
Some(APPLY_FIRST_PID),
"a promotion is only reachable through a process that actually replaced the old one"
);
}
#[tokio::test(start_paused = true)]
async fn to_info_reports_the_pending_fields_names_only() {
let dir = tempfile::tempdir().unwrap();
let (mut actor, _enforcer) = actor_over(&dir, &[app_with("web", |_| {})]);
let mut file = AppConfig::minimal("web", "./srv");
file.env = BTreeMap::from([("MODE".to_string(), "blue".to_string())]);
apply_config(
&mut actor,
vec![declared_app(file, &["name", "script", "env"])],
ResetDepth::None,
)
.await;
let entry = &actor.sheep[&0].entry;
assert!(
entry.pending.is_some(),
"the fixture must really park the change, or this case proves nothing"
);
let info = to_info(entry, &actor.smits);
assert_eq!(info.pending, Some(vec!["env".to_string()]));
}
#[tokio::test(start_paused = true)]
async fn to_info_carries_a_sheep_s_configured_memory_ceiling_in_bytes() {
const CEILING_BYTES: u64 = 43_000_001;
let dir = tempfile::tempdir().unwrap();
let (actor, _enforcer) = actor_over(
&dir,
&[app_with("web", |app| {
app.max_memory = Some(MemSize::from_bytes(CEILING_BYTES));
})],
);
let entry = &actor.sheep[&0].entry;
let info = to_info(entry, &actor.smits);
assert_eq!(info.max_memory, Some(CEILING_BYTES));
}
#[tokio::test(start_paused = true)]
async fn to_info_reports_none_for_a_dog_with_no_ceiling() {
let dir = tempfile::tempdir().unwrap();
let (actor, _enforcer) = actor_over(&dir, &[dog_app("watcher")]);
let entry = &actor.sheep[&0].entry;
let info = to_info(entry, &actor.smits);
assert_eq!(info.max_memory, None);
}
#[tokio::test(start_paused = true)]
async fn overridden_for_prefers_a_live_sibling_over_the_store() {
let dir = tempfile::tempdir().unwrap();
let (mut actor, _enforcer) = actor_over(&dir, &[app_with("web", |_| {})]);
actor.sheep.get_mut(&0).unwrap().entry.overridden = vec!["cwd".to_string()];
shep_core::overrides::put(
&actor.paths.overrides,
"web",
&established(
&["name", "script"],
vec![("max_restarts", serde_json::json!(9))],
),
)
.unwrap();
assert_eq!(actor.overridden_for("web"), vec!["cwd".to_string()]);
}
#[tokio::test(start_paused = true)]
async fn overridden_for_reads_the_store_when_no_sibling_exists() {
let dir = tempfile::tempdir().unwrap();
let (actor, _enforcer) = actor_over(&dir, &[]);
shep_core::overrides::put(
&actor.paths.overrides,
"web",
&established(
&["name", "script"],
vec![("max_restarts", serde_json::json!(9))],
),
)
.unwrap();
assert_eq!(
actor.overridden_for("web"),
vec!["max_restarts".to_string()]
);
assert_eq!(
actor.overridden_for("nobody-has-heard-of-this-app"),
Vec::<String>::new(),
"an unreadable-or-empty answer for a name the store has never seen"
);
}
#[tokio::test(start_paused = true)]
async fn to_info_reports_the_overridden_field_names_the_store_holds() {
let dir = tempfile::tempdir().unwrap();
let (mut actor, _enforcer) =
actor_over(&dir, &[app_with("web", |app| app.max_restarts = 7)]);
shep_core::overrides::put(
&actor.paths.overrides,
"web",
&established(
&["name", "script"],
vec![("max_restarts", serde_json::json!(7))],
),
)
.unwrap();
let file = AppConfig::minimal("web", "./srv");
apply_config(
&mut actor,
vec![declared_app(file, &["name", "script"])],
ResetDepth::None,
)
.await;
let entry = &actor.sheep[&0].entry;
assert_eq!(
entry.overridden,
vec!["max_restarts".to_string()],
"the cache must mirror what this load wrote back to the override store"
);
let info = to_info(entry, &actor.smits);
assert_eq!(info.overridden, Some(vec!["max_restarts".to_string()]));
}
#[tokio::test(start_paused = true)]
async fn to_info_never_carries_an_override_value() {
const SENTINEL: &str = "postgres://sentinel-value-that-must-never-appear";
let dir = tempfile::tempdir().unwrap();
let (mut actor, _enforcer) = actor_over(
&dir,
&[app_with("web", |app| {
app.env = BTreeMap::from([("DATABASE_URL".to_string(), SENTINEL.to_string())]);
})],
);
shep_core::overrides::put(
&actor.paths.overrides,
"web",
&established(
&["name", "script"],
vec![("env", serde_json::json!({ "DATABASE_URL": SENTINEL }))],
),
)
.unwrap();
let file = AppConfig::minimal("web", "./srv");
apply_config(
&mut actor,
vec![declared_app(file, &["name", "script"])],
ResetDepth::None,
)
.await;
let entry = &actor.sheep[&0].entry;
let info = to_info(entry, &actor.smits);
assert_eq!(
info.overridden,
Some(vec!["env".to_string()]),
"the name must still reach the operator"
);
let json = serde_json::to_string(&info).unwrap();
assert!(
!json.contains(SENTINEL),
"an override value reached the wire: {json}"
);
}
#[cfg(unix)]
#[tokio::test(start_paused = true)]
async fn promoting_a_user_change_re_resolves_credentials() {
let dir = tempfile::tempdir().unwrap();
let (mut actor, _enforcer) = actor_over(&dir, &[app_with("web", |_| {})]);
let mut file = AppConfig::minimal("web", "./srv");
file.user = Some(own_user_name());
apply_config(
&mut actor,
vec![declared_app(file, &["name", "script", "user"])],
ResetDepth::None,
)
.await;
assert!(
actor.sheep[&0].entry.pending.is_some(),
"the fixture must really park the change, or this case proves nothing"
);
actor.advance_reload("web", VecDeque::from([0]));
let wanted = Credentials {
uid: nix::unistd::geteuid().as_raw(),
gid: None,
};
assert_eq!(
actor.runner.spawned_as(0),
Some(wanted),
"the replacement must carry the identity the promoted `user` resolves to; `None` \
here is the fixture's stale resolution, which is the change being ignored"
);
let new_id = actor.reloads["web"]
.swap
.new_id
.expect("an overlapping reload spawns its replacement at once");
assert_eq!(
actor.sheep[&new_id].entry.credentials,
SpawnIdentity::Resolved(Some(wanted)),
"and the replacement records it, so the restart after this one reuses it"
);
assert_eq!(
actor.sheep[&0].entry.credentials,
SpawnIdentity::Resolved(None),
"while the drainee's own identity is untouched: it is still serving under it, and \
an abandoned swap must not leave it recorded as never looked up"
);
}
#[tokio::test(start_paused = true)]
async fn promoting_an_unrelated_change_keeps_the_resolved_identity() {
let dir = tempfile::tempdir().unwrap();
let (mut actor, _enforcer) = actor_over(
&dir,
&[app_with("web", |app| {
app.user = Some(NO_SUCH_USER.to_string())
})],
);
let settled = Credentials {
uid: 4242,
gid: None,
};
actor
.sheep
.get_mut(&0)
.expect("the fixture registers one instance")
.entry
.credentials = SpawnIdentity::Resolved(Some(settled));
let mut file = AppConfig::minimal("web", "./srv");
file.args = vec!["--port=8080".to_string()];
apply_config(
&mut actor,
vec![declared_app(file, &["name", "script", "args"])],
ResetDepth::None,
)
.await;
assert!(
actor.sheep[&0].entry.pending.is_some(),
"the fixture must really park the change, or this case proves nothing"
);
actor.advance_reload("web", VecDeque::from([0]));
assert_eq!(
actor.runner.spawn_count(),
1,
"the replacement must have been spawned at all: an identity re-resolved here could \
only refuse, and a refusal would abandon the reload"
);
assert_eq!(
actor.runner.spawned_as(0),
Some(settled),
"an `args` change is not an identity change, so the replacement runs as whoever the \
instance was already running as"
);
assert_eq!(
actor.sheep[&0].entry.credentials,
SpawnIdentity::Resolved(Some(settled)),
"and the stored resolution is untouched, so no passwd lookup was spent"
);
assert!(
actor.sheep[&0].entry.pending.is_some(),
"the drainee keeps what it was owed; the replacement is what carries it"
);
let new_id = actor.reloads["web"]
.swap
.new_id
.expect("an overlapping reload spawns its replacement at once");
assert_eq!(
actor.sheep[&new_id].entry.spec.config().args,
vec!["--port=8080".to_string()],
"the promotion itself must still have happened"
);
}
#[cfg(unix)]
#[tokio::test(start_paused = true)]
async fn a_sibling_that_has_not_promoted_yet_still_re_resolves_after_later_loads() {
let dir = tempfile::tempdir().unwrap();
let (mut actor, _enforcer) = actor_over(&dir, &[app_with("web", |app| app.instances = 3)]);
let settled = Credentials {
uid: 4242,
gid: None,
};
for id in [0, 1, 2] {
actor
.sheep
.get_mut(&id)
.expect("the fixture registers three")
.entry
.credentials = SpawnIdentity::Resolved(Some(settled));
}
let user_change = || {
let mut file = AppConfig::minimal("web", "./srv");
file.instances = 3;
file.user = Some(own_user_name());
vec![declared_app(file, &["name", "script", "instances", "user"])]
};
apply_config(&mut actor, user_change(), ResetDepth::None).await;
actor.respawn(0, true);
apply_config(&mut actor, user_change(), ResetDepth::None).await;
apply_config(&mut actor, user_change(), ResetDepth::None).await;
actor.respawn(1, true);
let wanted = Credentials {
uid: nix::unistd::geteuid().as_raw(),
gid: None,
};
assert_eq!(
actor.runner.spawned_as(1),
Some(wanted),
"instance 1 has still never applied the `user` change, so its promotion must \
re-resolve; the settled 4242 here is the change being silently dropped"
);
assert_eq!(
actor.sheep[&1].entry.spec.config().user,
Some(own_user_name()),
"and its own spec must record what it came up on"
);
}
#[tokio::test(start_paused = true)]
async fn an_abandoned_reload_leaves_the_parked_config_where_it_was() {
let dir = tempfile::tempdir().unwrap();
let (mut actor, _enforcer) = actor_over(&dir, &[app_with("web", |_| {})]);
let (ctl_tx, _ctl_rx) = mpsc::channel(SHEEP_CTL_CAPACITY);
actor.sheep.get_mut(&0).expect("the fixture's sheep").ctl = Some(ctl_tx);
let mut file = AppConfig::minimal("web", "./srv");
file.env = BTreeMap::from([("MODE".to_string(), "blue".to_string())]);
apply_config(
&mut actor,
vec![declared_app(file, &["name", "script", "env"])],
ResetDepth::None,
)
.await;
actor.advance_reload("web", VecDeque::from([0]));
actor.handle_reload_deadline("web", actor.reloads["web"].deadline);
assert!(actor.reloads.is_empty(), "the swap must really be off");
let entry = &actor.sheep[&0].entry;
assert_eq!(
entry.status,
ProcStatus::Online,
"the drainee is serving again, so it is the child spawned before the load"
);
assert!(
entry.spec.config().env.is_empty(),
"and its spec must still describe what that child was spawned from"
);
assert_eq!(
entry
.pending
.as_ref()
.expect("the config is still owed: no child ever came up on it")
.config()
.env
.get("MODE")
.map(String::as_str),
Some("blue")
);
}
#[tokio::test(start_paused = true)]
async fn an_abandoned_reload_leaves_the_drainees_identity_alone() {
let dir = tempfile::tempdir().unwrap();
let (mut actor, _enforcer) = actor_over(&dir, &[app_with("web", |_| {})]);
let settled = Credentials {
uid: 4242,
gid: None,
};
actor
.sheep
.get_mut(&0)
.expect("the fixture registers one instance")
.entry
.credentials = SpawnIdentity::Resolved(Some(settled));
let mut file = AppConfig::minimal("web", "./srv");
file.user = Some(NO_SUCH_USER.to_string());
apply_config(
&mut actor,
vec![declared_app(file, &["name", "script", "user"])],
ResetDepth::None,
)
.await;
actor.advance_reload("web", VecDeque::from([0]));
assert_eq!(
actor.runner.spawn_count(),
0,
"the fixture must really refuse the replacement, or this case proves nothing"
);
let entry = &actor.sheep[&0].entry;
assert_eq!(
entry.credentials,
SpawnIdentity::Resolved(Some(settled)),
"the drainee is still serving under this identity, so the abandoned swap must not \
record it as never looked up"
);
assert!(
entry.pending.is_some(),
"and it is still owed the config that swap was going to bring"
);
}
#[tokio::test(start_paused = true)]
async fn a_reload_orders_itself_by_the_config_its_replacement_will_carry() {
let dir = tempfile::tempdir().unwrap();
let (mut actor, _enforcer) =
actor_over(&dir, &[app_with("web", |app| app.wait_ready = true)]);
let mut file = AppConfig::minimal("web", "./srv");
file.wait_ready = false;
file.readiness_probe = Some(probe_config(ProbeKind::Tcp, "127.0.0.1:9"));
apply_config(
&mut actor,
vec![declared_app(
file,
&["name", "script", "wait_ready", "readiness_probe"],
)],
ResetDepth::None,
)
.await;
let entry = &actor.sheep[&0].entry;
assert!(
entry.spec.config().wait_ready && entry.spec.config().readiness_probe.is_some(),
"the fixture must really hold both at once, or this case proves nothing"
);
actor.advance_reload("web", VecDeque::from([0]));
let job = &actor.reloads["web"];
assert_eq!(
job.mode,
ReloadMode::Serial,
"the replacement is probe-gated, and a probe cannot say which of two overlapping \
instances answered it"
);
assert_eq!(
job.swap.phase,
ReloadPhase::DrainFirst,
"so the drain runs first and nothing is spawned yet"
);
assert_eq!(
actor.runner.spawn_count(),
0,
"an overlap here would put a second instance on the drainee's address"
);
}
#[tokio::test(start_paused = true)]
async fn a_scale_up_carries_the_parked_config_onto_the_instances_it_creates() {
let dir = tempfile::tempdir().unwrap();
let (mut actor, _enforcer) = actor_over(&dir, &[app_with("web", |app| app.instances = 2)]);
let mut file = AppConfig::minimal("web", "./srv");
file.instances = 2;
file.env = BTreeMap::from([("MODE".to_string(), "blue".to_string())]);
apply_config(
&mut actor,
vec![declared_app(file, &["name", "script", "instances", "env"])],
ResetDepth::None,
)
.await;
let (reply, mut answer) = oneshot::channel();
actor.handle_scale("web", 4, reply);
answer
.try_recv()
.expect("handle_scale answers before it returns")
.expect("the fixture has scripts enough to scale to four");
for id in [2, 3] {
assert_eq!(
actor.sheep[&id]
.entry
.pending
.as_ref()
.unwrap_or_else(|| panic!("instance {id} must be owed the parked config"))
.config()
.env
.get("MODE")
.map(String::as_str),
Some("blue"),
"an instance created during a parking window is owed the same config as its \
siblings"
);
assert!(
actor.sheep[&id].entry.spec.config().env.is_empty(),
"and its own spec still describes what it was actually spawned from"
);
}
}
#[tokio::test(start_paused = true)]
async fn a_scale_updates_the_count_inside_the_config_it_carries() {
let dir = tempfile::tempdir().unwrap();
let (mut actor, _enforcer) = actor_over(&dir, &[app_with("web", |app| app.instances = 2)]);
let mut file = AppConfig::minimal("web", "./srv");
file.instances = 2;
file.env = BTreeMap::from([("MODE".to_string(), "blue".to_string())]);
apply_config(
&mut actor,
vec![declared_app(file, &["name", "script", "instances", "env"])],
ResetDepth::None,
)
.await;
let (reply, mut answer) = oneshot::channel();
actor.handle_scale("web", 4, reply);
answer
.try_recv()
.expect("handle_scale answers before it returns")
.expect("the fixture has scripts enough to scale to four");
for id in 0..4 {
let entry = &actor.sheep[&id].entry;
assert_eq!(
entry
.pending
.as_ref()
.unwrap_or_else(|| panic!("instance {id} must be owed the parked config"))
.config()
.instances,
4,
"the count a scale achieved, not the one an earlier load parked"
);
assert!(
!to_info(entry, &actor.smits)
.pending
.unwrap_or_default()
.contains(&"instances".to_string()),
"a reload owes this instance nothing about the count"
);
}
}
#[cfg(unix)]
#[tokio::test(start_paused = true)]
async fn a_parked_config_and_its_reset_decision_survive_a_handover() {
let dir = tempfile::tempdir().unwrap();
let (mut actor, _enforcer) = actor_over(&dir, &[]);
let settled = Credentials {
uid: 4242,
gid: None,
};
let entry = ProcessEntry {
id: 7,
spec: app_with("web", |_| {}),
pending: Some(app_with("web", |app| app.user = Some(own_user_name()))),
pending_reidentifies: true,
overridden: Vec::new(),
instance: 0,
status: ProcStatus::Stopped,
pid: None,
restarts: 0,
started_at: None,
budget: RestartBudget::default(),
reload: ReloadState::None,
credentials: SpawnIdentity::Resolved(Some(settled)),
out_file: PathBuf::new(),
err_file: PathBuf::new(),
dog: None,
last_exit: None,
};
let carried =
CarriedSheep::from_entry(&entry, 0, CarriedFds::none(), false, None, false, None);
let crossed: CarriedSheep = serde_json::from_value(serde_json::to_value(&carried).unwrap())
.expect("this daemon reads what it writes");
actor
.install_adopted(without_handles(crossed), &Arc::new(AdoptedReaper::new()))
.expect("a registered-and-stopped sheep installs with nothing to adopt");
assert!(
actor.sheep[&7].entry.pending.is_some(),
"the successor must still owe this sheep the change its predecessor parked"
);
actor.respawn(7, true);
assert_eq!(
actor.runner.spawned_as(0),
Some(Credentials {
uid: nix::unistd::geteuid().as_raw(),
gid: None,
}),
"and promoting it must re-resolve: the settled 4242 here is the reset decision \
lost in the blob, which is the identity change silently dropped"
);
assert_eq!(
actor.sheep[&7].entry.spec.config().user,
Some(own_user_name()),
"and the promoted config is what the successor now records"
);
}
#[tokio::test(start_paused = true)]
async fn a_key_nobody_has_set_errors_the_sheep_without_a_spawn() {
let dir = tempfile::tempdir().unwrap();
let mut actor = actor_with_an_empty_flock(&dir, vec![ProcScript::never_exits()]);
let mut app = AppConfig::minimal("web", "./srv");
app.env
.insert("PW".to_string(), "{{secret:ABSENT}}".to_string());
let err = actor
.do_start(
vec![normalize(app).unwrap()],
None,
BatchPolicy::PerApp,
&BTreeSet::new(),
)
.unwrap_err();
assert!(matches!(err, SupervisorError::SpawnFailed(_)), "{err:?}");
let rendered = err.to_string();
assert!(
rendered.contains("ABSENT"),
"names the reference: {rendered}"
);
let entry = &actor.sheep.values().next().expect("one row").entry;
assert_eq!(entry.status, ProcStatus::Errored);
assert_eq!(
actor.runner.spawn_count(),
0,
"nothing may reach the runner"
);
}
#[test]
fn an_unreadable_store_warns_before_it_falls_back_to_an_empty_view() {
let dir = tempfile::tempdir().unwrap();
let actor = actor_with_an_empty_flock(&dir, vec![]);
std::fs::write(&actor.paths.secrets, "{ not json").unwrap();
let app = normalize(AppConfig::minimal("web", "./srv")).unwrap();
let logs = capture_logs(|| {
let view = actor.secret_view(&app);
let reference = shep_core::secrets::SecretRef {
namespace: None,
key: "K",
};
assert!(
matches!(
view.resolve(&reference),
shep_core::secrets::Resolution::MissingKey
),
"the fallback is an empty view, not a failed spawn"
);
});
assert!(logs.contains("WARN"), "loud enough to read: {logs}");
assert!(logs.contains("secrets.json"), "names the file: {logs}");
}
#[tokio::test(start_paused = true)]
async fn a_namespace_no_dog_has_pushed_to_leaves_the_sheep_waiting() {
let (events, _rx) = crate::bus::test_bus(64);
let runner = Arc::new(ScriptedRunner::new(vec![ProcScript::never_exits()]));
let dir = tempfile::tempdir().unwrap();
let handle = spawn_supervisor(SharedRunner(Arc::clone(&runner)), test_paths(&dir), events);
let mut app = AppConfig::minimal("web", "./srv");
app.env
.insert("PW".to_string(), "{{secret:vault/K}}".to_string());
let started = handle
.start(vec![normalize(app).unwrap()])
.await
.expect("a provider that has not reported yet is not a failed start");
assert_eq!(started[0].status, ProcStatus::WaitingRestart);
assert_eq!(handle.list().await[0].status, ProcStatus::WaitingRestart);
assert_eq!(runner.spawn_count(), 0, "nothing may reach the runner");
}
#[tokio::test(start_paused = true)]
async fn a_namespace_pushed_for_another_environment_leaves_the_sheep_waiting() {
let dir = tempfile::tempdir().unwrap();
let mut actor = actor_with_an_empty_flock(&dir, vec![ProcScript::never_exits()]);
actor
.provider_secrets
.put(
"vercel",
"production",
BTreeMap::from([("API_KEY".to_string(), "sk_live".to_string())]),
false,
)
.expect("no cache is written with persist off");
let mut app = AppConfig::minimal("web", "./srv");
app.environment = Some("staging".to_string());
app.env
.insert("K".to_string(), "{{secret:vercel/API_KEY}}".to_string());
actor
.do_start(
vec![normalize(app).unwrap()],
None,
BatchPolicy::PerApp,
&BTreeSet::new(),
)
.expect("a provider that has not pushed staging yet is not a failed start");
let entry = &actor.sheep.values().next().expect("one row").entry;
assert_eq!(entry.status, ProcStatus::WaitingRestart);
assert_eq!(
actor.runner.spawn_count(),
0,
"nothing may reach the runner"
);
}
#[tokio::test(start_paused = true)]
async fn a_key_absent_from_a_pushed_pair_errors_at_once() {
let dir = tempfile::tempdir().unwrap();
let mut actor = actor_with_an_empty_flock(&dir, vec![ProcScript::never_exits()]);
actor
.provider_secrets
.put(
"vercel",
"production",
BTreeMap::from([("OTHER".to_string(), "1".to_string())]),
false,
)
.expect("no cache is written with persist off");
let mut app = AppConfig::minimal("web", "./srv");
app.env
.insert("K".to_string(), "{{secret:vercel/API_KEY}}".to_string());
let err = actor
.do_start(
vec![normalize(app).unwrap()],
None,
BatchPolicy::PerApp,
&BTreeSet::new(),
)
.unwrap_err();
assert!(matches!(err, SupervisorError::SpawnFailed(_)), "{err:?}");
let entry = &actor.sheep.values().next().expect("one row").entry;
assert_eq!(entry.status, ProcStatus::Errored);
assert_eq!(
actor.runner.spawn_count(),
0,
"nothing may reach the runner"
);
}
#[tokio::test(start_paused = true)]
async fn a_namespace_that_never_arrives_errors_once_the_budget_runs_out() {
let (events, _rx) = crate::bus::test_bus(64);
let runner = ScriptedRunner::new(vec![]);
let dir = tempfile::tempdir().unwrap();
let handle = spawn_supervisor(runner, test_paths(&dir), events);
let mut app = AppConfig::minimal("web", "./srv");
app.env
.insert("PW".to_string(), "{{secret:vault/K}}".to_string());
app.max_restarts = 2;
app.restart_delay = Some(UpDuration::from_millis(100));
handle.start(vec![normalize(app).unwrap()]).await.unwrap();
assert_eq!(handle.list().await[0].status, ProcStatus::WaitingRestart);
tokio::time::sleep(Duration::from_millis(400)).await;
assert_eq!(
handle.list().await[0].status,
ProcStatus::Errored,
"the second refusal exhausts the budget"
);
}
#[tokio::test(start_paused = true)]
async fn a_value_in_the_store_lets_the_sheep_start() {
let (events, _rx) = crate::bus::test_bus(64);
let runner = Arc::new(ScriptedRunner::new(vec![ProcScript::never_exits()]));
let dir = tempfile::tempdir().unwrap();
let paths = test_paths(&dir);
shep_core::secrets::set(&paths.secrets, "DB_PASSWORD", "production", "hunter2").unwrap();
let handle = spawn_supervisor(SharedRunner(Arc::clone(&runner)), paths, events);
let mut app = AppConfig::minimal("web", "./srv");
app.env
.insert("PW".to_string(), "{{secret:DB_PASSWORD}}".to_string());
handle.start(vec![normalize(app).unwrap()]).await.unwrap();
assert_eq!(handle.list().await[0].status, ProcStatus::Online);
assert_eq!(runner.spawn_count(), 1);
}
#[tokio::test(start_paused = true)]
async fn a_sheeps_environment_decides_which_value_it_reads() {
let (events, _rx) = crate::bus::test_bus(64);
let runner = ScriptedRunner::new(vec![ProcScript::never_exits()]);
let dir = tempfile::tempdir().unwrap();
let paths = test_paths(&dir);
shep_core::secrets::set(&paths.secrets, "K", "staging", "v").unwrap();
let handle = spawn_supervisor(runner, paths, events);
let templated = |name: &str, environment: Option<&str>| {
let mut app = AppConfig::minimal(name, "./srv");
app.environment = environment.map(str::to_string);
app.env.insert("K".to_string(), "{{secret:K}}".to_string());
normalize(app).unwrap()
};
handle
.start(vec![templated("staged", Some("staging"))])
.await
.expect("the staging slot holds a value");
let err = handle
.start(vec![templated("live", None)])
.await
.unwrap_err();
assert!(
err.to_string().contains("production"),
"the host default is what the second one asked for: {err}"
);
}
#[test]
fn a_rearm_resolves_each_instance_against_its_own_environment() {
let dir = tempfile::tempdir().unwrap();
let actor = actor_with_an_empty_flock(&dir, vec![]);
shep_core::secrets::set(&actor.paths.secrets, "K", "production", "live").unwrap();
shep_core::secrets::set(&actor.paths.secrets, "K", "staging", "rehearsal").unwrap();
let entry_of = |id: u32, environment: &str| {
let mut entry = armed_entry(id, id, 4300 + id, app_with("web", |_| {}), &actor.paths);
entry.spec = app_with("web", |app| {
app.environment = Some(environment.to_string());
app.env.insert("K".to_string(), "{{secret:K}}".to_string());
});
entry
};
let promoted = entry_of(0, "staging");
let waiting = entry_of(1, "production");
let specs = actor.rearm_specs(&[&promoted, &waiting]);
assert_eq!(
specs[&0].env.get("K").map(String::as_str),
Some("rehearsal"),
"the promoted instance reads its own staging slot: {:?}",
specs[&0]
);
assert_eq!(
specs[&1].env.get("K").map(String::as_str),
Some("live"),
"the instance still on the old config keeps production: {:?}",
specs[&1]
);
}
#[tokio::test(start_paused = true)]
async fn a_batch_with_one_unresolvable_secret_registers_none_of_it() {
let (events, _rx) = crate::bus::test_bus(64);
let runner = Arc::new(ScriptedRunner::new(vec![ProcScript::never_exits()]));
let dir = tempfile::tempdir().unwrap();
let handle = spawn_supervisor(SharedRunner(Arc::clone(&runner)), test_paths(&dir), events);
let sound = normalize(AppConfig::minimal("first", "./srv")).unwrap();
let mut broken = AppConfig::minimal("second", "./srv");
broken
.env
.insert("PW".to_string(), "{{secret:TYPO}}".to_string());
let err = handle
.start(vec![sound, normalize(broken).unwrap()])
.await
.unwrap_err();
assert!(
handle.list().await.is_empty(),
"the app ahead of the refusal must not survive the batch"
);
assert_eq!(runner.spawn_count(), 0, "nothing may reach the runner");
assert!(matches!(err, SupervisorError::CannotStart(_)), "{err:?}");
assert!(
err.to_string().contains("TYPO"),
"names the reference: {err}"
);
}
async fn env_batch_harness() -> Harness {
let h = harness(vec![ProcScript::never_exits(); 2]);
start_app(&h, AppConfig::minimal("web", "./srv")).await;
h.ctx
.supervisor
.start_dog(dog_app("bark"), DogSource::BuiltIn)
.await
.unwrap();
h
}
fn stored_env(h: &Harness, name: &str) -> serde_json::Map<String, serde_json::Value> {
let record = shep_core::overrides::get(&h.ctx.paths.overrides, name)
.unwrap()
.expect("an override record");
record.fields["env"]
.as_object()
.expect("a flat env object")
.clone()
}
#[tokio::test(start_paused = true)]
async fn a_batch_writes_every_key_under_one_lock() {
let h = env_batch_harness().await;
let batch = h
.ctx
.supervisor
.set_sheep_env_batch(
"web".to_string(),
BTreeMap::from([
("A".to_string(), "1".to_string()),
("B".to_string(), "2".to_string()),
]),
false,
false,
)
.await
.unwrap()
.expect("web exists");
assert_eq!(batch.set, ["A", "B"]);
assert!(batch.collisions.is_empty());
let env = stored_env(&h, "web");
assert_eq!(env["A"], "1");
assert_eq!(env["B"], "2");
}
#[tokio::test(start_paused = true)]
async fn an_identical_value_is_unchanged_rather_than_a_collision() {
let h = env_batch_harness().await;
let entries = BTreeMap::from([("A".to_string(), "1".to_string())]);
h.ctx
.supervisor
.set_sheep_env_batch("web".to_string(), entries.clone(), false, false)
.await
.unwrap();
let batch = h
.ctx
.supervisor
.set_sheep_env_batch("web".to_string(), entries, false, false)
.await
.unwrap()
.expect("web exists");
assert!(batch.set.is_empty());
assert_eq!(batch.unchanged, ["A"]);
assert!(batch.collisions.is_empty());
}
#[tokio::test(start_paused = true)]
async fn a_collision_without_force_writes_nothing_at_all() {
let h = env_batch_harness().await;
h.ctx
.supervisor
.set_sheep_env_batch(
"web".to_string(),
BTreeMap::from([("A".to_string(), "1".to_string())]),
false,
false,
)
.await
.unwrap();
let batch = h
.ctx
.supervisor
.set_sheep_env_batch(
"web".to_string(),
BTreeMap::from([
("A".to_string(), "2".to_string()),
("B".to_string(), "9".to_string()),
]),
false,
false,
)
.await
.unwrap()
.expect("web exists");
assert_eq!(batch.collisions, ["A"]);
assert!(batch.set.is_empty());
assert!(batch.app.is_none());
let env = stored_env(&h, "web");
assert_eq!(env["A"], "1", "the colliding key kept its value");
assert!(
!env.contains_key("B"),
"the clean key was not written either"
);
}
#[tokio::test(start_paused = true)]
async fn force_overwrites_and_reports_the_collision_in_both_lists() {
let h = env_batch_harness().await;
h.ctx
.supervisor
.set_sheep_env_batch(
"web".to_string(),
BTreeMap::from([("A".to_string(), "1".to_string())]),
false,
false,
)
.await
.unwrap();
let batch = h
.ctx
.supervisor
.set_sheep_env_batch(
"web".to_string(),
BTreeMap::from([("A".to_string(), "2".to_string())]),
true,
false,
)
.await
.unwrap()
.expect("web exists");
assert_eq!(batch.set, ["A"]);
assert_eq!(batch.collisions, ["A"]);
assert_eq!(stored_env(&h, "web")["A"], "2");
}
#[tokio::test(start_paused = true)]
async fn a_dry_run_answers_and_writes_nothing() {
let h = env_batch_harness().await;
let batch = h
.ctx
.supervisor
.set_sheep_env_batch(
"web".to_string(),
BTreeMap::from([("A".to_string(), "1".to_string())]),
false,
true,
)
.await
.unwrap()
.expect("web exists");
assert_eq!(batch.set, ["A"]);
assert!(batch.app.is_none());
assert!(
shep_core::overrides::get(&h.ctx.paths.overrides, "web")
.unwrap()
.is_none(),
"a dry run left a store behind"
);
}
#[tokio::test(start_paused = true)]
async fn a_dry_run_refuses_what_the_real_send_would_refuse() {
let h = env_batch_harness().await;
let err = h
.ctx
.supervisor
.set_sheep_env_batch(
"web".to_string(),
BTreeMap::from([("SHEP_NAME".to_string(), "nope".to_string())]),
false,
true,
)
.await
.unwrap_err();
assert!(matches!(err, SupervisorError::InvalidEnv(_)), "{err:?}");
assert!(
shep_core::overrides::get(&h.ctx.paths.overrides, "web")
.unwrap()
.is_none(),
"a refused dry run left a store behind"
);
}
#[tokio::test(start_paused = true)]
async fn a_refused_collision_reports_rather_than_validating() {
let h = env_batch_harness().await;
h.ctx
.supervisor
.set_sheep_env_batch(
"web".to_string(),
BTreeMap::from([("A".to_string(), "1".to_string())]),
false,
false,
)
.await
.unwrap();
let batch = h
.ctx
.supervisor
.set_sheep_env_batch(
"web".to_string(),
BTreeMap::from([
("A".to_string(), "2".to_string()),
("SHEP_NAME".to_string(), "nope".to_string()),
]),
false,
false,
)
.await
.unwrap()
.expect("web exists");
assert_eq!(batch.collisions, ["A"]);
assert!(batch.set.is_empty());
assert!(batch.app.is_none());
}
#[tokio::test(start_paused = true)]
async fn a_batch_that_changes_nothing_parks_nothing() {
let h = env_batch_harness().await;
let entries = BTreeMap::from([("A".to_string(), "1".to_string())]);
h.ctx
.supervisor
.set_sheep_env_batch("web".to_string(), entries.clone(), false, false)
.await
.unwrap();
let batch = h
.ctx
.supervisor
.set_sheep_env_batch("web".to_string(), entries, false, false)
.await
.unwrap()
.expect("web exists");
assert!(batch.set.is_empty());
assert_eq!(batch.unchanged, ["A"]);
assert!(batch.app.is_none(), "nothing was written to record");
}
#[tokio::test(start_paused = true)]
async fn a_batch_refuses_a_dog_and_an_unknown_name() {
let h = env_batch_harness().await;
let entries = BTreeMap::from([("A".to_string(), "1".to_string())]);
assert!(
h.ctx
.supervisor
.set_sheep_env_batch("absent".to_string(), entries.clone(), false, false)
.await
.unwrap()
.is_none()
);
let err = h
.ctx
.supervisor
.set_sheep_env_batch("bark".to_string(), entries, false, false)
.await
.unwrap_err();
assert!(matches!(err, SupervisorError::IsADog(_)));
}
}