use core::cmp::Ordering;
use core::fmt;
use core::sync::atomic::{self, AtomicU64};
use core::time::Duration;
use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet, VecDeque};
use std::io;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use tokio::sync::{broadcast, mpsc, oneshot};
use shep_core::config::{AppConfig, ResolvedApp, normalize};
use shep_core::paths::ShepPaths;
use shep_core::protocol::{
ActionOutcome, ActionReply, BusEvent, DogSource, ExitInfo, LineOutcome, LineReply,
ProcessEventKind, ProcessInfo, SignalOutcome, SignalReply, Smit,
};
use shep_core::selector::ProcessSelector;
use shep_core::signals::OperatorSignal;
use shep_core::status::ProcStatus;
use crate::assemble::{assemble, instance_slots};
use crate::brain::{Decision, decide_on_exit};
use crate::channel::{ChildMessage, ShepherdMessage};
use crate::entry::{ProcessEntry, ReloadState, RestartBudget};
use crate::extras::{Extras, ExtrasRegistry};
use crate::kill::kill_process;
use crate::privilege::{self, Credentials};
use crate::probes::Prober;
use crate::probes::os::OsProber;
use crate::probes::ready::{Readiness, ReadinessSource, await_ready};
use crate::runner::{
ExitOutcome, FlushError, LogCtl, ProcIo, ProcessRunner, ReopenError, RunnerError,
RunningProcess, SpawnSpec, StdinWrite, check_log_ancestry, open_log_path,
};
const MAILBOX_CAPACITY: usize = 256;
const SHEEP_CTL_CAPACITY: usize = 4;
const SIGNAL_CAPACITY: usize = 16;
const RELOAD_DEADLINE_SLACK: Duration = Duration::from_secs(5);
const STDIN_WRITE_TIMEOUT: Duration = Duration::from_secs(2);
const MAX_ABANDONED_ACTION_REPLIES: usize = 64;
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub(crate) struct ConnId(u64);
impl ConnId {
pub(crate) fn next() -> Self {
static NEXT: AtomicU64 = AtomicU64::new(0);
Self(NEXT.fetch_add(1, atomic::Ordering::Relaxed))
}
}
type Smits = HashMap<String, (ConnId, String)>;
#[derive(Debug)]
pub(crate) enum Command {
Start {
apps: Vec<ResolvedApp>,
reply: oneshot::Sender<Result<Vec<ProcessInfo>, SupervisorError>>,
},
RegisterAtRest {
apps: Vec<ResolvedApp>,
reply: oneshot::Sender<Result<Vec<ProcessInfo>, SupervisorError>>,
},
StartDog {
app: Box<ResolvedApp>,
source: DogSource,
reply: oneshot::Sender<Result<ProcessInfo, SupervisorError>>,
},
Stop {
selector: ProcessSelector,
reply: oneshot::Sender<Result<Vec<ProcessInfo>, SupervisorError>>,
},
Restart {
selector: ProcessSelector,
origin: CommandOrigin,
reply: oneshot::Sender<Result<Vec<ProcessInfo>, SupervisorError>>,
},
ExtraRestart {
id: u32,
pid: u32,
},
Reload {
selector: ProcessSelector,
reply: oneshot::Sender<Result<Vec<ProcessInfo>, SupervisorError>>,
},
Delete {
selector: ProcessSelector,
reply: oneshot::Sender<Result<Vec<u32>, SupervisorError>>,
},
Scale {
name: String,
count: u32,
reply: oneshot::Sender<Result<Scaled, SupervisorError>>,
},
SetSmit {
conn: ConnId,
sheep: String,
smit: Option<Smit>,
reply: oneshot::Sender<Result<Vec<ProcessInfo>, SupervisorError>>,
},
ForgetSmits {
conn: ConnId,
reply: oneshot::Sender<()>,
},
List {
reply: oneshot::Sender<Vec<ProcessInfo>>,
},
Reopen {
selector: ProcessSelector,
reply: oneshot::Sender<Result<Vec<ProcessInfo>, SupervisorError>>,
},
Flush {
selector: ProcessSelector,
reply: oneshot::Sender<Result<Vec<ProcessInfo>, SupervisorError>>,
},
Trigger {
selector: ProcessSelector,
action: String,
params: Option<String>,
reply: oneshot::Sender<Result<Vec<ActionReply>, SupervisorError>>,
},
Signal {
selector: ProcessSelector,
sig: OperatorSignal,
reply: oneshot::Sender<Result<Vec<SignalReply>, SupervisorError>>,
},
SendLine {
selector: ProcessSelector,
line: String,
reply: oneshot::Sender<Result<Vec<LineReply>, SupervisorError>>,
},
Shutdown {
reply: oneshot::Sender<()>,
},
}
#[derive(Debug)]
pub(crate) enum Msg {
Command(Command),
Exited {
id: u32,
outcome: ExitOutcome,
},
RestartDue {
id: u32,
epoch: u64,
},
Ready {
id: u32,
},
ReloadDeadline {
name: String,
new_id: u32,
},
ActionReply {
id: u32,
action: String,
body: String,
stamp: Option<u64>,
},
ActionResult {
id: u32,
stamp: u64,
outcome: ActionOutcome,
},
ReadyResult {
id: u32,
epoch: u64,
manually: bool,
readiness: Readiness,
},
}
#[non_exhaustive]
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum SupervisorError {
NotFound,
SpawnFailed(String),
ReloadInFlight(String),
InvalidScale(String),
ReopenFailed(String),
FlushFailed(String),
EngineStopped,
}
impl fmt::Display for SupervisorError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::NotFound => f.write_str("selector matched no registered sheep"),
Self::SpawnFailed(msg) => write!(f, "spawn failed: {msg}"),
Self::ReloadInFlight(name) => write!(f, "{name} is already being reloaded"),
Self::InvalidScale(msg) => write!(f, "cannot scale: {msg}"),
Self::ReopenFailed(msg) => write!(f, "log reopen failed: {msg}"),
Self::FlushFailed(msg) => write!(f, "log flush failed: {msg}"),
Self::EngineStopped => f.write_str("supervisor engine has shut down"),
}
}
}
impl core::error::Error for SupervisorError {}
#[derive(Debug, Clone)]
pub struct SupervisorHandle {
tx: mpsc::Sender<Msg>,
}
impl SupervisorHandle {
pub async fn start(&self, apps: Vec<ResolvedApp>) -> Result<Vec<ProcessInfo>, SupervisorError> {
let (reply, rx) = oneshot::channel();
self.tx
.send(Msg::Command(Command::Start { apps, reply }))
.await
.map_err(|_| SupervisorError::EngineStopped)?;
rx.await.map_err(|_| SupervisorError::EngineStopped)?
}
pub(crate) async fn register_at_rest(
&self,
apps: Vec<ResolvedApp>,
) -> Result<Vec<ProcessInfo>, SupervisorError> {
let (reply, rx) = oneshot::channel();
self.tx
.send(Msg::Command(Command::RegisterAtRest { apps, reply }))
.await
.map_err(|_| SupervisorError::EngineStopped)?;
rx.await.map_err(|_| SupervisorError::EngineStopped)?
}
pub async fn start_dog(
&self,
app: ResolvedApp,
source: DogSource,
) -> Result<ProcessInfo, SupervisorError> {
let (reply, rx) = oneshot::channel();
self.tx
.send(Msg::Command(Command::StartDog {
app: Box::new(app),
source,
reply,
}))
.await
.map_err(|_| SupervisorError::EngineStopped)?;
rx.await.map_err(|_| SupervisorError::EngineStopped)?
}
pub(crate) async fn stop(
&self,
selector: ProcessSelector,
) -> Result<Vec<ProcessInfo>, SupervisorError> {
let (reply, rx) = oneshot::channel();
self.tx
.send(Msg::Command(Command::Stop { selector, reply }))
.await
.map_err(|_| SupervisorError::EngineStopped)?;
rx.await.map_err(|_| SupervisorError::EngineStopped)?
}
pub(crate) async fn restart(
&self,
selector: ProcessSelector,
) -> Result<Vec<ProcessInfo>, SupervisorError> {
self.restart_with_origin(selector, CommandOrigin::Operator)
.await
}
pub(crate) async fn restart_automatic(
&self,
selector: ProcessSelector,
) -> Result<Vec<ProcessInfo>, SupervisorError> {
self.restart_with_origin(selector, CommandOrigin::Automatic)
.await
}
async fn restart_with_origin(
&self,
selector: ProcessSelector,
origin: CommandOrigin,
) -> Result<Vec<ProcessInfo>, SupervisorError> {
let (reply, rx) = oneshot::channel();
self.tx
.send(Msg::Command(Command::Restart {
selector,
origin,
reply,
}))
.await
.map_err(|_| SupervisorError::EngineStopped)?;
rx.await.map_err(|_| SupervisorError::EngineStopped)?
}
pub(crate) async fn extra_restart(&self, id: u32, pid: u32) {
let _ = self
.tx
.send(Msg::Command(Command::ExtraRestart { id, pid }))
.await;
}
pub(crate) async fn reload(
&self,
selector: ProcessSelector,
) -> Result<Vec<ProcessInfo>, SupervisorError> {
let (reply, rx) = oneshot::channel();
self.tx
.send(Msg::Command(Command::Reload { selector, reply }))
.await
.map_err(|_| SupervisorError::EngineStopped)?;
rx.await.map_err(|_| SupervisorError::EngineStopped)?
}
pub(crate) async fn delete(
&self,
selector: ProcessSelector,
) -> Result<Vec<u32>, SupervisorError> {
let (reply, rx) = oneshot::channel();
self.tx
.send(Msg::Command(Command::Delete { selector, reply }))
.await
.map_err(|_| SupervisorError::EngineStopped)?;
rx.await.map_err(|_| SupervisorError::EngineStopped)?
}
pub(crate) async fn scale(&self, name: &str, count: u32) -> Result<Scaled, SupervisorError> {
let (reply, rx) = oneshot::channel();
self.tx
.send(Msg::Command(Command::Scale {
name: name.to_string(),
count,
reply,
}))
.await
.map_err(|_| SupervisorError::EngineStopped)?;
rx.await.map_err(|_| SupervisorError::EngineStopped)?
}
pub(crate) async fn set_smit(
&self,
conn: ConnId,
sheep: &str,
smit: Option<Smit>,
) -> Result<Vec<ProcessInfo>, SupervisorError> {
let (reply, rx) = oneshot::channel();
self.tx
.send(Msg::Command(Command::SetSmit {
conn,
sheep: sheep.to_string(),
smit,
reply,
}))
.await
.map_err(|_| SupervisorError::EngineStopped)?;
rx.await.map_err(|_| SupervisorError::EngineStopped)?
}
pub(crate) async fn forget_smits(&self, conn: ConnId) {
let (reply, rx) = oneshot::channel();
if self
.tx
.send(Msg::Command(Command::ForgetSmits { conn, reply }))
.await
.is_ok()
{
let _ = rx.await;
}
}
pub(crate) async fn reopen(
&self,
selector: ProcessSelector,
) -> Result<Vec<ProcessInfo>, SupervisorError> {
let (reply, rx) = oneshot::channel();
self.tx
.send(Msg::Command(Command::Reopen { selector, reply }))
.await
.map_err(|_| SupervisorError::EngineStopped)?;
rx.await.map_err(|_| SupervisorError::EngineStopped)?
}
pub(crate) async fn flush(
&self,
selector: ProcessSelector,
) -> Result<Vec<ProcessInfo>, SupervisorError> {
let (reply, rx) = oneshot::channel();
self.tx
.send(Msg::Command(Command::Flush { selector, reply }))
.await
.map_err(|_| SupervisorError::EngineStopped)?;
rx.await.map_err(|_| SupervisorError::EngineStopped)?
}
pub(crate) async fn trigger(
&self,
selector: ProcessSelector,
action: String,
params: Option<String>,
) -> Result<Vec<ActionReply>, SupervisorError> {
let (reply, rx) = oneshot::channel();
self.tx
.send(Msg::Command(Command::Trigger {
selector,
action,
params,
reply,
}))
.await
.map_err(|_| SupervisorError::EngineStopped)?;
rx.await.map_err(|_| SupervisorError::EngineStopped)?
}
pub(crate) async fn signal(
&self,
selector: ProcessSelector,
sig: OperatorSignal,
) -> Result<Vec<SignalReply>, SupervisorError> {
let (reply, rx) = oneshot::channel();
self.tx
.send(Msg::Command(Command::Signal {
selector,
sig,
reply,
}))
.await
.map_err(|_| SupervisorError::EngineStopped)?;
rx.await.map_err(|_| SupervisorError::EngineStopped)?
}
pub(crate) async fn send_line(
&self,
selector: ProcessSelector,
line: String,
) -> Result<Vec<LineReply>, SupervisorError> {
let (reply, rx) = oneshot::channel();
self.tx
.send(Msg::Command(Command::SendLine {
selector,
line,
reply,
}))
.await
.map_err(|_| SupervisorError::EngineStopped)?;
rx.await.map_err(|_| SupervisorError::EngineStopped)?
}
pub(crate) async fn list_checked(&self) -> Result<Vec<ProcessInfo>, SupervisorError> {
let (reply, rx) = oneshot::channel();
self.tx
.send(Msg::Command(Command::List { reply }))
.await
.map_err(|_| SupervisorError::EngineStopped)?;
rx.await.map_err(|_| SupervisorError::EngineStopped)
}
#[must_use]
pub async fn list(&self) -> Vec<ProcessInfo> {
self.list_checked()
.await
.expect("supervisor actor is no longer running")
}
pub async fn shutdown(&self) {
let (reply, rx) = oneshot::channel();
if self
.tx
.send(Msg::Command(Command::Shutdown { reply }))
.await
.is_ok()
{
let _ = rx.await;
}
}
}
#[derive(Debug)]
pub(crate) struct SupervisorBuilder<R: ProcessRunner> {
runner: R,
paths: ShepPaths,
events: broadcast::Sender<BusEvent>,
extras: Option<Extras>,
}
impl<R: ProcessRunner> SupervisorBuilder<R> {
pub(crate) fn new(runner: R, paths: ShepPaths, events: broadcast::Sender<BusEvent>) -> Self {
Self {
runner,
paths,
events,
extras: None,
}
}
#[must_use]
pub(crate) fn extras(mut self, extras: Extras) -> Self {
self.extras = Some(extras);
self
}
pub(crate) fn spawn(self) -> SupervisorHandle {
let (tx, rx) = mpsc::channel(MAILBOX_CAPACITY);
let actor = Actor {
runner: self.runner,
paths: self.paths,
events: self.events,
tx: tx.clone(),
sheep: HashMap::new(),
next_id: 0,
next_action_stamp: 0,
pending: Vec::new(),
shutting_down: false,
extras: self.extras,
registry: ExtrasRegistry::default(),
reloads: HashMap::new(),
smits: Smits::new(),
};
tokio::spawn(actor.run(rx));
SupervisorHandle { tx }
}
}
pub fn spawn_supervisor<R: ProcessRunner>(
runner: R,
paths: ShepPaths,
events: broadcast::Sender<BusEvent>,
) -> SupervisorHandle {
SupervisorBuilder::new(runner, paths, events).spawn()
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum SheepCtl {
Kill {
grace: Duration,
},
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum LadderCap {
Stop,
Drain,
}
impl LadderCap {
fn of(self, app: &AppConfig) -> Duration {
match self {
Self::Stop => app.kill_timeout,
Self::Drain => app.graceful_timeout,
}
.as_duration()
}
}
#[derive(Debug)]
struct ReloadJob {
queue: VecDeque<u32>,
swap: ReloadSwap,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
struct ReloadSwap {
old_id: u32,
new_id: u32,
phase: ReloadPhase,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum ReloadPhase {
AwaitReady,
DrainOld,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum ManualKind {
Stop,
Restart,
Delete,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum CommandOrigin {
Operator,
Automatic,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
struct PendingManual {
kind: ManualKind,
origin: CommandOrigin,
}
#[derive(Debug)]
struct PendingAction {
stamp: u64,
action: String,
waiter: Option<oneshot::Sender<String>>,
reply: oneshot::Sender<ActionOutcome>,
}
#[derive(Debug)]
struct AbandonedReply {
stamp: u64,
action: String,
}
#[derive(Debug, Default)]
struct ActionWaits {
live: Vec<PendingAction>,
abandoned: VecDeque<AbandonedReply>,
}
impl ActionWaits {
fn arm(&mut self, pending: PendingAction) {
self.live.push(pending);
}
fn answer(&mut self, action: &str, stamp: Option<u64>) -> Option<oneshot::Sender<String>> {
if let Some(stamp) = stamp {
if let Some(pending) = self
.live
.iter_mut()
.find(|pending| pending.stamp == stamp && pending.waiter.is_some())
{
return pending.waiter.take();
}
if let Some(owed) = self.abandoned.iter().position(|debt| debt.stamp == stamp) {
self.abandoned.remove(owed);
}
return None;
}
if let Some(owed) = self.abandoned.iter().position(|debt| debt.action == action) {
self.abandoned.remove(owed);
return None;
}
self.live
.iter_mut()
.find(|pending| pending.action == action && pending.waiter.is_some())
.and_then(|pending| pending.waiter.take())
}
fn resolve(&mut self, stamp: u64) -> Option<oneshot::Sender<ActionOutcome>> {
let at = self
.live
.iter()
.position(|pending| pending.stamp == stamp)?;
let pending = self.live.remove(at);
if pending.waiter.is_some() {
self.abandoned.push_back(AbandonedReply {
stamp: pending.stamp,
action: pending.action,
});
if self.abandoned.len() > MAX_ABANDONED_ACTION_REPLIES {
self.abandoned.pop_front();
}
}
Some(pending.reply)
}
fn abandon_all(&mut self) {
for pending in self.live.drain(..) {
let _ = pending.reply.send(ActionOutcome::NoChannel);
}
self.abandoned.clear();
}
}
#[derive(Debug)]
struct SheepSlot {
entry: ProcessEntry,
ctl: Option<mpsc::Sender<SheepCtl>>,
log_ctl: Option<mpsc::Sender<LogCtl>>,
to_child: Option<mpsc::Sender<ShepherdMessage>>,
signals: Option<mpsc::Sender<SignalRequest>>,
to_stdin: Option<mpsc::Sender<StdinWrite>>,
manual: Option<PendingManual>,
pending_delete: bool,
epoch: u64,
ready_tx: Option<oneshot::Sender<()>>,
actions: ActionWaits,
}
impl SheepSlot {
fn open_channel(&self) -> Option<&mpsc::Sender<ShepherdMessage>> {
self.to_child
.as_ref()
.filter(|to_child| !to_child.is_closed())
}
fn open_stdin(&self) -> Option<&mpsc::Sender<StdinWrite>> {
self.to_stdin
.as_ref()
.filter(|to_stdin| !to_stdin.is_closed())
}
}
#[derive(Debug)]
enum ReplyKind {
Info(oneshot::Sender<Result<Vec<ProcessInfo>, SupervisorError>>),
Ids(oneshot::Sender<Result<Vec<u32>, SupervisorError>>),
Shutdown(oneshot::Sender<()>),
}
#[derive(Debug)]
pub(crate) struct Scaled {
pub(crate) instances: Vec<ProcessInfo>,
pub(crate) app: ResolvedApp,
pub(crate) requested: u32,
pub(crate) shortfall: Option<String>,
}
impl Scaled {
pub(crate) fn achieved(&self) -> u32 {
u32::try_from(self.instances.len()).unwrap_or(u32::MAX)
}
}
#[derive(Debug)]
struct PendingReply {
remaining: HashSet<u32>,
results: Vec<ProcessInfo>,
reply: ReplyKind,
}
struct Actor<R: ProcessRunner> {
runner: R,
paths: ShepPaths,
events: broadcast::Sender<BusEvent>,
tx: mpsc::Sender<Msg>,
sheep: HashMap<u32, SheepSlot>,
next_id: u32,
next_action_stamp: u64,
pending: Vec<PendingReply>,
shutting_down: bool,
extras: Option<Extras>,
registry: ExtrasRegistry,
smits: Smits,
reloads: HashMap<String, ReloadJob>,
}
impl<R: ProcessRunner> Actor<R> {
async fn run(mut self, mut rx: mpsc::Receiver<Msg>) {
while let Some(msg) = rx.recv().await {
let should_break = match msg {
Msg::Command(cmd) => self.handle_command(cmd),
Msg::Exited { id, outcome } => self.handle_exited(id, outcome),
Msg::RestartDue { id, epoch } => {
self.handle_restart_due(id, epoch);
false
}
Msg::Ready { id } => {
self.handle_ready_signal(id);
false
}
Msg::ReloadDeadline { name, new_id } => {
self.handle_reload_deadline(&name, new_id);
false
}
Msg::ActionReply {
id,
action,
body,
stamp,
} => {
self.handle_action_reply(id, &action, body, stamp);
false
}
Msg::ActionResult { id, stamp, outcome } => {
self.handle_action_result(id, stamp, outcome);
false
}
Msg::ReadyResult {
id,
epoch,
manually,
readiness,
} => {
self.handle_ready_result(id, epoch, manually, readiness);
false
}
};
if should_break {
break;
}
}
}
fn handle_command(&mut self, cmd: Command) -> bool {
match cmd {
Command::Start { apps, reply } => {
let result = if self.shutting_down {
Err(SupervisorError::EngineStopped)
} else {
self.do_start(apps, None)
};
let _ = reply.send(result);
false
}
Command::RegisterAtRest { apps, reply } => {
let registered = apps.iter().map(|app| self.register_at_rest(app)).collect();
let _ = reply.send(Ok(registered));
false
}
Command::StartDog { app, source, reply } => {
let result = self.do_start_dog(*app, source);
let _ = reply.send(result);
false
}
Command::Scale { name, count, reply } => {
self.handle_scale(&name, count, reply);
false
}
Command::SetSmit {
conn,
sheep,
smit,
reply,
} => {
let _ = reply.send(self.handle_set_smit(conn, &sheep, smit));
false
}
Command::ForgetSmits { conn, reply } => {
self.smits.retain(|_, (painter, _)| *painter != conn);
let _ = reply.send(());
false
}
Command::List { reply } => {
let _ = reply.send(self.snapshot_all());
false
}
Command::Reopen { selector, reply } => {
self.handle_reopen(&selector, reply);
false
}
Command::Flush { selector, reply } => {
self.handle_flush(&selector, reply);
false
}
Command::Trigger {
selector,
action,
params,
reply,
} => {
self.begin_action(&selector, action, params, reply);
false
}
Command::Signal {
selector,
sig,
reply,
} => {
self.begin_signal(&selector, sig, reply);
false
}
Command::SendLine {
selector,
line,
reply,
} => {
self.begin_send_line(&selector, line, reply);
false
}
Command::Stop { selector, reply } => {
self.begin_manual(
selector,
ManualKind::Stop,
CommandOrigin::Operator,
ReplyKind::Info(reply),
);
false
}
Command::Restart {
selector,
origin,
reply,
} => {
if self.shutting_down {
send_reply(ReplyKind::Info(reply), Err(SupervisorError::EngineStopped));
} else {
self.begin_manual(
selector,
ManualKind::Restart,
origin,
ReplyKind::Info(reply),
);
}
false
}
Command::ExtraRestart { id, pid } => {
self.handle_extra_restart(id, pid);
false
}
Command::Reload { selector, reply } => {
if self.shutting_down {
let _ = reply.send(Err(SupervisorError::EngineStopped));
} else {
self.handle_reload(&selector, reply);
}
false
}
Command::Delete { selector, reply } => {
self.begin_manual(
selector,
ManualKind::Delete,
CommandOrigin::Operator,
ReplyKind::Ids(reply),
);
false
}
Command::Shutdown { reply } => self.begin_shutdown(reply),
}
}
fn do_start_dog(
&mut self,
app: ResolvedApp,
source: DogSource,
) -> Result<ProcessInfo, SupervisorError> {
if self.shutting_down {
return Err(SupervisorError::EngineStopped);
}
if let Some(slot) = self
.sheep
.values()
.find(|slot| slot.entry.spec.config().name == app.config().name)
{
return Ok(to_info(&slot.entry, &self.smits));
}
let started = self.do_start(vec![app], Some(source))?;
started
.into_iter()
.next()
.ok_or_else(|| SupervisorError::SpawnFailed("the dog registered no instance".into()))
}
fn do_start(
&mut self,
apps: Vec<ResolvedApp>,
dog: Option<DogSource>,
) -> Result<Vec<ProcessInfo>, SupervisorError> {
let mut results = Vec::new();
for app in apps {
let name = app.config().name.clone();
let credentials = privilege::resolve(app.config())
.map_err(|err| SupervisorError::SpawnFailed(err.to_string()))?;
let mut existing: Vec<u32> = self
.sheep
.values()
.filter(|slot| slot.entry.spec.config().name == name)
.map(|slot| slot.entry.instance)
.collect();
existing.sort_unstable();
let slots = instance_slots(&existing, app.config().instances);
for instance in slots {
match self.spawn_fresh(&app, instance, credentials, dog.clone()) {
Ok(info) => results.push(info),
Err(message) => return Err(SupervisorError::SpawnFailed(message)),
}
}
}
Ok(results)
}
fn register_at_rest(&mut self, app: &ResolvedApp) -> ProcessInfo {
let name = &app.config().name;
if let Some(slot) = self
.sheep
.values()
.find(|slot| &slot.entry.spec.config().name == name)
{
return to_info(&slot.entry, &self.smits);
}
let id = self.next_id;
self.next_id += 1;
let spec = assemble(app, 0, &self.paths, None);
let entry = ProcessEntry {
id,
spec: app.clone(),
instance: 0,
status: ProcStatus::Stopped,
pid: None,
restarts: 0,
started_at: None,
budget: RestartBudget::default(),
reload: ReloadState::None,
credentials: None,
out_file: spec.out_file.clone(),
err_file: spec.err_file.clone(),
dog: None,
last_exit: None,
};
let info = to_info(&entry, &self.smits);
self.sheep.insert(
id,
SheepSlot {
entry,
ctl: None,
log_ctl: None,
to_child: None,
signals: None,
to_stdin: None,
manual: None,
pending_delete: false,
epoch: 0,
ready_tx: None,
actions: ActionWaits::default(),
},
);
info
}
fn spawn_fresh(
&mut self,
app: &ResolvedApp,
instance: u32,
credentials: Option<Credentials>,
dog: Option<DogSource>,
) -> Result<ProcessInfo, String> {
let spec = assemble(app, instance, &self.paths, credentials);
let id = self.next_id;
self.next_id += 1;
let out_file = spec.out_file.clone();
let err_file = spec.err_file.clone();
let source = ReadinessSource::of(app.config())
.expect("ResolvedApp already passed ProbeTarget::parse in normalize");
let gated = !matches!(source, ReadinessSource::Heuristic);
match self.runner.spawn(&spec) {
Ok((proc, io)) => {
let pid = proc.pid();
let status = if gated {
ProcStatus::Starting
} else {
ProcStatus::Online
};
let entry = ProcessEntry {
id,
spec: app.clone(),
instance,
status,
pid: Some(pid),
restarts: 0,
started_at: Some(tokio::time::Instant::now()),
budget: RestartBudget::default(),
reload: ReloadState::None,
credentials,
out_file,
err_file,
dog,
last_exit: None,
};
let info = to_info(&entry, &self.smits);
let log_ctl = io.log_ctl.clone();
let to_child = io.to_child.clone();
let to_stdin = io.to_stdin.clone();
let handles = spawn_sheep_task::<R::Proc>(
id,
proc,
io,
app.clone(),
self.events.clone(),
self.tx.clone(),
);
let ready_tx = if gated {
Some(spawn_readiness_task(
id,
0,
true,
source,
app.config().listen_timeout.as_duration(),
spec_prober(&spec),
self.tx.clone(),
))
} else {
None
};
self.sheep.insert(
id,
SheepSlot {
entry,
ctl: Some(handles.ctl),
log_ctl: Some(log_ctl),
to_child: Some(to_child),
signals: Some(handles.signals),
to_stdin: Some(to_stdin),
manual: None,
pending_delete: false,
epoch: 0,
ready_tx,
actions: ActionWaits::default(),
},
);
self.emit(ProcessEventKind::Start, info.clone(), true);
if !gated {
self.went_online(id, info.clone(), true);
}
Ok(info)
}
Err(error) => {
let entry = ProcessEntry {
id,
spec: app.clone(),
instance,
status: ProcStatus::Errored,
pid: None,
restarts: 0,
started_at: None,
budget: RestartBudget::default(),
reload: ReloadState::None,
credentials,
out_file,
err_file,
dog,
last_exit: None,
};
let info = to_info(&entry, &self.smits);
self.sheep.insert(
id,
SheepSlot {
entry,
ctl: None,
log_ctl: None,
to_child: None,
signals: None,
to_stdin: None,
manual: None,
pending_delete: false,
epoch: 0,
ready_tx: None,
actions: ActionWaits::default(),
},
);
self.emit(ProcessEventKind::Errored, info, true);
Err(error.to_string())
}
}
}
fn respawn(&mut self, id: u32, manually: bool) -> ProcessInfo {
let slot = self.sheep.get(&id).expect("respawn: unknown id");
let app = slot.entry.spec.clone();
let instance = slot.entry.instance;
let credentials = slot.entry.credentials;
let next_epoch = slot.epoch + 1;
let spec = assemble(&app, instance, &self.paths, credentials);
let source = ReadinessSource::of(app.config())
.expect("ResolvedApp already passed ProbeTarget::parse in normalize");
let gated = !matches!(source, ReadinessSource::Heuristic);
match self.runner.spawn(&spec) {
Ok((proc, io)) => {
let pid = proc.pid();
let log_ctl = io.log_ctl.clone();
let to_child = io.to_child.clone();
let to_stdin = io.to_stdin.clone();
let handles = spawn_sheep_task::<R::Proc>(
id,
proc,
io,
app.clone(),
self.events.clone(),
self.tx.clone(),
);
let ready_tx = if gated {
Some(spawn_readiness_task(
id,
next_epoch,
manually,
source,
app.config().listen_timeout.as_duration(),
spec_prober(&spec),
self.tx.clone(),
))
} else {
None
};
let slot = self
.sheep
.get_mut(&id)
.expect("respawn: entry vanished mid-respawn");
slot.entry.status = if gated {
ProcStatus::Starting
} else {
ProcStatus::Online
};
slot.entry.pid = Some(pid);
slot.entry.started_at = Some(tokio::time::Instant::now());
slot.entry.restarts += 1;
slot.ctl = Some(handles.ctl);
slot.log_ctl = Some(log_ctl);
slot.to_child = Some(to_child);
slot.signals = Some(handles.signals);
slot.to_stdin = Some(to_stdin);
slot.epoch += 1;
debug_assert_eq!(slot.epoch, next_epoch);
slot.ready_tx = ready_tx;
let info = to_info(&slot.entry, &self.smits);
self.emit(ProcessEventKind::Restart, info.clone(), manually);
if !gated {
self.went_online(id, info.clone(), manually);
}
info
}
Err(_error) => {
let slot = self
.sheep
.get_mut(&id)
.expect("respawn: entry vanished mid-respawn");
slot.entry.status = ProcStatus::Errored;
slot.entry.pid = None;
slot.entry.started_at = None;
slot.entry.last_exit = None;
slot.ctl = None;
slot.to_child = None;
slot.signals = None;
slot.to_stdin = None;
slot.ready_tx = None;
let info = to_info(&slot.entry, &self.smits);
self.emit(ProcessEventKind::Errored, info.clone(), manually);
self.disarm_extras(id, &info.name);
info
}
}
}
fn claim_manual(&mut self, id: u32, manual: PendingManual, cap: LadderCap) {
let Some(slot) = self.sheep.get_mut(&id) else {
return;
};
match slot.manual.map(|in_flight| in_flight.origin) {
None => {
slot.manual = Some(manual);
if let Some(ctl) = &slot.ctl {
let grace = cap.of(slot.entry.spec.config());
let _ = ctl.try_send(SheepCtl::Kill { grace });
}
}
Some(CommandOrigin::Automatic) if manual.origin == CommandOrigin::Operator => {
slot.manual = Some(manual);
}
Some(CommandOrigin::Operator | CommandOrigin::Automatic) => {}
}
}
fn matching_ids(&self, selector: &ProcessSelector) -> Vec<u32> {
let exact = selector.is_exact();
let mut ids: Vec<u32> = self
.sheep
.iter()
.filter(|(_, slot)| exact || slot.entry.dog.is_none())
.filter_map(|(id, slot)| {
let config = slot.entry.spec.config();
selector
.matches(&config.name, *id, config.fold.as_deref())
.then_some(*id)
})
.collect();
ids.sort_unstable();
ids
}
fn begin_manual(
&mut self,
selector: ProcessSelector,
kind: ManualKind,
origin: CommandOrigin,
reply: ReplyKind,
) {
let matched = self.matching_ids(&selector);
if matched.is_empty() {
send_reply(reply, Err(SupervisorError::NotFound));
return;
}
self.begin_manual_ids(matched, kind, origin, reply);
}
fn begin_manual_ids(
&mut self,
matched: Vec<u32>,
kind: ManualKind,
origin: CommandOrigin,
reply: ReplyKind,
) {
let mut remaining = HashSet::new();
let mut results = Vec::new();
for id in matched {
let held_off_by_a_swap =
origin == CommandOrigin::Automatic && self.in_an_uncommitted_swap(id);
if held_off_by_a_swap {
tracing::debug!(
id,
?kind,
"automatic command dropped: this sheep is half of a swap that has not \
committed"
);
continue;
}
let is_running = self.sheep.get(&id).is_some_and(|slot| slot.ctl.is_some());
if is_running {
self.claim_manual(id, PendingManual { kind, origin }, LadderCap::Stop);
if kind == ManualKind::Delete {
if let Some(slot) = self.sheep.get_mut(&id) {
slot.pending_delete = true;
}
}
remaining.insert(id);
} else if let Some(info) = self.apply_immediate(id, kind, origin) {
results.push(info);
}
}
if remaining.is_empty() {
results.sort_unstable_by_key(|info| info.id);
send_reply(reply, Ok(results));
return;
}
self.pending.push(PendingReply {
remaining,
results,
reply,
});
}
fn apply_immediate(
&mut self,
id: u32,
kind: ManualKind,
origin: CommandOrigin,
) -> Option<ProcessInfo> {
let manually = origin == CommandOrigin::Operator;
match kind {
ManualKind::Stop => {
let slot = self.sheep.get_mut(&id)?;
match slot.entry.status {
ProcStatus::WaitingRestart | ProcStatus::Errored => {
slot.entry.status = ProcStatus::Stopped;
let info = to_info(&slot.entry, &self.smits);
self.emit(ProcessEventKind::Stop, info.clone(), manually);
self.disarm_extras(id, &info.name);
Some(info)
}
_ => Some(to_info(&slot.entry, &self.smits)),
}
}
ManualKind::Delete => {
let slot = self.sheep.remove(&id)?;
let info = to_info(&slot.entry, &self.smits);
self.emit(ProcessEventKind::Delete, info.clone(), manually);
self.disarm_extras(id, &info.name);
Some(info)
}
ManualKind::Restart => {
self.sheep.get_mut(&id)?.entry.budget.reset();
Some(self.respawn(id, manually))
}
}
}
fn handle_scale(
&mut self,
name: &str,
count: u32,
reply: oneshot::Sender<Result<Scaled, SupervisorError>>,
) {
let mut slots: Vec<(u32, u32)> = self
.sheep
.iter()
.filter(|(_, slot)| slot.entry.spec.config().name == name)
.map(|(id, slot)| (slot.entry.instance, *id))
.collect();
if slots.is_empty() {
let _ = reply.send(Err(SupervisorError::NotFound));
return;
}
slots.sort_unstable();
if count == 0 {
let _ = reply.send(Err(SupervisorError::InvalidScale(format!(
"an app runs at least one instance; use `shep delete {name}` to remove it"
))));
return;
}
if self
.sheep
.get(&slots[0].1)
.is_some_and(|slot| slot.entry.dog.is_some())
{
let _ = reply.send(Err(SupervisorError::InvalidScale(format!(
"{name} is a dog, and a dog runs one process"
))));
return;
}
if self.reloads.contains_key(name) {
let _ = reply.send(Err(SupervisorError::ReloadInFlight(name.to_string())));
return;
}
let leaving = slots
.iter()
.filter(|(_, id)| self.sheep.get(id).is_some_and(|slot| slot.pending_delete))
.count();
if leaving > 0 {
let _ = reply.send(Err(SupervisorError::InvalidScale(format!(
"{name} has {leaving} instance(s) still shutting down from an \
earlier command; wait for them to leave `shep flock` and scale \
again"
))));
return;
}
let mut config = self
.sheep
.get(&slots[0].1)
.expect("handle_scale: id read off this map a moment ago")
.entry
.spec
.config()
.clone();
config.instances = count;
let rescaled = match normalize(config) {
Ok(app) => app,
Err(err) => {
let _ = reply.send(Err(SupervisorError::InvalidScale(err.to_string())));
return;
}
};
let current = u32::try_from(slots.len()).unwrap_or(u32::MAX);
let credentials = self
.sheep
.get(&slots[0].1)
.expect("handle_scale: id read off this map a moment ago")
.entry
.credentials;
let mut failure = None;
let mut orphaned_by_failed_spawn = None;
let survivors: Vec<u32> = match count.cmp(¤t) {
Ordering::Equal => slots.iter().map(|(_, id)| *id).collect(),
Ordering::Greater => {
let existing: Vec<u32> = slots.iter().map(|(instance, _)| *instance).collect();
let mut ids: Vec<u32> = slots.iter().map(|(_, id)| *id).collect();
for instance in instance_slots(&existing, count - current) {
let attempted_id = self.next_id;
match self.spawn_fresh(&rescaled, instance, credentials, None) {
Ok(info) => ids.push(info.id),
Err(message) => {
orphaned_by_failed_spawn = Some(attempted_id);
failure = Some(message);
break;
}
}
}
ids
}
Ordering::Less => {
let cut = usize::try_from(count).unwrap_or(usize::MAX);
let (keep, remove) = slots.split_at(cut);
let removed: Vec<u32> = remove.iter().map(|(_, id)| *id).collect();
self.begin_manual_ids(
removed,
ManualKind::Delete,
CommandOrigin::Operator,
ReplyKind::Ids(oneshot::channel().0),
);
keep.iter().map(|(_, id)| *id).collect()
}
};
let achieved = u32::try_from(survivors.len()).unwrap_or(u32::MAX);
let stored = if achieved == count {
rescaled
} else {
let mut config = rescaled.config().clone();
config.instances = achieved;
match normalize(config) {
Ok(app) => app,
Err(err) => {
let _ = reply.send(Err(SupervisorError::InvalidScale(err.to_string())));
return;
}
}
};
for id in survivors.iter().chain(orphaned_by_failed_spawn.iter()) {
if let Some(slot) = self.sheep.get_mut(id) {
slot.entry.spec = stored.clone();
}
}
let mut instances: Vec<ProcessInfo> = survivors
.iter()
.filter_map(|id| {
self.sheep
.get(id)
.map(|slot| to_info(&slot.entry, &self.smits))
})
.collect();
instances.sort_unstable_by_key(|info| info.id);
let _ = reply.send(Ok(Scaled {
instances,
app: stored,
requested: count,
shortfall: failure,
}));
}
fn handle_reload(
&mut self,
selector: &ProcessSelector,
reply: oneshot::Sender<Result<Vec<ProcessInfo>, SupervisorError>>,
) {
let matched = self.matching_ids(selector);
if matched.is_empty() {
let _ = reply.send(Err(SupervisorError::NotFound));
return;
}
let in_flight = matched.iter().find_map(|id| {
let slot = self
.sheep
.get(id)
.expect("handle_reload: `matched` holds ids read off this map a moment ago");
let name = &slot.entry.spec.config().name;
self.reloads.contains_key(name).then(|| name.clone())
});
if let Some(name) = in_flight {
let _ = reply.send(Err(SupervisorError::ReloadInFlight(name)));
return;
}
let accepted: Vec<ProcessInfo> = matched
.iter()
.map(|id| {
let slot = self
.sheep
.get(id)
.expect("handle_reload: `matched` holds ids read off this map a moment ago");
to_info(&slot.entry, &self.smits)
})
.collect();
let mut queues: BTreeMap<String, Vec<(u32, u32)>> = BTreeMap::new();
for id in matched {
let entry = &self
.sheep
.get(&id)
.expect("handle_reload: `matched` holds ids read off this map a moment ago")
.entry;
if entry.status != ProcStatus::Online {
continue;
}
queues
.entry(entry.spec.config().name.clone())
.or_default()
.push((entry.instance, id));
}
let _ = reply.send(Ok(accepted));
for (name, mut instances) in queues {
instances.sort_unstable();
let queue = instances.into_iter().map(|(_, id)| id).collect();
self.advance_reload(&name, queue);
}
}
fn advance_reload(&mut self, name: &str, mut queue: VecDeque<u32>) {
if self.shutting_down {
self.reloads.remove(name);
return;
}
while let Some(old_id) = queue.pop_front() {
let slot = self.sheep.get(&old_id);
let replaceable = slot.is_some_and(|slot| {
slot.entry.status == ProcStatus::Online && slot.manual.is_none()
});
if !replaceable {
tracing::debug!(
name,
old_id,
status = ?slot.map(|slot| slot.entry.status),
"reload skipped an instance: it is gone, no longer online, or its next \
exit is already claimed"
);
continue;
}
match self.spawn_replacement(old_id) {
Ok(new_id) => {
self.reloads.insert(
name.to_string(),
ReloadJob {
queue,
swap: ReloadSwap {
old_id,
new_id,
phase: ReloadPhase::AwaitReady,
},
},
);
self.arm_reload_deadline(name, new_id);
return;
}
Err(error) => {
tracing::warn!(
name,
old_id,
error,
"reload abandoned: the replacement could not be spawned"
);
self.reloads.remove(name);
if let Some(slot) = self.sheep.get(&old_id) {
let info = to_info(&slot.entry, &self.smits);
self.emit(ProcessEventKind::ReloadAbandoned, info, true);
}
return;
}
}
}
self.reloads.remove(name);
}
fn spawn_replacement(&mut self, old_id: u32) -> Result<u32, String> {
let drainee = &self.sheep[&old_id].entry;
let app = drainee.spec.clone();
let instance = drainee.instance;
let credentials = drainee.credentials;
let restarts = drainee.restarts;
let dog = drainee.dog.clone();
let last_exit = drainee.last_exit;
let new_id = self.next_id;
self.next_id += 1;
let spec = assemble(&app, instance, &self.paths, credentials);
let out_file = spec.out_file.clone();
let err_file = spec.err_file.clone();
let source = ReadinessSource::of(app.config())
.expect("ResolvedApp already passed ProbeTarget::parse in normalize");
let drainee = self
.sheep
.get_mut(&old_id)
.expect("spawn_replacement: the drainee was read a moment ago");
drainee.entry.status = ProcStatus::Stopping;
drainee.entry.reload = ReloadState::Drainee { new_id };
match self.runner.spawn(&spec) {
Ok((proc, io)) => {
let pid = proc.pid();
let entry = ProcessEntry {
id: new_id,
spec: app.clone(),
instance,
status: ProcStatus::Starting,
pid: Some(pid),
restarts,
started_at: Some(tokio::time::Instant::now()),
budget: RestartBudget::default(),
reload: ReloadState::Replacement,
credentials,
out_file,
err_file,
dog,
last_exit,
};
let info = to_info(&entry, &self.smits);
let log_ctl = io.log_ctl.clone();
let to_child = io.to_child.clone();
let to_stdin = io.to_stdin.clone();
let handles = spawn_sheep_task::<R::Proc>(
new_id,
proc,
io,
app.clone(),
self.events.clone(),
self.tx.clone(),
);
let ready_tx = spawn_readiness_task(
new_id,
0,
true,
source,
app.config().listen_timeout.as_duration(),
spec_prober(&spec),
self.tx.clone(),
);
self.sheep.insert(
new_id,
SheepSlot {
entry,
ctl: Some(handles.ctl),
log_ctl: Some(log_ctl),
to_child: Some(to_child),
signals: Some(handles.signals),
to_stdin: Some(to_stdin),
manual: None,
pending_delete: false,
epoch: 0,
ready_tx: Some(ready_tx),
actions: ActionWaits::default(),
},
);
let drainee = to_info(&self.sheep[&old_id].entry, &self.smits);
self.emit(ProcessEventKind::Reload, drainee, true);
self.emit(ProcessEventKind::Start, info, true);
Ok(new_id)
}
Err(error) => {
let drainee = self
.sheep
.get_mut(&old_id)
.expect("spawn_replacement: the drainee was marked a moment ago");
drainee.entry.status = ProcStatus::Online;
drainee.entry.reload = ReloadState::None;
Err(error.to_string())
}
}
}
fn reload_ready_result(&mut self, new_id: u32, manually: bool, readiness: Readiness) {
let Some(name) = self.reload_of(new_id) else {
tracing::warn!(
id = new_id,
"a replacement resolved with no reload to belong to"
);
self.clear_reload(new_id);
let info = self.set_status(new_id, ProcStatus::Online);
self.went_online(new_id, info, manually);
return;
};
if readiness == Readiness::TimedOut {
let old_id = self.reloads[&name].swap.old_id;
if self.sheep.contains_key(&old_id) {
self.abort_reload(&name, "the replacement was not ready inside listen_timeout");
return;
}
tracing::warn!(
id = new_id,
"a replacement's readiness deadline elapsed with no old instance left to keep; \
marking online anyway"
);
}
let info = self.set_status(new_id, ProcStatus::Online);
self.went_online(new_id, info, manually);
self.begin_drain(&name);
}
fn begin_drain(&mut self, name: &str) {
let Some(job) = self.reloads.get_mut(name) else {
return;
};
job.swap.phase = ReloadPhase::DrainOld;
let old_id = job.swap.old_id;
if !self.sheep.contains_key(&old_id) {
self.finish_swap(name);
return;
}
self.claim_manual(
old_id,
PendingManual {
kind: ManualKind::Stop,
origin: CommandOrigin::Operator,
},
LadderCap::Drain,
);
}
fn finish_swap(&mut self, name: &str) {
let Some(job) = self.reloads.remove(name) else {
return;
};
let new_id = job.swap.new_id;
self.clear_reload(new_id);
let serving = self
.sheep
.get(&new_id)
.is_some_and(|slot| slot.entry.status == ProcStatus::Online);
if serving {
let info = to_info(&self.sheep[&new_id].entry, &self.smits);
self.emit(ProcessEventKind::Reloaded, info, true);
self.advance_reload(name, job.queue);
return;
}
tracing::warn!(
name,
new_id,
"reload abandoned: the replacement was no longer serving when the instance it \
replaced went"
);
if let Some(slot) = self.sheep.get(&new_id) {
let info = to_info(&slot.entry, &self.smits);
self.emit(ProcessEventKind::ReloadAbandoned, info, true);
}
}
fn abort_reload(&mut self, name: &str, reason: &str) {
let Some(job) = self.reloads.remove(name) else {
return;
};
debug_assert_eq!(
job.swap.phase,
ReloadPhase::AwaitReady,
"abort_reload: a committed swap has no old instance to go back to"
);
tracing::warn!(
name,
old_id = job.swap.old_id,
new_id = job.swap.new_id,
reason,
"reload abandoned"
);
let kept = self.sheep.get_mut(&job.swap.old_id).map(|drainee| {
drainee.entry.reload = ReloadState::None;
if drainee.ctl.is_some() && drainee.manual.is_none() {
drainee.entry.status = ProcStatus::Online;
}
to_info(&drainee.entry, &self.smits)
});
if let Some(info) = kept {
self.emit(ProcessEventKind::ReloadAbandoned, info, true);
}
let new_id = job.swap.new_id;
let Some(replacement) = self.sheep.get_mut(&new_id) else {
return;
};
replacement.entry.reload = ReloadState::None;
if replacement.ctl.is_none() {
return;
}
replacement.pending_delete = true;
self.claim_manual(
new_id,
PendingManual {
kind: ManualKind::Delete,
origin: CommandOrigin::Operator,
},
LadderCap::Stop,
);
}
fn arm_reload_deadline(&self, name: &str, new_id: u32) {
let app = self
.sheep
.get(&new_id)
.expect("arm_reload_deadline: the replacement was registered a moment ago")
.entry
.spec
.config();
let deadline = app.listen_timeout.as_duration()
+ app.graceful_timeout.as_duration()
+ RELOAD_DEADLINE_SLACK;
let tx = self.tx.clone();
let name = name.to_string();
tokio::spawn(async move {
tokio::time::sleep(deadline).await;
let _ = tx.send(Msg::ReloadDeadline { name, new_id }).await;
});
}
fn handle_reload_deadline(&mut self, name: &str, new_id: u32) {
let Some(job) = self.reloads.get(name) else {
return;
};
if job.swap.new_id != new_id {
return;
}
match job.swap.phase {
ReloadPhase::AwaitReady => {
self.abort_reload(
name,
"the swap passed its deadline with no readiness result",
);
}
ReloadPhase::DrainOld => {
let old_id = job.swap.old_id;
tracing::warn!(
name,
old_id,
new_id,
drainee_registered = self.sheep.contains_key(&old_id),
"reload abandoned: the swap passed its deadline, so the message that would \
have ended it is not coming"
);
self.reloads.remove(name);
self.clear_reload(new_id);
if let Some(slot) = self.sheep.get(&new_id) {
let info = to_info(&slot.entry, &self.smits);
self.emit(ProcessEventKind::ReloadAbandoned, info, true);
}
}
}
}
fn reap_drainee(&mut self, old_id: u32) -> bool {
let terminal = self.deregister_on_exit(old_id);
let Some(name) = self.reload_of(old_id) else {
return terminal;
};
match self.reloads[&name].swap.phase {
ReloadPhase::DrainOld => self.finish_swap(&name),
ReloadPhase::AwaitReady => {
self.reloads
.get_mut(&name)
.expect("reap_drainee: the phase was read a moment ago")
.swap
.phase = ReloadPhase::DrainOld;
}
}
terminal
}
fn deregister_on_exit(&mut self, id: u32) -> bool {
let mut removed = self
.sheep
.remove(&id)
.expect("deregister_on_exit: unknown id");
removed.entry.status = ProcStatus::Stopped;
removed.entry.reload = ReloadState::None;
let info = to_info(&removed.entry, &self.smits);
self.emit(ProcessEventKind::Delete, info.clone(), true);
self.disarm_extras(id, &info.name);
self.resolve_pending(id, info)
}
fn uncommitted_swap_of(&self, id: u32) -> Option<String> {
self.reloads
.iter()
.find(|(_, job)| {
job.swap.phase == ReloadPhase::AwaitReady
&& (job.swap.old_id == id || job.swap.new_id == id)
})
.map(|(name, _)| name.clone())
}
fn in_an_uncommitted_swap(&self, id: u32) -> bool {
self.uncommitted_swap_of(id).is_some()
}
fn reload_of(&self, id: u32) -> Option<String> {
self.reloads
.iter()
.find(|(_, job)| job.swap.old_id == id || job.swap.new_id == id)
.map(|(name, _)| name.clone())
}
fn clear_reload(&mut self, id: u32) {
if let Some(slot) = self.sheep.get_mut(&id) {
slot.entry.reload = ReloadState::None;
}
}
fn handle_reopen(
&self,
selector: &ProcessSelector,
reply: oneshot::Sender<Result<Vec<ProcessInfo>, SupervisorError>>,
) {
let mut matched: Vec<ProcessInfo> = Vec::new();
let mut paths: BTreeSet<PathBuf> = BTreeSet::new();
for id in self.matching_ids(selector) {
let slot = self
.sheep
.get(&id)
.expect("`matching_ids` answers with ids read off this map a moment ago");
paths.insert(slot.entry.out_file.clone());
paths.insert(slot.entry.err_file.clone());
matched.push(to_info(&slot.entry, &self.smits));
}
if matched.is_empty() {
let _ = reply.send(Err(SupervisorError::NotFound));
return;
}
let mut pumps: Vec<(ProcessInfo, mpsc::Sender<LogCtl>)> = self
.sheep
.values()
.filter(|slot| {
paths.contains(&slot.entry.out_file) || paths.contains(&slot.entry.err_file)
})
.filter_map(|slot| {
slot.log_ctl
.clone()
.map(|log_ctl| (to_info(&slot.entry, &self.smits), log_ctl))
})
.collect();
pumps.sort_unstable_by_key(|(info, _)| info.id);
spawn_reopen_task(matched, pumps, reply);
}
fn handle_flush(
&self,
selector: &ProcessSelector,
reply: oneshot::Sender<Result<Vec<ProcessInfo>, SupervisorError>>,
) {
let mut matched: Vec<ProcessInfo> = Vec::new();
let mut paths: BTreeSet<PathBuf> = BTreeSet::new();
for id in self.matching_ids(selector) {
let slot = self
.sheep
.get(&id)
.expect("`matching_ids` answers with ids read off this map a moment ago");
paths.insert(slot.entry.out_file.clone());
paths.insert(slot.entry.err_file.clone());
matched.push(to_info(&slot.entry, &self.smits));
}
if matched.is_empty() {
let _ = reply.send(Err(SupervisorError::NotFound));
return;
}
let mut pumps: Vec<(u32, mpsc::Sender<LogCtl>)> = self
.sheep
.iter()
.filter(|(_, slot)| {
paths.contains(&slot.entry.out_file) || paths.contains(&slot.entry.err_file)
})
.filter_map(|(id, slot)| slot.log_ctl.clone().map(|log_ctl| (*id, log_ctl)))
.collect();
pumps.sort_unstable_by_key(|&(id, _)| id);
let pumps = pumps.into_iter().map(|(_, log_ctl)| log_ctl).collect();
spawn_flush_task(matched, pumps, paths, reply);
}
fn begin_shutdown(&mut self, reply: oneshot::Sender<()>) -> bool {
self.shutting_down = true;
self.reloads.clear();
let online: HashSet<u32> = self
.sheep
.iter()
.filter(|(_, slot)| slot.ctl.is_some())
.map(|(&id, _)| id)
.collect();
if online.is_empty() {
let _ = reply.send(());
return true;
}
for &id in &online {
self.claim_manual(
id,
PendingManual {
kind: ManualKind::Stop,
origin: CommandOrigin::Operator,
},
LadderCap::Stop,
);
}
self.pending.push(PendingReply {
remaining: online,
results: Vec::new(),
reply: ReplyKind::Shutdown(reply),
});
false
}
fn handle_exited(&mut self, id: u32, outcome: ExitOutcome) -> bool {
let Some(slot) = self.sheep.get_mut(&id) else {
tracing::warn!(id, "Msg::Exited for an unregistered id");
return false;
};
slot.ctl = None;
slot.to_child = None;
slot.signals = None;
slot.to_stdin = None;
slot.actions.abandon_all();
slot.entry.pid = None;
slot.entry.last_exit = Some(outcome.into());
let manual = slot.manual.take();
let kind = manual.map(|pending| pending.kind);
let pending_delete = std::mem::take(&mut slot.pending_delete);
let reload = slot.entry.reload;
let started_at = slot.entry.started_at.take();
match reload {
ReloadState::Drainee { .. } => {
match self.uncommitted_swap_of(id) {
Some(name) if kind.is_some() => {
self.abort_reload(&name, "an operator's command reached the drainee first");
}
_ => return self.reap_drainee(id),
}
}
ReloadState::Replacement => {
self.clear_reload(id);
if let Some(name) = self.uncommitted_swap_of(id) {
self.abort_reload(&name, "the replacement exited before it was ready");
return self.deregister_on_exit(id);
}
if let Some(name) = self.reload_of(id) {
let old_id = self.reloads[&name].swap.old_id;
if !self.sheep.contains_key(&old_id) {
debug_assert_ne!(
self.sheep.get(&id).map(|slot| slot.entry.status),
Some(ProcStatus::Online),
"a swap committed by the drainee's death cannot have had a live \
replacement"
);
tracing::warn!(
name,
new_id = id,
"reload abandoned: the replacement exited before it was ready, with \
the instance it replaced already gone"
);
self.reloads.remove(&name);
if let Some(slot) = self.sheep.get(&id) {
let info = to_info(&slot.entry, &self.smits);
self.emit(ProcessEventKind::ReloadAbandoned, info, true);
}
}
}
}
ReloadState::None => {}
}
let Some(started_at) = started_at else {
tracing::warn!(
id,
"Msg::Exited for an entry with no started_at (duplicate?)"
);
if kind == Some(ManualKind::Delete) || pending_delete {
let mut removed = self.sheep.remove(&id).expect("checked above");
removed.entry.status = ProcStatus::Stopped;
let info = to_info(&removed.entry, &self.smits);
self.emit(ProcessEventKind::Delete, info.clone(), true);
self.disarm_extras(id, &info.name);
return self.resolve_pending(id, info);
}
let info = to_info(
&self.sheep.get(&id).expect("checked above").entry,
&self.smits,
);
return self.resolve_pending(id, info);
};
let uptime = tokio::time::Instant::now().saturating_duration_since(started_at);
if kind == Some(ManualKind::Restart) && !self.shutting_down && !pending_delete {
self.sheep
.get_mut(&id)
.expect("checked above")
.entry
.budget
.reset();
let manually = matches!(
manual,
Some(PendingManual {
origin: CommandOrigin::Operator,
..
})
);
let info = self.respawn(id, manually);
return self.resolve_pending(id, info);
}
let decision = {
let slot = self.sheep.get_mut(&id).expect("checked above");
decide_on_exit(
slot.entry.spec.config(),
&mut slot.entry.budget,
uptime,
outcome,
kind.is_some(),
)
};
let info = match decision {
Decision::Restart { delay } => {
let info = self.set_status(id, ProcStatus::WaitingRestart);
self.emit(ProcessEventKind::Exit, info.clone(), false);
let epoch = self.sheep.get(&id).expect("checked above").epoch;
self.schedule_restart(id, epoch, delay);
info
}
Decision::Errored => {
let info = self.set_status(id, ProcStatus::Errored);
self.emit(ProcessEventKind::Errored, info.clone(), kind.is_some());
self.disarm_extras(id, &info.name);
info
}
Decision::CleanStop if kind == Some(ManualKind::Delete) || pending_delete => {
let mut removed = self.sheep.remove(&id).expect("checked above");
removed.entry.status = ProcStatus::Stopped;
let info = to_info(&removed.entry, &self.smits);
self.emit(ProcessEventKind::Delete, info.clone(), true);
self.disarm_extras(id, &info.name);
info
}
Decision::CleanStop => {
let info = self.set_status(id, ProcStatus::Stopped);
self.disarm_extras(id, &info.name);
self.emit(
ProcessEventKind::Stop,
info.clone(),
kind == Some(ManualKind::Stop),
);
info
}
};
self.resolve_pending(id, info)
}
fn handle_extra_restart(&mut self, id: u32, pid: u32) {
if self.shutting_down {
tracing::debug!(id, pid, "extra restart dropped: engine is shutting down");
return;
}
let Some(slot) = self.sheep.get(&id) else {
tracing::debug!(id, pid, "extra restart dropped: no such sheep");
return;
};
if slot.entry.pid != Some(pid) {
tracing::debug!(
id,
pid,
current = slot.entry.pid,
"extra restart dropped: the reported pid is no longer this sheep's"
);
return;
}
if slot.entry.status != ProcStatus::Online {
tracing::debug!(
id,
pid,
status = %slot.entry.status,
"extra restart dropped: the sheep is no longer online"
);
return;
}
let (reply, _dropped) = oneshot::channel();
self.begin_manual(
ProcessSelector::Id(id),
ManualKind::Restart,
CommandOrigin::Automatic,
ReplyKind::Info(reply),
);
}
fn handle_restart_due(&mut self, id: u32, epoch: u64) {
if self.shutting_down {
return;
}
let Some(slot) = self.sheep.get(&id) else {
return;
};
if slot.entry.status != ProcStatus::WaitingRestart {
return;
}
if slot.epoch != epoch {
return;
}
self.respawn(id, false);
}
fn handle_ready_signal(&mut self, id: u32) {
let Some(slot) = self.sheep.get_mut(&id) else {
return;
};
if let Some(tx) = slot.ready_tx.take() {
let _ = tx.send(());
}
}
fn handle_ready_result(&mut self, id: u32, epoch: u64, manually: bool, readiness: Readiness) {
if self.shutting_down {
return;
}
let Some(slot) = self.sheep.get(&id) else {
return;
};
if slot.epoch != epoch {
return;
}
if slot.entry.status != ProcStatus::Starting {
return;
}
if matches!(slot.entry.reload, ReloadState::Replacement) {
self.reload_ready_result(id, manually, readiness);
return;
}
if readiness == Readiness::TimedOut {
tracing::warn!(id, "readiness deadline elapsed; marking online anyway");
}
let info = self.set_status(id, ProcStatus::Online);
self.went_online(id, info, manually);
}
fn begin_action(
&mut self,
selector: &ProcessSelector,
action: String,
params: Option<String>,
reply: oneshot::Sender<Result<Vec<ActionReply>, SupervisorError>>,
) {
let matched = self.matching_ids(selector);
if matched.is_empty() {
let _ = reply.send(Err(SupervisorError::NotFound));
return;
}
let mut refused = Vec::new();
let mut waits = Vec::new();
for id in matched {
let slot = self
.sheep
.get(&id)
.expect("begin_action: `matched` holds ids read off this map a moment ago");
let config = slot.entry.spec.config();
let name = config.name.clone();
let action_timeout = config.action_timeout.as_duration();
if matches!(slot.entry.reload, ReloadState::Drainee { .. }) {
refused.push(ActionReply {
id,
name,
outcome: ActionOutcome::Skipped,
});
continue;
}
let Some(to_child) = slot.open_channel().cloned() else {
refused.push(ActionReply {
id,
name,
outcome: ActionOutcome::NoChannel,
});
continue;
};
let answer =
self.arm_action(id, to_child, action.clone(), params.clone(), action_timeout);
waits.push((id, name, answer));
}
if waits.is_empty() {
let skipped = refused
.iter()
.filter(|row| row.outcome == ActionOutcome::Skipped)
.count();
tracing::warn!(
action,
matched = refused.len(),
skipped,
"no matched sheep could take this action; nothing was delivered"
);
let _ = reply.send(Ok(refused));
return;
}
spawn_trigger_task(refused, waits, reply);
}
fn begin_signal(
&mut self,
selector: &ProcessSelector,
sig: OperatorSignal,
reply: oneshot::Sender<Result<Vec<SignalReply>, SupervisorError>>,
) {
let matched = self.matching_ids(selector);
if matched.is_empty() {
let _ = reply.send(Err(SupervisorError::NotFound));
return;
}
let mut settled = Vec::new();
let mut waits = Vec::new();
for id in matched {
let slot = self
.sheep
.get(&id)
.expect("begin_signal: `matched` holds ids read off this map a moment ago");
let name = slot.entry.spec.config().name.clone();
let Some(signals) = slot.signals.clone() else {
settled.push(SignalReply {
id,
name,
outcome: SignalOutcome::NotRunning,
});
continue;
};
let (done, answer) = oneshot::channel();
if signals.try_send(SignalRequest { sig, done }).is_err() {
settled.push(SignalReply {
id,
name,
outcome: SignalOutcome::NotRunning,
});
continue;
}
waits.push((id, name, answer));
}
spawn_signal_task(settled, waits, reply);
}
fn begin_send_line(
&mut self,
selector: &ProcessSelector,
line: String,
reply: oneshot::Sender<Result<Vec<LineReply>, SupervisorError>>,
) {
let matched = self.matching_ids(selector);
if matched.is_empty() {
let _ = reply.send(Err(SupervisorError::NotFound));
return;
}
let mut settled = Vec::new();
let mut waits = Vec::new();
for id in matched {
let slot = self
.sheep
.get(&id)
.expect("begin_send_line: `matched` holds ids read off this map a moment ago");
let name = slot.entry.spec.config().name.clone();
let Some(to_stdin) = slot.open_stdin().cloned() else {
settled.push(LineReply {
id,
name,
outcome: LineOutcome::NoStdin,
});
continue;
};
let (done, answer) = oneshot::channel();
match to_stdin.try_send(StdinWrite {
line: line.clone(),
done,
}) {
Ok(()) => waits.push((id, name, answer)),
Err(mpsc::error::TrySendError::Full(_)) => settled.push(LineReply {
id,
name,
outcome: LineOutcome::NotWritten {
reason: "the app is not reading its stdin (its queue was \
already full when this line arrived)"
.to_string(),
},
}),
Err(mpsc::error::TrySendError::Closed(_)) => settled.push(LineReply {
id,
name,
outcome: LineOutcome::NoStdin,
}),
}
}
spawn_send_line_task(settled, waits, reply);
}
fn arm_action(
&mut self,
id: u32,
to_child: mpsc::Sender<ShepherdMessage>,
action: String,
params: Option<String>,
timeout: Duration,
) -> oneshot::Receiver<ActionOutcome> {
let (reply, answer) = oneshot::channel();
let stamp = self.next_action_stamp;
self.next_action_stamp += 1;
let waiter = spawn_action_task(
id,
stamp,
ShepherdMessage::Action {
name: action.clone(),
params,
id: stamp,
},
to_child,
timeout,
self.tx.clone(),
);
self.sheep
.get_mut(&id)
.expect("arm_action: the slot was read a moment ago")
.actions
.arm(PendingAction {
stamp,
action,
waiter: Some(waiter),
reply,
});
answer
}
fn handle_action_reply(&mut self, id: u32, action: &str, body: String, stamp: Option<u64>) {
let Some(slot) = self.sheep.get_mut(&id) else {
return;
};
if let Some(waiter) = slot.actions.answer(action, stamp) {
let _ = waiter.send(body);
}
}
fn handle_action_result(&mut self, id: u32, stamp: u64, outcome: ActionOutcome) {
let Some(slot) = self.sheep.get_mut(&id) else {
return;
};
if let Some(reply) = slot.actions.resolve(stamp) {
let _ = reply.send(outcome);
}
}
fn schedule_restart(&self, id: u32, epoch: u64, delay: Option<Duration>) {
let tx = self.tx.clone();
tokio::spawn(async move {
if let Some(delay) = delay {
tokio::time::sleep(delay).await;
}
let _ = tx.send(Msg::RestartDue { id, epoch }).await;
});
}
fn resolve_pending(&mut self, id: u32, info: ProcessInfo) -> bool {
let mut shutdown_completed = false;
let mut i = 0;
while i < self.pending.len() {
if self.pending[i].remaining.remove(&id) {
self.pending[i].results.push(info.clone());
}
if self.pending[i].remaining.is_empty() {
let mut pending = self.pending.remove(i);
pending.results.sort_unstable_by_key(|info| info.id);
if matches!(pending.reply, ReplyKind::Shutdown(_)) {
shutdown_completed = true;
}
send_reply(pending.reply, Ok(pending.results));
} else {
i += 1;
}
}
shutdown_completed
}
fn went_online(&mut self, id: u32, info: ProcessInfo, manually: bool) {
self.emit(ProcessEventKind::Online, info, manually);
self.arm_extras(id);
}
fn arm_extras(&mut self, id: u32) {
let Some(extras) = self.extras.as_ref() else {
return;
};
let Some(slot) = self.sheep.get(&id) else {
return;
};
let supervisor = SupervisorHandle {
tx: self.tx.clone(),
};
let spec = assemble(
&slot.entry.spec,
slot.entry.instance,
&self.paths,
slot.entry.credentials,
);
self.registry
.arm(&slot.entry, spec_prober(&spec), extras, &supervisor);
}
fn disarm_extras(&mut self, id: u32, name: &str) {
self.registry.disarm(id, name);
}
fn set_status(&mut self, id: u32, status: ProcStatus) -> ProcessInfo {
let slot = self.sheep.get_mut(&id).expect("set_status: unknown id");
slot.entry.status = status;
to_info(&slot.entry, &self.smits)
}
fn handle_set_smit(
&mut self,
conn: ConnId,
sheep: &str,
smit: Option<Smit>,
) -> Result<Vec<ProcessInfo>, SupervisorError> {
if !self
.sheep
.values()
.any(|slot| slot.entry.spec.config().name == sheep)
{
return Err(SupervisorError::NotFound);
}
match smit {
Some(smit) => {
self.smits
.insert(sheep.to_string(), (conn, smit.to_string()));
}
None => {
if self
.smits
.get(sheep)
.is_some_and(|(painter, _)| *painter == conn)
{
self.smits.remove(sheep);
}
}
}
Ok(self
.snapshot_all()
.into_iter()
.filter(|info| info.name == sheep)
.collect())
}
fn snapshot_all(&self) -> Vec<ProcessInfo> {
let mut entries: Vec<&ProcessEntry> = self.sheep.values().map(|slot| &slot.entry).collect();
entries.sort_unstable_by(|a, b| {
(a.spec.config().name.as_str(), a.instance, a.id).cmp(&(
b.spec.config().name.as_str(),
b.instance,
b.id,
))
});
entries
.into_iter()
.map(|entry| to_info(entry, &self.smits))
.collect()
}
fn emit(&self, event: ProcessEventKind, info: ProcessInfo, manually: bool) {
let _ = self.events.send(BusEvent::Process {
event,
info,
manually,
at_ms: crate::now_ms(),
});
}
}
fn send_reply(reply: ReplyKind, outcome: Result<Vec<ProcessInfo>, SupervisorError>) {
match reply {
ReplyKind::Info(tx) => {
let _ = tx.send(outcome);
}
ReplyKind::Ids(tx) => {
let ids = outcome.map(|infos| infos.into_iter().map(|info| info.id).collect());
let _ = tx.send(ids);
}
ReplyKind::Shutdown(tx) => {
let _ = tx.send(());
}
}
}
fn to_info(entry: &ProcessEntry, smits: &Smits) -> ProcessInfo {
let uptime_ms = entry.started_at.map_or(0, |started_at| {
tokio::time::Instant::now()
.saturating_duration_since(started_at)
.as_millis() as u64
});
ProcessInfo::builder(entry.id, entry.spec.config().name.clone(), entry.status)
.pid(entry.pid)
.restarts(entry.restarts)
.uptime_ms(uptime_ms)
.fold(entry.spec.config().fold.clone())
.out_file(Some(entry.out_file.to_string_lossy().into_owned()))
.err_file(Some(entry.err_file.to_string_lossy().into_owned()))
.cpu_percent(None)
.memory_bytes(None)
.dog(entry.dog.clone())
.last_exit(entry.last_exit)
.smit(
smits
.get(&entry.spec.config().name)
.map(|(_, smit)| smit.clone()),
)
.build()
}
impl From<ExitOutcome> for ExitInfo {
fn from(outcome: ExitOutcome) -> Self {
Self {
code: outcome.code,
signal: outcome.signal,
}
}
}
fn spec_prober(spec: &SpawnSpec) -> Arc<dyn Prober> {
Arc::new(OsProber::new(spec.cwd.clone(), spec.env.clone()))
}
fn spawn_readiness_task(
id: u32,
epoch: u64,
manually: bool,
source: ReadinessSource,
deadline: Duration,
prober: Arc<dyn Prober>,
actor_tx: mpsc::Sender<Msg>,
) -> oneshot::Sender<()> {
let (ready_tx, ready_rx) = oneshot::channel();
tokio::spawn(async move {
let readiness = await_ready(&source, deadline, ready_rx, prober).await;
let _ = actor_tx
.send(Msg::ReadyResult {
id,
epoch,
manually,
readiness,
})
.await;
});
ready_tx
}
fn spawn_action_task(
id: u32,
stamp: u64,
message: ShepherdMessage,
to_child: mpsc::Sender<ShepherdMessage>,
deadline: Duration,
actor_tx: mpsc::Sender<Msg>,
) -> oneshot::Sender<String> {
let (reply_tx, reply_rx) = oneshot::channel();
tokio::spawn(async move {
let delivered = tokio::time::timeout(deadline, async move {
if to_child.send(message).await.is_err() {
return None;
}
reply_rx.await.ok()
})
.await;
let outcome = match delivered {
Ok(Some(body)) => ActionOutcome::Replied { body },
Ok(None) => ActionOutcome::NoChannel,
Err(_elapsed) => ActionOutcome::TimedOut,
};
let _ = actor_tx
.send(Msg::ActionResult { id, stamp, outcome })
.await;
});
reply_tx
}
fn spawn_trigger_task(
mut rows: Vec<ActionReply>,
waits: Vec<(u32, String, oneshot::Receiver<ActionOutcome>)>,
reply: oneshot::Sender<Result<Vec<ActionReply>, SupervisorError>>,
) {
tokio::spawn(async move {
for (id, name, answer) in waits {
let outcome = answer.await.unwrap_or(ActionOutcome::NoChannel);
rows.push(ActionReply { id, name, outcome });
}
rows.sort_unstable_by_key(|row| row.id);
let _ = reply.send(Ok(rows));
});
}
type SignalWait = (u32, String, oneshot::Receiver<Result<(), RunnerError>>);
fn spawn_signal_task(
mut rows: Vec<SignalReply>,
waits: Vec<SignalWait>,
reply: oneshot::Sender<Result<Vec<SignalReply>, SupervisorError>>,
) {
tokio::spawn(async move {
for (id, name, answer) in waits {
let outcome = match answer.await {
Ok(Ok(())) => SignalOutcome::Delivered,
Ok(Err(err)) => SignalOutcome::Failed {
reason: err.to_string(),
},
Err(_dropped) => SignalOutcome::NotRunning,
};
rows.push(SignalReply { id, name, outcome });
}
rows.sort_unstable_by_key(|row| row.id);
let _ = reply.send(Ok(rows));
});
}
type LineWait = (u32, String, oneshot::Receiver<Result<(), RunnerError>>);
fn spawn_send_line_task(
settled: Vec<LineReply>,
waits: Vec<LineWait>,
reply: oneshot::Sender<Result<Vec<LineReply>, SupervisorError>>,
) {
tokio::spawn(async move {
let mut rows = settled;
rows.extend(
futures_util::future::join_all(waits.into_iter().map(
|(id, name, answer)| async move {
let outcome = match tokio::time::timeout(STDIN_WRITE_TIMEOUT, answer).await {
Ok(Ok(Ok(()))) => LineOutcome::Sent,
Ok(Ok(Err(err))) => LineOutcome::NotWritten {
reason: err.to_string(),
},
Ok(Err(_recv)) => LineOutcome::NoStdin,
Err(_elapsed) => LineOutcome::NotWritten {
reason: format!(
"the app did not read its stdin within {}s; this line \
may still land if it drains",
STDIN_WRITE_TIMEOUT.as_secs()
),
},
};
LineReply { id, name, outcome }
},
))
.await,
);
rows.sort_unstable_by_key(|row| row.id);
let _ = reply.send(Ok(rows));
});
}
fn spawn_reopen_task(
matched: Vec<ProcessInfo>,
pumps: Vec<(ProcessInfo, mpsc::Sender<LogCtl>)>,
reply: oneshot::Sender<Result<Vec<ProcessInfo>, SupervisorError>>,
) {
tokio::spawn(async move {
let mut failures = Vec::new();
for (info, log_ctl) in &pumps {
if let Err(error) = reopen_logs(log_ctl).await {
failures.push(format!("{} (id {}): {error}", info.name, info.id));
}
}
let _ = reply.send(if failures.is_empty() {
Ok(matched)
} else {
Err(SupervisorError::ReopenFailed(failures.join("; ")))
});
});
}
async fn reopen_logs(log_ctl: &mpsc::Sender<LogCtl>) -> Result<(), ReopenError> {
let (done, ack) = oneshot::channel();
if log_ctl.send(LogCtl::Reopen { done }).await.is_err() {
return Ok(());
}
ack.await.unwrap_or(Ok(()))
}
fn spawn_flush_task(
matched: Vec<ProcessInfo>,
pumps: Vec<mpsc::Sender<LogCtl>>,
paths: BTreeSet<PathBuf>,
reply: oneshot::Sender<Result<Vec<ProcessInfo>, SupervisorError>>,
) {
tokio::spawn(async move {
let mut failures = Vec::new();
for log_ctl in &pumps {
if let Err(error) = flush_logs(log_ctl).await {
failures.push(error.message);
}
}
for path in paths {
if let Err(error) = truncate_log(&path).await {
failures.push(error.message);
}
}
let _ = reply.send(if failures.is_empty() {
Ok(matched)
} else {
Err(SupervisorError::FlushFailed(failures.join("; ")))
});
});
}
async fn flush_logs(log_ctl: &mpsc::Sender<LogCtl>) -> Result<(), FlushError> {
let (done, ack) = oneshot::channel();
if log_ctl.send(LogCtl::Flush { done }).await.is_err() {
return Ok(());
}
ack.await.unwrap_or(Ok(()))
}
async fn truncate_log(path: &Path) -> Result<(), FlushError> {
let refused = |error: &dyn fmt::Display| FlushError {
message: format!("{}: {error}", path.display()),
};
if let Err(error) = check_log_ancestry(path) {
return Err(refused(&error));
}
let mut options = tokio::fs::OpenOptions::new();
options.write(true).truncate(true);
match open_log_path(&mut options, path).await {
Ok(_) => Ok(()),
Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(()),
Err(error) => Err(refused(&error)),
}
}
#[derive(Debug)]
struct SignalRequest {
sig: OperatorSignal,
done: oneshot::Sender<Result<(), RunnerError>>,
}
struct SheepHandles {
ctl: mpsc::Sender<SheepCtl>,
signals: mpsc::Sender<SignalRequest>,
}
fn spawn_sheep_task<P: RunningProcess>(
id: u32,
proc: P,
io: ProcIo,
app: ResolvedApp,
events: broadcast::Sender<BusEvent>,
actor_tx: mpsc::Sender<Msg>,
) -> SheepHandles {
let (ctl_tx, ctl_rx) = mpsc::channel(SHEEP_CTL_CAPACITY);
let (signal_tx, signal_rx) = mpsc::channel(SIGNAL_CAPACITY);
tokio::spawn(run_sheep(
id, proc, io, app, ctl_rx, signal_rx, events, actor_tx,
));
SheepHandles {
ctl: ctl_tx,
signals: signal_tx,
}
}
#[allow(clippy::too_many_arguments)]
async fn run_sheep<P: RunningProcess>(
id: u32,
mut proc: P,
io: ProcIo,
app: ResolvedApp,
mut ctl_rx: mpsc::Receiver<SheepCtl>,
mut signal_rx: mpsc::Receiver<SignalRequest>,
events: broadcast::Sender<BusEvent>,
actor_tx: mpsc::Sender<Msg>,
) {
let ProcIo {
mut logs,
mut from_child,
to_child,
log_ctl: _log_ctl,
to_stdin: _to_stdin,
} = io;
let mut ctl_open = true;
let mut logs_open = true;
let mut from_child_open = true;
let mut signals_open = true;
loop {
tokio::select! {
outcome = proc.wait() => {
let _ = actor_tx.send(Msg::Exited { id, outcome }).await;
break;
}
maybe_ctl = ctl_rx.recv(), if ctl_open => {
match maybe_ctl {
Some(SheepCtl::Kill { grace }) => {
let outcome =
kill_process(&mut proc, app.config(), Some(&to_child), grace).await;
let _ = actor_tx.send(Msg::Exited { id, outcome }).await;
break;
}
None => ctl_open = false,
}
}
maybe_signal = signal_rx.recv(), if signals_open => {
match maybe_signal {
Some(SignalRequest { sig, done }) => {
let _ = done.send(proc.signal_process(sig));
}
None => signals_open = false,
}
}
maybe_line = logs.recv(), if logs_open => {
match maybe_line {
Some(line) => {
let event = if line.err {
BusEvent::LogErr { id, line: line.line }
} else {
BusEvent::LogOut { id, line: line.line }
};
let _ = events.send(event);
}
None => logs_open = false,
}
}
maybe_msg = from_child.recv(), if from_child_open => {
match maybe_msg {
Some(message) => {
let _ = events.send(BusEvent::Channel {
id,
message: message.clone(),
});
match message {
ChildMessage::Ready => {
let _ = actor_tx.send(Msg::Ready { id }).await;
}
ChildMessage::Metric { name, value } => {
tracing::debug!(
id,
name,
value,
"child metric forwarded to the bus as channel.metric"
);
}
ChildMessage::ActionReply {
action,
body,
id: stamp,
} => {
let _ = actor_tx
.send(Msg::ActionReply { id, action, body, stamp })
.await;
}
}
}
None => from_child_open = false,
}
}
}
}
}
#[cfg(test)]
mod tests {
use shep_core::config::{AppConfig, ProbeConfig, ProbeKind, normalize};
use shep_core::protocol::DogSource;
use shep_core::status::ProcStatus;
use shep_core::values::UpDuration;
use super::*;
use crate::fake::{ProcScript, ScriptedRunner};
use crate::testing::{
Harness, RecordingEnforcer, SharedRunner, app_with, armed_entry, harness, idle_stats,
probe_config, test_paths,
};
use crate::cron::{DEFAULT_MAX_CRON_SLEEP, SystemClock};
use crate::extras::{ExtrasReports, spawn_extras_reporter};
use tokio::sync::watch;
use std::sync::atomic::{AtomicUsize, Ordering as AtomicOrdering};
async fn await_event(
rx: &mut tokio::sync::broadcast::Receiver<BusEvent>,
id: u32,
kind: ProcessEventKind,
) -> bool {
loop {
match rx.recv().await {
Ok(BusEvent::Process {
event,
info,
manually,
..
}) if info.id == id && event == kind => {
return manually;
}
Ok(_) => continue,
Err(tokio::sync::broadcast::error::RecvError::Lagged(_)) => continue,
Err(e) => panic!("event stream closed before {kind:?} for id {id}: {e}"),
}
}
}
async fn assert_no_event_within(
rx: &mut tokio::sync::broadcast::Receiver<BusEvent>,
id: u32,
kind: ProcessEventKind,
window: Duration,
) {
match tokio::time::timeout(window, await_event(rx, id, kind)).await {
Err(_elapsed) => {} Ok(_manually) => panic!("unexpected {kind:?} for id {id} within {window:?}"),
}
}
#[tokio::test(start_paused = true)]
async fn wait_ready_app_stays_starting_until_the_channel_signals() {
let (events, mut rx) = tokio::sync::broadcast::channel(64);
let runner = ScriptedRunner::new(vec![ProcScript::never_exits()]);
let dir = tempfile::tempdir().unwrap();
let handle = spawn_supervisor(runner, test_paths(&dir), events);
let mut app = AppConfig::minimal("web", "./srv");
app.wait_ready = true;
handle.start(vec![normalize(app).unwrap()]).await.unwrap();
assert_eq!(handle.list().await[0].status, ProcStatus::Starting);
assert_no_event_within(
&mut rx,
0,
ProcessEventKind::Online,
Duration::from_millis(500),
)
.await;
assert_eq!(handle.list().await[0].status, ProcStatus::Starting);
handle.tx.send(Msg::Ready { id: 0 }).await.unwrap();
tokio::time::timeout(
Duration::from_secs(1),
await_event(&mut rx, 0, ProcessEventKind::Online),
)
.await
.expect("Online once the channel signals");
assert_eq!(handle.list().await[0].status, ProcStatus::Online);
}
#[tokio::test]
async fn readiness_probe_app_stays_starting_until_the_probe_passes() {
let reserved = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
let addr = reserved.local_addr().unwrap();
drop(reserved);
let (events, mut rx) = tokio::sync::broadcast::channel(64);
let runner = ScriptedRunner::new(vec![ProcScript::never_exits()]);
let dir = tempfile::tempdir().unwrap();
let handle = spawn_supervisor(runner, test_paths(&dir), events);
let mut app = AppConfig::minimal("web", "./srv");
app.readiness_probe = Some(ProbeConfig {
interval: UpDuration::from_millis(50),
timeout: UpDuration::from_millis(200),
..probe_config(ProbeKind::Tcp, &addr.to_string())
});
handle.start(vec![normalize(app).unwrap()]).await.unwrap();
assert_eq!(handle.list().await[0].status, ProcStatus::Starting);
assert_no_event_within(
&mut rx,
0,
ProcessEventKind::Online,
Duration::from_millis(220),
)
.await;
assert_eq!(handle.list().await[0].status, ProcStatus::Starting);
let listener = tokio::net::TcpListener::bind(addr).await.unwrap();
let _accept = tokio::spawn(async move { while listener.accept().await.is_ok() {} });
tokio::time::timeout(
Duration::from_secs(2),
await_event(&mut rx, 0, ProcessEventKind::Online),
)
.await
.expect("Online once the probe starts passing");
assert_eq!(handle.list().await[0].status, ProcStatus::Online);
}
#[cfg(unix)]
#[tokio::test]
async fn an_exec_readiness_probe_sees_the_assembled_env_not_the_apps_own() {
let (events, mut rx) = tokio::sync::broadcast::channel(64);
let runner = ScriptedRunner::new(vec![ProcScript::never_exits()]);
let dir = tempfile::tempdir().unwrap();
let ready_file = dir.path().join("ready-0");
let handle = spawn_supervisor(runner, test_paths(&dir), events);
let mut app = AppConfig::minimal("web", "./srv");
app.readiness_probe = Some(ProbeConfig {
interval: UpDuration::from_millis(50),
timeout: UpDuration::from_millis(500),
..probe_config(
ProbeKind::Exec,
&format!(r#"test -f "{}/ready-$SHEP_INSTANCE""#, dir.path().display()),
)
});
app.listen_timeout = UpDuration::from_millis(60_000);
handle.start(vec![normalize(app).unwrap()]).await.unwrap();
assert_eq!(handle.list().await[0].status, ProcStatus::Starting);
assert_no_event_within(
&mut rx,
0,
ProcessEventKind::Online,
Duration::from_millis(220),
)
.await;
assert_eq!(handle.list().await[0].status, ProcStatus::Starting);
std::fs::write(&ready_file, b"").unwrap();
tokio::time::timeout(
Duration::from_secs(5),
await_event(&mut rx, 0, ProcessEventKind::Online),
)
.await
.expect("Online once the exec probe can resolve $SHEP_INSTANCE");
assert_eq!(handle.list().await[0].status, ProcStatus::Online);
}
#[tokio::test(start_paused = true)]
async fn a_gated_apps_online_carries_the_same_manually_flag_an_ungated_one_does() {
let (events, mut rx) = tokio::sync::broadcast::channel(64);
let runner = ScriptedRunner::new(vec![
ProcScript::never_exits(), ProcScript::never_exits(), ProcScript::never_exits(), ]);
let dir = tempfile::tempdir().unwrap();
let handle = spawn_supervisor(runner, test_paths(&dir), events);
let mut gated = AppConfig::minimal("gated", "./srv");
gated.wait_ready = true;
let ungated = AppConfig::minimal("ungated", "./srv");
handle
.start(vec![normalize(gated).unwrap(), normalize(ungated).unwrap()])
.await
.unwrap();
let ungated_manually = tokio::time::timeout(
Duration::from_secs(1),
await_event(&mut rx, 1, ProcessEventKind::Online),
)
.await
.expect("an ungated app is Online at spawn");
assert!(ungated_manually, "sanity: a Start is a manual event");
handle.tx.send(Msg::Ready { id: 0 }).await.unwrap();
let gated_manually = tokio::time::timeout(
Duration::from_secs(1),
await_event(&mut rx, 0, ProcessEventKind::Online),
)
.await
.expect("Online once the channel signals");
assert_eq!(
gated_manually, ungated_manually,
"the same `shep start` must report the same flag, gated or not"
);
let restarted = handle.restart(ProcessSelector::Id(0)).await.unwrap();
assert_eq!(restarted[0].status, ProcStatus::Starting);
handle.tx.send(Msg::Ready { id: 0 }).await.unwrap();
let restarted_manually = tokio::time::timeout(
Duration::from_secs(1),
await_event(&mut rx, 0, ProcessEventKind::Online),
)
.await
.expect("Online once the respawned sheep signals");
assert!(
restarted_manually,
"a manual Restart's Online must stay manual through the readiness gate"
);
}
#[tokio::test(start_paused = true)]
async fn an_operators_restart_is_reported_as_a_user_action() {
let (events, mut rx) = tokio::sync::broadcast::channel(64);
let runner = ScriptedRunner::new(vec![ProcScript::never_exits(); 2]);
let dir = tempfile::tempdir().unwrap();
let handle = spawn_supervisor(runner, test_paths(&dir), events);
handle
.start(vec![normalize(AppConfig::minimal("web", "./srv")).unwrap()])
.await
.unwrap();
let restarted = handle.restart(ProcessSelector::All).await.unwrap();
assert_eq!(restarted[0].restarts, 1, "the restart really respawned");
let manually = tokio::time::timeout(
Duration::from_secs(1),
await_event(&mut rx, 0, ProcessEventKind::Restart),
)
.await
.expect("the respawn `restart` performed");
assert!(
manually,
"a person typed `shep restart`; the bus must say a user action caused it"
);
}
#[tokio::test(start_paused = true)]
async fn a_gated_app_whose_deadline_elapses_goes_online_anyway() {
let (events, mut rx) = tokio::sync::broadcast::channel(64);
let runner = ScriptedRunner::new(vec![ProcScript::never_exits()]);
let dir = tempfile::tempdir().unwrap();
let handle = spawn_supervisor(runner, test_paths(&dir), events);
let mut app = AppConfig::minimal("web", "./srv");
app.wait_ready = true; handle.start(vec![normalize(app).unwrap()]).await.unwrap();
assert_eq!(handle.list().await[0].status, ProcStatus::Starting);
tokio::time::timeout(
Duration::from_secs(4), await_event(&mut rx, 0, ProcessEventKind::Online),
)
.await
.expect("Online once the readiness deadline elapses");
assert_eq!(handle.list().await[0].status, ProcStatus::Online);
}
#[tokio::test(start_paused = true)]
async fn a_gated_app_that_exits_while_starting_never_reaches_online_from_the_old_wait() {
let (events, mut rx) = tokio::sync::broadcast::channel(64);
let runner = ScriptedRunner::new(vec![
ProcScript::stable_then_exit(500, 1), ProcScript::never_exits(), ]);
let dir = tempfile::tempdir().unwrap();
let handle = spawn_supervisor(runner, test_paths(&dir), events);
let mut app = AppConfig::minimal("web", "./srv");
app.wait_ready = true; handle.start(vec![normalize(app).unwrap()]).await.unwrap();
assert_eq!(handle.list().await[0].status, ProcStatus::Starting);
tokio::time::timeout(
Duration::from_secs(1),
await_event(&mut rx, 0, ProcessEventKind::Restart),
)
.await
.expect("the automatic respawn after the unstable exit");
assert_eq!(handle.list().await[0].status, ProcStatus::Starting);
assert_eq!(handle.list().await[0].restarts, 1);
assert_no_event_within(
&mut rx,
0,
ProcessEventKind::Online,
Duration::from_millis(2_700),
)
.await;
assert_eq!(
handle.list().await[0].status,
ProcStatus::Starting,
"the OLD wait's stale TimedOut must not have marked the respawned process online"
);
tokio::time::timeout(
Duration::from_secs(2),
await_event(&mut rx, 0, ProcessEventKind::Online),
)
.await
.expect("the new process's own readiness deadline");
assert_eq!(handle.list().await[0].status, ProcStatus::Online);
}
#[tokio::test(start_paused = true)]
async fn a_gated_app_stopped_while_starting_ignores_the_old_wait() {
let (events, mut rx) = tokio::sync::broadcast::channel(64);
let runner = ScriptedRunner::new(vec![ProcScript::stable_then_exit(500, 1)]);
let dir = tempfile::tempdir().unwrap();
let handle = spawn_supervisor(runner, test_paths(&dir), events);
let mut app = AppConfig::minimal("web", "./srv");
app.wait_ready = true; app.autorestart = false; handle.start(vec![normalize(app).unwrap()]).await.unwrap();
assert_eq!(handle.list().await[0].status, ProcStatus::Starting);
tokio::time::timeout(
Duration::from_secs(1),
await_event(&mut rx, 0, ProcessEventKind::Stop),
)
.await
.expect("the natural exit at 500ms");
assert_eq!(handle.list().await[0].status, ProcStatus::Stopped);
assert_no_event_within(&mut rx, 0, ProcessEventKind::Online, Duration::from_secs(3)).await;
assert_eq!(handle.list().await[0].status, ProcStatus::Stopped);
}
#[tokio::test(start_paused = true)]
async fn a_gated_app_restarted_while_starting_ignores_the_old_wait() {
let (events, mut rx) = tokio::sync::broadcast::channel(64);
let runner =
ScriptedRunner::new(vec![ProcScript::never_exits(), ProcScript::never_exits()]);
let dir = tempfile::tempdir().unwrap();
let handle = spawn_supervisor(runner, test_paths(&dir), events);
let mut app = AppConfig::minimal("web", "./srv");
app.wait_ready = true; handle.start(vec![normalize(app).unwrap()]).await.unwrap();
assert_eq!(handle.list().await[0].status, ProcStatus::Starting);
tokio::time::sleep(Duration::from_millis(500)).await;
handle
.restart(ProcessSelector::Name("web".to_string()))
.await
.unwrap();
assert_eq!(
handle.list().await[0].status,
ProcStatus::Starting,
"the respawned process is gated too, so it must still be Starting"
);
assert_no_event_within(
&mut rx,
0,
ProcessEventKind::Online,
Duration::from_millis(2_700),
)
.await;
assert_eq!(
handle.list().await[0].status,
ProcStatus::Starting,
"the old wait's stale TimedOut must not have marked the new process online"
);
tokio::time::timeout(
Duration::from_secs(2),
await_event(&mut rx, 0, ProcessEventKind::Online),
)
.await
.expect("the new process's own readiness deadline");
assert_eq!(handle.list().await[0].status, ProcStatus::Online);
}
#[tokio::test(start_paused = true)]
async fn start_lists_online_instances() {
let (events, _rx) = tokio::sync::broadcast::channel(64);
let runner =
ScriptedRunner::new(vec![ProcScript::never_exits(), ProcScript::never_exits()]);
let dir = tempfile::tempdir().unwrap();
let handle = spawn_supervisor(runner, test_paths(&dir), events);
let mut app = AppConfig::minimal("web", "./srv");
app.instances = 2;
let infos = handle.start(vec![normalize(app).unwrap()]).await.unwrap();
assert_eq!(infos.len(), 2);
let list = handle.list().await;
assert_eq!(list.len(), 2);
assert!(list.iter().all(|i| i.status == ProcStatus::Online));
assert_eq!(list.iter().map(|i| i.id).collect::<Vec<_>>(), vec![0, 1]);
}
#[tokio::test(start_paused = true)]
async fn listed_log_paths_are_the_derived_defaults() {
let (events, _rx) = tokio::sync::broadcast::channel(64);
let runner = ScriptedRunner::new(vec![ProcScript::never_exits()]);
let dir = tempfile::tempdir().unwrap();
let paths = test_paths(&dir);
let logs = paths.logs.clone();
let handle = spawn_supervisor(runner, paths, events);
handle
.start(vec![normalize(AppConfig::minimal("web", "./srv")).unwrap()])
.await
.unwrap();
let list = handle.list().await;
assert_eq!(
list[0].out_file.as_deref(),
logs.join("web-0-out.log").to_str()
);
assert_eq!(
list[0].err_file.as_deref(),
logs.join("web-0-err.log").to_str()
);
}
#[tokio::test(start_paused = true)]
async fn listed_log_paths_honour_an_explicit_out_file() {
let (events, _rx) = tokio::sync::broadcast::channel(64);
let runner = ScriptedRunner::new(vec![ProcScript::never_exits()]);
let dir = tempfile::tempdir().unwrap();
let paths = test_paths(&dir);
let logs = paths.logs.clone();
let handle = spawn_supervisor(runner, paths, events);
let mut app = AppConfig::minimal("web", "./srv");
app.out_file = Some("/var/log/myapp.log".to_string());
handle.start(vec![normalize(app).unwrap()]).await.unwrap();
let list = handle.list().await;
assert_eq!(list[0].out_file.as_deref(), Some("/var/log/myapp.log"));
assert_eq!(
list[0].err_file.as_deref(),
logs.join("web-0-err.log").to_str(),
"err_file was not configured, so it must still be the default"
);
}
#[tokio::test(start_paused = true)]
async fn crash_loop_erroreds_after_budget_with_pinned_delays() {
let (events, mut rx) = tokio::sync::broadcast::channel(1024);
let runner = ScriptedRunner::new((0..16).map(|_| ProcScript::const_exit(1)).collect());
let dir = tempfile::tempdir().unwrap();
let handle = spawn_supervisor(runner, test_paths(&dir), events);
let mut app = AppConfig::minimal("crash", "./boom");
app.exp_backoff_restart_delay = Some("100".parse().unwrap());
handle.start(vec![normalize(app).unwrap()]).await.unwrap();
await_event(&mut rx, 0, ProcessEventKind::Errored).await;
let list = handle.list().await;
assert_eq!(list[0].status, ProcStatus::Errored);
assert_eq!(list[0].restarts, 15); assert_eq!(
list[0].last_exit,
Some(ExitInfo {
code: Some(1),
signal: None,
})
);
}
#[tokio::test(start_paused = true)]
async fn crash_loop_budget_check_fires_before_script_exhaustion_at_real_default() {
let (events, mut rx) = tokio::sync::broadcast::channel(1024);
let runner = ScriptedRunner::new((0..20).map(|_| ProcScript::const_exit(1)).collect());
let dir = tempfile::tempdir().unwrap();
let handle = spawn_supervisor(runner, test_paths(&dir), events);
let app = AppConfig::minimal("crash-default", "./boom"); handle.start(vec![normalize(app).unwrap()]).await.unwrap();
await_event(&mut rx, 0, ProcessEventKind::Errored).await;
let list = handle.list().await;
assert_eq!(list[0].status, ProcStatus::Errored);
assert_eq!(list[0].restarts, 15);
}
#[tokio::test(start_paused = true)]
async fn stable_run_resets_budget() {
let (events, mut rx) = tokio::sync::broadcast::channel(256);
let mut script = vec![
ProcScript::const_exit(1),
ProcScript::const_exit(1),
ProcScript::const_exit(1),
];
script.push(ProcScript::stable_then_exit(2000, 1)); script.extend((0..16).map(|_| ProcScript::const_exit(1)));
let runner = ScriptedRunner::new(script);
let dir = tempfile::tempdir().unwrap();
let handle = spawn_supervisor(runner, test_paths(&dir), events);
let app = AppConfig::minimal("flappy", "./f");
handle.start(vec![normalize(app).unwrap()]).await.unwrap();
await_event(&mut rx, 0, ProcessEventKind::Errored).await;
let list = handle.list().await;
assert_eq!(list[0].status, ProcStatus::Errored);
assert_eq!(list[0].restarts, 19);
}
#[tokio::test(start_paused = true)]
async fn manual_stop_prevents_restart() {
let (events, _rx) = tokio::sync::broadcast::channel(64);
let runner = ScriptedRunner::new(vec![ProcScript {
delay_ms: u64::MAX,
outcome: ExitOutcome {
code: None,
signal: None,
},
obeys_signal: true,
obeys_kill: true,
lamb_holds_the_pipe: false,
reads_stdin: true,
}]);
let dir = tempfile::tempdir().unwrap();
let handle = spawn_supervisor(runner, test_paths(&dir), events);
let app = AppConfig::minimal("svc", "./svc");
handle.start(vec![normalize(app).unwrap()]).await.unwrap();
let stopped = handle
.stop(ProcessSelector::Name("svc".to_string()))
.await
.unwrap();
assert_eq!(stopped[0].status, ProcStatus::Stopped); tokio::time::advance(std::time::Duration::from_secs(60)).await;
assert_eq!(handle.list().await[0].status, ProcStatus::Stopped);
}
#[tokio::test(start_paused = true)]
async fn stop_exit_codes_mean_clean_stop() {
let (events, mut rx) = tokio::sync::broadcast::channel(64);
let runner = ScriptedRunner::new(vec![ProcScript::const_exit(0)]);
let dir = tempfile::tempdir().unwrap();
let handle = spawn_supervisor(runner, test_paths(&dir), events);
let mut app = AppConfig::minimal("oneshot", "./job");
app.stop_exit_codes = vec![0];
handle.start(vec![normalize(app).unwrap()]).await.unwrap();
await_event(&mut rx, 0, ProcessEventKind::Stop).await;
assert_eq!(handle.list().await[0].status, ProcStatus::Stopped);
}
#[tokio::test(start_paused = true)]
async fn spawn_failure_surfaces_and_erroreds() {
let (events, _rx) = tokio::sync::broadcast::channel(64);
let runner = ScriptedRunner::new(vec![]); let dir = tempfile::tempdir().unwrap();
let handle = spawn_supervisor(runner, test_paths(&dir), events);
let app = AppConfig::minimal("ghost", "./missing");
let err = handle
.start(vec![normalize(app).unwrap()])
.await
.unwrap_err();
assert!(matches!(err, SupervisorError::SpawnFailed(_)));
assert_eq!(handle.list().await[0].status, ProcStatus::Errored);
}
#[tokio::test(start_paused = true)]
async fn an_unresolvable_user_fails_the_start() {
let dir = tempfile::tempdir().unwrap();
let (events, _rx) = tokio::sync::broadcast::channel(64);
let handle = spawn_supervisor(
ScriptedRunner::new(vec![ProcScript::never_exits()]),
test_paths(&dir),
events,
);
let mut app = AppConfig::minimal("svc", "./svc");
app.user = Some("definitely-not-a-real-shep-user".to_string());
let err = handle
.start(vec![normalize(app).unwrap()])
.await
.unwrap_err();
assert!(matches!(err, SupervisorError::SpawnFailed(_)));
}
#[tokio::test(start_paused = true)]
async fn delete_and_selectors_route() {
let (events, _rx) = tokio::sync::broadcast::channel(64);
let runner =
ScriptedRunner::new(vec![ProcScript::never_exits(), ProcScript::never_exits()]);
let dir = tempfile::tempdir().unwrap();
let handle = spawn_supervisor(runner, test_paths(&dir), events);
let mut a = AppConfig::minimal("api", "./a");
a.fold = Some("backend".to_string());
let b = AppConfig::minimal("web", "./w");
handle
.start(vec![normalize(a).unwrap(), normalize(b).unwrap()])
.await
.unwrap();
let hits = handle
.stop(ProcessSelector::Fold("backend".to_string()))
.await
.unwrap();
assert_eq!(hits.len(), 1);
assert_eq!(hits[0].name, "api");
let deleted = handle
.delete(ProcessSelector::Name("web".to_string()))
.await
.unwrap();
assert_eq!(deleted, vec![1]);
assert_eq!(handle.list().await.len(), 1);
}
#[tokio::test(start_paused = true)]
async fn manual_restart_resets_budget_and_respawns() {
let (events, _rx) = tokio::sync::broadcast::channel(64);
let runner = ScriptedRunner::new(vec![
ProcScript::const_exit(1),
ProcScript::const_exit(1),
ProcScript::never_exits(),
ProcScript::const_exit(1),
ProcScript::never_exits(),
]);
let dir = tempfile::tempdir().unwrap();
let handle = spawn_supervisor(runner, test_paths(&dir), events);
let mut app = AppConfig::minimal("svc", "./svc");
app.max_restarts = 3;
handle.start(vec![normalize(app).unwrap()]).await.unwrap();
loop {
let info = handle.list().await.remove(0);
if info.restarts == 2 && info.status == ProcStatus::Online {
break;
}
tokio::task::yield_now().await;
}
let restarted = handle
.restart(ProcessSelector::Name("svc".to_string()))
.await
.unwrap();
assert_eq!(restarted[0].status, ProcStatus::Online);
let mut settled = handle.list().await.remove(0);
for _ in 0..200 {
if settled.status == ProcStatus::Errored
|| (settled.status == ProcStatus::Online && settled.restarts == 4)
{
break;
}
tokio::task::yield_now().await;
settled = handle.list().await.remove(0);
}
assert_eq!(
(settled.status, settled.restarts),
(ProcStatus::Online, 4),
"an operator's restart left the two spent unstable exits on the \
books -- got {settled:?}"
);
}
#[tokio::test(start_paused = true)]
async fn an_automatic_restart_resets_the_budget_like_an_operators_does() {
let (events, _rx) = tokio::sync::broadcast::channel(64);
let runner = ScriptedRunner::new(vec![
ProcScript::const_exit(1),
ProcScript::const_exit(1),
ProcScript::never_exits(),
ProcScript::const_exit(1),
ProcScript::never_exits(),
]);
let dir = tempfile::tempdir().unwrap();
let handle = spawn_supervisor(runner, test_paths(&dir), events);
let mut app = AppConfig::minimal("svc", "./svc");
app.max_restarts = 3;
handle.start(vec![normalize(app).unwrap()]).await.unwrap();
loop {
let info = handle.list().await.remove(0);
if info.restarts == 2 && info.status == ProcStatus::Online {
break;
}
tokio::task::yield_now().await;
}
handle
.restart_automatic(ProcessSelector::Name("svc".to_string()))
.await
.unwrap();
let mut settled = handle.list().await.remove(0);
for _ in 0..200 {
if settled.status == ProcStatus::Errored
|| (settled.status == ProcStatus::Online && settled.restarts == 4)
{
break;
}
tokio::task::yield_now().await;
settled = handle.list().await.remove(0);
}
assert_eq!(
(settled.status, settled.restarts),
(ProcStatus::Online, 4),
"an automatic restart left the two spent unstable exits on the \
books -- got {settled:?}"
);
}
#[tokio::test(start_paused = true)]
async fn restarting_a_stopped_sheep_resets_the_budget() {
let (events, _rx) = tokio::sync::broadcast::channel(64);
let runner = ScriptedRunner::new(vec![
ProcScript::const_exit(1),
ProcScript::const_exit(1),
ProcScript::never_exits(),
ProcScript::const_exit(1),
ProcScript::never_exits(),
]);
let dir = tempfile::tempdir().unwrap();
let handle = spawn_supervisor(runner, test_paths(&dir), events);
let mut app = AppConfig::minimal("svc", "./svc");
app.max_restarts = 3;
handle.start(vec![normalize(app).unwrap()]).await.unwrap();
loop {
let info = handle.list().await.remove(0);
if info.restarts == 2 && info.status == ProcStatus::Online {
break;
}
tokio::task::yield_now().await;
}
let stopped = handle
.stop(ProcessSelector::Name("svc".to_string()))
.await
.unwrap();
assert_eq!(stopped[0].status, ProcStatus::Stopped);
let restarted = handle
.restart(ProcessSelector::Name("svc".to_string()))
.await
.unwrap();
assert_eq!(
(restarted[0].status, restarted[0].restarts),
(ProcStatus::Online, 3)
);
let mut settled = handle.list().await.remove(0);
for _ in 0..200 {
if settled.status == ProcStatus::Errored
|| (settled.status == ProcStatus::Online && settled.restarts == 4)
{
break;
}
tokio::task::yield_now().await;
settled = handle.list().await.remove(0);
}
assert_eq!(
(settled.status, settled.restarts),
(ProcStatus::Online, 4),
"restarting a stopped sheep left the two spent unstable exits on \
the books -- got {settled:?}"
);
}
#[tokio::test(start_paused = true)]
async fn shutdown_kills_all_and_stops_the_engine() {
let (events, _rx) = tokio::sync::broadcast::channel(64);
let runner =
ScriptedRunner::new(vec![ProcScript::never_exits(), ProcScript::never_exits()]);
let dir = tempfile::tempdir().unwrap();
let handle = spawn_supervisor(runner, test_paths(&dir), events);
let mut app = AppConfig::minimal("web", "./srv");
app.instances = 2;
handle.start(vec![normalize(app).unwrap()]).await.unwrap();
handle.shutdown().await; assert!(handle.list_checked().await.is_err());
}
async fn drain_kinds(
rx: &mut tokio::sync::broadcast::Receiver<BusEvent>,
) -> Vec<(u32, ProcessEventKind)> {
let mut out = Vec::new();
while let Ok(ev) = rx.try_recv() {
if let BusEvent::Process { event, info, .. } = ev {
out.push((info.id, event));
}
}
out
}
#[tokio::test(start_paused = true)]
async fn shutdown_ignores_a_pending_restart_timer() {
let (events, mut rx) = tokio::sync::broadcast::channel(1024);
let runner = ScriptedRunner::new(vec![
ProcScript::const_exit(1), ProcScript::ignores_signals(), ProcScript::never_exits(), ]);
let dir = tempfile::tempdir().unwrap();
let handle = spawn_supervisor(runner, test_paths(&dir), events);
let mut crash = AppConfig::minimal("crash", "./boom");
crash.exp_backoff_restart_delay = Some("500".parse().unwrap());
let web = AppConfig::minimal("web", "./srv");
handle
.start(vec![normalize(crash).unwrap(), normalize(web).unwrap()])
.await
.unwrap();
await_event(&mut rx, 0, ProcessEventKind::Exit).await;
handle.shutdown().await;
let seen = drain_kinds(&mut rx).await;
let ghost = seen
.iter()
.any(|(id, k)| *id == 0 && *k == ProcessEventKind::Restart);
assert!(
!ghost,
"GHOST RESPAWN during shutdown: events after shutdown = {seen:?}"
);
}
#[tokio::test(start_paused = true)]
async fn shutdown_ignores_a_pending_readiness_wait() {
let (events, mut rx) = tokio::sync::broadcast::channel(1024);
let runner = ScriptedRunner::new(vec![ProcScript::ignores_signals()]);
let dir = tempfile::tempdir().unwrap();
let handle = spawn_supervisor(runner, test_paths(&dir), events);
let mut app = AppConfig::minimal("gated", "./g");
app.wait_ready = true; app.listen_timeout = UpDuration::from_millis(1_000);
handle.start(vec![normalize(app).unwrap()]).await.unwrap();
assert_eq!(
handle.list().await[0].status,
ProcStatus::Starting,
"the readiness wait has to be armed for its result to be droppable"
);
handle.shutdown().await;
let seen = drain_kinds(&mut rx).await;
assert!(
seen.contains(&(0, ProcessEventKind::Stop)),
"the kill ladder must have outlasted the readiness deadline: events = {seen:?}"
);
assert!(
!seen.contains(&(0, ProcessEventKind::Online)),
"a readiness wait resolved during shutdown marked a dying sheep online: \
events = {seen:?}"
);
}
#[tokio::test(start_paused = true)]
async fn late_start_racing_shutdown_never_orphans() {
let (events, mut rx) = tokio::sync::broadcast::channel(1024);
let runner = ScriptedRunner::new(vec![
ProcScript::ignores_signals(), ProcScript::never_exits(), ]);
let dir = tempfile::tempdir().unwrap();
let handle = spawn_supervisor(runner, test_paths(&dir), events);
let web = AppConfig::minimal("web", "./srv");
handle.start(vec![normalize(web).unwrap()]).await.unwrap();
let h2 = handle.clone();
let late = tokio::spawn(async move {
let app = AppConfig::minimal("late", "./l");
h2.start(vec![normalize(app).unwrap()]).await
});
handle.shutdown().await;
let outcome = late.await.unwrap();
let seen = drain_kinds(&mut rx).await;
match outcome {
Err(SupervisorError::EngineStopped) => {} Ok(infos) => {
let late_id = infos[0].id;
assert!(
seen.iter().any(|(id, k)| *id == late_id
&& matches!(k, ProcessEventKind::Stop | ProcessEventKind::Exit)),
"late Start raced ahead of shutdown but was never killed: events = {seen:?}"
);
}
Err(other) => panic!("unexpected error from a late Start during shutdown: {other:?}"),
}
}
#[tokio::test(start_paused = true)]
async fn stale_restart_timer_never_short_circuits_a_newer_backoff() {
let (events, mut rx) = tokio::sync::broadcast::channel(1024);
let runner = ScriptedRunner::new(vec![
ProcScript::const_exit(1), ProcScript::stable_then_exit(1500, 1), ProcScript::never_exits(), ProcScript::never_exits(),
]);
let dir = tempfile::tempdir().unwrap();
let handle = spawn_supervisor(runner, test_paths(&dir), events);
let mut app = AppConfig::minimal("crash", "./boom");
app.exp_backoff_restart_delay = Some("2000".parse().unwrap());
app.min_uptime = "5000".parse().unwrap(); handle.start(vec![normalize(app).unwrap()]).await.unwrap();
await_event(&mut rx, 0, ProcessEventKind::Exit).await;
let out = handle.restart(ProcessSelector::All).await.unwrap();
assert_eq!(
out[0].status,
ProcStatus::Online,
"manual restart respawned"
);
tokio::time::advance(Duration::from_millis(2500)).await;
for _ in 0..20 {
tokio::task::yield_now().await;
}
let info = handle.list().await.remove(0);
assert_eq!(
(info.status, info.restarts),
(ProcStatus::WaitingRestart, 1),
"at t=2500 the sheep should still be waiting on its 3500ms backoff; \
got {info:?} -- the stale 2000ms timer fired early"
);
}
#[tokio::test(start_paused = true)]
async fn overlapping_stop_and_restart_agree_on_one_outcome() {
let (events, _rx) = tokio::sync::broadcast::channel(1024);
let runner = ScriptedRunner::new(vec![
ProcScript::ignores_signals(), ProcScript::never_exits(),
]);
let dir = tempfile::tempdir().unwrap();
let handle = spawn_supervisor(runner, test_paths(&dir), events);
let app = AppConfig::minimal("svc", "./svc");
handle.start(vec![normalize(app).unwrap()]).await.unwrap();
let h2 = handle.clone();
let stopper = tokio::spawn(async move { h2.stop(ProcessSelector::All).await });
for _ in 0..10 {
tokio::task::yield_now().await;
}
let restarted = handle.restart(ProcessSelector::All).await.unwrap();
let stopped = stopper.await.unwrap().unwrap();
assert_eq!(
stopped[0].status,
ProcStatus::Stopped,
"stop() reported a non-stopped sheep"
);
assert_eq!(
restarted[0].status,
ProcStatus::Stopped,
"restart() lost the race to the earlier stop() but got a different \
answer than the stop() caller -- the two callers disagree about \
what happened to the same sheep"
);
}
#[tokio::test(start_paused = true)]
async fn actor_never_blocks_behind_a_busy_kill_ladder() {
let (events, mut rx) = tokio::sync::broadcast::channel(1024);
let runner = ScriptedRunner::new(vec![
ProcScript::ignores_signals(), ProcScript::stable_then_exit(800, 0), ]);
let dir = tempfile::tempdir().unwrap();
let handle = spawn_supervisor(runner, test_paths(&dir), events);
let a = AppConfig::minimal("a", "./a");
let mut b = AppConfig::minimal("b", "./b");
b.autorestart = false;
handle
.start(vec![normalize(a).unwrap(), normalize(b).unwrap()])
.await
.unwrap();
let t0 = tokio::time::Instant::now();
let mut tasks = Vec::new();
for _ in 0..8 {
let h = handle.clone();
tasks.push(tokio::spawn(async move {
h.stop(ProcessSelector::Name("a".to_string())).await
}));
}
await_event(&mut rx, 1, ProcessEventKind::Stop).await;
let seen_at = t0.elapsed();
for t in tasks {
let _ = t.await;
}
assert!(
seen_at < Duration::from_millis(1000),
"sheep b's own exit was only processed at {seen_at:?} -- the actor \
was parked inside ctl.send() for sheep a's kill ladder"
);
}
#[tokio::test(start_paused = true)]
async fn mailbox_flood_during_a_kill_never_deadlocks() {
let (events, mut rx) = tokio::sync::broadcast::channel(4096);
let runner = ScriptedRunner::new(vec![ProcScript::ignores_signals()]);
let dir = tempfile::tempdir().unwrap();
let handle = spawn_supervisor(runner, test_paths(&dir), events);
let a = AppConfig::minimal("a", "./a");
handle.start(vec![normalize(a).unwrap()]).await.unwrap();
for _ in 0..8 {
let h = handle.clone();
tokio::spawn(async move {
let _ = h.stop(ProcessSelector::All).await;
});
}
for _ in 0..40 {
tokio::task::yield_now().await;
}
for _ in 0..400 {
let h = handle.clone();
tokio::spawn(async move {
let _ = h.list_checked().await;
});
}
for _ in 0..200 {
tokio::task::yield_now().await;
}
let r = tokio::time::timeout(
Duration::from_secs(600),
await_event(&mut rx, 0, ProcessEventKind::Stop),
)
.await;
assert!(
r.is_ok(),
"DEADLOCK: actor parked in ctl.send() while the sheep task is \
parked in actor_tx.send() -- the daemon never recovers"
);
}
#[tokio::test(start_paused = true)]
async fn delete_racing_shutdown_still_deregisters_the_sheep() {
let (events, _rx) = tokio::sync::broadcast::channel(1024);
let runner = ScriptedRunner::new(vec![
ProcScript::ignores_signals(), ProcScript::ignores_signals(), ]);
let dir = tempfile::tempdir().unwrap();
let handle = spawn_supervisor(runner, test_paths(&dir), events);
let target = AppConfig::minimal("svc", "./svc");
let mut decoy = AppConfig::minimal("decoy", "./decoy");
decoy.kill_timeout = "600000".parse().unwrap(); let started = handle
.start(vec![normalize(target).unwrap(), normalize(decoy).unwrap()])
.await
.unwrap();
let id = started
.iter()
.find(|info| info.name == "svc")
.expect("target sheep registered")
.id;
let h2 = handle.clone();
let shutter = tokio::spawn(async move { h2.shutdown().await });
for _ in 0..10 {
tokio::task::yield_now().await; }
let deleted = handle.delete(ProcessSelector::Id(id)).await.unwrap();
assert_eq!(deleted, vec![id], "the caller was told this id was deleted");
assert!(
handle.list().await.iter().all(|info| info.id != id),
"a Delete that raced a Shutdown must still deregister the sheep, \
not just tell its caller it did"
);
drop(shutter);
}
#[tokio::test(start_paused = true)]
async fn delete_racing_restart_still_deregisters_the_sheep() {
let (events, _rx) = tokio::sync::broadcast::channel(1024);
let runner = ScriptedRunner::new(vec![
ProcScript::ignores_signals(), ProcScript::never_exits(),
]);
let dir = tempfile::tempdir().unwrap();
let handle = spawn_supervisor(runner, test_paths(&dir), events);
let app = AppConfig::minimal("svc", "./svc");
let started = handle.start(vec![normalize(app).unwrap()]).await.unwrap();
let id = started[0].id;
let h2 = handle.clone();
let restarter = tokio::spawn(async move { h2.restart(ProcessSelector::All).await });
for _ in 0..10 {
tokio::task::yield_now().await; }
let deleted = handle.delete(ProcessSelector::Id(id)).await.unwrap();
let restarted = restarter.await.unwrap().unwrap();
assert_eq!(deleted, vec![id], "the caller was told this id was deleted");
assert_eq!(restarted[0].id, id);
assert_eq!(
restarted[0].status,
ProcStatus::Stopped,
"restart() must not report a respawned process once a racing \
Delete has claimed this id -- got {restarted:?}"
);
assert!(
handle.list().await.iter().all(|info| info.id != id),
"a Delete that raced a Restart must still deregister the sheep, \
not respawn a brand-new live process while telling its caller \
the sheep was deleted"
);
}
#[tokio::test(start_paused = true)]
async fn an_operators_stop_beats_an_automatic_restart_mid_ladder() {
let (events, _rx) = tokio::sync::broadcast::channel(1024);
let runner = ScriptedRunner::new(vec![
ProcScript::ignores_signals(), ProcScript::never_exits(), ]);
let dir = tempfile::tempdir().unwrap();
let handle = spawn_supervisor(runner, test_paths(&dir), events);
let app = AppConfig::minimal("svc", "./svc");
handle.start(vec![normalize(app).unwrap()]).await.unwrap();
let running = handle.list().await.remove(0);
let pid = running.pid.expect("an online sheep has a pid");
handle.extra_restart(running.id, pid).await;
let stopped = handle.stop(ProcessSelector::All).await.unwrap();
assert_eq!(stopped.len(), 1);
assert_eq!(stopped[0].id, running.id);
assert_eq!(
(stopped[0].status, stopped[0].restarts),
(ProcStatus::Stopped, 0),
"an operator's stop was silently converted into the automatic \
restart it raced -- got {stopped:?}"
);
let listed = handle.list().await;
assert_eq!(
(listed[0].status, listed[0].pid),
(ProcStatus::Stopped, None),
"the sheep an operator stopped is running again -- got {listed:?}"
);
}
#[tokio::test(start_paused = true)]
async fn an_operators_delete_beats_an_automatic_restart_mid_ladder() {
let (events, _rx) = tokio::sync::broadcast::channel(1024);
let runner = ScriptedRunner::new(vec![
ProcScript::ignores_signals(),
ProcScript::never_exits(),
]);
let dir = tempfile::tempdir().unwrap();
let handle = spawn_supervisor(runner, test_paths(&dir), events);
let app = AppConfig::minimal("svc", "./svc");
handle.start(vec![normalize(app).unwrap()]).await.unwrap();
let running = handle.list().await.remove(0);
let pid = running.pid.expect("an online sheep has a pid");
handle.extra_restart(running.id, pid).await;
let deleted = handle
.delete(ProcessSelector::Id(running.id))
.await
.unwrap();
assert_eq!(deleted, vec![running.id]);
assert!(
handle.list().await.is_empty(),
"a delete that raced an automatic restart must still deregister \
the sheep, not leave one behind for the restart to bring back"
);
}
#[tokio::test(start_paused = true)]
async fn an_operators_stop_still_shows_its_signal_as_the_last_exit() {
let (events, _rx) = tokio::sync::broadcast::channel(64);
let runner = ScriptedRunner::new(vec![ProcScript::never_exits()]);
let dir = tempfile::tempdir().unwrap();
let handle = spawn_supervisor(runner, test_paths(&dir), events);
let app = AppConfig::minimal("svc", "./svc");
handle.start(vec![normalize(app).unwrap()]).await.unwrap();
let stopped = handle.stop(ProcessSelector::All).await.unwrap();
assert_eq!(stopped.len(), 1);
assert_eq!(stopped[0].status, ProcStatus::Stopped);
assert_eq!(
stopped[0].last_exit,
Some(ExitInfo {
code: None,
signal: Some(15),
}),
"an operator's own stop must still show up as a last exit: {stopped:?}"
);
}
#[tokio::test(start_paused = true)]
async fn a_respawn_that_fails_to_spawn_clears_the_previous_exit_code() {
let (events, _rx) = tokio::sync::broadcast::channel(64);
let runner = ScriptedRunner::new(vec![ProcScript::const_exit(1)]);
let dir = tempfile::tempdir().unwrap();
let handle = spawn_supervisor(runner, test_paths(&dir), events);
let app = AppConfig::minimal("svc", "./svc");
handle.start(vec![normalize(app).unwrap()]).await.unwrap();
tokio::time::sleep(Duration::from_secs(30)).await;
let listing = handle.list().await;
assert_eq!(listing.len(), 1);
assert_eq!(
listing[0].status,
ProcStatus::Errored,
"a respawn that cannot spawn is still terminal: {listing:?}"
);
assert_eq!(
listing[0].last_exit, None,
"nothing exited on the failed respawn, so the earlier code must \
not still be showing: {listing:?}"
);
}
fn actor_with_stopping_drainee(
dir: &tempfile::TempDir,
pid: u32,
epoch: u64,
) -> (Actor<ScriptedRunner>, mpsc::Receiver<SheepCtl>) {
let paths = test_paths(dir);
let app = normalize(AppConfig::minimal("web", "./srv")).unwrap();
let mut entry = armed_entry(0, 0, pid, app, &paths);
entry.status = ProcStatus::Stopping;
let (ctl_tx, ctl_rx) = mpsc::channel(1);
let slot = SheepSlot {
entry,
ctl: Some(ctl_tx),
log_ctl: None,
to_child: None,
signals: None,
to_stdin: None,
manual: None,
pending_delete: false,
epoch,
ready_tx: None,
actions: ActionWaits::default(),
};
let mut sheep = HashMap::new();
sheep.insert(0, slot);
let (events, _events_rx) = broadcast::channel(16);
let (tx, _rx) = mpsc::channel(16);
let actor = Actor {
runner: ScriptedRunner::new(vec![]),
paths,
events,
tx,
sheep,
next_id: 1,
next_action_stamp: 0,
pending: Vec::new(),
shutting_down: false,
extras: None,
registry: ExtrasRegistry::default(),
reloads: HashMap::new(),
smits: Smits::new(),
};
(actor, ctl_rx)
}
fn actor_with_a_drainee_holding_a_signal_mailbox(
dir: &tempfile::TempDir,
) -> (Actor<ScriptedRunner>, mpsc::Receiver<SignalRequest>) {
let (mut actor, _mailbox) = actor_with_one_online_sheep(dir, vec![]);
let slot = actor.sheep.get_mut(&0).expect("the fixture registers id 0");
slot.entry.status = ProcStatus::Stopping;
slot.entry.reload = ReloadState::Drainee { new_id: 1 };
let (signals, signal_rx) = mpsc::channel(16);
slot.signals = Some(signals);
(actor, signal_rx)
}
#[test]
fn a_stopping_sheep_rejects_an_extra_restart() {
let dir = tempfile::tempdir().unwrap();
let (mut actor, mut ctl_rx) = actor_with_stopping_drainee(&dir, 4242, 0);
actor.handle_extra_restart(0, 4242);
let slot = actor.sheep.get(&0).expect("the sheep stays registered");
assert_eq!(
slot.entry.status,
ProcStatus::Stopping,
"a rejected extra restart must never touch status"
);
assert!(
slot.manual.is_none(),
"a Stopping sheep must never claim the manual marker off an extra restart"
);
assert!(
ctl_rx.try_recv().is_err(),
"a Stopping sheep must never receive a second Kill"
);
}
#[test]
fn a_stopping_sheep_rejects_a_restart_due() {
let dir = tempfile::tempdir().unwrap();
let (mut actor, _ctl_rx) = actor_with_stopping_drainee(&dir, 4242, 7);
actor.handle_restart_due(0, 7);
let slot = actor.sheep.get(&0).expect("the sheep stays registered");
assert_eq!(
slot.entry.status,
ProcStatus::Stopping,
"a rejected restart-due must never touch status"
);
assert_eq!(
slot.entry.restarts, 0,
"a rejected restart-due must never respawn"
);
}
const SWAP_WINDOW: Duration = Duration::from_secs(30);
async fn expect_event(
rx: &mut tokio::sync::broadcast::Receiver<BusEvent>,
id: u32,
kind: ProcessEventKind,
) {
assert!(
tokio::time::timeout(SWAP_WINDOW, await_event(rx, id, kind))
.await
.is_ok(),
"no {kind:?} for id {id} within {SWAP_WINDOW:?}"
);
}
async fn started(
dir: &tempfile::TempDir,
app: AppConfig,
scripts: Vec<ProcScript>,
) -> (
SupervisorHandle,
Arc<ScriptedRunner>,
tokio::sync::broadcast::Receiver<BusEvent>,
) {
let (events, rx) = tokio::sync::broadcast::channel(256);
let runner = Arc::new(ScriptedRunner::new(scripts));
let handle = spawn_supervisor(SharedRunner(Arc::clone(&runner)), test_paths(dir), events);
handle.start(vec![normalize(app).unwrap()]).await.unwrap();
(handle, runner, rx)
}
fn actor_with_one_online_sheep(
dir: &tempfile::TempDir,
scripts: Vec<ProcScript>,
) -> (Actor<ScriptedRunner>, mpsc::Receiver<Msg>) {
let paths = test_paths(dir);
let app = normalize(AppConfig::minimal("web", "./srv")).unwrap();
let mut sheep = HashMap::new();
sheep.insert(
0,
SheepSlot {
entry: armed_entry(0, 0, 1111, app, &paths),
ctl: None,
log_ctl: None,
to_child: None,
signals: None,
to_stdin: None,
manual: None,
pending_delete: false,
epoch: 0,
ready_tx: None,
actions: ActionWaits::default(),
},
);
let (events, _events_rx) = broadcast::channel(64);
let (tx, rx) = mpsc::channel(MAILBOX_CAPACITY);
let actor = Actor {
runner: ScriptedRunner::new(scripts),
paths,
events,
tx,
sheep,
next_id: 1,
next_action_stamp: 0,
pending: Vec::new(),
shutting_down: false,
extras: None,
registry: ExtrasRegistry::default(),
reloads: HashMap::new(),
smits: Smits::new(),
};
(actor, rx)
}
const SHEEP_ID: u32 = 0;
const DOG_ID: u32 = 1;
fn actor_with_a_sheep_and_a_dog(
dir: &tempfile::TempDir,
) -> (Actor<ScriptedRunner>, mpsc::Receiver<Msg>) {
let paths = test_paths(dir);
let mut sheep = HashMap::new();
for (id, name, dog) in [
(SHEEP_ID, "web", None),
(DOG_ID, "bark", Some(DogSource::BuiltIn)),
] {
let app = app_with(name, |config| config.fold = Some("svc".to_string()));
let mut entry = armed_entry(id, 0, 1111 + id, app, &paths);
entry.dog = dog;
sheep.insert(
id,
SheepSlot {
entry,
ctl: None,
log_ctl: None,
to_child: None,
signals: None,
to_stdin: None,
manual: None,
pending_delete: false,
epoch: 0,
ready_tx: None,
actions: ActionWaits::default(),
},
);
}
let (events, _events_rx) = broadcast::channel(64);
let (tx, rx) = mpsc::channel(MAILBOX_CAPACITY);
let actor = Actor {
runner: ScriptedRunner::new(Vec::new()),
paths,
events,
tx,
sheep,
next_id: DOG_ID + 1,
next_action_stamp: 0,
pending: Vec::new(),
shutting_down: false,
extras: None,
registry: ExtrasRegistry::default(),
reloads: HashMap::new(),
smits: Smits::new(),
};
(actor, rx)
}
#[test]
fn a_wildcard_passes_a_dog_by_and_its_own_name_still_reaches_it() {
let dir = tempfile::tempdir().unwrap();
let (actor, _mailbox) = actor_with_a_sheep_and_a_dog(&dir);
assert_eq!(
actor.matching_ids(&ProcessSelector::All),
vec![SHEEP_ID],
"`all` is the flock, not the kennel"
);
assert_eq!(
actor.matching_ids(&ProcessSelector::parse("/^(web|bark)$/").unwrap()),
vec![SHEEP_ID],
"a sweep that spells both names out is still a sweep"
);
assert_eq!(
actor.matching_ids(&ProcessSelector::Fold("svc".into())),
vec![SHEEP_ID],
"a dog shares its fold with the flock and is still not swept by it"
);
assert_eq!(
actor.matching_ids(&ProcessSelector::Name("bark".into())),
vec![DOG_ID]
);
assert_eq!(
actor.matching_ids(&ProcessSelector::Id(DOG_ID)),
vec![DOG_ID]
);
}
#[test]
fn a_listing_reports_where_a_dog_came_from() {
let dir = tempfile::tempdir().unwrap();
let (actor, _mailbox) = actor_with_a_sheep_and_a_dog(&dir);
assert_eq!(
to_info(&actor.sheep[&DOG_ID].entry, &actor.smits).dog,
Some(DogSource::BuiltIn)
);
assert_eq!(
to_info(&actor.sheep[&SHEEP_ID].entry, &actor.smits).dog,
None
);
}
async fn start_app(h: &Harness, app: AppConfig) -> Vec<ProcessInfo> {
h.ctx
.supervisor
.start(vec![normalize(app).unwrap()])
.await
.unwrap()
}
async fn instance_slots_of(h: &Harness, name: &str) -> Vec<u32> {
let mut slots: Vec<u32> = h
.ctx
.supervisor
.list()
.await
.iter()
.filter(|info| info.name == name)
.map(|info| {
let out = info
.out_file
.as_deref()
.expect("a listed sheep has a log path");
let stem = std::path::Path::new(out)
.file_name()
.and_then(std::ffi::OsStr::to_str)
.and_then(|file| file.strip_suffix("-out.log"))
.and_then(|stem| stem.strip_prefix(&format!("{name}-")))
.expect("a derived log path is `<name>-<instance>-out.log`");
stem.parse().expect("the instance slot is a number")
})
.collect();
slots.sort_unstable();
slots
}
async fn settle_to(h: &Harness, name: &str, count: usize) {
let settled = tokio::time::timeout(SWAP_WINDOW, async {
loop {
let live = h
.ctx
.supervisor
.list()
.await
.iter()
.filter(|info| info.name == name)
.count();
if live == count {
return;
}
tokio::task::yield_now().await;
}
})
.await;
assert!(settled.is_ok(), "{name} never settled to {count} instances");
}
fn actor_with_a_scaled_app(
dir: &tempfile::TempDir,
instances: u32,
scripts: Vec<ProcScript>,
) -> Actor<ScriptedRunner> {
let paths = test_paths(dir);
let app = normalize(AppConfig {
instances,
..AppConfig::minimal("web", "./srv")
})
.unwrap();
let mut sheep = HashMap::new();
for instance in 0..instances {
sheep.insert(
instance,
SheepSlot {
entry: armed_entry(instance, instance, 1111 + instance, app.clone(), &paths),
ctl: None,
log_ctl: None,
to_child: None,
signals: None,
to_stdin: None,
manual: None,
pending_delete: false,
epoch: 0,
ready_tx: None,
actions: ActionWaits::default(),
},
);
}
let (events, _events_rx) = broadcast::channel(64);
let (tx, _rx) = mpsc::channel(MAILBOX_CAPACITY);
Actor {
runner: ScriptedRunner::new(scripts),
paths,
events,
tx,
sheep,
next_id: instances,
next_action_stamp: 0,
pending: Vec::new(),
shutting_down: false,
extras: None,
registry: ExtrasRegistry::default(),
reloads: HashMap::new(),
smits: Smits::new(),
}
}
fn stored_instance_counts(actor: &Actor<ScriptedRunner>) -> Vec<u32> {
let mut ids: Vec<u32> = actor.sheep.keys().copied().collect();
ids.sort_unstable();
ids.iter()
.map(|id| actor.sheep[id].entry.spec.config().instances)
.collect()
}
fn reload_job_for(name: &str) -> ReloadJob {
let _ = name;
ReloadJob {
queue: VecDeque::new(),
swap: ReloadSwap {
old_id: 0,
new_id: 1,
phase: ReloadPhase::AwaitReady,
},
}
}
#[tokio::test(start_paused = true)]
async fn scaling_up_fills_the_lowest_free_slots() {
let h = harness(vec![ProcScript::never_exits(); 4]);
start_app(
&h,
AppConfig {
instances: 2,
..AppConfig::minimal("web", "./srv")
},
)
.await;
let scaled = h.ctx.supervisor.scale("web", 4).await.unwrap();
assert_eq!(scaled.instances.len(), 4);
assert_eq!(instance_slots_of(&h, "web").await, vec![0, 1, 2, 3]);
}
#[tokio::test(start_paused = true)]
async fn scaling_down_removes_the_highest_slots_so_a_round_trip_returns() {
let h = harness(vec![ProcScript::never_exits(); 4]);
start_app(
&h,
AppConfig {
instances: 2,
..AppConfig::minimal("web", "./srv")
},
)
.await;
h.ctx.supervisor.scale("web", 4).await.unwrap();
let scaled = h.ctx.supervisor.scale("web", 2).await.unwrap();
assert_eq!(scaled.instances.len(), 2);
settle_to(&h, "web", 2).await;
assert_eq!(instance_slots_of(&h, "web").await, vec![0, 1]);
}
#[tokio::test(start_paused = true)]
async fn scaling_to_the_current_count_is_a_no_op() {
let h = harness(vec![ProcScript::never_exits(); 2]);
start_app(
&h,
AppConfig {
instances: 2,
..AppConfig::minimal("web", "./srv")
},
)
.await;
let before = h.ctx.supervisor.list().await;
let scaled = h.ctx.supervisor.scale("web", 2).await.unwrap();
assert_eq!(scaled.instances.len(), 2);
let after = h.ctx.supervisor.list().await;
assert_eq!(
after.iter().map(|i| i.id).collect::<Vec<_>>(),
before.iter().map(|i| i.id).collect::<Vec<_>>(),
"a no-op scale replaced processes"
);
}
#[tokio::test(start_paused = true)]
async fn scaling_to_zero_is_refused_and_names_delete() {
let h = harness(vec![ProcScript::never_exits()]);
start_app(&h, AppConfig::minimal("web", "./srv")).await;
let err = h.ctx.supervisor.scale("web", 0).await.unwrap_err();
let SupervisorError::InvalidScale(message) = err else {
panic!("expected InvalidScale, got {err:?}");
};
assert!(message.contains("delete"), "{message}");
}
#[tokio::test(start_paused = true)]
async fn scaling_an_unregistered_app_is_not_found() {
let h = harness(vec![]);
assert_eq!(
h.ctx.supervisor.scale("ghost", 2).await.unwrap_err(),
SupervisorError::NotFound
);
}
#[tokio::test(start_paused = true)]
async fn a_dog_cannot_be_scaled() {
let dir = tempfile::tempdir().unwrap();
let (mut actor, _mailbox) = actor_with_a_sheep_and_a_dog(&dir);
let (reply, answer) = oneshot::channel();
actor.handle_command(Command::Scale {
name: "bark".to_string(),
count: 2,
reply,
});
let SupervisorError::InvalidScale(message) = answer.await.unwrap().unwrap_err() else {
panic!("expected InvalidScale");
};
assert!(message.contains("dog"), "{message}");
}
#[tokio::test(start_paused = true)]
async fn an_app_mid_reload_refuses_a_scale() {
let dir = tempfile::tempdir().unwrap();
let mut actor = actor_with_a_scaled_app(&dir, 2, vec![]);
actor
.reloads
.insert("web".to_string(), reload_job_for("web"));
let (reply, answer) = oneshot::channel();
actor.handle_command(Command::Scale {
name: "web".to_string(),
count: 4,
reply,
});
assert_eq!(
answer.await.unwrap().unwrap_err(),
SupervisorError::ReloadInFlight("web".to_string())
);
}
#[tokio::test(start_paused = true)]
async fn a_scale_is_refused_while_an_earlier_ones_departures_are_still_leaving() {
let h = harness(vec![
ProcScript::never_exits(),
ProcScript::never_reports_its_exit(),
ProcScript::never_reports_its_exit(),
ProcScript::never_reports_its_exit(),
]);
start_app(
&h,
AppConfig {
instances: 4,
..AppConfig::minimal("web", "./srv")
},
)
.await;
let down = h.ctx.supervisor.scale("web", 1).await.unwrap();
assert_eq!(down.instances.len(), 1);
let err = h.ctx.supervisor.scale("web", 4).await.unwrap_err();
let SupervisorError::InvalidScale(message) = err else {
panic!("expected InvalidScale, got {err:?}");
};
assert!(
message.contains("3 instance(s) still shutting down"),
"the refusal has to say how much of the flock is still moving: {message}"
);
assert!(
message.contains("shep flock"),
"the refusal has to name what to wait for: {message}"
);
}
#[tokio::test(start_paused = true)]
async fn a_scale_is_accepted_again_once_the_departures_have_left() {
let h = harness(vec![ProcScript::never_exits(); 7]);
start_app(
&h,
AppConfig {
instances: 4,
..AppConfig::minimal("web", "./srv")
},
)
.await;
h.ctx.supervisor.scale("web", 1).await.unwrap();
settle_to(&h, "web", 1).await;
let up = h.ctx.supervisor.scale("web", 4).await.unwrap();
assert_eq!(up.instances.len(), 4);
assert_eq!(up.shortfall, None);
assert_eq!(instance_slots_of(&h, "web").await, vec![0, 1, 2, 3]);
}
#[tokio::test(start_paused = true)]
async fn a_scale_updates_the_stored_instance_count_on_every_slot() {
let dir = tempfile::tempdir().unwrap();
let mut actor = actor_with_a_scaled_app(&dir, 2, vec![ProcScript::never_exits(); 2]);
let (reply, answer) = oneshot::channel();
actor.handle_command(Command::Scale {
name: "web".to_string(),
count: 4,
reply,
});
let scaled = answer.await.unwrap().unwrap();
assert_eq!(scaled.app.config().instances, 4);
assert_eq!(stored_instance_counts(&actor), vec![4, 4, 4, 4]);
}
#[tokio::test(start_paused = true)]
async fn a_partial_scale_up_stores_the_count_it_achieved() {
let dir = tempfile::tempdir().unwrap();
let mut actor = actor_with_a_scaled_app(&dir, 1, vec![ProcScript::never_exits()]);
let (reply, answer) = oneshot::channel();
actor.handle_command(Command::Scale {
name: "web".to_string(),
count: 3,
reply,
});
let scaled = answer.await.unwrap().expect(
"a partial scale-up is a partial success: an `Err` here would take \
the achieved config with it and leave the roll pre-scale",
);
assert_eq!(scaled.requested, 3);
assert_eq!(scaled.achieved(), 2);
assert!(
scaled.shortfall.is_some(),
"the shortfall has to survive the reply, or nothing downstream can \
tell the operator they got two of three"
);
assert_eq!(scaled.app.config().instances, 2);
assert_eq!(
stored_instance_counts(&actor),
vec![2, 2, 2],
"the flock achieved two, so every registered slot — including the \
errored attempt — must say two"
);
}
#[tokio::test]
async fn a_listing_groups_an_apps_instances_under_its_name() {
let h = harness(vec![ProcScript::never_exits(); 4]);
start_app(
&h,
AppConfig {
instances: 2,
..AppConfig::minimal("zebra", "./z")
},
)
.await;
start_app(
&h,
AppConfig {
instances: 2,
..AppConfig::minimal("alpha", "./a")
},
)
.await;
let listed = h.ctx.supervisor.list().await;
let names: Vec<&str> = listed.iter().map(|i| i.name.as_str()).collect();
assert_eq!(names, ["alpha", "alpha", "zebra", "zebra"]);
let ids: Vec<u32> = listed.iter().map(|i| i.id).collect();
assert_ne!(
ids,
{
let mut sorted = ids.clone();
sorted.sort_unstable();
sorted
},
"the fixture must make id order and name order disagree, or it proves nothing"
);
}
fn dog_app(name: &str) -> ResolvedApp {
normalize(AppConfig::minimal(name, "/nonexistent/shep")).unwrap()
}
fn dog_row(listed: &[ProcessInfo], id: u32) -> ProcessInfo {
listed
.iter()
.find(|info| info.id == id)
.unwrap_or_else(|| panic!("id {id} left the flock: {listed:?}"))
.clone()
}
#[tokio::test(start_paused = true)]
async fn a_dog_that_restarts_is_still_a_dog() {
let dir = tempfile::tempdir().unwrap();
let (events, mut rx) = broadcast::channel(64);
let runner =
ScriptedRunner::new(vec![ProcScript::const_exit(1), ProcScript::never_exits()]);
let handle = spawn_supervisor(runner, test_paths(&dir), events);
let dog = handle
.start_dog(dog_app("bark"), DogSource::BuiltIn)
.await
.unwrap();
assert_eq!(dog.dog, Some(DogSource::BuiltIn));
expect_event(&mut rx, dog.id, ProcessEventKind::Restart).await;
let after = dog_row(&handle.list().await, dog.id);
assert_eq!(
after.restarts, 1,
"the ordinary restart path, not a dog one"
);
assert_eq!(after.dog, Some(DogSource::BuiltIn));
}
#[tokio::test(start_paused = true)]
async fn a_dog_that_cannot_be_spawned_is_still_a_dog() {
let dir = tempfile::tempdir().unwrap();
let (events, _rx) = broadcast::channel(64);
let handle = spawn_supervisor(ScriptedRunner::new(Vec::new()), test_paths(&dir), events);
let failed = handle
.start_dog(dog_app("bark"), DogSource::BuiltIn)
.await
.expect_err("a spawn with no script behind it cannot succeed");
assert!(matches!(failed, SupervisorError::SpawnFailed(_)));
let listed = handle.list().await;
let errored = dog_row(&listed, 0);
assert_eq!(errored.status, ProcStatus::Errored);
assert_eq!(errored.dog, Some(DogSource::BuiltIn));
}
#[tokio::test(start_paused = true)]
async fn a_reloaded_dog_is_still_a_dog() {
let dir = tempfile::tempdir().unwrap();
let (events, mut rx) = broadcast::channel(256);
let runner = ScriptedRunner::new(vec![ProcScript::never_exits(); 3]);
let handle = spawn_supervisor(runner, test_paths(&dir), events);
let dog = handle
.start_dog(dog_app("bark"), DogSource::BuiltIn)
.await
.unwrap();
handle
.reload(ProcessSelector::Name("bark".to_string()))
.await
.expect("a reload that names the dog is accepted");
let replacement = dog.id + 1;
expect_event(&mut rx, replacement, ProcessEventKind::Reloaded).await;
let listed = handle.list().await;
assert_eq!(
listed.len(),
1,
"the swap is over, not in flight: {listed:?}"
);
assert_eq!(
dog_row(&listed, replacement).dog,
Some(DogSource::BuiltIn),
"the half that arrived is the same dog the half that left was"
);
}
#[tokio::test(start_paused = true)]
async fn a_dog_is_refused_once_a_shutdown_has_begun() {
let dir = tempfile::tempdir().unwrap();
let (mut actor, _mailbox) =
actor_with_one_online_sheep(&dir, vec![ProcScript::never_exits()]);
actor.shutting_down = true;
let (reply, rx) = oneshot::channel();
actor.handle_command(Command::StartDog {
app: Box::new(dog_app("bark")),
source: DogSource::BuiltIn,
reply,
});
assert_eq!(rx.await, Ok(Err(SupervisorError::EngineStopped)));
assert_eq!(actor.sheep.len(), 1, "nothing new was registered");
}
#[tokio::test(start_paused = true)]
async fn enabling_a_dog_twice_starts_one_process() {
let dir = tempfile::tempdir().unwrap();
let (events, _rx) = broadcast::channel(64);
let runner = ScriptedRunner::new(vec![ProcScript::never_exits(); 2]);
let handle = spawn_supervisor(runner, test_paths(&dir), events);
let first = handle
.start_dog(dog_app("bark"), DogSource::BuiltIn)
.await
.unwrap();
let second = handle
.start_dog(dog_app("bark"), DogSource::BuiltIn)
.await
.unwrap();
assert_eq!(first.id, second.id);
assert_eq!(second.pid, first.pid, "the same process, not a fresh one");
let listed = handle.list().await;
assert_eq!(listed.iter().filter(|i| i.name == "bark").count(), 1);
}
struct RefusesOneSpawn {
inner: ScriptedRunner,
refuse: usize,
attempts: Arc<AtomicUsize>,
}
impl fmt::Debug for RefusesOneSpawn {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("RefusesOneSpawn").finish_non_exhaustive()
}
}
impl ProcessRunner for RefusesOneSpawn {
type Proc = crate::fake::FakeProc;
fn spawn(
&self,
spec: &SpawnSpec,
) -> Result<(Self::Proc, ProcIo), crate::runner::RunnerError> {
let nth = self.attempts.fetch_add(1, AtomicOrdering::SeqCst);
if nth == self.refuse {
return Err(crate::runner::RunnerError::SpawnFailed(
"refused by the fixture".to_string(),
));
}
self.inner.spawn(spec)
}
}
#[tokio::test(start_paused = true)]
async fn a_swap_puts_each_half_of_a_reload_on_the_entry_that_owns_it() {
let dir = tempfile::tempdir().unwrap();
let (mut actor, _mailbox) =
actor_with_one_online_sheep(&dir, vec![ProcScript::never_exits()]);
let new_id = actor
.spawn_replacement(0)
.expect("the fixture's one script covers this spawn");
assert_ne!(new_id, 0, "a replacement never reuses the drainee's id");
let drainee = &actor.sheep[&0].entry;
let replacement = &actor.sheep[&new_id].entry;
assert_eq!(drainee.status, ProcStatus::Stopping);
assert_eq!(drainee.reload, ReloadState::Drainee { new_id });
assert_ne!(
replacement.status,
ProcStatus::Stopping,
"`Stopping` belongs to the instance going away, not the one arriving"
);
assert_eq!(replacement.status, ProcStatus::Starting);
assert_eq!(replacement.reload, ReloadState::Replacement);
assert_eq!(
replacement.instance, drainee.instance,
"a replacement takes the drainee's instance slot, or an app deriving \
its port from it binds a different one and nothing overlaps"
);
}
#[tokio::test(start_paused = true)]
async fn a_reload_is_refused_once_a_shutdown_has_begun() {
let dir = tempfile::tempdir().unwrap();
let (mut actor, _mailbox) =
actor_with_one_online_sheep(&dir, vec![ProcScript::never_exits()]);
actor.shutting_down = true;
let (reply, rx) = oneshot::channel();
actor.handle_command(Command::Reload {
selector: ProcessSelector::All,
reply,
});
assert_eq!(rx.await, Ok(Err(SupervisorError::EngineStopped)));
actor.advance_reload("web", VecDeque::from([0]));
assert_eq!(actor.sheep.len(), 1, "nothing new was registered");
assert_eq!(actor.sheep[&0].entry.status, ProcStatus::Online);
assert_eq!(actor.sheep[&0].entry.reload, ReloadState::None);
assert!(actor.reloads.is_empty(), "no job was started");
}
#[tokio::test(start_paused = true)]
async fn a_reload_gives_the_replacement_a_new_id_in_the_drainees_slot() {
let dir = tempfile::tempdir().unwrap();
let (handle, runner, mut rx) = started(
&dir,
AppConfig::minimal("web", "./srv"),
vec![ProcScript::never_exits(), ProcScript::never_exits()],
)
.await;
let before = handle.list().await;
let accepted = handle
.reload(ProcessSelector::Name("web".to_string()))
.await
.expect("the reload is accepted");
assert_eq!(
accepted.iter().map(|info| info.id).collect::<Vec<_>>(),
vec![0],
"the answer is the flock as it stood when the reload was accepted"
);
expect_event(&mut rx, 1, ProcessEventKind::Online).await;
expect_event(&mut rx, 0, ProcessEventKind::Delete).await;
let after = handle.list().await;
assert_eq!(after.len(), 1, "the drainee's registration goes with it");
assert_eq!(after[0].id, 1);
assert_eq!(after[0].status, ProcStatus::Online);
assert_eq!(after[0].out_file, before[0].out_file);
assert_eq!(after[0].err_file, before[0].err_file);
assert_eq!(
runner.kill_counts().len(),
2,
"one original and one replacement, and nothing else"
);
}
#[tokio::test(start_paused = true)]
async fn a_reload_stops_counting_the_drainee_as_running_before_its_replacement_starts() {
let dir = tempfile::tempdir().unwrap();
let (handle, _runner, mut rx) = started(
&dir,
AppConfig::minimal("web", "./srv"),
vec![ProcScript::never_exits(), ProcScript::never_exits()],
)
.await;
handle
.reload(ProcessSelector::Name("web".to_string()))
.await
.expect("the reload is accepted");
expect_event(&mut rx, 1, ProcessEventKind::Start).await;
let mid = handle.list().await;
assert_eq!(mid.len(), 2, "both entries are registered mid-swap");
assert_eq!(mid[0].status, ProcStatus::Stopping, "the drainee");
assert_eq!(mid[1].status, ProcStatus::Starting, "its replacement");
let running = mid
.iter()
.filter(|info| {
matches!(
info.status,
ProcStatus::Online | ProcStatus::Starting | ProcStatus::WaitingRestart
)
})
.count();
assert_eq!(
running, 1,
"a one-instance app must never count as two running instances"
);
}
#[tokio::test(start_paused = true)]
async fn a_reload_of_an_app_with_no_readiness_signal_completes_at_its_deadline() {
let dir = tempfile::tempdir().unwrap();
let mut app = AppConfig::minimal("web", "./srv");
app.listen_timeout = UpDuration::from_millis(2_500);
let (handle, _runner, mut rx) = started(
&dir,
app,
vec![ProcScript::never_exits(), ProcScript::never_exits()],
)
.await;
let start = tokio::time::Instant::now();
handle
.reload(ProcessSelector::Name("web".to_string()))
.await
.expect("the reload is accepted");
expect_event(&mut rx, 1, ProcessEventKind::Online).await;
assert_eq!(
tokio::time::Instant::now() - start,
Duration::from_millis(2_500)
);
expect_event(&mut rx, 0, ProcessEventKind::Delete).await;
let after = handle.list().await;
assert_eq!(after.len(), 1);
assert_eq!(after[0].id, 1);
assert_eq!(after[0].status, ProcStatus::Online);
}
#[tokio::test(start_paused = true)]
async fn a_reload_commits_the_moment_the_replacement_signals_ready() {
let dir = tempfile::tempdir().unwrap();
let mut app = AppConfig::minimal("web", "./srv");
app.wait_ready = true;
let (handle, _runner, mut rx) = started(
&dir,
app,
vec![ProcScript::never_exits(), ProcScript::never_exits()],
)
.await;
handle.tx.send(Msg::Ready { id: 0 }).await.unwrap();
expect_event(&mut rx, 0, ProcessEventKind::Online).await;
let start = tokio::time::Instant::now();
handle
.reload(ProcessSelector::Name("web".to_string()))
.await
.expect("the reload is accepted");
expect_event(&mut rx, 1, ProcessEventKind::Start).await;
handle.tx.send(Msg::Ready { id: 1 }).await.unwrap();
expect_event(&mut rx, 1, ProcessEventKind::Online).await;
assert!(
tokio::time::Instant::now() - start < Duration::from_millis(3_000),
"the swap committed at the signal, not at `listen_timeout`"
);
expect_event(&mut rx, 0, ProcessEventKind::Delete).await;
assert_eq!(handle.list().await[0].id, 1);
}
#[tokio::test(start_paused = true)]
async fn a_replacement_that_never_becomes_ready_leaves_the_old_instance_serving() {
let dir = tempfile::tempdir().unwrap();
let mut app = AppConfig::minimal("web", "./srv");
app.wait_ready = true; let (handle, runner, mut rx) = started(
&dir,
app,
vec![ProcScript::never_exits(), ProcScript::never_exits()],
)
.await;
handle.tx.send(Msg::Ready { id: 0 }).await.unwrap();
expect_event(&mut rx, 0, ProcessEventKind::Online).await;
handle
.reload(ProcessSelector::Name("web".to_string()))
.await
.expect("the reload is accepted");
expect_event(&mut rx, 1, ProcessEventKind::Delete).await;
let after = handle.list().await;
assert_eq!(after.len(), 1, "the abandoned replacement is deregistered");
assert_eq!(after[0].id, 0);
assert_eq!(
after[0].status,
ProcStatus::Online,
"the instance that can serve keeps serving"
);
assert_eq!(
runner.signals(1),
vec![15],
"the replacement went through the stop ladder rather than being dropped"
);
assert_eq!(runner.kill_counts(), vec![0, 0], "neither needed SIGKILL");
}
#[tokio::test(start_paused = true)]
async fn a_reload_drains_the_old_instance_under_graceful_timeout() {
let dir = tempfile::tempdir().unwrap();
let (handle, runner, mut rx) = started(
&dir,
AppConfig::minimal("web", "./srv"),
vec![ProcScript::ignores_signals(), ProcScript::never_exits()],
)
.await;
let start = tokio::time::Instant::now();
handle
.reload(ProcessSelector::Name("web".to_string()))
.await
.expect("the reload is accepted");
expect_event(&mut rx, 1, ProcessEventKind::Online).await;
expect_event(&mut rx, 0, ProcessEventKind::Delete).await;
assert_eq!(
tokio::time::Instant::now() - start,
Duration::from_millis(11_000)
);
assert_eq!(
runner.kill_counts(),
vec![1, 0],
"only the defiant drainee reached the SIGKILL rung"
);
}
#[tokio::test(start_paused = true)]
async fn a_reload_replaces_a_clustered_apps_instances_one_at_a_time() {
let dir = tempfile::tempdir().unwrap();
let mut app = AppConfig::minimal("web", "./srv");
app.instances = 2;
let (handle, runner, mut rx) = started(&dir, app, vec![ProcScript::never_exits(); 4]).await;
handle
.reload(ProcessSelector::Name("web".to_string()))
.await
.expect("the reload is accepted");
expect_event(&mut rx, 2, ProcessEventKind::Start).await;
assert_eq!(
runner.kill_counts().len(),
3,
"only the first instance's replacement exists yet"
);
assert_eq!(
handle
.list()
.await
.iter()
.filter(|info| info.status == ProcStatus::Stopping)
.count(),
1,
"one instance is being replaced, and the other is untouched"
);
expect_event(&mut rx, 0, ProcessEventKind::Delete).await;
expect_event(&mut rx, 3, ProcessEventKind::Start).await;
assert_eq!(runner.kill_counts().len(), 4);
expect_event(&mut rx, 1, ProcessEventKind::Delete).await;
let after = handle.list().await;
assert_eq!(
after.iter().map(|info| info.id).collect::<Vec<_>>(),
vec![2, 3]
);
assert!(after.iter().all(|info| info.status == ProcStatus::Online));
}
#[tokio::test(start_paused = true)]
async fn a_replacement_that_cannot_be_spawned_leaves_every_instance_alone() {
let dir = tempfile::tempdir().unwrap();
let mut app = AppConfig::minimal("web", "./srv");
app.instances = 2;
let attempts = Arc::new(AtomicUsize::new(0));
let runner = RefusesOneSpawn {
inner: ScriptedRunner::new(vec![ProcScript::never_exits(); 4]),
refuse: 2,
attempts: Arc::clone(&attempts),
};
let (events, mut rx) = tokio::sync::broadcast::channel(256);
let handle = spawn_supervisor(runner, test_paths(&dir), events);
handle.start(vec![normalize(app).unwrap()]).await.unwrap();
handle
.reload(ProcessSelector::Name("web".to_string()))
.await
.expect("the reload is accepted before anything is spawned");
assert_no_event_within(&mut rx, 3, ProcessEventKind::Start, Duration::from_secs(10)).await;
let after = handle.list().await;
assert_eq!(
after.iter().map(|info| info.id).collect::<Vec<_>>(),
vec![0, 1]
);
assert!(
after.iter().all(|info| info.status == ProcStatus::Online),
"both old instances keep serving: {after:?}"
);
assert_eq!(
attempts.load(AtomicOrdering::SeqCst),
3,
"two originals and one refused replacement, and no second attempt"
);
}
#[tokio::test(start_paused = true)]
async fn a_second_reload_of_an_app_already_reloading_is_refused() {
let dir = tempfile::tempdir().unwrap();
let mut app = AppConfig::minimal("web", "./srv");
app.instances = 2;
let (handle, runner, mut rx) = started(&dir, app, vec![ProcScript::never_exits(); 4]).await;
handle
.reload(ProcessSelector::Name("web".to_string()))
.await
.expect("the first reload is accepted");
expect_event(&mut rx, 2, ProcessEventKind::Start).await;
let refused = handle.reload(ProcessSelector::All).await;
assert_eq!(
refused,
Err(SupervisorError::ReloadInFlight("web".to_string()))
);
assert_eq!(
runner.kill_counts().len(),
3,
"no second replacement was spawned"
);
assert_eq!(
handle.list().await.len(),
3,
"one drainee, its replacement, and the instance untouched so far"
);
}
#[tokio::test(start_paused = true)]
async fn a_reload_of_a_sheep_that_is_not_online_is_a_no_op_success() {
let dir = tempfile::tempdir().unwrap();
let (handle, runner, _rx) = started(
&dir,
AppConfig::minimal("web", "./srv"),
vec![ProcScript::never_exits()],
)
.await;
handle.stop(ProcessSelector::All).await.unwrap();
let reloaded = handle
.reload(ProcessSelector::All)
.await
.expect("a reload that has nothing to replace still succeeds");
assert_eq!(reloaded.len(), 1);
assert_eq!(reloaded[0].status, ProcStatus::Stopped);
assert_eq!(handle.list().await.len(), 1, "nothing was registered");
assert_eq!(runner.kill_counts().len(), 1, "nothing was spawned");
}
#[tokio::test(start_paused = true)]
async fn a_reload_skips_an_instance_whose_kill_ladder_is_already_running() {
let dir = tempfile::tempdir().unwrap();
let (handle, runner, mut rx) = started(
&dir,
AppConfig::minimal("web", "./srv"),
vec![ProcScript::ignores_signals(), ProcScript::never_exits()],
)
.await;
let pid = handle.list().await[0].pid.expect("a live sheep has a pid");
handle.extra_restart(0, pid).await;
let reloaded = handle
.reload(ProcessSelector::Name("web".to_string()))
.await
.expect("a reload with nothing left to replace still succeeds");
assert_eq!(reloaded.len(), 1, "the reload still answers for the match");
assert_eq!(
handle.list().await.len(),
1,
"no replacement was registered against an instance on its way out"
);
assert_eq!(runner.kill_counts().len(), 1, "nothing was spawned");
expect_event(&mut rx, 0, ProcessEventKind::Restart).await;
let after = handle.list().await;
assert_eq!(after.len(), 1);
assert_eq!(after[0].id, 0);
assert_eq!(
runner.kill_counts().len(),
2,
"the original and its respawn"
);
}
#[tokio::test(start_paused = true)]
async fn a_report_raised_against_a_drainee_never_takes_it_off_the_reload() {
let dir = tempfile::tempdir().unwrap();
let (handle, runner, mut rx) = started(
&dir,
AppConfig::minimal("web", "./srv"),
vec![ProcScript::never_exits(); 3],
)
.await;
let pid = handle.list().await[0].pid.expect("a live sheep has a pid");
handle
.reload(ProcessSelector::Name("web".to_string()))
.await
.expect("the reload is accepted");
expect_event(&mut rx, 1, ProcessEventKind::Start).await;
handle.extra_restart(0, pid).await;
assert_no_event_within(
&mut rx,
0,
ProcessEventKind::Restart,
Duration::from_millis(1_000),
)
.await;
assert_eq!(handle.list().await[0].status, ProcStatus::Stopping);
expect_event(&mut rx, 1, ProcessEventKind::Online).await;
expect_event(&mut rx, 0, ProcessEventKind::Delete).await;
assert_eq!(
runner.kill_counts().len(),
2,
"the report never caused a spawn"
);
}
#[tokio::test(start_paused = true)]
async fn a_drainee_whose_liveness_probe_fails_is_reaped_rather_than_restarted() {
let dir = tempfile::tempdir().unwrap();
let reserved = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
let addr = reserved.local_addr().unwrap();
drop(reserved);
let mut app = AppConfig::minimal("web", "./srv");
app.wait_ready = true;
app.listen_timeout = UpDuration::from_millis(60_000);
app.liveness_probe = Some(ProbeConfig {
interval: UpDuration::from_millis(1_000),
timeout: UpDuration::from_millis(500),
failure_threshold: 1,
..probe_config(ProbeKind::Tcp, &addr.to_string())
});
let (events, mut rx) = tokio::sync::broadcast::channel(256);
let runner = Arc::new(ScriptedRunner::new(vec![ProcScript::never_exits(); 4]));
let (breaches_tx, breaches_rx) = mpsc::channel(8);
let (liveness_tx, liveness_rx) = mpsc::channel(8);
let handle =
SupervisorBuilder::new(SharedRunner(Arc::clone(&runner)), test_paths(&dir), events)
.extras(Extras {
clock: Arc::new(SystemClock),
enforcer: Arc::new(RecordingEnforcer::default()),
max_cron_sleep: DEFAULT_MAX_CRON_SLEEP,
reports: ExtrasReports {
breaches: breaches_tx,
liveness: liveness_tx,
},
stats: idle_stats(),
})
.spawn();
let _reporter = spawn_extras_reporter(breaches_rx, liveness_rx, handle.clone());
handle.start(vec![normalize(app).unwrap()]).await.unwrap();
handle.tx.send(Msg::Ready { id: 0 }).await.unwrap();
expect_event(&mut rx, 0, ProcessEventKind::Online).await;
handle
.reload(ProcessSelector::Name("web".to_string()))
.await
.expect("the reload is accepted");
expect_event(&mut rx, 1, ProcessEventKind::Start).await;
assert_no_event_within(
&mut rx,
0,
ProcessEventKind::Restart,
Duration::from_secs(5),
)
.await;
assert_eq!(handle.list().await[0].status, ProcStatus::Stopping);
assert_eq!(
runner.kill_counts().len(),
2,
"the failing probe never caused a spawn"
);
handle.tx.send(Msg::Ready { id: 1 }).await.unwrap();
expect_event(&mut rx, 1, ProcessEventKind::Online).await;
expect_event(&mut rx, 0, ProcessEventKind::Delete).await;
let after = handle.list().await;
assert_eq!(after.len(), 1, "the drainee left no registration behind");
assert_eq!(after[0].id, 1);
expect_event(&mut rx, 1, ProcessEventKind::Restart).await;
}
#[tokio::test(start_paused = true)]
async fn a_drainee_that_exits_on_its_own_is_never_restarted() {
let dir = tempfile::tempdir().unwrap();
let (handle, runner, mut rx) = started(
&dir,
AppConfig::minimal("web", "./srv"),
vec![
ProcScript::stable_then_exit(1_000, 1),
ProcScript::never_exits(),
],
)
.await;
handle
.reload(ProcessSelector::Name("web".to_string()))
.await
.expect("the reload is accepted");
expect_event(&mut rx, 1, ProcessEventKind::Start).await;
expect_event(&mut rx, 0, ProcessEventKind::Delete).await;
expect_event(&mut rx, 1, ProcessEventKind::Online).await;
let after = handle.list().await;
assert_eq!(after.len(), 1, "the drainee left no registration behind");
assert_eq!(after[0].id, 1);
assert_eq!(after[0].status, ProcStatus::Online);
assert_eq!(runner.kill_counts().len(), 2, "the drainee never respawned");
}
#[tokio::test(start_paused = true)]
async fn an_operators_stop_mid_reload_leaves_the_app_stopped_and_registered() {
let dir = tempfile::tempdir().unwrap();
let (handle, runner, mut rx) = started(
&dir,
AppConfig::minimal("web", "./srv"),
vec![ProcScript::never_exits(), ProcScript::ignores_signals()],
)
.await;
handle
.reload(ProcessSelector::Name("web".to_string()))
.await
.expect("the reload is accepted");
expect_event(&mut rx, 1, ProcessEventKind::Start).await;
let stopped = handle
.stop(ProcessSelector::Name("web".to_string()))
.await
.expect("the stop reaches both entries");
assert_eq!(stopped.len(), 2, "a stop answers for every id it matched");
let after = handle.list().await;
assert_eq!(
after.iter().map(|info| info.id).collect::<Vec<_>>(),
vec![0],
"the instance stays registered; only the abandoned replacement goes"
);
assert_eq!(after[0].status, ProcStatus::Stopped);
assert_eq!(runner.kill_counts().len(), 2, "nothing was spawned again");
}
#[tokio::test(start_paused = true)]
async fn a_replacement_is_kept_when_the_deadline_elapses_with_no_old_instance_left() {
let dir = tempfile::tempdir().unwrap();
let mut app = AppConfig::minimal("web", "./srv");
app.wait_ready = true; let (handle, runner, mut rx) = started(
&dir,
app,
vec![
ProcScript::stable_then_exit(1_000, 1),
ProcScript::never_exits(),
],
)
.await;
handle.tx.send(Msg::Ready { id: 0 }).await.unwrap();
expect_event(&mut rx, 0, ProcessEventKind::Online).await;
handle
.reload(ProcessSelector::Name("web".to_string()))
.await
.expect("the reload is accepted");
expect_event(&mut rx, 0, ProcessEventKind::Delete).await;
expect_event(&mut rx, 1, ProcessEventKind::Online).await;
let after = handle.list().await;
assert_eq!(after.len(), 1, "the app still has its instance");
assert_eq!(after[0].id, 1);
assert_eq!(after[0].status, ProcStatus::Online);
assert_eq!(runner.kill_counts(), vec![0, 0], "neither was SIGKILLed");
}
#[tokio::test(start_paused = true)]
async fn an_abandoned_reload_never_reports_a_dying_drainee_as_online() {
let dir = tempfile::tempdir().unwrap();
let mut app = AppConfig::minimal("web", "./srv");
app.wait_ready = true;
let (handle, _runner, mut rx) = started(
&dir,
app,
vec![ProcScript::ignores_signals(), ProcScript::never_exits()],
)
.await;
handle.tx.send(Msg::Ready { id: 0 }).await.unwrap();
expect_event(&mut rx, 0, ProcessEventKind::Online).await;
handle
.reload(ProcessSelector::Name("web".to_string()))
.await
.expect("the reload is accepted");
expect_event(&mut rx, 1, ProcessEventKind::Start).await;
let stopper = {
let handle = handle.clone();
tokio::spawn(async move { handle.stop(ProcessSelector::Name("web".to_string())).await })
};
expect_event(&mut rx, 1, ProcessEventKind::Delete).await;
let mid = handle.list().await;
assert_eq!(mid.len(), 1, "only the abandoned replacement has gone");
assert_eq!(mid[0].id, 0);
assert_eq!(
mid[0].status,
ProcStatus::Stopping,
"a drainee an operator already claimed is not back to serving"
);
let stopped = stopper.await.unwrap().expect("the stop is answered");
assert_eq!(stopped.len(), 2, "a stop answers for every id it matched");
assert_eq!(handle.list().await[0].status, ProcStatus::Stopped);
}
#[tokio::test(start_paused = true)]
async fn an_automatic_restart_never_lands_on_either_half_of_a_swap() {
let dir = tempfile::tempdir().unwrap();
let mut app = AppConfig::minimal("web", "./srv");
app.wait_ready = true;
let (handle, runner, mut rx) = started(
&dir,
app,
vec![ProcScript::never_exits(), ProcScript::never_exits()],
)
.await;
handle.tx.send(Msg::Ready { id: 0 }).await.unwrap();
expect_event(&mut rx, 0, ProcessEventKind::Online).await;
handle
.reload(ProcessSelector::Name("web".to_string()))
.await
.expect("the reload is accepted");
expect_event(&mut rx, 1, ProcessEventKind::Start).await;
let restarted = handle
.restart_automatic(ProcessSelector::Name("web".to_string()))
.await
.expect("the selector matches both halves of the swap");
assert!(
restarted.is_empty(),
"neither half of a swap is an automatic restart's to take"
);
handle.tx.send(Msg::Ready { id: 1 }).await.unwrap();
expect_event(&mut rx, 1, ProcessEventKind::Online).await;
expect_event(&mut rx, 0, ProcessEventKind::Delete).await;
let after = handle.list().await;
assert_eq!(after.len(), 1);
assert_eq!(after[0].id, 1, "the replacement, not a restarted drainee");
assert_eq!(after[0].status, ProcStatus::Online);
assert_eq!(after[0].restarts, 0, "nothing counted a restart");
assert_eq!(
runner.kill_counts().len(),
2,
"one original and one replacement, and nothing else"
);
}
#[tokio::test(start_paused = true)]
async fn a_reload_ends_when_its_replacement_goes_with_the_drainee_already_gone() {
let dir = tempfile::tempdir().unwrap();
let mut app = AppConfig::minimal("web", "./srv");
app.wait_ready = true; let (handle, _runner, mut rx) = started(
&dir,
app,
vec![
ProcScript::stable_then_exit(1_000, 1),
ProcScript::never_exits(),
],
)
.await;
handle.tx.send(Msg::Ready { id: 0 }).await.unwrap();
expect_event(&mut rx, 0, ProcessEventKind::Online).await;
handle
.reload(ProcessSelector::Name("web".to_string()))
.await
.expect("the reload is accepted");
expect_event(&mut rx, 0, ProcessEventKind::Delete).await;
handle
.stop(ProcessSelector::Name("web".to_string()))
.await
.expect("the stop reaches the replacement");
let seen = events_through(&mut rx, 1, ProcessEventKind::Stop).await;
assert!(
at(&seen, 1, ProcessEventKind::ReloadAbandoned) < at(&seen, 1, ProcessEventKind::Stop),
"the reload gives up before the exit that ended it is reported, so a \
subscriber reads the two in the order they happened: {seen:?}"
);
let again = handle
.reload(ProcessSelector::Name("web".to_string()))
.await
.map(|infos| infos.iter().map(|info| info.id).collect::<Vec<_>>());
assert_eq!(
again,
Ok(vec![1]),
"the reload is over, so the app is reloadable again"
);
}
#[tokio::test(start_paused = true)]
async fn a_reload_carries_the_drainees_restart_count_to_its_replacement() {
let dir = tempfile::tempdir().unwrap();
let (handle, _runner, mut rx) = started(
&dir,
AppConfig::minimal("web", "./srv"),
vec![ProcScript::never_exits(); 3],
)
.await;
handle
.restart(ProcessSelector::Name("web".to_string()))
.await
.unwrap();
assert_eq!(handle.list().await[0].restarts, 1);
handle
.reload(ProcessSelector::Name("web".to_string()))
.await
.expect("the reload is accepted");
expect_event(&mut rx, 1, ProcessEventKind::Online).await;
expect_event(&mut rx, 0, ProcessEventKind::Delete).await;
let after = handle.list().await;
assert_eq!(after[0].id, 1);
assert_eq!(after[0].restarts, 1);
}
#[tokio::test(start_paused = true)]
async fn a_reload_carries_the_drainees_last_exit_to_its_replacement() {
let dir = tempfile::tempdir().unwrap();
let (handle, _runner, mut rx) = started(
&dir,
AppConfig::minimal("web", "./srv"),
vec![ProcScript::never_exits(); 3],
)
.await;
handle
.restart(ProcessSelector::Name("web".to_string()))
.await
.unwrap();
let restarted = handle.list().await;
let last_exit = restarted[0].last_exit;
assert!(
last_exit.is_some(),
"a manual restart is itself an exit, so this must not be None: {restarted:?}"
);
handle
.reload(ProcessSelector::Name("web".to_string()))
.await
.expect("the reload is accepted");
expect_event(&mut rx, 1, ProcessEventKind::Online).await;
expect_event(&mut rx, 0, ProcessEventKind::Delete).await;
let after = handle.list().await;
assert_eq!(after[0].id, 1);
assert_eq!(
after[0].last_exit, last_exit,
"a reload is not an exit -- the replacement must inherit the drainee's \
last_exit rather than reset it to None"
);
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
struct Seen {
id: u32,
kind: ProcessEventKind,
status: ProcStatus,
manually: bool,
}
async fn events_through(
rx: &mut tokio::sync::broadcast::Receiver<BusEvent>,
id: u32,
kind: ProcessEventKind,
) -> Vec<Seen> {
let collect = async {
let mut seen = Vec::new();
loop {
match rx.recv().await {
Ok(BusEvent::Process {
event,
info,
manually,
..
}) => {
seen.push(Seen {
id: info.id,
kind: event,
status: info.status,
manually,
});
if info.id == id && event == kind {
return seen;
}
}
Ok(_) => continue,
Err(tokio::sync::broadcast::error::RecvError::Lagged(n)) => {
panic!("the event stream lagged by {n}; no ordering claim survives that")
}
Err(e) => panic!("event stream closed before {kind:?} for id {id}: {e}"),
}
}
};
tokio::time::timeout(SWAP_WINDOW, collect)
.await
.unwrap_or_else(|_| panic!("no {kind:?} for id {id} within {SWAP_WINDOW:?}"))
}
fn at(seen: &[Seen], id: u32, kind: ProcessEventKind) -> usize {
seen.iter()
.position(|e| e.id == id && e.kind == kind)
.unwrap_or_else(|| panic!("no {kind:?} for id {id} in {seen:?}"))
}
#[tokio::test(start_paused = true)]
async fn a_completed_swap_reports_itself_on_the_bus() {
let dir = tempfile::tempdir().unwrap();
let (handle, _runner, mut rx) = started(
&dir,
AppConfig::minimal("web", "./srv"),
vec![ProcScript::never_exits(); 3],
)
.await;
handle
.reload(ProcessSelector::Name("web".to_string()))
.await
.expect("the reload is accepted");
let seen = events_through(&mut rx, 1, ProcessEventKind::Reloaded).await;
assert!(
at(&seen, 0, ProcessEventKind::Reload) < at(&seen, 1, ProcessEventKind::Start),
"the instance being replaced is named before its replacement starts: {seen:?}"
);
assert_eq!(
seen[at(&seen, 0, ProcessEventKind::Reload)],
Seen {
id: 0,
kind: ProcessEventKind::Reload,
status: ProcStatus::Stopping,
manually: true,
}
);
assert!(
at(&seen, 0, ProcessEventKind::Delete) < at(&seen, 1, ProcessEventKind::Reloaded),
"a swap is not over until the instance it replaced is gone: {seen:?}"
);
assert_eq!(
seen[at(&seen, 1, ProcessEventKind::Reloaded)],
Seen {
id: 1,
kind: ProcessEventKind::Reloaded,
status: ProcStatus::Online,
manually: true,
}
);
}
#[tokio::test(start_paused = true)]
async fn a_swap_is_not_announced_for_a_replacement_that_is_no_longer_serving() {
let dir = tempfile::tempdir().unwrap();
let mut app = AppConfig::minimal("web", "./srv");
app.autorestart = false; let (handle, _runner, mut rx) = started(
&dir,
app,
vec![
ProcScript::ignores_signals(),
ProcScript::stable_then_exit(5_000, 1),
],
)
.await;
handle
.reload(ProcessSelector::Name("web".to_string()))
.await
.expect("the reload is accepted");
let seen = events_through(&mut rx, 1, ProcessEventKind::ReloadAbandoned).await;
assert_eq!(
seen[at(&seen, 1, ProcessEventKind::ReloadAbandoned)],
Seen {
id: 1,
kind: ProcessEventKind::ReloadAbandoned,
status: ProcStatus::Stopped,
manually: true,
},
"a replacement that died inside the drain window is what the \
abandonment names, carrying the status it actually reached"
);
assert!(
!seen.iter().any(|e| e.kind == ProcessEventKind::Reloaded),
"no swap succeeded, so nothing may say one did: {seen:?}"
);
}
#[tokio::test(start_paused = true)]
async fn a_swap_whose_drainee_never_reports_its_exit_gives_up_on_its_own() {
let dir = tempfile::tempdir().unwrap();
let (handle, runner, mut rx) = started(
&dir,
AppConfig::minimal("web", "./srv"),
vec![
ProcScript::never_reports_its_exit(),
ProcScript::never_exits(),
ProcScript::never_exits(),
],
)
.await;
handle
.reload(ProcessSelector::Name("web".to_string()))
.await
.expect("the reload is accepted");
let seen = events_through(&mut rx, 1, ProcessEventKind::ReloadAbandoned).await;
assert_eq!(
seen[at(&seen, 1, ProcessEventKind::ReloadAbandoned)],
Seen {
id: 1,
kind: ProcessEventKind::ReloadAbandoned,
status: ProcStatus::Online,
manually: true,
},
"the replacement took the slot over and is what is left holding it"
);
assert_eq!(
runner.kill_counts(),
vec![1, 0],
"the drain did reach `SIGKILL`; what never came back was the exit"
);
let again = handle
.reload(ProcessSelector::Name("web".to_string()))
.await
.map(|infos| infos.iter().map(|info| info.id).collect::<Vec<_>>());
assert_eq!(
again,
Ok(vec![0, 1]),
"a wedged instance must not refuse the app's next reload"
);
}
#[tokio::test(start_paused = true)]
async fn a_deadline_from_a_finished_swap_never_ends_the_one_that_followed_it() {
let dir = tempfile::tempdir().unwrap();
let mut app = AppConfig::minimal("web", "./srv");
app.instances = 2;
let (handle, _runner, mut rx) = started(
&dir,
app,
vec![
ProcScript::ignores_signals(),
ProcScript::ignores_signals(),
ProcScript::never_exits(),
ProcScript::never_exits(),
],
)
.await;
handle
.reload(ProcessSelector::Name("web".to_string()))
.await
.expect("the reload is accepted");
expect_event(&mut rx, 2, ProcessEventKind::Reloaded).await;
expect_event(&mut rx, 3, ProcessEventKind::Reloaded).await;
let after = handle.list().await;
assert_eq!(
after.iter().map(|info| info.id).collect::<Vec<_>>(),
vec![2, 3],
"both instances were replaced: {after:?}"
);
assert!(after.iter().all(|info| info.status == ProcStatus::Online));
}
#[tokio::test(start_paused = true)]
async fn a_deadline_before_the_commit_puts_the_instance_being_replaced_back() {
let dir = tempfile::tempdir().unwrap();
let (mut actor, _mailbox) =
actor_with_one_online_sheep(&dir, vec![ProcScript::never_exits()]);
let (ctl_tx, _ctl_rx) = mpsc::channel(SHEEP_CTL_CAPACITY);
actor.sheep.get_mut(&0).expect("the fixture's sheep").ctl = Some(ctl_tx);
let new_id = actor
.spawn_replacement(0)
.expect("the fixture's one script covers this spawn");
actor.reloads.insert(
"web".to_string(),
ReloadJob {
queue: VecDeque::new(),
swap: ReloadSwap {
old_id: 0,
new_id,
phase: ReloadPhase::AwaitReady,
},
},
);
actor.handle_reload_deadline("web", new_id);
assert!(
actor.reloads.is_empty(),
"the job is gone, so the app is reloadable again"
);
let drainee = &actor.sheep[&0];
assert_eq!(
drainee.entry.status,
ProcStatus::Online,
"nothing was ever killed, so the instance being replaced goes back to serving"
);
assert_eq!(drainee.entry.reload, ReloadState::None);
assert_eq!(
actor.sheep[&new_id].manual.map(|pending| pending.kind),
Some(ManualKind::Delete),
"the replacement that never proved itself is taken back down"
);
}
#[tokio::test(start_paused = true)]
async fn an_abandoned_reload_says_so_on_the_bus() {
let dir = tempfile::tempdir().unwrap();
let mut app = AppConfig::minimal("web", "./srv");
app.wait_ready = true; let (handle, _runner, mut rx) =
started(&dir, app, vec![ProcScript::never_exits(); 2]).await;
handle.tx.send(Msg::Ready { id: 0 }).await.unwrap();
expect_event(&mut rx, 0, ProcessEventKind::Online).await;
handle
.reload(ProcessSelector::Name("web".to_string()))
.await
.expect("the reload is accepted");
let seen = events_through(&mut rx, 0, ProcessEventKind::ReloadAbandoned).await;
assert_eq!(
seen[at(&seen, 0, ProcessEventKind::ReloadAbandoned)],
Seen {
id: 0,
kind: ProcessEventKind::ReloadAbandoned,
status: ProcStatus::Online,
manually: true,
},
"the abandoned reload's own instance is still the one serving"
);
}
#[tokio::test(start_paused = true)]
async fn a_reload_whose_replacement_cannot_spawn_says_so_on_the_bus() {
let dir = tempfile::tempdir().unwrap();
let (handle, _runner, mut rx) = started(
&dir,
AppConfig::minimal("web", "./srv"),
vec![ProcScript::never_exits()],
)
.await;
handle
.reload(ProcessSelector::Name("web".to_string()))
.await
.expect("the reload is accepted before anything is spawned");
let seen = events_through(&mut rx, 0, ProcessEventKind::ReloadAbandoned).await;
assert_eq!(
seen[at(&seen, 0, ProcessEventKind::ReloadAbandoned)],
Seen {
id: 0,
kind: ProcessEventKind::ReloadAbandoned,
status: ProcStatus::Online,
manually: true,
},
"a failed spawn leaves the instance it was replacing serving"
);
let after = handle.list().await;
assert_eq!(after.len(), 1, "no replacement was registered: {after:?}");
assert_eq!(after[0].id, 0);
}
#[tokio::test(start_paused = true)]
async fn a_live_sheep_keeps_holding_its_log_control_sender() {
let runner = ScriptedRunner::new(vec![ProcScript::never_exits()]);
let (proc, io) = runner.spawn(&log_ctl_spec()).unwrap();
assert!(runner.log_ctl_live(0), "sanity: the fake starts it live");
let (events, _rx) = tokio::sync::broadcast::channel(64);
let (_ctl_tx, ctl_rx) = mpsc::channel(8);
let (_signal_tx, signal_rx) = mpsc::channel(8);
let (actor_tx, _actor_rx) = mpsc::channel(8);
let app = normalize(AppConfig::minimal("svc", "./svc")).unwrap();
tokio::spawn(run_sheep(
7, proc, io, app, ctl_rx, signal_rx, events, actor_tx,
));
for _ in 0..100 {
tokio::task::yield_now().await;
}
assert!(
runner.log_ctl_live(0),
"run_sheep dropped ProcIo::log_ctl while its sheep was still \
running: against the real runner that closes the read ends of \
the child's stdout and stderr"
);
}
#[tokio::test(start_paused = true)]
async fn a_pump_is_reaped_when_its_sheep_ends_even_with_a_lamb_on_the_pipe() {
let (events, mut rx) = tokio::sync::broadcast::channel(64);
let runner = Arc::new(ScriptedRunner::new(vec![
ProcScript::stable_then_exit(1_000, 0).with_a_lamb_holding_the_pipe(),
]));
let dir = tempfile::tempdir().unwrap();
let handle = spawn_supervisor(SharedRunner(Arc::clone(&runner)), test_paths(&dir), events);
let mut app = AppConfig::minimal("web", "./srv");
app.autorestart = false;
handle.start(vec![normalize(app).unwrap()]).await.unwrap();
assert!(
runner.log_ctl_live(0),
"sanity: a running sheep has a live pump"
);
await_event(&mut rx, 0, ProcessEventKind::Stop).await;
assert_eq!(
handle.list().await.len(),
1,
"sanity: the sheep is still registered, so its slot still holds a \
clone of the control sender"
);
let reaped = tokio::time::timeout(Duration::from_secs(5), async {
while runner.log_ctl_live(0) {
tokio::time::sleep(Duration::from_millis(10)).await;
}
})
.await;
assert!(
reaped.is_ok(),
"the pump outlived its sheep task, holding both log files and \
both pipe read ends open"
);
}
#[tokio::test(start_paused = true)]
async fn a_writer_task_is_reaped_when_its_sheep_exits() {
let dir = tempfile::tempdir().unwrap();
let mut app = AppConfig::minimal("web", "./srv");
app.autorestart = false;
app.channel = true;
let (handle, runner, mut rx) =
started(&dir, app, vec![ProcScript::stable_then_exit(1_000, 0)]).await;
let mut io = runner.io_handles(0);
assert_eq!(
io.to_child_rx.try_recv(),
Err(mpsc::error::TryRecvError::Empty),
"sanity: a running sheep's channel is open and quiet, so the \
`None` below is a close and not a channel that never opened"
);
await_event(&mut rx, 0, ProcessEventKind::Stop).await;
assert_eq!(
handle.list().await.len(),
1,
"sanity: the sheep is still registered, so nothing but the \
clearing can have let go of the slot's clone"
);
let reaped = tokio::time::timeout(Duration::from_secs(5), io.to_child_rx.recv()).await;
assert_eq!(
reaped.ok(),
Some(None),
"the writer task outlived its sheep, parked on `recv()` and \
holding the daemon's end of the shepherd channel"
);
}
#[tokio::test(start_paused = true)]
async fn every_spawn_leaves_the_daemons_end_of_the_channel_on_the_slot() {
let dir = tempfile::tempdir().unwrap();
let (mut actor, _mailbox) = actor_with_one_online_sheep(
&dir,
vec![
ProcScript::never_exits(),
ProcScript::never_exits(),
ProcScript::never_exits(),
],
);
let mut app = AppConfig::minimal("api", "./api");
app.autorestart = false;
let (reply, _answer) = oneshot::channel();
actor.handle_command(Command::Start {
apps: vec![normalize(app).unwrap()],
reply,
});
assert!(
actor.sheep[&1].to_child.is_some(),
"a fresh spawn's slot holds the daemon's end of its shepherd channel"
);
actor.handle_exited(
1,
ExitOutcome {
code: Some(0),
signal: None,
},
);
assert!(
actor.sheep[&1].to_child.is_none(),
"the clone goes with the process it was cloned for"
);
actor.respawn(1, false);
assert!(
actor.sheep[&1].to_child.is_some(),
"a respawn hands the slot the new process's channel, not the dead \
one's"
);
actor.advance_reload("web", VecDeque::from([0]));
let new_id = actor.reloads["web"].swap.new_id;
assert!(
actor.sheep[&new_id].to_child.is_some(),
"a reload's replacement holds the daemon's end of its own channel"
);
}
const ACTION_TIMEOUT: Duration = Duration::from_secs(20);
const ACTION_WINDOW: Duration = Duration::from_secs(120);
fn actor_with_an_open_channel(
dir: &tempfile::TempDir,
) -> (
Actor<ScriptedRunner>,
mpsc::Receiver<Msg>,
mpsc::Receiver<ShepherdMessage>,
) {
let (mut actor, mailbox) = actor_with_one_online_sheep(dir, vec![]);
let (to_child, child_rx) = mpsc::channel(16);
actor
.sheep
.get_mut(&0)
.expect("the fixture registers id 0")
.to_child = Some(to_child);
(actor, mailbox, child_rx)
}
fn trigger_action(
actor: &mut Actor<ScriptedRunner>,
action: &str,
) -> oneshot::Receiver<ActionOutcome> {
let to_child = actor.sheep[&0]
.to_child
.clone()
.expect("the fixture's sheep holds the daemon's end of a channel");
actor.arm_action(0, to_child, action.to_string(), None, ACTION_TIMEOUT)
}
async fn settle_action(
actor: &mut Actor<ScriptedRunner>,
mailbox: &mut mpsc::Receiver<Msg>,
) -> ActionOutcome {
let msg = tokio::time::timeout(ACTION_WINDOW, mailbox.recv())
.await
.expect("an action wait reported nothing within the window")
.expect("the actor's mailbox closed");
match msg {
Msg::ActionResult { id, stamp, outcome } => {
actor.handle_action_result(id, stamp, outcome.clone());
outcome
}
other => panic!("expected an action result, got {other:?}"),
}
}
async fn sent_action(child_rx: &mut mpsc::Receiver<ShepherdMessage>) -> ShepherdMessage {
tokio::time::timeout(ACTION_WINDOW, child_rx.recv())
.await
.expect("nothing reached the child's end of the channel")
.expect("the child's end of the channel closed")
}
#[tokio::test(start_paused = true)]
async fn a_triggered_action_answers_with_the_apps_reply() {
let dir = tempfile::tempdir().unwrap();
let mut app = AppConfig::minimal("web", "./srv");
app.channel = true;
let (handle, runner, _events) = started(&dir, app, vec![ProcScript::never_exits()]).await;
let mut io = runner.io_handles(0);
let triggered = tokio::spawn(async move {
handle
.trigger(
ProcessSelector::Name("web".to_string()),
"gc".to_string(),
Some("--full".to_string()),
)
.await
});
assert_eq!(
sent_action(&mut io.to_child_rx).await,
ShepherdMessage::Action {
name: "gc".to_string(),
params: Some("--full".to_string()),
id: 0,
},
"the action reaches the child's end of the channel as it was asked for"
);
io.from_child_tx
.send(ChildMessage::ActionReply {
action: "gc".to_string(),
body: "swept 3".to_string(),
id: Some(0),
})
.await
.unwrap();
assert_eq!(
triggered.await.unwrap(),
Ok(vec![ActionReply {
id: 0,
name: "web".to_string(),
outcome: ActionOutcome::Replied {
body: "swept 3".to_string()
},
}]),
"the app's reply body is what the caller is answered with"
);
}
#[tokio::test(start_paused = true)]
async fn a_ready_on_the_channel_reaches_both_the_bus_and_the_readiness_wait() {
let dir = tempfile::tempdir().unwrap();
let mut app = AppConfig::minimal("web", "./srv");
app.channel = true;
app.wait_ready = true;
let (handle, runner, mut events) =
started(&dir, app, vec![ProcScript::never_exits()]).await;
let io = runner.io_handles(0);
io.from_child_tx.send(ChildMessage::Ready).await.unwrap();
let seen = tokio::time::timeout(ACTION_WINDOW, async {
loop {
if let BusEvent::Channel { id, message } = events.recv().await.unwrap() {
break (id, message);
}
}
})
.await
.expect("no channel event within the window");
assert_eq!(seen, (0, ChildMessage::Ready));
let listed = handle.list().await;
assert_eq!(listed[0].status, ProcStatus::Online);
}
#[tokio::test(start_paused = true)]
async fn a_metric_on_the_channel_reaches_the_bus_with_its_name_and_value() {
let dir = tempfile::tempdir().unwrap();
let mut app = AppConfig::minimal("web", "./srv");
app.channel = true;
let (_handle, runner, mut events) =
started(&dir, app, vec![ProcScript::never_exits()]).await;
let io = runner.io_handles(0);
io.from_child_tx
.send(ChildMessage::Metric {
name: "rps".to_string(),
value: 42.0,
})
.await
.unwrap();
let seen = tokio::time::timeout(ACTION_WINDOW, async {
loop {
if let BusEvent::Channel { id, message } = events.recv().await.unwrap() {
break (id, message);
}
}
})
.await
.expect("no channel event within the window");
assert_eq!(
seen,
(
0,
ChildMessage::Metric {
name: "rps".to_string(),
value: 42.0,
}
)
);
}
#[tokio::test(start_paused = true)]
async fn an_action_reply_no_trigger_is_waiting_for_still_reaches_the_bus() {
let dir = tempfile::tempdir().unwrap();
let mut app = AppConfig::minimal("web", "./srv");
app.channel = true;
let (_handle, runner, mut events) =
started(&dir, app, vec![ProcScript::never_exits()]).await;
let io = runner.io_handles(0);
io.from_child_tx
.send(ChildMessage::ActionReply {
action: "gc".to_string(),
body: "unprompted".to_string(),
id: None,
})
.await
.unwrap();
let seen = tokio::time::timeout(ACTION_WINDOW, async {
loop {
if let BusEvent::Channel { message, .. } = events.recv().await.unwrap() {
break message;
}
}
})
.await
.expect("no channel event within the window");
let ChildMessage::ActionReply { body, .. } = seen else {
panic!("expected an action reply, got {seen:?}");
};
assert_eq!(body, "unprompted");
}
#[tokio::test(start_paused = true)]
async fn a_triggered_action_times_out_when_the_app_never_answers() {
let dir = tempfile::tempdir().unwrap();
let mut app = AppConfig::minimal("api", "./api");
app.channel = true;
let (handle, runner, _events) = started(&dir, app, vec![ProcScript::never_exits()]).await;
let mut io = runner.io_handles(0);
assert_eq!(
handle
.trigger(
ProcessSelector::Name("api".to_string()),
"stats".to_string(),
None,
)
.await,
Ok(vec![ActionReply {
id: 0,
name: "api".to_string(),
outcome: ActionOutcome::TimedOut,
}]),
"an app that says nothing is reported as saying nothing, not waited on forever"
);
assert_eq!(
sent_action(&mut io.to_child_rx).await,
ShepherdMessage::Action {
name: "stats".to_string(),
params: None,
id: 0,
},
"the timeout is an app that did not answer, not an action that was never sent"
);
}
#[test]
fn a_stamped_reply_wakes_its_own_wait_even_with_a_debt_outstanding() {
let mut waits = ActionWaits::default();
let (t1_reply, _t1_out) = oneshot::channel();
let (t1_waiter, _t1_body) = oneshot::channel();
waits.arm(PendingAction {
stamp: 1,
action: "gc".to_string(),
waiter: Some(t1_waiter),
reply: t1_reply,
});
assert!(waits.resolve(1).is_some(), "T1 must have been live");
let (t2_reply, _t2_out) = oneshot::channel();
let (t2_waiter, t2_body) = oneshot::channel();
waits.arm(PendingAction {
stamp: 2,
action: "gc".to_string(),
waiter: Some(t2_waiter),
reply: t2_reply,
});
let woken = waits
.answer("gc", Some(2))
.expect("a reply stamped with the live wait's own stamp must reach it");
woken.send("collected".to_string()).unwrap();
assert_eq!(t2_body.blocking_recv().unwrap(), "collected");
}
#[test]
fn an_unstamped_reply_still_settles_the_oldest_debt_first() {
let mut waits = ActionWaits::default();
let (t1_reply, _t1_out) = oneshot::channel();
let (t1_waiter, _t1_body) = oneshot::channel();
waits.arm(PendingAction {
stamp: 1,
action: "gc".to_string(),
waiter: Some(t1_waiter),
reply: t1_reply,
});
waits.resolve(1);
let (t2_reply, _t2_out) = oneshot::channel();
let (t2_waiter, _t2_body) = oneshot::channel();
waits.arm(PendingAction {
stamp: 2,
action: "gc".to_string(),
waiter: Some(t2_waiter),
reply: t2_reply,
});
assert!(
waits.answer("gc", None).is_none(),
"an unstamped reply pays the debt, exactly as it did before stamping"
);
assert!(
waits.answer("gc", None).is_some(),
"and the next one reaches the live wait, exactly as it did before"
);
}
#[test]
fn a_stamped_reply_for_a_dead_wait_does_not_reach_a_live_one() {
let mut waits = ActionWaits::default();
let (t1_reply, _t1_out) = oneshot::channel();
let (t1_waiter, _t1_body) = oneshot::channel();
waits.arm(PendingAction {
stamp: 1,
action: "gc".to_string(),
waiter: Some(t1_waiter),
reply: t1_reply,
});
waits.resolve(1);
let (t2_reply, _t2_out) = oneshot::channel();
let (t2_waiter, _t2_body) = oneshot::channel();
waits.arm(PendingAction {
stamp: 2,
action: "gc".to_string(),
waiter: Some(t2_waiter),
reply: t2_reply,
});
assert!(
waits.answer("gc", Some(1)).is_none(),
"T1's own late reply belongs to T1's debt, not to T2"
);
assert!(
waits.answer("gc", Some(2)).is_some(),
"and T2 is still waiting for its own"
);
}
#[tokio::test(start_paused = true)]
async fn a_reply_owed_by_a_timed_out_action_never_answers_the_next_one() {
let dir = tempfile::tempdir().unwrap();
let (mut actor, mut mailbox, mut child_rx) = actor_with_an_open_channel(&dir);
let first = trigger_action(&mut actor, "gc");
sent_action(&mut child_rx).await;
assert_eq!(
settle_action(&mut actor, &mut mailbox).await,
ActionOutcome::TimedOut
);
assert_eq!(first.await.unwrap(), ActionOutcome::TimedOut);
let second = trigger_action(&mut actor, "gc");
sent_action(&mut child_rx).await;
actor.handle_action_reply(0, "gc", "swept 3".to_string(), None);
assert_eq!(
settle_action(&mut actor, &mut mailbox).await,
ActionOutcome::TimedOut,
"a reply the first `gc` was owed was handed to the second one"
);
assert_eq!(second.await.unwrap(), ActionOutcome::TimedOut);
let third = trigger_action(&mut actor, "gc");
sent_action(&mut child_rx).await;
actor.handle_action_reply(0, "gc", "swept 7".to_string(), None);
actor.handle_action_reply(0, "gc", "swept 11".to_string(), None);
assert_eq!(
settle_action(&mut actor, &mut mailbox).await,
ActionOutcome::Replied {
body: "swept 11".to_string()
},
"the debts outlived the replies that settled them"
);
assert_eq!(
third.await.unwrap(),
ActionOutcome::Replied {
body: "swept 11".to_string()
}
);
}
#[tokio::test(start_paused = true)]
async fn a_second_reply_to_an_answered_action_is_dropped() {
let dir = tempfile::tempdir().unwrap();
let (mut actor, mut mailbox, mut child_rx) = actor_with_an_open_channel(&dir);
let answered = trigger_action(&mut actor, "gc");
sent_action(&mut child_rx).await;
actor.handle_action_reply(0, "gc", "swept 3".to_string(), None);
assert_eq!(
settle_action(&mut actor, &mut mailbox).await,
ActionOutcome::Replied {
body: "swept 3".to_string()
}
);
assert_eq!(
answered.await.unwrap(),
ActionOutcome::Replied {
body: "swept 3".to_string()
}
);
actor.handle_action_reply(0, "gc", "spent".to_string(), None);
let next = trigger_action(&mut actor, "gc");
sent_action(&mut child_rx).await;
assert_eq!(
settle_action(&mut actor, &mut mailbox).await,
ActionOutcome::TimedOut,
"a spare reply was kept and used to answer a wait that came after it"
);
assert_eq!(next.await.unwrap(), ActionOutcome::TimedOut);
}
#[tokio::test(start_paused = true)]
async fn two_waits_for_one_action_are_answered_in_the_order_they_were_asked() {
let dir = tempfile::tempdir().unwrap();
let (mut actor, mut mailbox, mut child_rx) = actor_with_an_open_channel(&dir);
let first = trigger_action(&mut actor, "gc");
sent_action(&mut child_rx).await;
let second = trigger_action(&mut actor, "gc");
sent_action(&mut child_rx).await;
actor.handle_action_reply(0, "gc", "swept 3".to_string(), None);
actor.handle_action_reply(0, "gc", "swept 7".to_string(), None);
settle_action(&mut actor, &mut mailbox).await;
settle_action(&mut actor, &mut mailbox).await;
assert_eq!(
first.await.unwrap(),
ActionOutcome::Replied {
body: "swept 3".to_string()
},
"the earlier trigger was answered with the later reply"
);
assert_eq!(
second.await.unwrap(),
ActionOutcome::Replied {
body: "swept 7".to_string()
},
"the second reply was dropped and its wait left waiting"
);
}
#[tokio::test(start_paused = true)]
async fn an_action_that_cannot_even_be_delivered_still_ends() {
let dir = tempfile::tempdir().unwrap();
let (mut actor, mut mailbox) = actor_with_one_online_sheep(&dir, vec![]);
let (to_child, _wedged) = mpsc::channel(1);
to_child.try_send(ShepherdMessage::Shutdown).unwrap();
actor
.sheep
.get_mut(&0)
.expect("the fixture registers id 0")
.to_child = Some(to_child);
let answer = trigger_action(&mut actor, "gc");
assert_eq!(
settle_action(&mut actor, &mut mailbox).await,
ActionOutcome::TimedOut,
"an action that never got onto the channel left its wait parked"
);
assert_eq!(answer.await.unwrap(), ActionOutcome::TimedOut);
}
fn register_sheep(
actor: &mut Actor<ScriptedRunner>,
dir: &tempfile::TempDir,
name: &str,
to_child: Option<mpsc::Sender<ShepherdMessage>>,
) -> u32 {
let id = actor.next_id;
actor.next_id += 1;
let paths = test_paths(dir);
let app = normalize(AppConfig::minimal(name, "./srv")).unwrap();
actor.sheep.insert(
id,
SheepSlot {
entry: armed_entry(id, 0, 2000 + id, app, &paths),
ctl: None,
log_ctl: None,
to_child,
signals: None,
to_stdin: None,
manual: None,
pending_delete: false,
epoch: 0,
ready_tx: None,
actions: ActionWaits::default(),
},
);
id
}
fn trigger_flock(
actor: &mut Actor<ScriptedRunner>,
selector: ProcessSelector,
action: &str,
) -> oneshot::Receiver<Result<Vec<ActionReply>, SupervisorError>> {
let (reply, answer) = oneshot::channel();
actor.handle_command(Command::Trigger {
selector,
action: action.to_string(),
params: None,
reply,
});
answer
}
async fn triggered(
answer: oneshot::Receiver<Result<Vec<ActionReply>, SupervisorError>>,
) -> Result<Vec<ActionReply>, SupervisorError> {
tokio::time::timeout(ACTION_WINDOW, answer)
.await
.expect("a trigger reported nothing within the window")
.expect("the trigger's reply channel was dropped")
}
fn row(id: u32, name: &str, outcome: ActionOutcome) -> ActionReply {
ActionReply {
id,
name: name.to_string(),
outcome,
}
}
#[tokio::test(start_paused = true)]
async fn a_trigger_answers_every_sheep_it_matched_before_it_answers_at_all() {
let dir = tempfile::tempdir().unwrap();
let (mut actor, mut mailbox, mut child_rx) = actor_with_an_open_channel(&dir);
register_sheep(&mut actor, &dir, "api", None);
let (silent_tx, mut silent_rx) = mpsc::channel(16);
register_sheep(&mut actor, &dir, "worker", Some(silent_tx));
let mut answer = trigger_flock(&mut actor, ProcessSelector::All, "gc");
sent_action(&mut child_rx).await;
sent_action(&mut silent_rx).await;
actor.handle_action_reply(0, "gc", "swept 3".to_string(), None);
assert_eq!(
settle_action(&mut actor, &mut mailbox).await,
ActionOutcome::Replied {
body: "swept 3".to_string()
}
);
tokio::task::yield_now().await;
assert!(
matches!(answer.try_recv(), Err(oneshot::error::TryRecvError::Empty)),
"a trigger answered while a sheep it matched was still being waited on"
);
assert_eq!(
settle_action(&mut actor, &mut mailbox).await,
ActionOutcome::TimedOut
);
assert_eq!(
triggered(answer).await,
Ok(vec![
row(
0,
"web",
ActionOutcome::Replied {
body: "swept 3".to_string()
}
),
row(1, "api", ActionOutcome::NoChannel),
row(2, "worker", ActionOutcome::TimedOut),
])
);
}
#[tokio::test(start_paused = true)]
async fn a_sheep_with_no_channel_is_refused_in_its_own_row() {
let dir = tempfile::tempdir().unwrap();
let (mut actor, mut mailbox, mut child_rx) = actor_with_an_open_channel(&dir);
register_sheep(&mut actor, &dir, "api", None);
let answer = trigger_flock(&mut actor, ProcessSelector::All, "gc");
sent_action(&mut child_rx).await;
actor.handle_action_reply(0, "gc", "swept 3".to_string(), None);
settle_action(&mut actor, &mut mailbox).await;
assert_eq!(
triggered(answer).await,
Ok(vec![
row(
0,
"web",
ActionOutcome::Replied {
body: "swept 3".to_string()
}
),
row(1, "api", ActionOutcome::NoChannel),
])
);
assert!(
matches!(mailbox.try_recv(), Err(mpsc::error::TryRecvError::Empty)),
"a sheep with no channel armed a wait anyway"
);
}
#[tokio::test(start_paused = true)]
async fn a_sheep_whose_channel_has_no_far_end_is_refused_too() {
let dir = tempfile::tempdir().unwrap();
let (mut actor, mut mailbox) = actor_with_one_online_sheep(&dir, vec![]);
let (to_child, receiver) = mpsc::channel(16);
actor
.sheep
.get_mut(&0)
.expect("the fixture registers id 0")
.to_child = Some(to_child);
drop(receiver);
assert_eq!(
triggered(trigger_flock(&mut actor, ProcessSelector::All, "gc")).await,
Ok(vec![row(0, "web", ActionOutcome::NoChannel)])
);
assert!(
matches!(mailbox.try_recv(), Err(mpsc::error::TryRecvError::Empty)),
"a sheep whose channel has no far end armed a wait anyway"
);
}
#[tokio::test(start_paused = true)]
async fn a_trigger_no_sheep_can_take_is_a_success_that_says_so() {
let dir = tempfile::tempdir().unwrap();
let (mut actor, mut mailbox) = actor_with_one_online_sheep(&dir, vec![]);
register_sheep(&mut actor, &dir, "api", None);
assert_eq!(
triggered(trigger_flock(&mut actor, ProcessSelector::All, "gc")).await,
Ok(vec![
row(0, "web", ActionOutcome::NoChannel),
row(1, "api", ActionOutcome::NoChannel),
])
);
assert!(
matches!(mailbox.try_recv(), Err(mpsc::error::TryRecvError::Empty)),
"a flock with nothing to deliver to armed a wait anyway"
);
}
#[tokio::test(start_paused = true)]
async fn a_reload_drainee_is_skipped_and_its_replacement_answers() {
let dir = tempfile::tempdir().unwrap();
let (mut actor, mut mailbox, mut child_rx) = actor_with_an_open_channel(&dir);
let (replacement_tx, mut replacement_rx) = mpsc::channel(16);
let new_id = register_sheep(&mut actor, &dir, "web", Some(replacement_tx));
let drainee = actor.sheep.get_mut(&0).expect("the fixture registers id 0");
drainee.entry.status = ProcStatus::Stopping;
drainee.entry.reload = ReloadState::Drainee { new_id };
actor
.sheep
.get_mut(&new_id)
.expect("the replacement was just registered")
.entry
.reload = ReloadState::Replacement;
let answer = trigger_flock(&mut actor, ProcessSelector::Name("web".to_string()), "gc");
sent_action(&mut replacement_rx).await;
actor.handle_action_reply(new_id, "gc", "swept 3".to_string(), None);
settle_action(&mut actor, &mut mailbox).await;
assert_eq!(
triggered(answer).await,
Ok(vec![
row(0, "web", ActionOutcome::Skipped),
row(
new_id,
"web",
ActionOutcome::Replied {
body: "swept 3".to_string()
}
),
])
);
assert!(
matches!(child_rx.try_recv(), Err(mpsc::error::TryRecvError::Empty)),
"an action was delivered to a process that is on its way out"
);
}
#[tokio::test(start_paused = true)]
async fn a_trigger_matching_no_sheep_is_an_error() {
let dir = tempfile::tempdir().unwrap();
let (mut actor, _mailbox) = actor_with_one_online_sheep(&dir, vec![]);
assert_eq!(
triggered(trigger_flock(
&mut actor,
ProcessSelector::Name("ghost".to_string()),
"gc"
))
.await,
Err(SupervisorError::NotFound)
);
}
#[tokio::test(start_paused = true)]
async fn a_sheep_exiting_answers_every_action_waiting_on_it() {
let dir = tempfile::tempdir().unwrap();
let (mut actor, mut mailbox, mut child_rx) = actor_with_an_open_channel(&dir);
let timed_out = trigger_action(&mut actor, "gc");
sent_action(&mut child_rx).await;
settle_action(&mut actor, &mut mailbox).await;
assert_eq!(timed_out.await.unwrap(), ActionOutcome::TimedOut);
let waiting = trigger_action(&mut actor, "stats");
sent_action(&mut child_rx).await;
actor.handle_exited(
0,
ExitOutcome {
code: Some(0),
signal: None,
},
);
let answered = tokio::time::timeout(ACTION_WINDOW, waiting)
.await
.expect("a wait outlived the process it was waiting on")
.unwrap();
assert_eq!(
answered,
ActionOutcome::NoChannel,
"a wait outlived the process it was waiting on"
);
assert!(
actor.sheep[&0].actions.abandoned.is_empty(),
"a debt owed by a process that has exited outlived it, and would \
have swallowed a reply from whatever runs under this id next"
);
}
struct SilentPumpRunner {
inner: ScriptedRunner,
seen: watch::Sender<u32>,
}
impl SilentPumpRunner {
fn new(scripts: Vec<ProcScript>) -> (Self, watch::Receiver<u32>) {
let (seen, requests) = watch::channel(0);
(
Self {
inner: ScriptedRunner::new(scripts),
seen,
},
requests,
)
}
}
impl fmt::Debug for SilentPumpRunner {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("SilentPumpRunner").finish_non_exhaustive()
}
}
impl ProcessRunner for SilentPumpRunner {
type Proc = crate::fake::FakeProc;
fn spawn(
&self,
spec: &SpawnSpec,
) -> Result<(Self::Proc, ProcIo), crate::runner::RunnerError> {
let (proc, mut io) = self.inner.spawn(spec)?;
let (tx, mut rx) = mpsc::channel(SHEEP_CTL_CAPACITY);
io.log_ctl = tx;
let seen = self.seen.clone();
tokio::spawn(async move {
let mut held = Vec::new();
while let Some(request) = rx.recv().await {
held.push(request);
seen.send_modify(|count| *count += 1);
}
});
Ok((proc, io))
}
}
#[tokio::test(start_paused = true)]
async fn the_actor_keeps_answering_while_a_reopen_waits_on_a_silent_pump() {
let (events, _rx) = tokio::sync::broadcast::channel(64);
let (runner, mut requests) = SilentPumpRunner::new(vec![ProcScript::never_exits()]);
let dir = tempfile::tempdir().unwrap();
let handle = spawn_supervisor(runner, test_paths(&dir), events);
handle
.start(vec![normalize(AppConfig::minimal("web", "./srv")).unwrap()])
.await
.unwrap();
let reopening = tokio::spawn({
let handle = handle.clone();
async move { handle.reopen(ProcessSelector::All).await }
});
tokio::time::timeout(Duration::from_secs(5), requests.wait_for(|seen| *seen == 1))
.await
.expect("the reopen must reach the pump")
.expect("the runner outlives this wait, so its sender cannot have closed");
let listed = tokio::time::timeout(Duration::from_secs(5), handle.list())
.await
.expect("the actor must keep answering while a reopen is outstanding");
assert_eq!(listed.len(), 1);
assert!(
!reopening.is_finished(),
"sanity: nothing can acknowledge this reopen, so `list` answering \
above is not just the reopen having finished first"
);
reopening.abort();
}
#[tokio::test(start_paused = true)]
async fn a_reopen_reaches_every_matched_sheep_and_no_others() {
let (events, _rx) = tokio::sync::broadcast::channel(64);
let runner = Arc::new(ScriptedRunner::new(vec![
ProcScript::never_exits(),
ProcScript::never_exits(),
ProcScript::never_exits(),
]));
let dir = tempfile::tempdir().unwrap();
let handle = spawn_supervisor(SharedRunner(Arc::clone(&runner)), test_paths(&dir), events);
let mut web = AppConfig::minimal("web", "./srv");
web.instances = 2;
handle
.start(vec![
normalize(web).unwrap(),
normalize(AppConfig::minimal("api", "./api")).unwrap(),
])
.await
.unwrap();
let reopened = handle
.reopen(ProcessSelector::Name("web".to_string()))
.await
.unwrap();
assert_eq!(
reopened.iter().map(|info| info.id).collect::<Vec<_>>(),
vec![0, 1],
"the reply must carry both `web` instances, id-sorted, and no `api`"
);
assert_eq!(runner.reopens(0), 1, "web's first instance");
assert_eq!(runner.reopens(1), 1, "web's second instance");
assert_eq!(runner.reopens(2), 0, "api was never named");
}
#[tokio::test(start_paused = true)]
async fn a_reopen_after_a_restart_reaches_the_pump_the_restart_spawned() {
let (events, _rx) = tokio::sync::broadcast::channel(64);
let runner = Arc::new(ScriptedRunner::new(vec![
ProcScript::never_exits(),
ProcScript::never_exits(),
]));
let dir = tempfile::tempdir().unwrap();
let handle = spawn_supervisor(SharedRunner(Arc::clone(&runner)), test_paths(&dir), events);
handle
.start(vec![normalize(AppConfig::minimal("web", "./srv")).unwrap()])
.await
.unwrap();
let restarted = handle
.restart(ProcessSelector::Name("web".to_string()))
.await
.unwrap();
assert_eq!(
(restarted[0].restarts, restarted[0].status),
(1, ProcStatus::Online),
"the sheep must be back up on a second process before the reopen"
);
let reopened =
tokio::time::timeout(Duration::from_secs(5), handle.reopen(ProcessSelector::All))
.await
.expect("a live pump must answer rather than leave the reopen waiting")
.expect("a running sheep's reopen must succeed");
assert_eq!(reopened.len(), 1);
assert_eq!(
runner.reopens(1),
1,
"the reopen must reach the pump the restart spawned"
);
assert_eq!(
runner.reopens(0),
0,
"the pre-restart pump belongs to a process that is gone; a reopen \
sent there reaches nothing and is reported as a success"
);
}
#[tokio::test(start_paused = true)]
async fn a_reopen_matching_nothing_is_not_found() {
let (events, _rx) = tokio::sync::broadcast::channel(64);
let runner = ScriptedRunner::new(vec![]);
let dir = tempfile::tempdir().unwrap();
let handle = spawn_supervisor(runner, test_paths(&dir), events);
assert_eq!(
handle.reopen(ProcessSelector::All).await,
Err(SupervisorError::NotFound)
);
}
#[tokio::test(start_paused = true)]
async fn a_stopped_sheep_is_a_no_op_success() {
let (events, _rx) = tokio::sync::broadcast::channel(64);
let runner = ScriptedRunner::new(vec![ProcScript::never_exits()]);
let dir = tempfile::tempdir().unwrap();
let handle = spawn_supervisor(runner, test_paths(&dir), events);
let mut app = AppConfig::minimal("web", "./srv");
app.autorestart = false;
handle.start(vec![normalize(app).unwrap()]).await.unwrap();
handle
.stop(ProcessSelector::Name("web".to_string()))
.await
.unwrap();
assert_eq!(handle.list().await[0].status, ProcStatus::Stopped);
let reopened =
tokio::time::timeout(Duration::from_secs(5), handle.reopen(ProcessSelector::All))
.await
.expect("a reopen aimed at a stopped sheep must not wait for an acknowledgement")
.expect("a stopped sheep has nothing to reopen, which is a success");
assert_eq!(reopened.len(), 1);
assert_eq!(reopened[0].status, ProcStatus::Stopped);
}
#[tokio::test(start_paused = true)]
async fn a_reopen_whose_pump_is_already_gone_returns_at_once() {
let (tx, rx) = mpsc::channel(1);
drop(rx);
let outcome = tokio::time::timeout(Duration::from_secs(5), reopen_logs(&tx))
.await
.expect("a failed send must end the reopen, not leave it waiting");
assert_eq!(
outcome,
Ok(()),
"a pump that was never reached reopened nothing, which is a no-op \
success rather than a reopen that failed"
);
}
#[tokio::test(start_paused = true)]
async fn a_pump_that_ends_mid_request_still_ends_the_reopen() {
let (tx, mut rx) = mpsc::channel(1);
tokio::spawn(async move {
let request = rx.recv().await.expect("the reopen must reach the pump");
drop(request); });
let outcome = tokio::time::timeout(Duration::from_secs(5), reopen_logs(&tx))
.await
.expect("a dropped acknowledgement must end the reopen, not leave it waiting");
assert_eq!(
outcome,
Ok(()),
"a pump that ended mid-request reopened nothing, which is the same \
no-op success a failed send is"
);
}
#[tokio::test(start_paused = true)]
async fn a_pump_that_ends_mid_request_still_ends_the_flush() {
let (tx, mut rx) = mpsc::channel(1);
tokio::spawn(async move {
let request = rx.recv().await.expect("the flush must reach the pump");
drop(request); });
let outcome = tokio::time::timeout(Duration::from_secs(5), flush_logs(&tx))
.await
.expect("a dropped acknowledgement must end the flush, not leave it waiting");
assert_eq!(
outcome,
Ok(()),
"a pump that ended mid-request owes no bytes, which is the same \
no-op success a failed send is"
);
}
const PUMP_REFUSAL: &str = "/gone/web-out.log: No such file or directory";
const REFUSING_SHEEP: &str = "web";
#[derive(Debug)]
struct FailingPumpRunner {
inner: Arc<ScriptedRunner>,
}
impl FailingPumpRunner {
fn new(inner: Arc<ScriptedRunner>) -> Self {
Self { inner }
}
}
impl ProcessRunner for FailingPumpRunner {
type Proc = crate::fake::FakeProc;
fn spawn(
&self,
spec: &SpawnSpec,
) -> Result<(Self::Proc, ProcIo), crate::runner::RunnerError> {
let (proc, mut io) = self.inner.spawn(spec)?;
if spec.name != REFUSING_SHEEP {
return Ok((proc, io));
}
let (tx, mut rx) = mpsc::channel(SHEEP_CTL_CAPACITY);
io.log_ctl = tx;
tokio::spawn(async move {
while let Some(ctl) = rx.recv().await {
match ctl {
LogCtl::Reopen { done } => {
let _ = done.send(Err(ReopenError {
message: PUMP_REFUSAL.to_string(),
}));
}
LogCtl::Flush { done } => {
let _ = done.send(Err(FlushError {
message: PUMP_REFUSAL.to_string(),
}));
}
}
}
});
Ok((proc, io))
}
}
#[tokio::test(start_paused = true)]
async fn a_pump_that_could_not_reopen_fails_the_request_and_names_its_sheep() {
let (events, _rx) = tokio::sync::broadcast::channel(64);
let scripted = Arc::new(ScriptedRunner::new(vec![
ProcScript::never_exits(),
ProcScript::never_exits(),
]));
let dir = tempfile::tempdir().unwrap();
let handle = spawn_supervisor(
FailingPumpRunner::new(Arc::clone(&scripted)),
test_paths(&dir),
events,
);
handle
.start(vec![
normalize(AppConfig::minimal("web", "./srv")).unwrap(),
normalize(AppConfig::minimal("api", "./api")).unwrap(),
])
.await
.unwrap();
let error =
tokio::time::timeout(Duration::from_secs(5), handle.reopen(ProcessSelector::All))
.await
.expect("a pump that answers must not leave the reopen waiting")
.expect_err("a reopen a pump could not carry out must not answer Ok");
assert_eq!(
error,
SupervisorError::ReopenFailed(format!("web (id 0): could not reopen {PUMP_REFUSAL}")),
"the failure must carry the sheep and the path, and only the sheep that failed"
);
assert_eq!(
scripted.reopens(1),
1,
"the healthy sheep must still have been reopened"
);
}
#[tokio::test(start_paused = true)]
async fn a_signal_reaches_the_sheeps_own_process_and_not_its_group() {
let dir = tempfile::tempdir().unwrap();
let (handle, runner, _events) = started(
&dir,
AppConfig::minimal("web", "./srv"),
vec![ProcScript::never_exits()],
)
.await;
let rows = handle
.signal(ProcessSelector::Id(0), OperatorSignal::Hup)
.await
.unwrap();
assert_eq!(
rows,
vec![SignalReply {
id: 0,
name: "web".to_string(),
outcome: SignalOutcome::Delivered,
}]
);
assert_eq!(runner.process_signals(0), vec![OperatorSignal::Hup]);
assert!(
runner.signals(0).is_empty(),
"shep signal must not reach the process group"
);
}
#[tokio::test(start_paused = true)]
async fn a_stopped_sheep_answers_not_running_rather_than_delivered() {
let dir = tempfile::tempdir().unwrap();
let (handle, _runner, _events) = started(
&dir,
AppConfig::minimal("web", "./srv"),
vec![ProcScript::never_exits()],
)
.await;
handle.stop(ProcessSelector::Id(0)).await.unwrap();
let rows = handle
.signal(ProcessSelector::Id(0), OperatorSignal::Hup)
.await
.unwrap();
assert_eq!(rows.len(), 1);
assert_eq!(rows[0].outcome, SignalOutcome::NotRunning);
}
#[tokio::test(start_paused = true)]
async fn a_selector_that_matches_nothing_is_not_found() {
let h = harness(vec![]);
let err = h
.ctx
.supervisor
.signal(
ProcessSelector::Name("ghost".to_string()),
OperatorSignal::Hup,
)
.await
.unwrap_err();
assert_eq!(err, SupervisorError::NotFound);
}
#[tokio::test(start_paused = true)]
async fn a_reload_drainee_is_signalled_like_any_other_live_sheep() {
let dir = tempfile::tempdir().unwrap();
let (mut actor, mut signal_rx) = actor_with_a_drainee_holding_a_signal_mailbox(&dir);
let (reply, answer) = oneshot::channel();
actor.handle_command(Command::Signal {
selector: ProcessSelector::Id(0),
sig: OperatorSignal::Hup,
reply,
});
let request = tokio::time::timeout(ACTION_WINDOW, signal_rx.recv())
.await
.expect("no signal reached the drainee's mailbox within the window")
.expect("the drainee's signal mailbox closed");
assert_eq!(request.sig, OperatorSignal::Hup);
let _ = request.done.send(Ok(()));
let rows = tokio::time::timeout(ACTION_WINDOW, answer)
.await
.expect("the signal reported nothing within the window")
.expect("the signal's reply channel was dropped")
.unwrap();
assert_eq!(rows[0].outcome, SignalOutcome::Delivered);
}
#[tokio::test(start_paused = true)]
async fn a_line_reaches_a_sheep_that_asked_for_stdin() {
let dir = tempfile::tempdir().unwrap();
let mut app = AppConfig::minimal("repl", "./repl");
app.stdin = true;
let (handle, runner, _events) = started(&dir, app, vec![ProcScript::never_exits()]).await;
let rows = handle
.send_line(ProcessSelector::Id(0), "reload-config".to_string())
.await
.unwrap();
assert_eq!(
rows,
vec![LineReply {
id: 0,
name: "repl".to_string(),
outcome: LineOutcome::Sent,
}]
);
assert_eq!(runner.stdin_lines(0), vec!["reload-config".to_string()]);
}
#[tokio::test(start_paused = true)]
async fn a_sheep_without_stdin_answers_no_stdin() {
let dir = tempfile::tempdir().unwrap();
let (handle, _runner, _events) = started(
&dir,
AppConfig::minimal("web", "./srv"),
vec![ProcScript::never_exits()],
)
.await;
let rows = handle
.send_line(ProcessSelector::Id(0), "hello".to_string())
.await
.unwrap();
assert_eq!(rows[0].outcome, LineOutcome::NoStdin);
}
#[tokio::test(start_paused = true)]
async fn a_mixed_flock_reports_per_sheep_rather_than_failing() {
let dir = tempfile::tempdir().unwrap();
let mut piped = AppConfig::minimal("repl", "./repl");
piped.stdin = true;
let (handle, runner, _events) = started(
&dir,
piped,
vec![ProcScript::never_exits(), ProcScript::never_exits()],
)
.await;
handle
.start(vec![normalize(AppConfig::minimal("web", "./srv")).unwrap()])
.await
.unwrap();
let rows = handle
.send_line(ProcessSelector::All, "hello".to_string())
.await
.unwrap();
let outcome = |id| rows.iter().find(|r| r.id == id).unwrap().outcome.clone();
assert_eq!(outcome(0), LineOutcome::Sent);
assert_eq!(outcome(1), LineOutcome::NoStdin);
assert_eq!(runner.stdin_lines(1), Vec::<String>::new());
assert!(rows.windows(2).all(|w| w[0].id < w[1].id));
}
#[tokio::test(start_paused = true)]
async fn a_write_that_never_lands_times_out_and_says_so() {
let dir = tempfile::tempdir().unwrap();
let mut app = AppConfig::minimal("stuck", "./stuck");
app.stdin = true;
let (handle, _runner, _events) =
started(&dir, app, vec![ProcScript::never_reads_its_stdin()]).await;
let rows = tokio::time::timeout(
STDIN_WRITE_TIMEOUT * 4,
handle.send_line(ProcessSelector::Id(0), "hello".to_string()),
)
.await
.expect("send_line did not honour its own bound")
.unwrap();
let LineOutcome::NotWritten { reason } = rows[0].outcome.clone() else {
panic!("expected NotWritten, got {:?}", rows[0].outcome);
};
assert!(reason.contains("read"), "{reason}");
}
#[tokio::test(start_paused = true)]
async fn a_flock_of_wedged_sheep_is_bounded_once_and_not_once_each() {
let dir = tempfile::tempdir().unwrap();
let mut app = AppConfig::minimal("stuck", "./stuck");
app.stdin = true;
app.instances = 3;
let (handle, _runner, _events) =
started(&dir, app, vec![ProcScript::never_reads_its_stdin(); 3]).await;
let started_at = tokio::time::Instant::now();
let rows = handle
.send_line(ProcessSelector::All, "hello".to_string())
.await
.unwrap();
let elapsed = started_at.elapsed();
assert_eq!(rows.len(), 3);
assert!(
rows.iter()
.all(|row| matches!(row.outcome, LineOutcome::NotWritten { .. })),
"{rows:?}"
);
assert!(
elapsed < STDIN_WRITE_TIMEOUT * 2,
"three wedged sheep cost {elapsed:?}; the bound is per-CALL, not per-sheep"
);
}
#[tokio::test(start_paused = true)]
async fn a_selector_that_matches_nothing_is_not_found_for_send_line() {
let h = harness(vec![]);
assert_eq!(
h.ctx
.supervisor
.send_line(ProcessSelector::Name("ghost".to_string()), "x".to_string())
.await
.unwrap_err(),
SupervisorError::NotFound
);
}
#[tokio::test(start_paused = true)]
async fn a_flush_reaches_every_matched_sheep_and_no_others() {
let (events, _rx) = tokio::sync::broadcast::channel(64);
let runner = Arc::new(ScriptedRunner::new(vec![
ProcScript::never_exits(),
ProcScript::never_exits(),
ProcScript::never_exits(),
]));
let dir = tempfile::tempdir().unwrap();
let handle = spawn_supervisor(SharedRunner(Arc::clone(&runner)), test_paths(&dir), events);
let mut web = AppConfig::minimal("web", "./srv");
web.instances = 2;
handle
.start(vec![
normalize(web).unwrap(),
normalize(AppConfig::minimal("api", "./api")).unwrap(),
])
.await
.unwrap();
let flushed = handle
.flush(ProcessSelector::Name("web".to_string()))
.await
.unwrap();
assert_eq!(
flushed.iter().map(|info| info.id).collect::<Vec<_>>(),
vec![0, 1],
"the reply must carry both `web` instances, id-sorted, and no `api`"
);
assert_eq!(runner.flushes(0), 1, "web's first instance");
assert_eq!(runner.flushes(1), 1, "web's second instance");
assert_eq!(runner.flushes(2), 0, "api was never named");
assert_eq!(
runner.reopens(0),
0,
"a flush must push `LogCtl::Flush`, never `LogCtl::Reopen` — a \
flush wired to the neighbouring variant would swap the flock's \
handles and empty nothing"
);
}
#[tokio::test(start_paused = true)]
async fn the_actor_keeps_answering_while_a_flush_waits_on_a_silent_pump() {
let (events, _rx) = tokio::sync::broadcast::channel(64);
let (runner, mut requests) = SilentPumpRunner::new(vec![ProcScript::never_exits()]);
let dir = tempfile::tempdir().unwrap();
let handle = spawn_supervisor(runner, test_paths(&dir), events);
handle
.start(vec![normalize(AppConfig::minimal("web", "./srv")).unwrap()])
.await
.unwrap();
let flushing = tokio::spawn({
let handle = handle.clone();
async move { handle.flush(ProcessSelector::All).await }
});
tokio::time::timeout(Duration::from_secs(5), requests.wait_for(|seen| *seen == 1))
.await
.expect("the flush must reach the pump")
.expect("the runner outlives this wait, so its sender cannot have closed");
let listed = tokio::time::timeout(Duration::from_secs(5), handle.list())
.await
.expect("the actor must keep answering while a flush is outstanding");
assert_eq!(listed.len(), 1);
assert!(
!flushing.is_finished(),
"sanity: nothing can acknowledge this flush, so `list` answering \
above is not just the flush having finished first"
);
flushing.abort();
}
const LATE_LINE: &str = "landed-while-the-flush-was-being-answered\n";
const LATE_WRITING_SHEEP: &str = "latecomer";
struct LateWritingPumpRunner {
inner: ScriptedRunner,
}
impl LateWritingPumpRunner {
fn new(inner: ScriptedRunner) -> Self {
Self { inner }
}
}
impl fmt::Debug for LateWritingPumpRunner {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("LateWritingPumpRunner")
.finish_non_exhaustive()
}
}
impl ProcessRunner for LateWritingPumpRunner {
type Proc = crate::fake::FakeProc;
fn spawn(
&self,
spec: &SpawnSpec,
) -> Result<(Self::Proc, ProcIo), crate::runner::RunnerError> {
let (proc, mut io) = self.inner.spawn(spec)?;
if spec.name != LATE_WRITING_SHEEP {
return Ok((proc, io));
}
let (tx, mut rx) = mpsc::channel(SHEEP_CTL_CAPACITY);
io.log_ctl = tx;
let out_file = spec.out_file.clone();
tokio::spawn(async move {
while let Some(ctl) = rx.recv().await {
match ctl {
LogCtl::Flush { done } => {
if let Some(parent) = out_file.parent() {
std::fs::create_dir_all(parent).unwrap();
}
let mut file = std::fs::OpenOptions::new()
.create(true)
.append(true)
.open(&out_file)
.unwrap();
std::io::Write::write_all(&mut file, LATE_LINE.as_bytes()).unwrap();
let _ = done.send(Ok(()));
}
LogCtl::Reopen { done } => {
let _ = done.send(Ok(()));
}
}
}
});
Ok((proc, io))
}
}
#[tokio::test(start_paused = true)]
async fn a_flush_truncates_only_after_its_pump_has_answered() {
let (events, _rx) = tokio::sync::broadcast::channel(64);
let dir = tempfile::tempdir().unwrap();
let handle = spawn_supervisor(
LateWritingPumpRunner::new(ScriptedRunner::new(vec![ProcScript::never_exits()])),
test_paths(&dir),
events,
);
handle
.start(vec![
normalize(AppConfig::minimal(LATE_WRITING_SHEEP, "./srv")).unwrap(),
])
.await
.unwrap();
let out_file = PathBuf::from(
handle.list().await[0]
.out_file
.clone()
.expect("the daemon reports its own resolved log paths"),
);
handle.flush(ProcessSelector::All).await.unwrap();
assert!(
out_file.exists(),
"the pump never wrote, so this flush never reached one: {}",
out_file.display()
);
assert_eq!(
std::fs::read_to_string(&out_file).unwrap(),
"",
"a line the pump landed as it answered the flush must not survive \
the truncate that follows it"
);
}
#[tokio::test(start_paused = true)]
async fn a_stopped_sheeps_log_file_is_truncated_too() {
let (events, _rx) = tokio::sync::broadcast::channel(64);
let runner = ScriptedRunner::new(vec![ProcScript::never_exits()]);
let dir = tempfile::tempdir().unwrap();
let handle = spawn_supervisor(runner, test_paths(&dir), events);
let mut app = AppConfig::minimal("web", "./srv");
app.autorestart = false;
handle.start(vec![normalize(app).unwrap()]).await.unwrap();
handle
.stop(ProcessSelector::Name("web".to_string()))
.await
.unwrap();
let listed = handle.list().await;
assert_eq!(listed[0].status, ProcStatus::Stopped);
let out_file = PathBuf::from(listed[0].out_file.clone().unwrap());
std::fs::create_dir_all(out_file.parent().unwrap()).unwrap();
std::fs::write(&out_file, "what the sheep logged before it stopped\n").unwrap();
let flushed =
tokio::time::timeout(Duration::from_secs(5), handle.flush(ProcessSelector::All))
.await
.expect("a flush aimed at a stopped sheep must not wait for an acknowledgement")
.expect("a stopped sheep has no pump to flush, which is not a failure");
assert_eq!(flushed.len(), 1);
assert_eq!(flushed[0].status, ProcStatus::Stopped);
assert_eq!(
std::fs::read_to_string(&out_file).unwrap(),
"",
"a stopped sheep's log is still readable, so it is still emptied"
);
}
#[tokio::test(start_paused = true)]
async fn instances_sharing_one_log_path_answer_one_row_each() {
let (events, _rx) = tokio::sync::broadcast::channel(64);
let runner =
ScriptedRunner::new(vec![ProcScript::never_exits(), ProcScript::never_exits()]);
let dir = tempfile::tempdir().unwrap();
let handle = spawn_supervisor(runner, test_paths(&dir), events);
let mut web = AppConfig::minimal("web", "./srv");
web.instances = 2;
web.merge_logs = true;
handle.start(vec![normalize(web).unwrap()]).await.unwrap();
let listed = handle.list().await;
assert_eq!(
listed[0].out_file, listed[1].out_file,
"fixture check: `merge_logs` must really point both instances at \
one path, or this case proves nothing"
);
let shared = PathBuf::from(listed[0].out_file.clone().unwrap());
std::fs::create_dir_all(shared.parent().unwrap()).unwrap();
std::fs::write(&shared, "both instances wrote here\n").unwrap();
let flushed = handle.flush(ProcessSelector::All).await.unwrap();
assert_eq!(
flushed.iter().map(|info| info.id).collect::<Vec<_>>(),
vec![0, 1],
"one row per sheep, not per file emptied"
);
assert_eq!(std::fs::read_to_string(&shared).unwrap(), "");
}
#[tokio::test(start_paused = true)]
async fn a_sibling_sharing_a_path_is_flushed_even_when_the_selector_skips_it() {
let (events, _rx) = tokio::sync::broadcast::channel(64);
let dir = tempfile::tempdir().unwrap();
let handle = spawn_supervisor(
LateWritingPumpRunner::new(ScriptedRunner::new(vec![
ProcScript::never_exits(),
ProcScript::never_exits(),
])),
test_paths(&dir),
events,
);
let shared = dir.path().join("shared-out.log");
let mut named = AppConfig::minimal("web", "./srv");
named.out_file = Some(shared.display().to_string());
let mut sibling = AppConfig::minimal(LATE_WRITING_SHEEP, "./api");
sibling.out_file = Some(shared.display().to_string());
handle
.start(vec![normalize(named).unwrap(), normalize(sibling).unwrap()])
.await
.unwrap();
let listed = handle.list().await;
assert_eq!(
listed[0].out_file, listed[1].out_file,
"fixture check: both apps must really resolve to one path, or \
this case proves nothing"
);
let flushed = handle
.flush(ProcessSelector::Name("web".to_string()))
.await
.unwrap();
assert_eq!(
flushed.iter().map(|info| info.id).collect::<Vec<_>>(),
vec![0],
"the reply answers the selector: the sibling's file was emptied \
too, but the operator never named the sibling and it is not a \
row here"
);
assert!(
shared.exists(),
"the sibling's pump writes as it answers a flush, so a path that \
is not there means it was never asked: {}",
shared.display()
);
assert_eq!(
std::fs::read_to_string(&shared).unwrap(),
"",
"a line the unmatched sibling landed as it answered must not \
survive the truncate of the path it shares"
);
}
#[tokio::test(start_paused = true)]
async fn a_pump_that_could_not_flush_fails_the_request() {
let (events, _rx) = tokio::sync::broadcast::channel(64);
let scripted = Arc::new(ScriptedRunner::new(vec![
ProcScript::never_exits(),
ProcScript::never_exits(),
]));
let dir = tempfile::tempdir().unwrap();
let handle = spawn_supervisor(
FailingPumpRunner::new(Arc::clone(&scripted)),
test_paths(&dir),
events,
);
handle
.start(vec![
normalize(AppConfig::minimal(REFUSING_SHEEP, "./srv")).unwrap(),
normalize(AppConfig::minimal("api", "./api")).unwrap(),
])
.await
.unwrap();
let error =
tokio::time::timeout(Duration::from_secs(5), handle.flush(ProcessSelector::All))
.await
.expect("a pump that answers must not leave the flush waiting")
.expect_err("a flush a pump could not carry out must not answer Ok");
assert_eq!(
error,
SupervisorError::FlushFailed(PUMP_REFUSAL.to_string()),
"the failure must carry the path, and only the path that failed"
);
assert_eq!(
scripted.flushes(1),
1,
"the healthy sheep must still have been flushed"
);
}
#[tokio::test(start_paused = true)]
async fn a_flush_matching_nothing_is_not_found() {
let (events, _rx) = tokio::sync::broadcast::channel(64);
let dir = tempfile::tempdir().unwrap();
let handle = spawn_supervisor(ScriptedRunner::new(vec![]), test_paths(&dir), events);
assert_eq!(
handle.flush(ProcessSelector::All).await,
Err(SupervisorError::NotFound)
);
}
#[tokio::test]
async fn truncating_a_path_that_is_not_there_creates_nothing_and_succeeds() {
let dir = tempfile::tempdir().unwrap();
let missing = dir.path().join("never-started-out.log");
assert_eq!(truncate_log(&missing).await, Ok(()));
assert!(
!missing.exists(),
"a flush must not create the log file it did not find"
);
}
#[tokio::test]
async fn truncating_a_path_that_is_a_directory_reports_the_failure() {
let dir = tempfile::tempdir().unwrap();
let blocked = dir.path().join("web-out.log");
std::fs::create_dir(&blocked).unwrap();
let error = truncate_log(&blocked)
.await
.expect_err("a path that could not be truncated must not answer Ok");
assert!(
error
.message
.starts_with(&format!("{}: ", blocked.display())),
"the failure must name the path it could not empty: {error}"
);
}
#[cfg(unix)]
#[tokio::test]
async fn truncating_a_symlinked_log_path_refuses_and_leaves_its_target_alone() {
let dir = tempfile::tempdir().unwrap();
let target = dir.path().join("precious.txt");
let link = dir.path().join("web-out.log");
std::fs::write(&target, b"do not empty me").unwrap();
std::os::unix::fs::symlink(&target, &link).unwrap();
let error = truncate_log(&link)
.await
.expect_err("a symlinked log path must not be truncated through");
assert_eq!(
std::fs::read(&target).unwrap(),
b"do not empty me",
"the symlink's target must still hold every byte it did"
);
assert!(
std::fs::symlink_metadata(&link)
.unwrap()
.file_type()
.is_symlink(),
"the refusal must leave the symlink itself in place, not replace it"
);
assert_eq!(
error.message,
format!("{}: {}", link.display(), crate::runner::SYMLINK_REFUSED),
"the failure must name the path and say the word symlink: {error}"
);
}
#[tokio::test(start_paused = true)]
async fn a_flush_naming_a_replacement_still_drains_the_drainee_sharing_its_path() {
let dir = tempfile::tempdir().unwrap();
let (handle, runner, mut rx) = started(
&dir,
AppConfig::minimal("web", "./srv"),
vec![ProcScript::never_exits(), ProcScript::never_exits()],
)
.await;
handle
.reload(ProcessSelector::Name("web".to_string()))
.await
.expect("the reload is accepted");
expect_event(&mut rx, 1, ProcessEventKind::Start).await;
let mid = handle.list().await;
assert_eq!(
mid.len(),
2,
"fixture check: both halves of the swap must be registered, or \
there is no shared path to widen to"
);
assert_eq!(
mid[0].out_file, mid[1].out_file,
"fixture check: one instance slot must really give both entries \
one out path, or this case proves nothing"
);
assert_eq!(
mid[0].err_file, mid[1].err_file,
"fixture check: and one err path"
);
let flushed = handle.flush(ProcessSelector::Id(1)).await.unwrap();
assert_eq!(
flushed.iter().map(|info| info.id).collect::<Vec<_>>(),
vec![1],
"the reply answers the selector: the drainee's pump was drained \
too, but the operator named only the replacement"
);
assert_eq!(
runner.flushes(0),
1,
"the drainee's pump is what this case exists for — it is still \
holding the file the truncate is about to empty"
);
assert_eq!(
runner.flushes(1),
1,
"the replacement's pump, which the selector did name"
);
}
#[tokio::test(start_paused = true)]
async fn a_reopen_naming_a_replacement_still_reaches_the_drainee_sharing_its_path() {
let dir = tempfile::tempdir().unwrap();
let (handle, runner, mut rx) = started(
&dir,
AppConfig::minimal("web", "./srv"),
vec![ProcScript::never_exits(), ProcScript::never_exits()],
)
.await;
handle
.reload(ProcessSelector::Name("web".to_string()))
.await
.expect("the reload is accepted");
expect_event(&mut rx, 1, ProcessEventKind::Start).await;
let mid = handle.list().await;
assert_eq!(
mid.len(),
2,
"fixture check: both halves of the swap must be registered"
);
assert_eq!(
mid[0].out_file, mid[1].out_file,
"fixture check: one instance slot must really give both entries \
one out path, or this case proves nothing"
);
assert_eq!(
mid[0].err_file, mid[1].err_file,
"fixture check: and one err path"
);
let reopened = handle.reopen(ProcessSelector::Id(1)).await.unwrap();
assert_eq!(
reopened.iter().map(|info| info.id).collect::<Vec<_>>(),
vec![1],
"the reply answers the selector, the same way `flush`'s does"
);
assert_eq!(
runner.reopens(0),
1,
"the drainee's pump is what this case exists for — unasked, it \
keeps the renamed inode open and goes on filling the archive"
);
assert_eq!(
runner.reopens(1),
1,
"the replacement's pump, which the selector did name"
);
assert_eq!(
runner.flushes(0),
0,
"a reopen must push `LogCtl::Reopen`, never `LogCtl::Flush` — the \
neighbouring variant would land the drainee's owed bytes and \
leave it on the renamed inode regardless"
);
}
fn log_ctl_spec() -> SpawnSpec {
SpawnSpec {
name: "svc".to_string(),
program: "./svc".to_string(),
args: Vec::new(),
cwd: None,
env: std::collections::BTreeMap::new(),
out_file: std::path::PathBuf::from("out.log"),
err_file: std::path::PathBuf::from("err.log"),
channel: false,
stdin: false,
credentials: None,
}
}
#[derive(Debug, Clone, Copy)]
enum Step {
List,
StopAll,
RestartAll,
DeleteFirst,
StartOne,
Report,
StaleReport,
}
fn step_strategy() -> impl proptest::strategy::Strategy<Value = Step> {
proptest::prop_oneof![
proptest::strategy::Just(Step::List),
proptest::strategy::Just(Step::StopAll),
proptest::strategy::Just(Step::RestartAll),
proptest::strategy::Just(Step::DeleteFirst),
proptest::strategy::Just(Step::StartOne),
proptest::strategy::Just(Step::Report),
proptest::strategy::Just(Step::StaleReport),
]
}
#[derive(Debug, Clone, Copy)]
enum Gate {
Ungated,
Channel(u64),
}
fn gate_strategy() -> impl proptest::strategy::Strategy<Value = Gate> {
use proptest::strategy::Strategy as _; proptest::prop_oneof![
2 => proptest::strategy::Just(Gate::Ungated),
1 => (1u64..4_000u64).prop_map(Gate::Channel),
]
}
fn script_strategy() -> impl proptest::strategy::Strategy<Value = ProcScript> {
proptest::prop_oneof![
6 => proptest::strategy::Just(ProcScript::never_exits()),
2 => proptest::strategy::Just(ProcScript::const_exit(1)),
1 => proptest::strategy::Just(ProcScript::stable_then_exit(2_000, 0)),
1 => proptest::strategy::Just(ProcScript::ignores_signals()),
]
}
const SCRIPT_POOL: usize = 512;
const QUIET_WINDOW: Duration = Duration::from_secs(60);
const EVENT_BUDGET: usize = 3 * SCRIPT_POOL;
proptest::proptest! {
#![proptest_config(crate::testing::proptest_config(128))]
#[test]
fn supervisor_upholds_its_invariants_under_any_interleaving(
steps in proptest::collection::vec(step_strategy(), 1..10),
gates in proptest::collection::vec(gate_strategy(), 1..10),
scripts in proptest::collection::vec(script_strategy(), SCRIPT_POOL..SCRIPT_POOL + 1),
) {
let runtime = tokio::runtime::Builder::new_current_thread()
.enable_all()
.start_paused(true)
.build()
.unwrap();
let dir = tempfile::tempdir().unwrap();
runtime.block_on(async move {
let (events, mut rx) = tokio::sync::broadcast::channel(8192);
let handle = spawn_supervisor(
ScriptedRunner::new(scripts),
test_paths(&dir),
events,
);
let mut started = 0u32;
let mut highest_restarts = std::collections::HashMap::<u32, u32>::new();
for step in steps {
match step {
Step::StartOne => {
let gate = gates[started as usize % gates.len()];
started += 1;
let mut app = AppConfig::minimal(&format!("sheep-{started}"), "./s");
if let Gate::Channel(ms) = gate {
app.wait_ready = true;
app.listen_timeout = UpDuration::from_millis(ms);
}
let _ = handle.start(vec![normalize(app).unwrap()]).await;
}
Step::StopAll => {
if let Ok(stopped) = handle.stop(ProcessSelector::All).await {
for info in stopped {
proptest::prop_assert_eq!(info.status, ProcStatus::Stopped);
}
}
}
Step::RestartAll => {
let _ = handle.restart(ProcessSelector::All).await;
}
Step::DeleteFirst => {
if let Some(first) = handle.list().await.first() {
let id = first.id;
if let Ok(deleted) = handle.delete(ProcessSelector::Id(id)).await {
proptest::prop_assert_eq!(deleted, vec![id]);
}
proptest::prop_assert!(
handle.list().await.iter().all(|i| i.id != id)
);
}
}
Step::Report => {
if let Some(first) = handle.list().await.first()
&& let Some(pid) = first.pid
{
handle.extra_restart(first.id, pid).await;
}
}
Step::StaleReport => {
if let Some(first) = handle.list().await.first() {
let stale = first.pid.unwrap_or(0).wrapping_add(1);
handle.extra_restart(first.id, stale).await;
}
}
Step::List => {}
}
let listed = handle.list().await;
let ids: Vec<u32> = listed.iter().map(|i| i.id).collect();
let mut sorted = ids.clone();
sorted.sort_unstable();
sorted.dedup();
proptest::prop_assert_eq!(&ids, &sorted);
for info in &listed {
let seen = highest_restarts.entry(info.id).or_default();
proptest::prop_assert!(info.restarts >= *seen);
*seen = info.restarts;
proptest::prop_assert!(matches!(
info.status,
ProcStatus::Starting | ProcStatus::Online | ProcStatus::Stopping
| ProcStatus::Stopped | ProcStatus::Errored | ProcStatus::WaitingRestart
));
}
}
let mut observed = Vec::new();
loop {
match tokio::time::timeout(QUIET_WINDOW, rx.recv()).await {
Ok(Ok(event)) => {
observed.push(event);
proptest::prop_assert!(
observed.len() <= EVENT_BUDGET,
"the flock never reached steady state: {} transitions after \
the last command",
observed.len()
);
}
Ok(Err(tokio::sync::broadcast::error::RecvError::Lagged(skipped))) => {
return Err(proptest::test_runner::TestCaseError::fail(format!(
"event stream lagged by {skipped}: the invariants below cannot \
be read off a stream with holes in it"
)));
}
Ok(Err(tokio::sync::broadcast::error::RecvError::Closed)) => break,
Err(_elapsed) => break, }
}
let mut live = std::collections::HashSet::<u32>::new();
let mut event_restarts = std::collections::HashMap::<u32, u32>::new();
for event in observed {
let BusEvent::Process { event, info, .. } = event else {
continue;
};
match event {
ProcessEventKind::Start => {
proptest::prop_assert!(
live.insert(info.id),
"two live spawns for id {}",
info.id
);
}
ProcessEventKind::Restart => {
live.insert(info.id);
}
ProcessEventKind::Online => {
proptest::prop_assert!(
live.contains(&info.id),
"id {} was marked online with no live process: a readiness \
wait resolved onto a sheep that had already gone terminal",
info.id
);
}
ProcessEventKind::Exit
| ProcessEventKind::Stop
| ProcessEventKind::Errored
| ProcessEventKind::Delete => {
live.remove(&info.id);
}
_ => {}
}
let seen = event_restarts.entry(info.id).or_default();
proptest::prop_assert!(
info.restarts >= *seen,
"restart count for id {} went backwards: {} after {}",
info.id,
info.restarts,
*seen
);
*seen = info.restarts;
}
Ok::<(), proptest::test_runner::TestCaseError>(())
})?;
}
}
}