use core::fmt;
use core::time::Duration;
use std::collections::{HashMap, HashSet};
use std::sync::Arc;
use tokio::sync::mpsc;
use tokio::task::JoinHandle;
use shep_core::config::{AppConfig, CronSchedule, ProbeTarget};
use shep_core::values::UpDuration;
use crate::cron::{Clock, SystemClock, spawn_cron_worker};
use crate::entry::ProcessEntry;
use crate::limits::sample::{MemorySampler, SysinfoSampler};
use crate::limits::stats::StatsState;
use crate::limits::{LimitBreach, LimitEnforcer, PollingEnforcer};
use crate::probes::{LivenessFailure, Prober, spawn_liveness_task};
use crate::supervisor::SupervisorHandle;
use crate::watch::{
DEFAULT_WATCH_DELAY, MIN_WATCH_DELAY, WatchFilter, own_log_ignores, spawn_watch_group,
};
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) struct LivenessReport {
pub id: u32,
pub pid: u32,
pub epoch: u64,
}
#[derive(Debug, Clone)]
pub struct ExtrasReports {
pub breaches: mpsc::Sender<LimitBreach>,
pub liveness: mpsc::Sender<LivenessReport>,
}
pub struct Extras {
pub clock: Arc<dyn Clock>,
pub enforcer: Arc<dyn LimitEnforcer>,
pub max_cron_sleep: Duration,
pub reports: ExtrasReports,
pub stats: Arc<StatsState>,
}
impl Extras {
#[must_use]
pub fn real(reports: ExtrasReports, max_cron_sleep: Duration) -> Self {
let sampler: Arc<dyn MemorySampler> = Arc::new(SysinfoSampler::new());
let stats = Arc::new(StatsState::new(Arc::clone(&sampler)));
let enforcer =
PollingEnforcer::start(sampler, reports.breaches.clone(), Arc::clone(&stats));
Self {
clock: Arc::new(SystemClock),
enforcer: Arc::new(enforcer),
max_cron_sleep,
reports,
stats,
}
}
}
impl fmt::Debug for Extras {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("Extras")
.field("clock", &"<dyn Clock>")
.field("enforcer", &"<dyn LimitEnforcer>")
.field("max_cron_sleep", &self.max_cron_sleep)
.finish_non_exhaustive()
}
}
pub fn spawn_extras_reporter(
mut breaches: mpsc::Receiver<LimitBreach>,
mut liveness: mpsc::Receiver<LivenessReport>,
supervisor: SupervisorHandle,
) -> JoinHandle<()> {
tokio::spawn(async move {
let mut breaches_open = true;
let mut liveness_open = true;
while breaches_open || liveness_open {
tokio::select! {
maybe_breach = breaches.recv(), if breaches_open => match maybe_breach {
Some(breach) => {
tracing::warn!(
id = breach.id,
pid = breach.root_pid,
observed = %breach.observed,
limit = %breach.limit,
"process tree exceeded its max_memory; restarting"
);
supervisor
.extra_restart(
breach.id,
breach.root_pid,
None,
Some(breach.observed),
)
.await;
}
None => breaches_open = false,
},
maybe_failure = liveness.recv(), if liveness_open => match maybe_failure {
Some(report) => {
tracing::warn!(
id = report.id,
pid = report.pid,
"liveness probe hit its failure_threshold; restarting"
);
supervisor
.extra_restart(report.id, report.pid, Some(report.epoch), None)
.await;
}
None => liveness_open = false,
},
}
}
})
}
#[derive(Debug, Default)]
pub struct ExtrasRegistry {
groups: HashMap<String, NameExtras>,
instances: HashMap<u32, InstanceExtras>,
liveness_epochs: HashMap<u32, u64>,
}
#[derive(Debug, Default)]
struct NameExtras {
cron: Option<JoinHandle<()>>,
watch: Option<JoinHandle<()>>,
members: HashSet<u32>,
}
impl NameExtras {
fn abort(self) {
if let Some(cron) = self.cron {
cron.abort();
}
if let Some(watch) = self.watch {
watch.abort();
}
}
}
struct InstanceExtras {
stats: Arc<StatsState>,
limit: Option<Arc<dyn LimitEnforcer>>,
liveness: Option<JoinHandle<()>>,
}
impl InstanceExtras {
fn watched_only(stats: Arc<StatsState>) -> Self {
Self {
stats,
limit: None,
liveness: None,
}
}
fn disarm(self, id: u32) {
self.stats.unwatch(id);
if let Some(enforcer) = self.limit {
enforcer.disarm(id);
}
if let Some(liveness) = self.liveness {
liveness.abort();
}
}
}
impl fmt::Debug for InstanceExtras {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("InstanceExtras")
.field("limit_armed", &self.limit.is_some())
.field("liveness", &self.liveness)
.finish_non_exhaustive()
}
}
impl ExtrasRegistry {
pub fn arm(
&mut self,
entry: &ProcessEntry,
prober: Arc<dyn Prober>,
extras: &Extras,
supervisor: &SupervisorHandle,
) {
let config = entry.spec.config();
let id = entry.id;
self.disarm_instance(id);
let liveness_epoch = self.liveness_epochs.entry(id).or_insert(0);
*liveness_epoch += 1;
let liveness_epoch = *liveness_epoch;
if let Some(instance) = arm_instance(entry, prober, extras, liveness_epoch) {
self.instances.insert(id, instance);
}
if config.cron_restart.is_none() && !config.watch {
return;
}
let group = self.groups.entry(config.name.clone()).or_default();
group.members.insert(id);
if group.cron.as_ref().is_none_or(JoinHandle::is_finished) {
group.cron = arm_cron(config, extras, supervisor);
}
if group.watch.as_ref().is_none_or(JoinHandle::is_finished) {
group.watch = arm_watch(entry, supervisor);
}
}
pub fn disarm(&mut self, id: u32, name: &str) {
self.disarm_instance(id);
self.liveness_epochs.remove(&id);
let Some(group) = self.groups.get_mut(name) else {
return;
};
if !group.members.remove(&id) || !group.members.is_empty() {
return;
}
if let Some(group) = self.groups.remove(name) {
group.abort();
}
}
pub fn rearm_name(
&mut self,
name: &str,
entries: &[&ProcessEntry],
prober: impl Fn(&ProcessEntry) -> Arc<dyn Prober>,
extras: &Extras,
supervisor: &SupervisorHandle,
) {
if let Some(group) = self.groups.remove(name) {
group.abort();
}
for entry in entries {
self.arm(entry, prober(entry), extras, supervisor);
}
}
fn disarm_instance(&mut self, id: u32) {
if let Some(instance) = self.instances.remove(&id) {
instance.disarm(id);
}
}
pub(crate) fn liveness_epoch(&self, id: u32) -> u64 {
self.liveness_epochs.get(&id).copied().unwrap_or(0)
}
#[cfg(test)]
pub(crate) fn group_members(&self, name: &str) -> Option<Vec<u32>> {
self.groups.get(name).map(|group| {
let mut members: Vec<u32> = group.members.iter().copied().collect();
members.sort_unstable();
members
})
}
}
impl Drop for ExtrasRegistry {
fn drop(&mut self) {
for (id, instance) in self.instances.drain() {
instance.disarm(id);
}
for (_name, group) in self.groups.drain() {
group.abort();
}
}
}
fn arm_instance(
entry: &ProcessEntry,
prober: Arc<dyn Prober>,
extras: &Extras,
liveness_epoch: u64,
) -> Option<InstanceExtras> {
let config = entry.spec.config();
let wants_anything = config.max_memory.is_some() || config.liveness_probe.is_some();
let Some(pid) = entry.pid else {
if wants_anything {
tracing::warn!(
id = entry.id,
"arming a sheep with no pid; its memory limit and liveness probe stay disarmed"
);
}
return None;
};
extras.stats.watch(entry.id, pid);
let mut instance = InstanceExtras::watched_only(Arc::clone(&extras.stats));
if let Some(limit) = config.max_memory {
extras.enforcer.arm(entry.id, pid, limit);
instance.limit = Some(Arc::clone(&extras.enforcer));
}
if let Some(probe) = config.liveness_probe.as_ref() {
match ProbeTarget::parse(probe) {
Ok(target) => {
let (raw_tx, mut raw_rx) = mpsc::channel::<LivenessFailure>(1);
let reports_liveness = extras.reports.liveness.clone();
tokio::spawn(async move {
if let Some(failure) = raw_rx.recv().await {
let _ = reports_liveness
.send(LivenessReport {
id: failure.id,
pid: failure.pid,
epoch: liveness_epoch,
})
.await;
}
});
instance.liveness = Some(spawn_liveness_task(
entry.id,
pid,
probe.clone(),
target,
prober,
raw_tx,
));
}
Err(err) => {
tracing::warn!(
id = entry.id,
name = config.name.as_str(),
%err,
"liveness_probe target could not be parsed; arming no liveness probe"
);
}
}
}
Some(instance)
}
fn arm_cron(
config: &AppConfig,
extras: &Extras,
supervisor: &SupervisorHandle,
) -> Option<JoinHandle<()>> {
let pattern = config.cron_restart.as_ref()?;
let schedule = match CronSchedule::parse(pattern, config.cron_timezone.as_deref()) {
Ok(schedule) => schedule,
Err(err) => {
tracing::warn!(
name = config.name.as_str(),
pattern = pattern.as_str(),
%err,
"cron_restart pattern could not be parsed; arming no cron worker"
);
return None;
}
};
Some(spawn_cron_worker(
config.name.clone(),
schedule,
Arc::clone(&extras.clock),
supervisor.clone(),
extras.max_cron_sleep,
))
}
fn arm_watch(entry: &ProcessEntry, supervisor: &SupervisorHandle) -> Option<JoinHandle<()>> {
let config = entry.spec.config();
if !config.watch {
return None;
}
let Some(cwd) = config.cwd.as_deref() else {
tracing::warn!(
name = config.name.as_str(),
"watch is on but the app names no cwd; arming no watch"
);
return None;
};
let root = match std::fs::canonicalize(cwd) {
Ok(root) => root,
Err(err) => {
tracing::warn!(
name = config.name.as_str(),
path = cwd,
%err,
"watch root could not be resolved; arming no watch"
);
return None;
}
};
let mut ignores = config.ignore_watch.clone();
ignores.extend(own_log_ignores(
&root,
[entry.out_file.as_path(), entry.err_file.as_path()],
));
let filter = match WatchFilter::new(&config.watch_options, &ignores) {
Ok(filter) => filter,
Err(err) => {
tracing::warn!(
name = config.name.as_str(),
%err,
"watch globs could not be compiled; arming no watch"
);
return None;
}
};
match spawn_watch_group(
config.name.clone(),
root,
filter,
watch_delay_for(config),
supervisor.clone(),
) {
Ok(handle) => Some(handle),
Err(err) => {
tracing::warn!(
name = config.name.as_str(),
%err,
"the OS watch could not be started; arming no watch"
);
None
}
}
}
fn watch_delay_for(config: &AppConfig) -> Duration {
config
.watch_delay
.map(UpDuration::as_duration)
.unwrap_or(DEFAULT_WATCH_DELAY)
.max(MIN_WATCH_DELAY)
}
#[cfg(test)]
mod tests {
use std::collections::HashSet;
use chrono::{DateTime, Utc};
use tokio::sync::broadcast;
use super::*;
use crate::bus::SharedEvent;
use crate::cron::DEFAULT_MAX_CRON_SLEEP;
use crate::fake::{ProcScript, ScriptedRunner};
use crate::limits::PollingEnforcer;
use crate::limits::sample::ProcessRss;
use crate::probes::ProbeFailure;
use crate::supervisor::spawn_supervisor;
use crate::testing::{
ArmCall, Harness, RecordingEnforcer, ScriptedProber, ScriptedSampler, TestClock, app_with,
armed_entry, capture_logs, harness, harness_with_extras, idle_stats, probe_config,
test_paths, touch,
};
use crate::watch::real_time;
use shep_core::config::{ProbeConfig, ProbeKind};
use shep_core::protocol::{BusEvent, ProcessEventKind, ProcessInfo};
use shep_core::selector::ProcessSelector;
use shep_core::status::ProcStatus;
use shep_core::values::{MemSize, UpDuration};
const EVENT_WAIT: Duration = Duration::from_secs(120);
const PAST_THE_NEXT_OCCURRENCE: Duration = Duration::from_secs(3_700);
const LIVENESS_DEADLINE: Duration = Duration::from_secs(10);
const PROBE_INTERVAL: UpDuration = UpDuration::from_millis(1_000);
fn dt(s: &str) -> DateTime<Utc> {
s.parse().expect("valid RFC3339 timestamp")
}
struct Rig {
extras: Extras,
enforcer: Arc<RecordingEnforcer>,
clock: Arc<TestClock>,
liveness: mpsc::Receiver<LivenessReport>,
_breaches: mpsc::Receiver<LimitBreach>,
}
fn rig(max_cron_sleep: Duration) -> Rig {
let clock = Arc::new(TestClock::starting_at(dt("2026-01-01T00:00:00Z")));
let enforcer = Arc::new(RecordingEnforcer::default());
let (breach_tx, breaches) = mpsc::channel(8);
let (live_tx, liveness) = mpsc::channel(8);
Rig {
extras: Extras {
clock: Arc::clone(&clock) as Arc<dyn Clock>,
enforcer: Arc::clone(&enforcer) as Arc<dyn LimitEnforcer>,
max_cron_sleep,
reports: ExtrasReports {
breaches: breach_tx,
liveness: live_tx,
},
stats: idle_stats(),
},
enforcer,
clock,
liveness,
_breaches: breaches,
}
}
fn spawn_test_fixture() -> (
SupervisorHandle,
broadcast::Receiver<SharedEvent>,
tempfile::TempDir,
) {
let dir = tempfile::tempdir().unwrap();
let (events, rx) = crate::bus::test_bus(64);
let runner = ScriptedRunner::new(vec![ProcScript::never_exits(); 12]);
let handle = spawn_supervisor(runner, test_paths(&dir), events);
(handle, rx, dir)
}
fn idle_prober() -> Arc<dyn Prober> {
Arc::new(ScriptedProber::new(vec![]))
}
fn failing_prober() -> Arc<dyn Prober> {
Arc::new(ScriptedProber::new(vec![Err(ProbeFailure::Timeout)]))
}
async fn settle_finished(task: &JoinHandle<()>) {
for _ in 0..100 {
if task.is_finished() {
return;
}
tokio::task::yield_now().await;
}
panic!("the task never finished");
}
async fn expect_restart(
rx: &mut broadcast::Receiver<SharedEvent>,
name: &str,
window: Duration,
) -> ProcessInfo {
expect_restart_event(rx, name, window).await.0
}
async fn expect_restart_event(
rx: &mut broadcast::Receiver<SharedEvent>,
name: &str,
window: Duration,
) -> (ProcessInfo, bool) {
let restart = async {
loop {
match rx.recv().await.map(|event| event.to_event()) {
Ok(BusEvent::Process {
event: ProcessEventKind::Restart,
info,
manually,
..
}) if info.name == name => return (info, manually),
Ok(_) => continue,
Err(broadcast::error::RecvError::Lagged(_)) => continue,
Err(err) => panic!("event stream closed before a restart of {name}: {err}"),
}
}
};
match tokio::time::timeout(window, restart).await {
Ok(observed) => observed,
Err(_) => panic!("timed out waiting for a restart of {name}"),
}
}
async fn assert_no_restart_within(
rx: &mut broadcast::Receiver<SharedEvent>,
name: &str,
window: Duration,
) {
let deadline = tokio::time::Instant::now() + window;
loop {
let remaining = deadline.saturating_duration_since(tokio::time::Instant::now());
match tokio::time::timeout(remaining, rx.recv())
.await
.map(|received| received.map(|event| event.to_event()))
{
Err(_) => return, Ok(Ok(BusEvent::Process {
event: ProcessEventKind::Restart,
info,
..
})) if info.name == name => {
panic!(
"unexpected restart of {name} observed (restarts={})",
info.restarts
);
}
Ok(Ok(_)) => continue,
Ok(Err(broadcast::error::RecvError::Lagged(skipped))) => {
panic!(
"event stream lagged by {skipped} while checking for no restart of \
{name}: a skipped event may have been the restart this forbids"
)
}
Ok(Err(err)) => {
panic!("event channel closed while checking for no restart of {name}: {err}")
}
}
}
}
async fn expect_liveness(
rx: &mut mpsc::Receiver<LivenessReport>,
window: Duration,
) -> LivenessReport {
match tokio::time::timeout(window, rx.recv()).await {
Ok(Some(failure)) => failure,
Ok(None) => panic!("the liveness channel closed before a failure arrived"),
Err(_) => panic!("timed out waiting for a liveness failure"),
}
}
async fn assert_no_liveness_within(rx: &mut mpsc::Receiver<LivenessReport>, window: Duration) {
match tokio::time::timeout(window, rx.recv()).await {
Err(_) => {} Ok(Some(failure)) => panic!("unexpected liveness failure observed: {failure:?}"),
Ok(None) => panic!("the liveness channel disconnected while checking for silence"),
}
}
async fn cross_one_hour() {
for _ in 0..120 {
tokio::time::advance(Duration::from_secs(30)).await;
}
}
#[tokio::test(start_paused = true)]
async fn extras_debug_names_the_seams_by_role_and_the_sleep_bound_by_value() {
let rig = rig(Duration::from_secs(300));
assert_eq!(
format!("{:?}", rig.extras),
r#"Extras { clock: "<dyn Clock>", enforcer: "<dyn LimitEnforcer>", max_cron_sleep: 300s, .. }"#
);
}
#[tokio::test(start_paused = true)]
async fn instance_extras_debug_reports_an_arming_without_naming_the_enforcer() {
let dir = tempfile::tempdir().unwrap();
let paths = test_paths(&dir);
let (handle, _rx, _fixture) = spawn_test_fixture();
let rig = rig(DEFAULT_MAX_CRON_SLEEP);
let mut registry = ExtrasRegistry::default();
let capped = app_with("web", |app| app.max_memory = Some(MemSize::from_bytes(500)));
let uncapped = app_with("api", |app| app.max_memory = None);
registry.arm(
&armed_entry(0, 0, 1000, capped, &paths),
idle_prober(),
&rig.extras,
&handle,
);
registry.arm(
&armed_entry(1, 0, 1001, uncapped, &paths),
idle_prober(),
&rig.extras,
&handle,
);
assert_eq!(
format!("{:?}", registry.instances[&0]),
"InstanceExtras { limit_armed: true, liveness: None, .. }"
);
assert_eq!(
format!("{:?}", registry.instances[&1]),
"InstanceExtras { limit_armed: false, liveness: None, .. }"
);
}
#[tokio::test(start_paused = true)]
async fn real_extras_wire_the_enforcer_to_the_reports_channel() {
let (breach_tx, mut breaches) = mpsc::channel(4);
let (live_tx, _liveness) = mpsc::channel(4);
let extras = Extras::real(
ExtrasReports {
breaches: breach_tx,
liveness: live_tx,
},
Duration::from_secs(300),
);
assert_eq!(
format!("{extras:?}"),
r#"Extras { clock: "<dyn Clock>", enforcer: "<dyn LimitEnforcer>", max_cron_sleep: 300s, .. }"#,
"`real` must carry the sleep bound it was handed, not re-derive one"
);
let limit = MemSize::from_bytes(1);
extras.enforcer.arm(3, std::process::id(), limit);
let breach = match tokio::time::timeout(EVENT_WAIT, breaches.recv()).await {
Ok(Some(breach)) => breach,
Ok(None) => panic!("the breach channel closed before a breach arrived"),
Err(_) => panic!("timed out waiting for a breach from the real enforcer"),
};
assert_eq!(breach.id, 3);
assert_eq!(breach.root_pid, std::process::id());
assert_eq!(breach.limit, limit);
}
#[tokio::test(start_paused = true)]
async fn an_app_configuring_no_extras_arms_nothing() {
let dir = tempfile::tempdir().unwrap();
let paths = test_paths(&dir);
let root = tempfile::tempdir().unwrap();
let (handle, _rx, _fixture) = spawn_test_fixture();
let rig = rig(DEFAULT_MAX_CRON_SLEEP);
let mut registry = ExtrasRegistry::default();
let app = app_with("web", |app| {
app.cwd = Some(root.path().display().to_string());
});
let entry = armed_entry(0, 0, 1000, app, &paths);
registry.arm(&entry, idle_prober(), &rig.extras, &handle);
assert!(
registry.groups.is_empty(),
"an app with neither cron_restart nor watch must arm no name-group tasks"
);
assert_eq!(
format!("{:?}", registry.instances[&0]),
"InstanceExtras { limit_armed: false, liveness: None, .. }",
"an app with neither max_memory nor liveness_probe must arm nothing beyond sampling"
);
assert!(
rig.enforcer.arms().is_empty(),
"an app with no max_memory must not reach the enforcer at all"
);
}
#[tokio::test(start_paused = true)]
async fn every_sheep_with_a_pid_is_watched_even_with_no_limit_and_no_probe() {
let dir = tempfile::tempdir().unwrap();
let paths = test_paths(&dir);
let (handle, _rx, _fixture) = spawn_test_fixture();
let rig = rig(DEFAULT_MAX_CRON_SLEEP);
let mut registry = ExtrasRegistry::default();
let app = app_with("web", |app| {
app.max_memory = None;
app.liveness_probe = None;
});
registry.arm(
&armed_entry(7, 0, 4242, app, &paths),
idle_prober(),
&rig.extras,
&handle,
);
assert_eq!(
rig.extras.stats.watched_for_test(),
vec![(7, 4242)],
"an app with neither max_memory nor a liveness_probe is the ORDINARY case, and a \
listing reporting `-` for every one of them is what this split exists to fix"
);
assert!(
rig.enforcer.arms().is_empty(),
"sampling is not enforcement: an app with no ceiling must still not be armed"
);
registry.disarm(7, "web");
assert!(rig.extras.stats.watched_for_test().is_empty());
}
#[tokio::test(start_paused = true)]
async fn a_cron_only_app_with_a_real_cwd_arms_no_watch() {
let dir = tempfile::tempdir().unwrap();
let paths = test_paths(&dir);
let root = tempfile::tempdir().unwrap();
let (handle, _rx, _fixture) = spawn_test_fixture();
let rig = rig(DEFAULT_MAX_CRON_SLEEP);
let mut registry = ExtrasRegistry::default();
let app = app_with("web", |app| {
app.cron_restart = Some("0 * * * *".to_string());
app.cwd = Some(root.path().display().to_string());
});
registry.arm(
&armed_entry(0, 0, 1000, app, &paths),
idle_prober(),
&rig.extras,
&handle,
);
let group = ®istry.groups["web"];
assert!(group.cron.is_some(), "the cron worker is what armed here");
assert!(
group.watch.is_none(),
"an app that did not ask to be watched must get no watcher on its cwd"
);
}
#[tokio::test(start_paused = true)]
async fn a_cron_pattern_is_resolved_in_the_apps_own_timezone() {
let dir = tempfile::tempdir().unwrap();
let paths = test_paths(&dir);
let (handle, mut rx, _fixture) = spawn_test_fixture();
let rig = rig(DEFAULT_MAX_CRON_SLEEP);
let mut registry = ExtrasRegistry::default();
let app = app_with("web", |app| {
app.cron_restart = Some("0 5 * * *".to_string());
app.cron_timezone = Some("Etc/GMT+5".to_string());
});
handle.start(vec![app.clone()]).await.unwrap();
registry.arm(
&armed_entry(0, 0, 1000, app, &paths),
idle_prober(),
&rig.extras,
&handle,
);
tokio::task::yield_now().await;
assert_no_restart_within(&mut rx, "web", Duration::from_secs(6 * 3_600)).await;
expect_restart(&mut rx, "web", Duration::from_secs(6 * 3_600)).await;
}
#[tokio::test(start_paused = true)]
async fn a_watch_root_that_will_not_resolve_costs_the_app_its_watch_and_nothing_else() {
let dir = tempfile::tempdir().unwrap();
let paths = test_paths(&dir);
let root = tempfile::tempdir().unwrap();
let (handle, _rx, _fixture) = spawn_test_fixture();
let rig = rig(DEFAULT_MAX_CRON_SLEEP);
let mut registry = ExtrasRegistry::default();
let app = app_with("web", |app| {
app.watch = true;
app.cwd = Some(root.path().join("no-such-directory").display().to_string());
app.cron_restart = Some("0 * * * *".to_string());
});
registry.arm(
&armed_entry(0, 0, 1000, app, &paths),
idle_prober(),
&rig.extras,
&handle,
);
let group = ®istry.groups["web"];
assert!(group.watch.is_none(), "the watch root cannot have resolved");
assert!(
group.cron.is_some(),
"an unresolvable watch root must not cost this app its cron worker too"
);
}
#[tokio::test(start_paused = true)]
async fn a_watch_root_that_will_not_resolve_says_in_the_log_which_app_lost_its_watch() {
let dir = tempfile::tempdir().unwrap();
let paths = test_paths(&dir);
let root = tempfile::tempdir().unwrap();
let (handle, _rx, _fixture) = spawn_test_fixture();
let rig = rig(DEFAULT_MAX_CRON_SLEEP);
let mut registry = ExtrasRegistry::default();
let app = app_with("unwatchable", |app| {
app.watch = true;
app.cwd = Some(root.path().join("no-such-directory").display().to_string());
});
let entry = armed_entry(0, 0, 1000, app, &paths);
let records = capture_logs(|| {
registry.arm(&entry, idle_prober(), &rig.extras, &handle);
});
assert!(
registry.groups["unwatchable"].watch.is_none(),
"precondition: the watch root cannot have resolved"
);
assert!(
records.contains("watch root could not be resolved"),
"arming no watch must be reported, not swallowed: {records:?}"
);
assert!(
records.contains("WARN"),
"an app silently losing its watch is a warning, not a debug detail: {records:?}"
);
assert!(
records.contains(r#"name="unwatchable""#),
"the record must name the app that lost its watch: {records:?}"
);
}
#[tokio::test(start_paused = true)]
async fn a_second_instance_of_a_name_arms_no_second_cron_worker() {
let dir = tempfile::tempdir().unwrap();
let paths = test_paths(&dir);
let (handle, mut rx, _fixture) = spawn_test_fixture();
let rig = rig(Duration::from_secs(600));
let mut registry = ExtrasRegistry::default();
let app = app_with("web", |app| {
app.cron_restart = Some("0 * * * *".to_string());
app.instances = 2;
});
handle.start(vec![app.clone()]).await.unwrap();
for (id, instance, pid) in [(0, 0, 1000), (1, 1, 1001)] {
let entry = armed_entry(id, instance, pid, app.clone(), &paths);
registry.arm(&entry, idle_prober(), &rig.extras, &handle);
tokio::task::yield_now().await;
}
assert_eq!(registry.groups.len(), 1, "one group, not one per instance");
assert_eq!(
registry.groups["web"].members,
HashSet::from([0, 1]),
"both instances must be recorded as keeping the group alive"
);
cross_one_hour().await;
expect_restart(&mut rx, "web", EVENT_WAIT).await;
assert!(
rig.clock.reads() < 20,
"one cron worker reads the clock ~13 times over this hour; two read ~26 (got {})",
rig.clock.reads()
);
}
#[tokio::test(start_paused = true)]
async fn a_respawn_re_arms_the_enforcer_with_the_new_pid() {
let dir = tempfile::tempdir().unwrap();
let paths = test_paths(&dir);
let (handle, _rx, _fixture) = spawn_test_fixture();
let rig = rig(DEFAULT_MAX_CRON_SLEEP);
let mut registry = ExtrasRegistry::default();
let limit = MemSize::from_bytes(500);
let app = app_with("web", |app| app.max_memory = Some(limit));
registry.arm(
&armed_entry(4, 0, 1000, app.clone(), &paths),
idle_prober(),
&rig.extras,
&handle,
);
registry.arm(
&armed_entry(4, 0, 2000, app, &paths),
idle_prober(),
&rig.extras,
&handle,
);
assert_eq!(
rig.enforcer.arms(),
vec![
ArmCall {
id: 4,
root_pid: 1000,
limit,
},
ArmCall {
id: 4,
root_pid: 2000,
limit,
},
]
);
assert_eq!(
rig.enforcer.disarms(),
vec![4],
"a re-arm must undo the previous arming rather than leaking it"
);
}
#[tokio::test(start_paused = true)]
async fn only_the_last_instance_leaving_stops_the_name_groups_cron_worker() {
let dir = tempfile::tempdir().unwrap();
let paths = test_paths(&dir);
let (handle, mut rx, _fixture) = spawn_test_fixture();
let rig = rig(DEFAULT_MAX_CRON_SLEEP);
let mut registry = ExtrasRegistry::default();
let app = app_with("web", |app| {
app.cron_restart = Some("0 * * * *".to_string());
app.instances = 2;
});
handle.start(vec![app.clone()]).await.unwrap();
for (id, instance, pid) in [(0, 0, 1000), (1, 1, 1001)] {
let entry = armed_entry(id, instance, pid, app.clone(), &paths);
registry.arm(&entry, idle_prober(), &rig.extras, &handle);
tokio::task::yield_now().await;
}
registry.disarm(0, "web");
assert_eq!(
registry.groups["web"].members,
HashSet::from([1]),
"a non-last disarm drops only its own membership"
);
cross_one_hour().await;
expect_restart(&mut rx, "web", EVENT_WAIT).await;
expect_restart(&mut rx, "web", EVENT_WAIT).await;
registry.disarm(1, "web");
assert!(
!registry.groups.contains_key("web"),
"the last instance leaving must take the group with it"
);
assert_no_restart_within(&mut rx, "web", PAST_THE_NEXT_OCCURRENCE).await;
}
#[tokio::test(start_paused = true)]
async fn disarming_an_id_that_was_never_armed_leaves_the_group_alone() {
let dir = tempfile::tempdir().unwrap();
let paths = test_paths(&dir);
let (handle, mut rx, _fixture) = spawn_test_fixture();
let rig = rig(DEFAULT_MAX_CRON_SLEEP);
let mut registry = ExtrasRegistry::default();
let app = app_with("web", |app| {
app.cron_restart = Some("0 * * * *".to_string())
});
handle.start(vec![app.clone()]).await.unwrap();
registry.arm(
&armed_entry(0, 0, 1000, app, &paths),
idle_prober(),
&rig.extras,
&handle,
);
tokio::task::yield_now().await;
registry.disarm(99, "web"); registry.disarm(0, "other");
assert_eq!(registry.groups["web"].members, HashSet::from([0]));
cross_one_hour().await;
expect_restart(&mut rx, "web", EVENT_WAIT).await;
}
#[tokio::test(start_paused = true)]
async fn a_cron_worker_that_ended_on_its_own_is_rebuilt_on_the_next_arm() {
let dir = tempfile::tempdir().unwrap();
let paths = test_paths(&dir);
let (handle, _rx, _fixture) = spawn_test_fixture();
let rig = rig(DEFAULT_MAX_CRON_SLEEP);
let mut registry = ExtrasRegistry::default();
let app = app_with("web", |app| {
app.cron_restart = Some("0 0 30 2 *".to_string());
});
registry.arm(
&armed_entry(0, 0, 1000, app.clone(), &paths),
idle_prober(),
&rig.extras,
&handle,
);
settle_finished(
registry.groups["web"]
.cron
.as_ref()
.expect("the first arm spawns a worker"),
)
.await;
assert_eq!(
rig.clock.reads(),
1,
"a worker on a pattern with no occurrence reads the clock once and ends"
);
registry.arm(
&armed_entry(0, 0, 2000, app, &paths),
idle_prober(),
&rig.extras,
&handle,
);
settle_finished(
registry.groups["web"]
.cron
.as_ref()
.expect("the re-arm must leave a worker behind"),
)
.await;
assert_eq!(
rig.clock.reads(),
2,
"a re-arm must rebuild a name-group task that ended on its own"
);
}
#[tokio::test(start_paused = true)]
async fn an_instance_whose_watch_could_not_be_armed_still_joins_its_group() {
let dir = tempfile::tempdir().unwrap();
let paths = test_paths(&dir);
let (handle, _rx, _fixture) = spawn_test_fixture();
let rig = rig(DEFAULT_MAX_CRON_SLEEP);
let mut registry = ExtrasRegistry::default();
let limit = MemSize::from_bytes(500);
let missing = dir.path().join("no-such-directory");
let app = app_with("web", |app| {
app.watch = true;
app.cwd = Some(missing.display().to_string());
app.max_memory = Some(limit);
});
registry.arm(
&armed_entry(0, 0, 1000, app, &paths),
idle_prober(),
&rig.extras,
&handle,
);
let group = registry
.groups
.get("web")
.expect("a watched app is a member of its group whether or not the watch armed");
assert!(group.watch.is_none(), "this app's watch cannot have armed");
assert_eq!(group.members, HashSet::from([0]));
assert_eq!(
rig.enforcer.arms(),
vec![ArmCall {
id: 0,
root_pid: 1000,
limit,
}],
"a watch that could not be armed must not take the memory limit with it"
);
}
#[tokio::test(start_paused = true)]
async fn dropping_the_registry_stops_the_liveness_loop_it_armed() {
let dir = tempfile::tempdir().unwrap();
let paths = test_paths(&dir);
let (handle, _rx, _fixture) = spawn_test_fixture();
let mut rig = rig(DEFAULT_MAX_CRON_SLEEP);
let app = app_with("web", |app| {
app.liveness_probe = Some(ProbeConfig {
failure_threshold: 1,
..probe_config(ProbeKind::Tcp, "localhost:5432")
});
});
let interval = app
.config()
.liveness_probe
.as_ref()
.expect("the fixture just set one")
.interval
.as_duration();
let mut kept = ExtrasRegistry::default();
kept.arm(
&armed_entry(8, 1, 5678, app.clone(), &paths),
failing_prober(),
&rig.extras,
&handle,
);
let mut discarded = ExtrasRegistry::default();
discarded.arm(
&armed_entry(7, 0, 1234, app, &paths),
failing_prober(),
&rig.extras,
&handle,
);
drop(discarded);
let failure = expect_liveness(&mut rig.liveness, EVENT_WAIT).await;
assert_eq!(
failure,
LivenessReport {
id: 8,
pid: 5678,
epoch: 1
},
"only the registry that is still alive may report"
);
assert_no_liveness_within(&mut rig.liveness, interval * 3).await;
}
#[tokio::test(start_paused = true)]
async fn dropping_the_registry_stops_the_name_group_worker_it_armed() {
let dir = tempfile::tempdir().unwrap();
let paths = test_paths(&dir);
let (handle, mut rx, _fixture) = spawn_test_fixture();
let rig = rig(DEFAULT_MAX_CRON_SLEEP);
let hourly = |name: &str| {
app_with(name, |app| {
app.cron_restart = Some("0 * * * *".to_string());
})
};
handle
.start(vec![hourly("kept"), hourly("dropped")])
.await
.unwrap();
let mut kept = ExtrasRegistry::default();
kept.arm(
&armed_entry(0, 0, 1000, hourly("kept"), &paths),
idle_prober(),
&rig.extras,
&handle,
);
let mut discarded = ExtrasRegistry::default();
discarded.arm(
&armed_entry(1, 0, 1001, hourly("dropped"), &paths),
idle_prober(),
&rig.extras,
&handle,
);
tokio::task::yield_now().await;
drop(discarded);
cross_one_hour().await;
expect_restart(&mut rx, "kept", EVENT_WAIT).await;
assert_no_restart_within(&mut rx, "dropped", PAST_THE_NEXT_OCCURRENCE).await;
}
#[tokio::test(start_paused = true)]
async fn disarming_an_id_stops_its_liveness_loop() {
let dir = tempfile::tempdir().unwrap();
let paths = test_paths(&dir);
let (handle, _rx, _fixture) = spawn_test_fixture();
let mut rig = rig(DEFAULT_MAX_CRON_SLEEP);
let mut registry = ExtrasRegistry::default();
let app = app_with("web", |app| {
app.instances = 2;
app.liveness_probe = Some(ProbeConfig {
failure_threshold: 1,
..probe_config(ProbeKind::Tcp, "localhost:5432")
});
});
let interval = app
.config()
.liveness_probe
.as_ref()
.expect("the fixture just set one")
.interval
.as_duration();
for (id, instance, pid) in [(0, 0, 1000), (1, 1, 1001)] {
registry.arm(
&armed_entry(id, instance, pid, app.clone(), &paths),
failing_prober(),
&rig.extras,
&handle,
);
}
registry.disarm(0, "web");
let failure = expect_liveness(&mut rig.liveness, EVENT_WAIT).await;
assert_eq!(
failure,
LivenessReport {
id: 1,
pid: 1001,
epoch: 1
},
"only the instance that is still armed may report"
);
assert_no_liveness_within(&mut rig.liveness, interval * 3).await;
}
#[tokio::test(start_paused = true)]
async fn a_liveness_threshold_reports_this_instances_id_and_pid() {
let dir = tempfile::tempdir().unwrap();
let paths = test_paths(&dir);
let (handle, _rx, _fixture) = spawn_test_fixture();
let mut rig = rig(DEFAULT_MAX_CRON_SLEEP);
let mut registry = ExtrasRegistry::default();
let app = app_with("web", |app| {
app.liveness_probe = Some(ProbeConfig {
failure_threshold: 2,
..probe_config(ProbeKind::Tcp, "localhost:5432")
});
});
let interval = app
.config()
.liveness_probe
.as_ref()
.expect("the fixture just set one")
.interval
.as_duration();
registry.arm(
&armed_entry(7, 0, 1234, app, &paths),
Arc::new(ScriptedProber::new(vec![Err(ProbeFailure::Timeout)])),
&rig.extras,
&handle,
);
let failure = expect_liveness(&mut rig.liveness, EVENT_WAIT).await;
assert_eq!(
failure,
LivenessReport {
id: 7,
pid: 1234,
epoch: 1
}
);
assert_no_liveness_within(&mut rig.liveness, interval * 3).await;
}
#[test]
fn a_zero_watch_delay_is_floored_before_it_reaches_the_debouncer() {
let mut app = AppConfig::minimal("web", "./srv");
app.watch_delay = Some(UpDuration::from_millis(0));
assert_eq!(watch_delay_for(&app), MIN_WATCH_DELAY);
}
#[test]
fn a_watch_delay_the_config_layer_accepts_is_never_clamped() {
let mut app = AppConfig::minimal("web", "./srv");
app.watch_delay = Some(UpDuration::from_millis(1));
assert_eq!(watch_delay_for(&app), Duration::from_millis(1));
app.watch_delay = Some(UpDuration::from_millis(20));
assert_eq!(watch_delay_for(&app), Duration::from_millis(20));
app.watch_delay = None;
assert_eq!(watch_delay_for(&app), DEFAULT_WATCH_DELAY);
}
#[tokio::test(start_paused = true)]
async fn a_breach_naming_the_running_pid_restarts_that_sheep() {
let (handle, mut rx, _fixture) = spawn_test_fixture();
handle
.start(vec![app_with("web", |app| {
app.max_memory = Some(MemSize::from_bytes(500));
})])
.await
.unwrap();
let pid = handle.list().await[0].pid.expect("a live sheep has a pid");
let (breach_tx, breach_rx) = mpsc::channel(4);
let (_live_tx, live_rx) = mpsc::channel(4);
let _reporter = spawn_extras_reporter(breach_rx, live_rx, handle.clone());
breach_tx
.send(LimitBreach {
id: 0,
root_pid: pid,
observed: MemSize::from_bytes(900),
limit: MemSize::from_bytes(500),
})
.await
.unwrap();
let (info, manually) = expect_restart_event(&mut rx, "web", EVENT_WAIT).await;
assert_eq!(info.id, 0);
assert_eq!(info.restarts, 1);
assert!(
!manually,
"nobody typed this: a memory breach is the daemon's own doing"
);
}
#[tokio::test(start_paused = true)]
async fn a_liveness_failure_naming_the_running_pid_restarts_that_sheep() {
let (handle, mut rx, _fixture) = spawn_test_fixture();
handle.start(vec![app_with("web", |_| {})]).await.unwrap();
let pid = handle.list().await[0].pid.expect("a live sheep has a pid");
let (_breach_tx, breach_rx) = mpsc::channel(4);
let (live_tx, live_rx) = mpsc::channel(4);
let _reporter = spawn_extras_reporter(breach_rx, live_rx, handle.clone());
live_tx
.send(LivenessReport {
id: 0,
pid,
epoch: 0,
})
.await
.unwrap();
let (info, manually) = expect_restart_event(&mut rx, "web", EVENT_WAIT).await;
assert_eq!(info.id, 0);
assert_eq!(info.restarts, 1);
assert!(
!manually,
"nobody typed this: a liveness failure is the daemon's own doing"
);
}
#[tokio::test(start_paused = true)]
async fn an_extra_restart_for_an_unknown_id_leaves_the_engine_running() {
let (handle, _rx, _fixture) = spawn_test_fixture();
handle.start(vec![app_with("web", |_| {})]).await.unwrap();
handle.extra_restart(99, 4242, None, None).await;
assert_eq!(
handle.list().await.len(),
1,
"the actor must still be serving after a report for an id it does not know"
);
}
#[tokio::test(start_paused = true)]
async fn an_extra_restart_for_a_sheep_that_is_still_starting_restarts_nothing() {
let (handle, mut rx, _fixture) = spawn_test_fixture();
let app = app_with("web", |app| {
app.wait_ready = true;
app.listen_timeout = UpDuration::from_millis(6 * 60 * 60 * 1_000);
});
handle.start(vec![app]).await.unwrap();
let listing = handle.list().await;
assert_eq!(
listing[0].status,
ProcStatus::Starting,
"this case is only meaningful while the sheep is gated on readiness"
);
let pid = listing[0].pid.expect("a spawned sheep has a pid");
handle.extra_restart(0, pid, None, None).await;
assert_no_restart_within(&mut rx, "web", Duration::from_secs(30)).await;
assert_eq!(handle.list().await[0].restarts, 0);
}
#[tokio::test(start_paused = true)]
async fn a_breach_for_a_stopped_sheep_restarts_nothing() {
let (handle, mut rx, _fixture) = spawn_test_fixture();
handle.start(vec![app_with("web", |_| {})]).await.unwrap();
let pid = handle.list().await[0].pid.expect("a live sheep has a pid");
let (breach_tx, breach_rx) = mpsc::channel(4);
let (_live_tx, live_rx) = mpsc::channel(4);
let _reporter = spawn_extras_reporter(breach_rx, live_rx, handle.clone());
handle
.stop(ProcessSelector::Id(0))
.await
.expect("the sheep stops");
breach_tx
.send(LimitBreach {
id: 0,
root_pid: pid,
observed: MemSize::from_bytes(900),
limit: MemSize::from_bytes(500),
})
.await
.unwrap();
assert_no_restart_within(&mut rx, "web", Duration::from_secs(30)).await;
let listing = handle.list().await;
assert_eq!(listing[0].status, ProcStatus::Stopped);
assert_eq!(listing[0].restarts, 0);
}
#[tokio::test(start_paused = true)]
async fn a_breach_carrying_the_previous_pid_restarts_nothing() {
let (handle, mut rx, _fixture) = spawn_test_fixture();
handle
.start(vec![app_with("web", |app| {
app.max_memory = Some(MemSize::from_bytes(500));
})])
.await
.unwrap();
let stale_pid = handle.list().await[0].pid.expect("a live sheep has a pid");
let (breach_tx, breach_rx) = mpsc::channel(4);
let (_live_tx, live_rx) = mpsc::channel(4);
let _reporter = spawn_extras_reporter(breach_rx, live_rx, handle.clone());
handle
.restart(ProcessSelector::Id(0))
.await
.expect("the sheep restarts");
expect_restart(&mut rx, "web", EVENT_WAIT).await;
let live_pid = handle.list().await[0].pid.expect("a live sheep has a pid");
assert_ne!(stale_pid, live_pid, "the respawn must have a new pid");
let breach = |root_pid| LimitBreach {
id: 0,
root_pid,
observed: MemSize::from_bytes(900),
limit: MemSize::from_bytes(500),
};
breach_tx.send(breach(stale_pid)).await.unwrap();
assert_no_restart_within(&mut rx, "web", Duration::from_secs(30)).await;
assert_eq!(
handle.list().await[0].restarts,
1,
"a stale breach must not bump the restart count"
);
breach_tx.send(breach(live_pid)).await.unwrap();
let info = expect_restart(&mut rx, "web", EVENT_WAIT).await;
assert_eq!(
info.restarts, 2,
"the same reporter, fed the current pid, must restart"
);
}
#[tokio::test(start_paused = true)]
async fn stopping_the_last_instance_stops_its_cron_worker() {
let clock = Arc::new(TestClock::starting_at(dt("2026-01-01T00:00:00Z")));
let h = harness_with_extras(vec![ProcScript::never_exits(); 12], |reports| Extras {
clock: Arc::clone(&clock) as Arc<dyn Clock>,
enforcer: Arc::new(RecordingEnforcer::default()),
max_cron_sleep: DEFAULT_MAX_CRON_SLEEP,
reports,
stats: idle_stats(),
});
let mut rx = h.ctx.events.subscribe();
h.ctx
.supervisor
.start(vec![app_with("web", |app| {
app.cron_restart = Some("0 * * * *".to_string());
})])
.await
.unwrap();
cross_one_hour().await;
expect_restart(&mut rx, "web", EVENT_WAIT).await;
h.ctx
.supervisor
.stop(ProcessSelector::Name("web".to_string()))
.await
.expect("the sheep stops");
assert_no_restart_within(&mut rx, "web", PAST_THE_NEXT_OCCURRENCE).await;
assert_eq!(h.ctx.supervisor.list().await[0].status, ProcStatus::Stopped);
}
async fn expect_process_event(
rx: &mut broadcast::Receiver<SharedEvent>,
id: u32,
kind: ProcessEventKind,
window: Duration,
) {
let wanted = async {
loop {
match rx.recv().await.map(|event| event.to_event()) {
Ok(BusEvent::Process { event, info, .. }) if event == kind && info.id == id => {
return;
}
Ok(_) => continue,
Err(broadcast::error::RecvError::Lagged(_)) => continue,
Err(err) => panic!("event stream closed before {kind:?} for id {id}: {err}"),
}
}
};
if tokio::time::timeout(window, wanted).await.is_err() {
panic!("timed out waiting for {kind:?} for id {id}");
}
}
#[tokio::test(start_paused = true)]
async fn a_reload_leaves_the_name_groups_cron_worker_where_it_was() {
let clock = Arc::new(TestClock::starting_at(dt("2026-01-01T00:00:00Z")));
let h = harness_with_extras(vec![ProcScript::never_exits(); 2], |reports| Extras {
clock: Arc::clone(&clock) as Arc<dyn Clock>,
enforcer: Arc::new(RecordingEnforcer::default()),
max_cron_sleep: Duration::from_secs(600),
reports,
stats: idle_stats(),
});
let mut rx = h.ctx.events.subscribe();
h.ctx
.supervisor
.start(vec![app_with("web", |app| {
app.cron_restart = Some("0 * * * *".to_string());
})])
.await
.unwrap();
expect_process_event(&mut rx, 0, ProcessEventKind::Online, EVENT_WAIT).await;
tokio::task::yield_now().await;
let reads_before = clock.reads();
assert_eq!(
reads_before, 1,
"fixture check: one armed cron worker takes exactly one reading, \
so a rebuild's second one is a countable difference rather than \
noise — and a count of 0 here would mean the worker had not run \
yet, which would make the claim below vacuous"
);
h.ctx
.supervisor
.reload(ProcessSelector::Name("web".to_string()))
.await
.expect("the reload is accepted");
expect_process_event(&mut rx, 1, ProcessEventKind::Online, EVENT_WAIT).await;
expect_process_event(&mut rx, 0, ProcessEventKind::Delete, EVENT_WAIT).await;
let listed = h.ctx.supervisor.list().await;
assert_eq!(
listed.iter().map(|info| info.id).collect::<Vec<_>>(),
vec![1],
"fixture check: the swap must have run to completion, or there was \
never an overlap for the ordering to matter in"
);
assert_eq!(
clock.reads(),
reads_before,
"the name group's cron worker must have been left where it was: a \
rebuilt one reads the clock again to derive its next occurrence"
);
}
fn backoff_harness(clock: &Arc<TestClock>) -> Harness {
harness_with_extras(
{
let mut scripts = vec![ProcScript::never_exits()];
scripts.push(ProcScript::const_exit(1));
scripts.extend([ProcScript::never_exits(); 8]);
scripts
},
|reports| Extras {
clock: Arc::clone(clock) as Arc<dyn Clock>,
enforcer: Arc::new(RecordingEnforcer::default()),
max_cron_sleep: DEFAULT_MAX_CRON_SLEEP,
reports,
stats: idle_stats(),
},
)
}
fn backoff_app() -> shep_core::config::ResolvedApp {
app_with("web", |app| {
app.cron_restart = Some("0 * * * *".to_string());
app.restart_delay = Some(UpDuration::from_millis(3 * 60 * 60 * 1_000));
})
}
async fn settle_into(supervisor: &SupervisorHandle, id: u32, status: ProcStatus) {
for _ in 0..200 {
tokio::task::yield_now().await;
let listing = supervisor.list().await;
if listing
.iter()
.any(|info| info.id == id && info.status == status)
{
return;
}
}
panic!("id {id} never reached {status:?}");
}
#[tokio::test(start_paused = true)]
async fn stopping_a_sheep_mid_backoff_still_stops_its_cron_worker() {
let clock = Arc::new(TestClock::starting_at(dt("2026-01-01T00:00:00Z")));
let h = backoff_harness(&clock);
let mut rx = h.ctx.events.subscribe();
h.ctx.supervisor.start(vec![backoff_app()]).await.unwrap();
cross_one_hour().await;
expect_restart(&mut rx, "web", EVENT_WAIT).await;
settle_into(&h.ctx.supervisor, 0, ProcStatus::WaitingRestart).await;
h.ctx
.supervisor
.stop(ProcessSelector::Name("web".to_string()))
.await
.expect("the sheep stops");
assert_eq!(h.ctx.supervisor.list().await[0].status, ProcStatus::Stopped);
assert_no_restart_within(&mut rx, "web", PAST_THE_NEXT_OCCURRENCE).await;
}
#[tokio::test(start_paused = true)]
async fn deleting_a_sheep_mid_backoff_takes_its_cron_worker_with_it() {
let clock = Arc::new(TestClock::starting_at(dt("2026-01-01T00:00:00Z")));
let h = backoff_harness(&clock);
let mut rx = h.ctx.events.subscribe();
h.ctx.supervisor.start(vec![backoff_app()]).await.unwrap();
cross_one_hour().await;
expect_restart(&mut rx, "web", EVENT_WAIT).await;
settle_into(&h.ctx.supervisor, 0, ProcStatus::WaitingRestart).await;
h.ctx
.supervisor
.delete(ProcessSelector::Name("web".to_string()))
.await
.expect("the sheep is deleted");
assert!(h.ctx.supervisor.list().await.is_empty());
let reads_after_delete = clock.reads();
assert_no_restart_within(&mut rx, "web", PAST_THE_NEXT_OCCURRENCE).await;
assert_eq!(
clock.reads(),
reads_after_delete,
"a deleted sheep's cron worker must stop reading the clock, not merely stop finding sheep"
);
}
#[tokio::test(start_paused = true)]
async fn a_cron_restart_is_never_reported_as_a_user_action() {
let clock = Arc::new(TestClock::starting_at(dt("2026-01-01T00:00:00Z")));
let h = backoff_harness(&clock);
let mut rx = h.ctx.events.subscribe();
h.ctx.supervisor.start(vec![backoff_app()]).await.unwrap();
cross_one_hour().await;
let (running, manually) = expect_restart_event(&mut rx, "web", EVENT_WAIT).await;
assert_eq!(running.restarts, 1);
assert!(
!manually,
"a cron occurrence is nobody typing `shep restart`"
);
settle_into(&h.ctx.supervisor, 0, ProcStatus::WaitingRestart).await;
cross_one_hour().await;
let (backing_off, manually) = expect_restart_event(&mut rx, "web", EVENT_WAIT).await;
assert_eq!(backing_off.restarts, 2);
assert!(
!manually,
"the same occurrence must answer the same way through `apply_immediate`"
);
}
#[tokio::test(start_paused = true)]
async fn a_respawn_that_cannot_spawn_stops_the_name_groups_cron_worker() {
let clock = Arc::new(TestClock::starting_at(dt("2026-01-01T00:00:00Z")));
let h = harness_with_extras(vec![ProcScript::never_exits()], |reports| Extras {
clock: Arc::clone(&clock) as Arc<dyn Clock>,
enforcer: Arc::new(RecordingEnforcer::default()),
max_cron_sleep: DEFAULT_MAX_CRON_SLEEP,
reports,
stats: idle_stats(),
});
let mut rx = h.ctx.events.subscribe();
h.ctx
.supervisor
.start(vec![app_with("web", |app| {
app.cron_restart = Some("0 * * * *".to_string());
})])
.await
.unwrap();
cross_one_hour().await;
settle_into(&h.ctx.supervisor, 0, ProcStatus::Errored).await;
let reads_after_error = clock.reads();
assert_no_restart_within(&mut rx, "web", PAST_THE_NEXT_OCCURRENCE).await;
assert_eq!(
clock.reads(),
reads_after_error,
"an errored sheep's cron worker must stop reading the clock"
);
}
#[tokio::test(start_paused = true)]
async fn the_actor_arms_the_memory_limit_against_the_spawned_pid() {
let mut h = harness_with_extras(vec![ProcScript::never_exits(); 4], |reports| {
let sampler: Arc<dyn MemorySampler> =
Arc::new(ScriptedSampler::new(vec![vec![ProcessRss {
pid: 1000,
parent: None,
bytes: 900,
cpu_ms: 0,
}]]));
let stats = Arc::new(StatsState::new(Arc::clone(&sampler)));
Extras {
clock: Arc::new(TestClock::starting_at(dt("2026-01-01T00:00:00Z"))),
enforcer: Arc::new(PollingEnforcer::start(
sampler,
reports.breaches.clone(),
Arc::clone(&stats),
)),
max_cron_sleep: DEFAULT_MAX_CRON_SLEEP,
reports,
stats,
}
});
h.ctx
.supervisor
.start(vec![app_with("web", |app| {
app.max_memory = Some(MemSize::from_bytes(500));
})])
.await
.unwrap();
let pid = h.ctx.supervisor.list().await[0]
.pid
.expect("a live sheep has a pid");
let breach = match tokio::time::timeout(EVENT_WAIT, h.breaches.recv()).await {
Ok(Some(breach)) => breach,
Ok(None) => panic!("the breach channel closed before a breach arrived"),
Err(_) => panic!("timed out waiting for a breach"),
};
assert_eq!(breach.id, 0);
assert_eq!(
breach.root_pid, pid,
"the enforcer must be armed against the pid the sheep is actually running as"
);
assert_eq!(breach.observed.bytes(), 900);
}
#[tokio::test(start_paused = true)]
async fn the_actor_arms_a_readiness_gated_app_once_it_comes_online() {
let mut h = harness_with_extras(vec![ProcScript::never_exits(); 4], |reports| {
let sampler: Arc<dyn MemorySampler> =
Arc::new(ScriptedSampler::new(vec![vec![ProcessRss {
pid: 1000,
parent: None,
bytes: 900,
cpu_ms: 0,
}]]));
let stats = Arc::new(StatsState::new(Arc::clone(&sampler)));
Extras {
clock: Arc::new(TestClock::starting_at(dt("2026-01-01T00:00:00Z"))),
enforcer: Arc::new(PollingEnforcer::start(
sampler,
reports.breaches.clone(),
Arc::clone(&stats),
)),
max_cron_sleep: DEFAULT_MAX_CRON_SLEEP,
reports,
stats,
}
});
h.ctx
.supervisor
.start(vec![app_with("web", |app| {
app.wait_ready = true;
app.max_memory = Some(MemSize::from_bytes(500));
})])
.await
.unwrap();
let listing = h.ctx.supervisor.list().await;
assert_eq!(
listing[0].status,
ProcStatus::Starting,
"a gated app must not be Online when its Start reply lands"
);
let pid = listing[0].pid.expect("a spawned sheep has a pid");
let breach = match tokio::time::timeout(EVENT_WAIT, h.breaches.recv()).await {
Ok(Some(breach)) => breach,
Ok(None) => panic!("the breach channel closed before a breach arrived"),
Err(_) => panic!("timed out waiting for a breach"),
};
assert_eq!(breach.id, 0);
assert_eq!(
breach.root_pid, pid,
"the gated path must arm against the pid the sheep is running as"
);
}
#[tokio::test]
async fn the_actor_arms_the_liveness_loop_against_the_spawned_pid() {
let reserved = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
let addr = reserved.local_addr().unwrap();
drop(reserved);
let mut h = harness(vec![ProcScript::never_exits(); 4]);
h.ctx
.supervisor
.start(vec![app_with("web", |app| {
app.liveness_probe = Some(ProbeConfig {
failure_threshold: 1,
interval: PROBE_INTERVAL,
timeout: UpDuration::from_millis(500),
..probe_config(ProbeKind::Tcp, &addr.to_string())
});
})])
.await
.unwrap();
let pid = h.ctx.supervisor.list().await[0]
.pid
.expect("a live sheep has a pid");
let failure = expect_liveness(&mut h.liveness, LIVENESS_DEADLINE).await;
assert_eq!(
failure,
LivenessReport {
id: 0,
pid,
epoch: 1
}
);
}
#[tokio::test(start_paused = true)]
async fn rearm_name_replaces_a_live_group_task() {
let dir = tempfile::tempdir().unwrap();
let paths = test_paths(&dir);
let root = tempfile::tempdir().unwrap();
let (handle, _rx, _fixture) = spawn_test_fixture();
let rig = rig(DEFAULT_MAX_CRON_SLEEP);
let mut registry = ExtrasRegistry::default();
let app = app_with("web", |app| {
app.cron_restart = Some("0 * * * *".to_string());
app.watch = true;
app.cwd = Some(root.path().display().to_string());
});
handle.start(vec![app.clone()]).await.unwrap();
let entry = armed_entry(0, 0, 1000, app.clone(), &paths);
registry.arm(&entry, idle_prober(), &rig.extras, &handle);
tokio::task::yield_now().await;
let before_cron = registry.groups["web"].cron.as_ref().unwrap().abort_handle();
let before_watch = registry.groups["web"]
.watch
.as_ref()
.unwrap()
.abort_handle();
registry.rearm_name("web", &[&entry], |_| idle_prober(), &rig.extras, &handle);
tokio::task::yield_now().await;
let after_cron = registry.groups["web"].cron.as_ref().unwrap().abort_handle();
let after_watch = registry.groups["web"]
.watch
.as_ref()
.unwrap()
.abort_handle();
assert_ne!(
before_cron.id(),
after_cron.id(),
"the cron worker survived a rearm"
);
assert_ne!(
before_watch.id(),
after_watch.id(),
"the watch task survived a rearm"
);
}
#[tokio::test(start_paused = true)]
async fn rearm_name_leaves_the_group_armed() {
let dir = tempfile::tempdir().unwrap();
let paths = test_paths(&dir);
let root = tempfile::tempdir().unwrap();
let (handle, _rx, _fixture) = spawn_test_fixture();
let rig = rig(DEFAULT_MAX_CRON_SLEEP);
let mut registry = ExtrasRegistry::default();
let app = app_with("web", |app| {
app.cron_restart = Some("0 * * * *".to_string());
app.watch = true;
app.cwd = Some(root.path().display().to_string());
});
handle.start(vec![app.clone()]).await.unwrap();
let entry = armed_entry(0, 0, 1000, app.clone(), &paths);
registry.arm(&entry, idle_prober(), &rig.extras, &handle);
tokio::task::yield_now().await;
registry.rearm_name("web", &[&entry], |_| idle_prober(), &rig.extras, &handle);
tokio::task::yield_now().await;
let group = ®istry.groups["web"];
assert!(
group.cron.as_ref().is_some_and(|cron| !cron.is_finished()),
"the group must still have a live cron worker after a rearm"
);
assert!(
group
.watch
.as_ref()
.is_some_and(|watch| !watch.is_finished()),
"the group must still have a live watch task after a rearm"
);
}
#[tokio::test(start_paused = true)]
async fn rearm_name_leaves_another_apps_group_alone() {
let dir = tempfile::tempdir().unwrap();
let paths = test_paths(&dir);
let web_root = tempfile::tempdir().unwrap();
let worker_root = tempfile::tempdir().unwrap();
let (handle, _rx, _fixture) = spawn_test_fixture();
let rig = rig(DEFAULT_MAX_CRON_SLEEP);
let mut registry = ExtrasRegistry::default();
let web = app_with("web", |app| {
app.watch = true;
app.cwd = Some(web_root.path().display().to_string());
});
let worker = app_with("worker", |app| {
app.watch = true;
app.cwd = Some(worker_root.path().display().to_string());
});
handle
.start(vec![web.clone(), worker.clone()])
.await
.unwrap();
let web_entry = armed_entry(0, 0, 1000, web.clone(), &paths);
let worker_entry = armed_entry(1, 0, 1001, worker.clone(), &paths);
registry.arm(&web_entry, idle_prober(), &rig.extras, &handle);
registry.arm(&worker_entry, idle_prober(), &rig.extras, &handle);
tokio::task::yield_now().await;
let worker_watch_before = registry.groups["worker"]
.watch
.as_ref()
.unwrap()
.abort_handle();
registry.rearm_name(
"web",
&[&web_entry],
|_| idle_prober(),
&rig.extras,
&handle,
);
tokio::task::yield_now().await;
let worker_watch_after = registry.groups["worker"]
.watch
.as_ref()
.unwrap()
.abort_handle();
assert_eq!(
worker_watch_before.id(),
worker_watch_after.id(),
"rearming \"web\" must not touch \"worker\"'s group"
);
}
#[tokio::test(start_paused = true)]
async fn rearm_name_rebuilds_a_multi_instance_group_exactly_once() {
let dir = tempfile::tempdir().unwrap();
let paths = test_paths(&dir);
let root = tempfile::tempdir().unwrap();
let (handle, _rx, _fixture) = spawn_test_fixture();
let rig = rig(DEFAULT_MAX_CRON_SLEEP);
let mut registry = ExtrasRegistry::default();
let app = app_with("web", |app| {
app.instances = 2;
app.cron_restart = Some("0 * * * *".to_string());
app.watch = true;
app.cwd = Some(root.path().display().to_string());
});
handle.start(vec![app.clone()]).await.unwrap();
let entry_a = armed_entry(0, 0, 1000, app.clone(), &paths);
let entry_b = armed_entry(1, 1, 1001, app.clone(), &paths);
registry.arm(&entry_a, idle_prober(), &rig.extras, &handle);
registry.arm(&entry_b, idle_prober(), &rig.extras, &handle);
tokio::task::yield_now().await;
let before_cron = registry.groups["web"].cron.as_ref().unwrap().abort_handle();
let before_watch = registry.groups["web"]
.watch
.as_ref()
.unwrap()
.abort_handle();
let probed = std::sync::Mutex::new(Vec::new());
registry.rearm_name(
"web",
&[&entry_a, &entry_b],
|entry| {
probed
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.push(entry.id);
idle_prober()
},
&rig.extras,
&handle,
);
tokio::task::yield_now().await;
assert_eq!(
probed
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.as_slice(),
[0, 1],
"each instance must get its own prober, in id order"
);
let group = ®istry.groups["web"];
assert_eq!(
group.members,
HashSet::from([0, 1]),
"both instances must still be members after a rearm, not just the last one arm'd"
);
assert!(
group.cron.is_some(),
"the group must hold exactly one cron task, not zero"
);
assert!(
group.watch.is_some(),
"the group must hold exactly one watch task, not zero"
);
let after_cron = group.cron.as_ref().unwrap().abort_handle();
let after_watch = group.watch.as_ref().unwrap().abort_handle();
assert_ne!(
before_cron.id(),
after_cron.id(),
"the cron worker must be rebuilt by the rearm, not left over"
);
assert_ne!(
before_watch.id(),
after_watch.id(),
"the watch task must be rebuilt by the rearm, not left over"
);
}
mod slow {
use super::*;
#[tokio::test(start_paused = true)]
async fn a_replacement_arming_before_the_drainee_disarms_keeps_the_groups_own_tasks() {
let dir = tempfile::tempdir().unwrap();
let paths = test_paths(&dir);
let root = tempfile::tempdir().unwrap();
let (handle, _rx, _fixture) = spawn_test_fixture();
let rig = rig(DEFAULT_MAX_CRON_SLEEP);
let mut registry = ExtrasRegistry::default();
let app = app_with("web", |app| {
app.cron_restart = Some("0 * * * *".to_string());
app.watch = true;
app.cwd = Some(root.path().display().to_string());
});
handle.start(vec![app.clone()]).await.unwrap();
registry.arm(
&armed_entry(0, 0, 1000, app.clone(), &paths),
idle_prober(),
&rig.extras,
&handle,
);
tokio::task::yield_now().await;
let group = ®istry.groups["web"];
let cron = group
.cron
.as_ref()
.expect("fixture check: the cron worker must have armed")
.abort_handle();
let watch = group
.watch
.as_ref()
.expect("fixture check: the watch must have armed")
.abort_handle();
let reads_before = rig.clock.reads();
registry.arm(
&armed_entry(1, 0, 2000, app, &paths),
idle_prober(),
&rig.extras,
&handle,
);
registry.disarm(0, "web");
tokio::task::yield_now().await;
let group = ®istry.groups["web"];
assert_eq!(
group.members,
HashSet::from([1]),
"fixture check: the drainee must really have left a group the \
replacement had already joined — this reads the same under either \
ordering, which is why it cannot be the claim that matters"
);
assert_eq!(
group.cron.as_ref().map(JoinHandle::id),
Some(cron.id()),
"the group must still hold the cron worker it was armed with, not \
an identical one put back in its place"
);
assert_eq!(
group.watch.as_ref().map(JoinHandle::id),
Some(watch.id()),
"and the watch it was armed with, whose rebuild means re-registering \
the OS watcher"
);
assert_eq!(
rig.clock.reads(),
reads_before,
"a surviving cron worker performs no startup work; a rebuilt one \
reads the clock again to derive its next occurrence"
);
}
#[tokio::test(start_paused = true)]
async fn a_group_disarmed_before_its_first_occurrence_leaves_no_worker_behind() {
let dir = tempfile::tempdir().unwrap();
let paths = test_paths(&dir);
let root = tempfile::tempdir().unwrap();
let (handle, mut rx, _fixture) = spawn_test_fixture();
let rig = rig(DEFAULT_MAX_CRON_SLEEP);
let mut registry = ExtrasRegistry::default();
let app = app_with("web", |app| {
app.cron_restart = Some("0 * * * *".to_string());
app.watch = true;
app.cwd = Some(root.path().display().to_string());
app.max_memory = Some(MemSize::from_bytes(1024));
});
handle.start(vec![app.clone()]).await.unwrap();
registry.arm(
&armed_entry(0, 0, 1000, app, &paths),
idle_prober(),
&rig.extras,
&handle,
);
let group = ®istry.groups["web"];
assert_eq!(group.members, HashSet::from([0]));
assert!(group.cron.is_some(), "the cron worker must have armed");
assert!(group.watch.is_some(), "the watch must have armed");
assert_eq!(rig.enforcer.arms().len(), 1);
registry.disarm(0, "web");
assert!(
registry.groups.is_empty(),
"a group whose only member left before its first occurrence must go with it"
);
assert!(
registry.instances.is_empty(),
"the same disarm must take the instance's own extras too"
);
assert_eq!(rig.enforcer.disarms(), vec![0]);
assert_no_restart_within(&mut rx, "web", PAST_THE_NEXT_OCCURRENCE).await;
}
#[tokio::test]
async fn a_watch_that_ended_on_its_own_is_rebuilt_on_the_next_arm() {
let home = tempfile::tempdir().unwrap();
let paths = test_paths(&home);
let root = tempfile::tempdir().unwrap();
let (handle, mut rx, _fixture) = spawn_test_fixture();
let rig = rig(DEFAULT_MAX_CRON_SLEEP);
let mut registry = ExtrasRegistry::default();
let app = app_with("web", |app| {
app.watch = true;
app.cwd = Some(root.path().display().to_string());
app.watch_delay = Some(UpDuration::from_millis(
real_time::TEST_DELAY.as_millis() as u64
));
});
handle.start(vec![app.clone()]).await.unwrap();
registry.arm(
&armed_entry(0, 0, 1000, app.clone(), &paths),
idle_prober(),
&rig.extras,
&handle,
);
let armed = registry.groups["web"]
.watch
.as_ref()
.expect("the first arm registers a watcher");
armed.abort();
settle_finished(armed).await;
registry.arm(
&armed_entry(0, 0, 2000, app, &paths),
idle_prober(),
&rig.extras,
&handle,
);
touch(root.path(), "trigger.txt").unwrap();
let info = expect_restart(&mut rx, "web", real_time::SMOKE_DEADLINE).await;
assert_eq!(info.restarts, 1);
}
#[tokio::test]
async fn a_watched_app_restarts_on_a_save_and_goes_quiet_once_disarmed() {
let home = tempfile::tempdir().unwrap();
let paths = test_paths(&home);
let root = tempfile::tempdir().unwrap();
let (handle, mut rx, _fixture) = spawn_test_fixture();
let rig = rig(DEFAULT_MAX_CRON_SLEEP);
let mut registry = ExtrasRegistry::default();
let app = app_with("web", |app| {
app.watch = true;
app.cwd = Some(root.path().display().to_string());
app.watch_delay = Some(UpDuration::from_millis(
real_time::TEST_DELAY.as_millis() as u64
));
});
handle.start(vec![app.clone()]).await.unwrap();
registry.arm(
&armed_entry(0, 0, 1000, app, &paths),
idle_prober(),
&rig.extras,
&handle,
);
touch(root.path(), "trigger.txt").unwrap();
let info = expect_restart(&mut rx, "web", real_time::SMOKE_DEADLINE).await;
assert_eq!(info.restarts, 1);
registry.disarm(0, "web");
touch(root.path(), "after-disarm.txt").unwrap();
assert_no_restart_within(&mut rx, "web", real_time::NO_EVENT_WINDOW).await;
}
#[tokio::test]
async fn a_watched_app_naming_no_delay_still_restarts_on_a_save() {
let home = tempfile::tempdir().unwrap();
let paths = test_paths(&home);
let root = tempfile::tempdir().unwrap();
let (handle, mut rx, _fixture) = spawn_test_fixture();
let rig = rig(DEFAULT_MAX_CRON_SLEEP);
let mut registry = ExtrasRegistry::default();
let app = app_with("web", |app| {
app.watch = true;
app.cwd = Some(root.path().display().to_string());
});
handle.start(vec![app.clone()]).await.unwrap();
registry.arm(
&armed_entry(0, 0, 1000, app, &paths),
idle_prober(),
&rig.extras,
&handle,
);
touch(root.path(), "trigger.txt").unwrap();
let info = expect_restart(&mut rx, "web", real_time::SMOKE_DEADLINE).await;
assert_eq!(info.restarts, 1);
}
#[tokio::test]
async fn a_watch_restart_is_not_reported_as_a_user_action() {
let home = tempfile::tempdir().unwrap();
let paths = test_paths(&home);
let root = tempfile::tempdir().unwrap();
let (handle, mut rx, _fixture) = spawn_test_fixture();
let rig = rig(DEFAULT_MAX_CRON_SLEEP);
let mut registry = ExtrasRegistry::default();
let app = app_with("web", |app| {
app.watch = true;
app.cwd = Some(root.path().display().to_string());
app.watch_delay = Some(UpDuration::from_millis(
real_time::TEST_DELAY.as_millis() as u64
));
});
handle.start(vec![app.clone()]).await.unwrap();
registry.arm(
&armed_entry(0, 0, 1000, app, &paths),
idle_prober(),
&rig.extras,
&handle,
);
touch(root.path(), "trigger.txt").unwrap();
let (info, manually) =
expect_restart_event(&mut rx, "web", real_time::SMOKE_DEADLINE).await;
assert_eq!(info.restarts, 1);
assert!(
!manually,
"a file changing under a watched tree is not a user action"
);
}
#[tokio::test]
async fn a_watched_app_ignores_the_paths_its_ignore_watch_names() {
let home = tempfile::tempdir().unwrap();
let paths = test_paths(&home);
let root = tempfile::tempdir().unwrap();
let (handle, mut rx, _fixture) = spawn_test_fixture();
let rig = rig(DEFAULT_MAX_CRON_SLEEP);
let mut registry = ExtrasRegistry::default();
let app = app_with("web", |app| {
app.watch = true;
app.cwd = Some(root.path().display().to_string());
app.ignore_watch = vec!["ignored.txt".to_string()];
app.watch_delay = Some(UpDuration::from_millis(
real_time::TEST_DELAY.as_millis() as u64
));
});
handle.start(vec![app.clone()]).await.unwrap();
registry.arm(
&armed_entry(0, 0, 1000, app, &paths),
idle_prober(),
&rig.extras,
&handle,
);
touch(root.path(), "ignored.txt").unwrap();
assert_no_restart_within(&mut rx, "web", real_time::NO_EVENT_WINDOW).await;
touch(root.path(), "trigger.txt").unwrap();
let info = expect_restart(&mut rx, "web", real_time::SMOKE_DEADLINE).await;
assert_eq!(info.restarts, 1);
}
#[tokio::test]
async fn a_watched_app_ignores_its_own_log_writes() {
let home = tempfile::tempdir().unwrap();
let paths = test_paths(&home);
let root = tempfile::tempdir().unwrap();
let (handle, mut rx, _fixture) = spawn_test_fixture();
let rig = rig(DEFAULT_MAX_CRON_SLEEP);
let mut registry = ExtrasRegistry::default();
let app = app_with("web", |app| {
app.watch = true;
app.cwd = Some(root.path().display().to_string());
app.out_file = Some(root.path().join("app-out.txt").display().to_string());
app.err_file = Some(root.path().join("app-err.txt").display().to_string());
app.watch_delay = Some(UpDuration::from_millis(
real_time::TEST_DELAY.as_millis() as u64
));
});
handle.start(vec![app.clone()]).await.unwrap();
registry.arm(
&armed_entry(0, 0, 1000, app, &paths),
idle_prober(),
&rig.extras,
&handle,
);
touch(root.path(), "app-out.txt").unwrap();
touch(root.path(), "app-err.txt").unwrap();
assert_no_restart_within(&mut rx, "web", real_time::NO_EVENT_WINDOW).await;
touch(root.path(), "trigger.txt").unwrap();
let info = expect_restart(&mut rx, "web", real_time::SMOKE_DEADLINE).await;
assert_eq!(info.restarts, 1);
}
#[cfg(unix)]
#[tokio::test]
async fn each_instances_liveness_probe_runs_with_its_own_assembled_env() {
let markers = tempfile::tempdir().unwrap();
std::fs::write(markers.path().join("live-0"), b"").unwrap();
let mut h = harness(vec![ProcScript::never_exits(); 4]);
h.ctx
.supervisor
.start(vec![app_with("web", |app| {
app.instances = 2;
app.liveness_probe = Some(ProbeConfig {
failure_threshold: 1,
interval: PROBE_INTERVAL,
timeout: UpDuration::from_millis(5_000),
..probe_config(
ProbeKind::Exec,
&format!(
r#"test -f "{}/live-$SHEP_INSTANCE""#,
markers.path().display()
),
)
});
})])
.await
.unwrap();
let listing = h.ctx.supervisor.list().await;
assert!(
listing[1]
.out_file
.as_ref()
.is_some_and(|path| path.ends_with("web-1-out.log")),
"id 1 must be instance 1: {:?}",
listing[1].out_file
);
let instance_one_pid = listing[1].pid.expect("a live sheep has a pid");
let failure = expect_liveness(&mut h.liveness, LIVENESS_DEADLINE).await;
assert_eq!(
failure,
LivenessReport {
id: 1,
pid: instance_one_pid,
epoch: 1,
},
"only the instance whose own marker is missing may report"
);
assert_no_liveness_within(&mut h.liveness, PROBE_INTERVAL.as_duration() * 3).await;
}
}
}