use super::*;
use crate::sweep::{LambSignal, LambSnapshot, LambSweep, SignalError};
#[tokio::test(start_paused = true)]
async fn stopping_the_last_instance_stops_its_cron_worker() {
let clock = Arc::new(TestClock::starting_at(dt("2026-01-01T00:00:00Z")));
let h = harness_with_extras(vec![ProcScript::never_exits(); 12], |reports| Extras {
clock: Arc::clone(&clock) as Arc<dyn Clock>,
enforcer: Arc::new(RecordingEnforcer::default()),
lamb_sweep: idle_sweep(),
max_cron_sleep: DEFAULT_MAX_CRON_SLEEP,
reports,
stats: idle_stats(),
});
let mut rx = h.ctx.events.subscribe();
h.ctx
.supervisor
.start(vec![app_with("web", |app| {
app.cron_restart = Some("0 * * * *".to_string());
})])
.await
.unwrap();
wait_for_cron_worker(&clock).await;
cross_one_hour().await;
expect_restart(&mut rx, "web", EVENT_WAIT).await;
h.ctx
.supervisor
.stop(ProcessSelector::Name("web".to_string()))
.await
.expect("the sheep stops");
assert_no_restart_within(&mut rx, "web", PAST_THE_NEXT_OCCURRENCE).await;
assert_eq!(h.ctx.supervisor.list().await[0].status, ProcStatus::Stopped);
}
async fn expect_process_event(
rx: &mut broadcast::Receiver<SharedEvent>,
id: u32,
kind: ProcessEventKind,
window: Duration,
) {
let wanted = async {
loop {
match rx.recv().await.map(|event| event.to_event()) {
Ok(BusEvent::Process { event, info, .. }) if event == kind && info.id == id => {
return;
}
Ok(_) => continue,
Err(broadcast::error::RecvError::Lagged(_)) => continue,
Err(err) => panic!("event stream closed before {kind:?} for id {id}: {err}"),
}
}
};
if tokio::time::timeout(window, wanted).await.is_err() {
panic!("timed out waiting for {kind:?} for id {id}");
}
}
#[tokio::test(start_paused = true)]
async fn a_reload_leaves_the_name_groups_cron_worker_where_it_was() {
let clock = Arc::new(TestClock::starting_at(dt("2026-01-01T00:00:00Z")));
let h = harness_with_extras(vec![ProcScript::never_exits(); 2], |reports| Extras {
clock: Arc::clone(&clock) as Arc<dyn Clock>,
enforcer: Arc::new(RecordingEnforcer::default()),
lamb_sweep: idle_sweep(),
max_cron_sleep: Duration::from_secs(600),
reports,
stats: idle_stats(),
});
let mut rx = h.ctx.events.subscribe();
h.ctx
.supervisor
.start(vec![app_with("web", |app| {
app.cron_restart = Some("0 * * * *".to_string());
})])
.await
.unwrap();
expect_process_event(&mut rx, 0, ProcessEventKind::Online, EVENT_WAIT).await;
tokio::task::yield_now().await;
let reads_before = clock.reads();
assert_eq!(
reads_before, 1,
"fixture check: one armed cron worker takes exactly one reading, \
so a rebuild's second one is a countable difference rather than \
noise — and a count of 0 here would mean the worker had not run \
yet, which would make the claim below vacuous"
);
h.ctx
.supervisor
.reload(ProcessSelector::Name("web".to_string()))
.await
.expect("the reload is accepted");
expect_process_event(&mut rx, 1, ProcessEventKind::Online, EVENT_WAIT).await;
expect_process_event(&mut rx, 0, ProcessEventKind::Delete, EVENT_WAIT).await;
let listed = h.ctx.supervisor.list().await;
assert_eq!(
listed.iter().map(|info| info.id).collect::<Vec<_>>(),
vec![1],
"fixture check: the swap must have run to completion, or there was \
never an overlap for the ordering to matter in"
);
assert_eq!(
clock.reads(),
reads_before,
"the name group's cron worker must have been left where it was: a \
rebuilt one reads the clock again to derive its next occurrence"
);
}
fn backoff_harness(clock: &Arc<TestClock>) -> Harness {
harness_with_extras(
{
let mut scripts = vec![ProcScript::never_exits()];
scripts.push(ProcScript::const_exit(1));
scripts.extend([ProcScript::never_exits(); 8]);
scripts
},
|reports| Extras {
clock: Arc::clone(clock) as Arc<dyn Clock>,
enforcer: Arc::new(RecordingEnforcer::default()),
lamb_sweep: idle_sweep(),
max_cron_sleep: DEFAULT_MAX_CRON_SLEEP,
reports,
stats: idle_stats(),
},
)
}
fn backoff_app() -> shep_core::config::ResolvedApp {
app_with("web", |app| {
app.cron_restart = Some("0 * * * *".to_string());
app.restart_delay = Some(UpDuration::from_millis(3 * 60 * 60 * 1_000));
})
}
const SETTLE_DEADLINE: Duration = Duration::from_secs(10);
const SETTLE_VIRTUAL_LIMIT: Duration = Duration::from_secs(5);
async fn wait_for_cron_worker(clock: &TestClock) {
let started = std::time::Instant::now();
while clock.reads() == 0 {
assert!(
started.elapsed() < SETTLE_DEADLINE,
"the cron worker never took its first clock reading"
);
tokio::time::sleep(Duration::from_millis(1)).await;
}
}
async fn settle_into(supervisor: &SupervisorHandle, id: u32, status: ProcStatus) {
let started = std::time::Instant::now();
let virtual_start = tokio::time::Instant::now();
let mut last = Vec::new();
while started.elapsed() < SETTLE_DEADLINE && virtual_start.elapsed() < SETTLE_VIRTUAL_LIMIT {
tokio::time::sleep(Duration::from_millis(1)).await;
last = supervisor.list().await;
if last
.iter()
.any(|info| info.id == id && info.status == status)
{
return;
}
}
let seen = last
.iter()
.filter(|info| info.id == id)
.map(|info| {
format!(
"status={:?} restarts={} pid={:?}",
info.status, info.restarts, info.pid
)
})
.collect::<Vec<_>>()
.join("; ");
panic!(
"id {id} never reached {status:?} (last listing: {})",
if seen.is_empty() {
"no entry with this id".to_string()
} else {
seen
}
);
}
#[tokio::test(start_paused = true)]
async fn stopping_a_sheep_mid_backoff_still_stops_its_cron_worker() {
let clock = Arc::new(TestClock::starting_at(dt("2026-01-01T00:00:00Z")));
let h = backoff_harness(&clock);
let mut rx = h.ctx.events.subscribe();
h.ctx.supervisor.start(vec![backoff_app()]).await.unwrap();
wait_for_cron_worker(&clock).await;
cross_one_hour().await;
expect_restart(&mut rx, "web", EVENT_WAIT).await;
settle_into(&h.ctx.supervisor, 0, ProcStatus::WaitingRestart).await;
h.ctx
.supervisor
.stop(ProcessSelector::Name("web".to_string()))
.await
.expect("the sheep stops");
assert_eq!(h.ctx.supervisor.list().await[0].status, ProcStatus::Stopped);
assert_no_restart_within(&mut rx, "web", PAST_THE_NEXT_OCCURRENCE).await;
}
#[tokio::test(start_paused = true)]
async fn deleting_a_sheep_mid_backoff_takes_its_cron_worker_with_it() {
let clock = Arc::new(TestClock::starting_at(dt("2026-01-01T00:00:00Z")));
let h = backoff_harness(&clock);
let mut rx = h.ctx.events.subscribe();
h.ctx.supervisor.start(vec![backoff_app()]).await.unwrap();
wait_for_cron_worker(&clock).await;
cross_one_hour().await;
expect_restart(&mut rx, "web", EVENT_WAIT).await;
settle_into(&h.ctx.supervisor, 0, ProcStatus::WaitingRestart).await;
h.ctx
.supervisor
.delete(ProcessSelector::Name("web".to_string()))
.await
.expect("the sheep is deleted");
assert!(h.ctx.supervisor.list().await.is_empty());
let reads_after_delete = clock.reads();
assert_no_restart_within(&mut rx, "web", PAST_THE_NEXT_OCCURRENCE).await;
assert_eq!(
clock.reads(),
reads_after_delete,
"a deleted sheep's cron worker must stop reading the clock, not merely stop finding sheep"
);
}
#[tokio::test(start_paused = true)]
async fn a_cron_restart_is_never_reported_as_a_user_action() {
let clock = Arc::new(TestClock::starting_at(dt("2026-01-01T00:00:00Z")));
let h = backoff_harness(&clock);
let mut rx = h.ctx.events.subscribe();
h.ctx.supervisor.start(vec![backoff_app()]).await.unwrap();
wait_for_cron_worker(&clock).await;
cross_one_hour().await;
let (running, manually) = expect_restart_event(&mut rx, "web", EVENT_WAIT).await;
assert_eq!(running.restarts, 1);
assert!(
!manually,
"a cron occurrence is nobody typing `shep restart`"
);
settle_into(&h.ctx.supervisor, 0, ProcStatus::WaitingRestart).await;
cross_one_hour().await;
let (backing_off, manually) = expect_restart_event(&mut rx, "web", EVENT_WAIT).await;
assert_eq!(backing_off.restarts, 2);
assert!(
!manually,
"the same occurrence must answer the same way through `apply_immediate`"
);
}
#[tokio::test(start_paused = true)]
async fn a_respawn_that_cannot_spawn_stops_the_name_groups_cron_worker() {
let clock = Arc::new(TestClock::starting_at(dt("2026-01-01T00:00:00Z")));
let h = harness_with_extras(vec![ProcScript::never_exits()], |reports| Extras {
clock: Arc::clone(&clock) as Arc<dyn Clock>,
enforcer: Arc::new(RecordingEnforcer::default()),
lamb_sweep: idle_sweep(),
max_cron_sleep: DEFAULT_MAX_CRON_SLEEP,
reports,
stats: idle_stats(),
});
let mut rx = h.ctx.events.subscribe();
h.ctx
.supervisor
.start(vec![app_with("web", |app| {
app.cron_restart = Some("0 * * * *".to_string());
})])
.await
.unwrap();
wait_for_cron_worker(&clock).await;
cross_one_hour().await;
settle_into(&h.ctx.supervisor, 0, ProcStatus::Errored).await;
let reads_after_error = clock.reads();
assert_no_restart_within(&mut rx, "web", PAST_THE_NEXT_OCCURRENCE).await;
assert_eq!(
clock.reads(),
reads_after_error,
"an errored sheep's cron worker must stop reading the clock"
);
}
#[tokio::test(start_paused = true)]
async fn the_actor_arms_the_memory_limit_against_the_spawned_pid() {
let mut h = harness_with_extras(vec![ProcScript::never_exits(); 4], |reports| {
let sampler: Arc<dyn MemorySampler> =
Arc::new(ScriptedSampler::new(vec![vec![ProcessRss {
pid: 1000,
parent: None,
bytes: 900,
cpu_ms: 0,
}]]));
let stats = Arc::new(StatsState::new(Arc::clone(&sampler)));
Extras {
clock: Arc::new(TestClock::starting_at(dt("2026-01-01T00:00:00Z"))),
enforcer: Arc::new(PollingEnforcer::start(
sampler,
reports.breaches.clone(),
Arc::clone(&stats),
)),
lamb_sweep: idle_sweep(),
max_cron_sleep: DEFAULT_MAX_CRON_SLEEP,
reports,
stats,
}
});
h.ctx
.supervisor
.start(vec![app_with("web", |app| {
app.max_memory = Some(MemSize::from_bytes(500));
})])
.await
.unwrap();
let pid = h.ctx.supervisor.list().await[0]
.pid
.expect("a live sheep has a pid");
let breach = match tokio::time::timeout(EVENT_WAIT, h.breaches.recv()).await {
Ok(Some(breach)) => breach,
Ok(None) => panic!("the breach channel closed before a breach arrived"),
Err(_) => panic!("timed out waiting for a breach"),
};
assert_eq!(breach.id, 0);
assert_eq!(
breach.root_pid, pid,
"the enforcer must be armed against the pid the sheep is actually running as"
);
assert_eq!(breach.observed.bytes(), 900);
}
#[tokio::test(start_paused = true)]
async fn the_actor_arms_a_readiness_gated_app_once_it_comes_online() {
let mut h = harness_with_extras(vec![ProcScript::never_exits(); 4], |reports| {
let sampler: Arc<dyn MemorySampler> =
Arc::new(ScriptedSampler::new(vec![vec![ProcessRss {
pid: 1000,
parent: None,
bytes: 900,
cpu_ms: 0,
}]]));
let stats = Arc::new(StatsState::new(Arc::clone(&sampler)));
Extras {
clock: Arc::new(TestClock::starting_at(dt("2026-01-01T00:00:00Z"))),
enforcer: Arc::new(PollingEnforcer::start(
sampler,
reports.breaches.clone(),
Arc::clone(&stats),
)),
lamb_sweep: idle_sweep(),
max_cron_sleep: DEFAULT_MAX_CRON_SLEEP,
reports,
stats,
}
});
h.ctx
.supervisor
.start(vec![app_with("web", |app| {
app.wait_ready = true;
app.max_memory = Some(MemSize::from_bytes(500));
})])
.await
.unwrap();
let listing = h.ctx.supervisor.list().await;
assert_eq!(
listing[0].status,
ProcStatus::Starting,
"a gated app must not be Online when its Start reply lands"
);
let pid = listing[0].pid.expect("a spawned sheep has a pid");
let breach = match tokio::time::timeout(EVENT_WAIT, h.breaches.recv()).await {
Ok(Some(breach)) => breach,
Ok(None) => panic!("the breach channel closed before a breach arrived"),
Err(_) => panic!("timed out waiting for a breach"),
};
assert_eq!(breach.id, 0);
assert_eq!(
breach.root_pid, pid,
"the gated path must arm against the pid the sheep is running as"
);
}
#[tokio::test]
async fn the_actor_arms_the_liveness_loop_against_the_spawned_pid() {
let reserved = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
let addr = reserved.local_addr().unwrap();
drop(reserved);
let mut h = harness(vec![ProcScript::never_exits(); 4]);
h.ctx
.supervisor
.start(vec![app_with("web", |app| {
app.liveness_probe = Some(ProbeConfig {
failure_threshold: 1,
interval: PROBE_INTERVAL,
timeout: UpDuration::from_millis(500),
..probe_config(ProbeKind::Tcp, &addr.to_string())
});
})])
.await
.unwrap();
let pid = h.ctx.supervisor.list().await[0]
.pid
.expect("a live sheep has a pid");
let failure = expect_liveness(&mut h.liveness, LIVENESS_DEADLINE).await;
assert_eq!(
failure,
LivenessReport {
id: 0,
pid,
epoch: 1
}
);
}
struct SlowWalk(Arc<dyn LambSweep>);
impl LambSweep for SlowWalk {
fn snapshot(&self, root_pid: u32) -> LambSnapshot {
std::thread::sleep(Duration::from_millis(100));
self.0.snapshot(root_pid)
}
fn last_snapshot(&self, root_pid: u32) -> Option<LambSnapshot> {
self.0.last_snapshot(root_pid)
}
fn survivors(&self, snapshot: &LambSnapshot) -> Vec<u32> {
self.0.survivors(snapshot)
}
fn descendants(&self, roots: &[u32]) -> LambSnapshot {
self.0.descendants(roots)
}
fn signal(&self, pid: u32, signal: LambSignal) -> Result<(), SignalError> {
self.0.signal(pid, signal)
}
}
#[tokio::test(start_paused = true)]
async fn a_respawn_waits_out_a_slow_lamb_walk_before_it_errors() {
let clock = Arc::new(TestClock::starting_at(dt("2026-01-01T00:00:00Z")));
let h = harness_with_extras(vec![ProcScript::never_exits()], |reports| Extras {
clock: Arc::clone(&clock) as Arc<dyn Clock>,
enforcer: Arc::new(RecordingEnforcer::default()),
lamb_sweep: Arc::new(SlowWalk(idle_sweep())),
max_cron_sleep: DEFAULT_MAX_CRON_SLEEP,
reports,
stats: idle_stats(),
});
h.ctx
.supervisor
.start(vec![app_with("web", |app| {
app.cron_restart = Some("0 * * * *".to_string());
})])
.await
.unwrap();
wait_for_cron_worker(&clock).await;
cross_one_hour().await;
settle_into(&h.ctx.supervisor, 0, ProcStatus::Errored).await;
}