use core::time::Duration;
use std::sync::Arc;
use chrono::{DateTime, Utc};
use shep_core::config::CronSchedule;
use shep_core::selector::ProcessSelector;
use crate::supervisor::{SupervisorError, SupervisorHandle};
pub trait Clock: Send + Sync + 'static {
fn now_utc(&self) -> DateTime<Utc>;
}
#[derive(Debug, Clone, Copy, Default)]
pub struct SystemClock;
impl Clock for SystemClock {
fn now_utc(&self) -> DateTime<Utc> {
Utc::now()
}
}
#[allow(dead_code)]
pub(crate) const DEFAULT_MAX_CRON_SLEEP: Duration = Duration::from_secs(60);
const MIN_MAX_SLEEP: Duration = Duration::from_millis(1_000);
pub fn spawn_cron_worker(
name: String,
schedule: CronSchedule,
clock: Arc<dyn Clock>,
supervisor: SupervisorHandle,
max_sleep: Duration,
) -> tokio::task::JoinHandle<()> {
let max_sleep = max_sleep.max(MIN_MAX_SLEEP);
tokio::spawn(async move {
loop {
let now = clock.now_utc();
let next = match schedule.next_after(now) {
Ok(Some(next)) => next,
Ok(None) => {
tracing::info!(
name,
pattern = schedule.pattern(),
"cron_restart pattern has no further occurrence; worker ending"
);
return;
}
Err(err) => {
tracing::warn!(
name,
pattern = schedule.pattern(),
%err,
"cron schedule could not resolve its next occurrence; worker ending"
);
return;
}
};
let until_next = (next - now).to_std().unwrap_or(Duration::ZERO);
tokio::time::sleep(until_next.min(max_sleep)).await;
if clock.now_utc() >= next {
match supervisor
.restart_automatic(ProcessSelector::Name(name.clone()))
.await
{
Ok(_) => {}
Err(SupervisorError::NotFound) => {
tracing::debug!(name, "cron fired but no sheep by this name is registered");
}
Err(err @ SupervisorError::SpawnFailed(_)) => {
tracing::warn!(name, %err, "cron-triggered restart failed to spawn");
}
Err(
err @ (SupervisorError::ReopenFailed(_)
| SupervisorError::FlushFailed(_)
| SupervisorError::ReloadInFlight(_)
| SupervisorError::InvalidScale(_)
| SupervisorError::CannotStart(_)),
) => {
tracing::warn!(name, %err, "cron-triggered restart reported an unrelated failure");
}
Err(err @ SupervisorError::EngineStopped) => {
tracing::warn!(name, %err, "supervisor engine has shut down; cron worker ending");
return;
}
}
}
}
})
}
#[cfg(test)]
mod tests {
use chrono::{DateTime, Utc};
use tokio::sync::broadcast;
use super::*;
use crate::bus::SharedEvent;
use crate::fake::{ProcScript, ScriptedRunner};
use crate::supervisor::spawn_supervisor;
use crate::testing::{TestClock, test_paths};
use shep_core::config::{AppConfig, normalize};
use shep_core::protocol::{BusEvent, ProcessEventKind, ProcessInfo};
use shep_core::status::ProcStatus;
#[test]
fn clock_is_dyn_compatible() {
let _: &dyn Clock = &SystemClock;
}
const EVENT_WAIT: Duration = Duration::from_secs(30);
fn dt(s: &str) -> DateTime<Utc> {
s.parse().expect("valid RFC3339 timestamp")
}
fn spawn_test_fixture() -> (
SupervisorHandle,
broadcast::Receiver<SharedEvent>,
tempfile::TempDir,
) {
spawn_test_fixture_with(vec![ProcScript::never_exits(); 8])
}
fn spawn_test_fixture_with(
scripts: Vec<ProcScript>,
) -> (
SupervisorHandle,
broadcast::Receiver<SharedEvent>,
tempfile::TempDir,
) {
let dir = tempfile::tempdir().unwrap();
let (events, rx) = crate::bus::test_bus(64);
let runner = ScriptedRunner::new(scripts);
let handle = spawn_supervisor(runner, test_paths(&dir), events);
(handle, rx, dir)
}
async fn start_named(handle: &SupervisorHandle, name: &str) {
let app = AppConfig::minimal(name, "./srv");
handle.start(vec![normalize(app).unwrap()]).await.unwrap();
}
async fn spawn_worker_and_settle(
name: &str,
schedule: CronSchedule,
clock: Arc<dyn Clock>,
handle: &SupervisorHandle,
max_sleep: Duration,
) -> tokio::task::JoinHandle<()> {
let worker =
spawn_cron_worker(name.to_string(), schedule, clock, handle.clone(), max_sleep);
tokio::task::yield_now().await;
worker
}
async fn expect_restart(rx: &mut broadcast::Receiver<SharedEvent>, name: &str) -> ProcessInfo {
loop {
match tokio::time::timeout(EVENT_WAIT, rx.recv())
.await
.map(|received| received.map(|event| event.to_event()))
{
Ok(Ok(BusEvent::Process {
event: ProcessEventKind::Restart,
info,
..
})) if info.name == name => return info,
Ok(Ok(_)) => continue,
Ok(Err(broadcast::error::RecvError::Lagged(_))) => continue,
Ok(Err(err)) => panic!("event stream closed before a restart for {name}: {err}"),
Err(_) => panic!("timed out waiting for a cron restart of {name}"),
}
}
}
fn assert_no_restart_pending(rx: &mut broadcast::Receiver<SharedEvent>, name: &str) {
loop {
match rx.try_recv().map(|event| event.to_event()) {
Ok(BusEvent::Process {
event: ProcessEventKind::Restart,
info,
..
}) if info.name == name => {
panic!("unexpected cron restart of {name} observed");
}
Ok(_) => continue,
Err(broadcast::error::TryRecvError::Empty) => return,
Err(err) => panic!("event channel error while checking for no restart: {err}"),
}
}
}
async fn assert_no_restart_within(
rx: &mut broadcast::Receiver<SharedEvent>,
name: &str,
window: Duration,
) {
let deadline = tokio::time::Instant::now() + window;
loop {
let remaining = deadline.saturating_duration_since(tokio::time::Instant::now());
match tokio::time::timeout(remaining, rx.recv())
.await
.map(|received| received.map(|event| event.to_event()))
{
Err(_) => return, Ok(Ok(BusEvent::Process {
event: ProcessEventKind::Restart,
info,
..
})) if info.name == name => {
panic!(
"unexpected cron restart of {name} observed (restarts={})",
info.restarts
);
}
Ok(Ok(_)) => continue,
Ok(Err(broadcast::error::RecvError::Lagged(_))) => continue,
Ok(Err(err)) => {
panic!("event channel closed while checking for no restart of {name}: {err}")
}
}
}
}
#[tokio::test(start_paused = true)]
async fn fires_at_the_top_of_three_successive_hours() {
let (handle, mut rx, _dir) = spawn_test_fixture();
let name = "web";
start_named(&handle, name).await;
let clock = Arc::new(TestClock::starting_at(dt("2026-01-01T00:00:00Z")));
let schedule = CronSchedule::parse("0 * * * *", None).unwrap();
let worker =
spawn_worker_and_settle(name, schedule, clock, &handle, DEFAULT_MAX_CRON_SLEEP).await;
let mut observed = Vec::new();
for _ in 0..3 {
for _ in 0..120 {
tokio::time::advance(Duration::from_secs(30)).await;
}
let info = expect_restart(&mut rx, name).await;
observed.push(info.restarts);
}
assert_eq!(observed, vec![1, 2, 3]);
worker.abort();
}
#[tokio::test(start_paused = true)]
async fn thirty_second_steps_across_one_hour_yield_exactly_one_restart() {
let (handle, mut rx, _dir) = spawn_test_fixture();
let name = "web";
start_named(&handle, name).await;
let clock = Arc::new(TestClock::starting_at(dt("2026-01-01T00:00:30Z")));
let schedule = CronSchedule::parse("0 * * * *", None).unwrap();
let worker =
spawn_worker_and_settle(name, schedule, clock, &handle, DEFAULT_MAX_CRON_SLEEP).await;
for _ in 0..120 {
tokio::time::advance(Duration::from_secs(30)).await;
}
let info = expect_restart(&mut rx, name).await;
assert_eq!(info.restarts, 1);
assert_no_restart_pending(&mut rx, name);
worker.abort();
}
#[tokio::test(start_paused = true)]
async fn one_jump_past_six_occurrences_yields_exactly_one_restart() {
let (handle, mut rx, _dir) = spawn_test_fixture();
let name = "web";
start_named(&handle, name).await;
let clock = Arc::new(TestClock::starting_at(dt("2026-01-01T00:00:00Z")));
let schedule = CronSchedule::parse("0 * * * *", None).unwrap();
let worker =
spawn_worker_and_settle(name, schedule, clock, &handle, DEFAULT_MAX_CRON_SLEEP).await;
tokio::time::advance(Duration::from_secs(6 * 3600 + 30)).await;
let info = expect_restart(&mut rx, name).await;
assert_eq!(info.restarts, 1);
assert_no_restart_pending(&mut rx, name);
worker.abort();
}
#[tokio::test(start_paused = true)]
async fn exhausted_pattern_ends_the_task_without_restarting() {
let (handle, mut rx, _dir) = spawn_test_fixture();
let name = "web";
let schedule = CronSchedule::parse("0 0 30 2 *", None).unwrap();
let clock = Arc::new(TestClock::starting_at(dt("2026-01-01T00:00:00Z")));
let worker =
spawn_worker_and_settle(name, schedule, clock, &handle, DEFAULT_MAX_CRON_SLEEP).await;
tokio::time::timeout(EVENT_WAIT, worker)
.await
.expect("worker did not end for an exhausted schedule")
.expect("worker task panicked");
assert_no_restart_pending(&mut rx, name);
}
#[tokio::test(start_paused = true)]
async fn the_worker_ends_when_the_supervisor_engine_has_stopped() {
let (handle, _rx, _dir) = spawn_test_fixture_with(Vec::new());
let name = "web";
handle.shutdown().await;
assert_eq!(
handle
.restart_automatic(ProcessSelector::Name(name.to_string()))
.await
.unwrap_err(),
SupervisorError::EngineStopped
);
let clock = Arc::new(TestClock::starting_at(dt("2026-01-01T00:00:00Z")));
let schedule = CronSchedule::parse("0 * * * *", None).unwrap();
let worker =
spawn_worker_and_settle(name, schedule, clock, &handle, DEFAULT_MAX_CRON_SLEEP).await;
for _ in 0..120 {
tokio::time::advance(Duration::from_secs(30)).await;
}
tokio::time::timeout(EVENT_WAIT, worker)
.await
.expect("the worker did not end after the engine shut down")
.expect("worker task panicked");
}
#[tokio::test(start_paused = true)]
async fn abort_stops_the_worker_after_observing_one_restart() {
let (handle, mut rx, _dir) = spawn_test_fixture();
let name = "web";
start_named(&handle, name).await;
let clock = Arc::new(TestClock::starting_at(dt("2026-01-01T00:00:00Z")));
let schedule = CronSchedule::parse("0 * * * *", None).unwrap();
let worker =
spawn_worker_and_settle(name, schedule, clock, &handle, DEFAULT_MAX_CRON_SLEEP).await;
tokio::time::advance(Duration::from_secs(3600)).await;
let info = expect_restart(&mut rx, name).await;
assert_eq!(info.restarts, 1);
worker.abort();
tokio::time::advance(Duration::from_secs(3600)).await;
assert_no_restart_within(&mut rx, name, Duration::from_secs(10)).await;
}
#[tokio::test(start_paused = true)]
async fn a_per_minute_pattern_fires_once_on_the_max_sleep_boundary() {
let (handle, mut rx, _dir) = spawn_test_fixture();
let name = "web";
start_named(&handle, name).await;
let clock = Arc::new(TestClock::starting_at(dt("2026-01-01T00:00:00Z")));
let schedule = CronSchedule::parse("* * * * *", None).unwrap();
let start = tokio::time::Instant::now();
let worker =
spawn_worker_and_settle(name, schedule, clock, &handle, DEFAULT_MAX_CRON_SLEEP).await;
assert_no_restart_within(&mut rx, name, Duration::from_millis(59_999)).await;
let first = expect_restart(&mut rx, name).await;
assert_eq!(first.restarts, 1);
assert_eq!(
tokio::time::Instant::now() - start,
Duration::from_secs(60),
"the occurrence and the sleep cap coincide here, so the restart belongs at \
exactly 00:01:00 -- neither dropped nor early"
);
assert_no_restart_within(&mut rx, name, Duration::from_millis(59_999)).await;
let second = expect_restart(&mut rx, name).await;
assert_eq!(second.restarts, 2);
assert_eq!(
tokio::time::Instant::now() - start,
Duration::from_secs(120),
"the schedule must survive its own boundary and fire the next occurrence too"
);
worker.abort();
}
#[tokio::test(start_paused = true)]
async fn ten_minute_cap_reads_the_clock_fewer_than_twenty_times() {
let (handle, mut rx, _dir) = spawn_test_fixture();
let name = "web";
start_named(&handle, name).await;
let clock = Arc::new(TestClock::starting_at(dt("2026-01-01T00:00:00Z")));
let schedule = CronSchedule::parse("0 * * * *", None).unwrap();
let worker = spawn_worker_and_settle(
name,
schedule,
Arc::clone(&clock) as Arc<dyn Clock>,
&handle,
Duration::from_secs(600),
)
.await;
for _ in 0..120 {
tokio::time::advance(Duration::from_secs(30)).await;
}
let info = expect_restart(&mut rx, name).await;
assert_eq!(info.restarts, 1);
assert!(
clock.reads() < 20,
"expected fewer than 20 clock reads with a 10-minute max_sleep honored, got {}",
clock.reads()
);
worker.abort();
}
#[tokio::test(start_paused = true)]
async fn a_sub_second_max_sleep_is_floored_instead_of_waking_every_millisecond() {
let (handle, _rx, _dir) = spawn_test_fixture_with(Vec::new());
let name = "web";
let clock = Arc::new(TestClock::starting_at(dt("2026-01-01T00:00:00Z")));
let schedule = CronSchedule::parse("0 * * * *", None).unwrap();
let worker = spawn_worker_and_settle(
name,
schedule,
Arc::clone(&clock) as Arc<dyn Clock>,
&handle,
Duration::from_millis(1),
)
.await;
tokio::time::sleep(MIN_MAX_SLEEP * 5).await;
assert!(
clock.reads() < 20,
"max_sleep must be floored at MIN_MAX_SLEEP: five floored sleeps cost about \
two clock reads each, got {}",
clock.reads()
);
worker.abort();
}
#[tokio::test(start_paused = true)]
async fn an_operators_stop_beats_a_cron_triggered_restart_mid_ladder() {
let (handle, mut rx, _dir) = spawn_test_fixture_with(vec![
ProcScript::ignores_signals(), ProcScript::never_exits(), ProcScript::never_exits(), ProcScript::never_exits(), ]);
let name = "web";
let mut app = AppConfig::minimal(name, "./srv");
app.instances = 2;
handle.start(vec![normalize(app).unwrap()]).await.unwrap();
let listed = handle.list().await;
let (held, released) = (listed[0].id, listed[1].id);
let clock = Arc::new(TestClock::starting_at(dt("2026-01-01T00:00:00Z")));
let schedule = CronSchedule::parse("0 * * * *", None).unwrap();
let worker =
spawn_worker_and_settle(name, schedule, clock, &handle, DEFAULT_MAX_CRON_SLEEP).await;
tokio::time::advance(Duration::from_secs(3600)).await;
let restarted = expect_restart(&mut rx, name).await;
assert_eq!(
(restarted.id, restarted.restarts),
(released, 1),
"the occurrence never reached the actor, so the stop below would \
race nothing -- got {restarted:?}"
);
worker.abort();
let stopped = handle.stop(ProcessSelector::Id(held)).await.unwrap();
assert_eq!(stopped.len(), 1);
assert_eq!(
(stopped[0].id, stopped[0].status, stopped[0].restarts),
(held, ProcStatus::Stopped, 0),
"an operator's stop was silently converted into the cron-triggered \
restart it raced -- got {stopped:?}"
);
let listed = handle.list().await;
assert_eq!(
(listed[0].id, listed[0].status, listed[0].pid),
(held, ProcStatus::Stopped, None),
"the sheep an operator stopped is running again -- got {listed:?}"
);
assert_eq!(
(listed[1].id, listed[1].status),
(released, ProcStatus::Online),
"the instance the operator did not name must still be up, \
restarted by the occurrence -- got {listed:?}"
);
}
}