use std::{sync::Arc, time::Duration};
use async_trait::async_trait;
use tokio::sync::{Mutex, mpsc, watch};
use crate::clock::{Clock, ManualClock};
use super::*;
type TickResult = Result<WorkerTick, ()>;
struct ControlledWorker(Mutex<mpsc::UnboundedReceiver<TickResult>>);
#[async_trait]
impl Worker for ControlledWorker {
fn name(&self) -> &str {
"observed"
}
async fn run_once(&self, _context: WorkerContext) -> Result<WorkerTick, BoxWorkerError> {
self.0
.lock()
.await
.recv()
.await
.expect("the test must provide a tick result")
.map_err(|()| std::io::Error::other("private tick failure").into())
}
}
fn controlled() -> (ControlledWorker, mpsc::UnboundedSender<TickResult>) {
let (sender, receiver) = mpsc::unbounded_channel();
(ControlledWorker(Mutex::new(receiver)), sender)
}
async fn phase(
receiver: &mut watch::Receiver<WorkerStatus>,
expected: WorkerPhase,
) -> WorkerStatus {
loop {
let status = *receiver.borrow_and_update();
if status.phase() == expected {
return status;
}
receiver
.changed()
.await
.expect("runner stopped before the expected phase");
}
}
async fn advance(clock: &ManualClock, duration: Duration) {
clock.advance(duration).unwrap();
tokio::time::advance(duration).await;
}
#[tokio::test(start_paused = true)]
async fn status_tracks_running_and_waiting_until_graceful_shutdown() {
let clock = ManualClock::new();
let started_at = clock.now();
let wakeup = WorkerWakeup::new();
let mut options = WorkerRunOptions::new()
.with_clock(clock.shared())
.with_wakeup(wakeup.clone());
let mut status = options.subscribe_status();
let second_reader = options.subscribe_status();
assert_eq!(status.borrow().phase(), WorkerPhase::Pending);
let config = WorkerSettings::default()
.with_startup_delay(Duration::from_secs(2))
.with_poll_interval(Duration::from_secs(10))
.with_min_queue_interval(Duration::from_secs(1))
.build()
.unwrap();
let (trigger, shutdown) = shutdown_channel();
let (worker, commands) = controlled();
let task = tokio::spawn(run_worker_until_shutdown_with_options(
worker, config, shutdown, options,
));
assert_eq!(
phase(&mut status, WorkerPhase::StartupDelay)
.await
.phase_since(),
started_at
);
advance(&clock, Duration::from_secs(2)).await;
let running = phase(&mut status, WorkerPhase::Running).await;
assert_eq!(running.last_tick_started_at(), Some(clock.now()));
assert_eq!(running.last_tick_finished_at(), None);
assert_eq!(running.last_tick_outcome(), None);
advance(&clock, Duration::from_secs(3)).await;
commands.send(Ok(WorkerTick::new(1))).unwrap();
let waiting = phase(&mut status, WorkerPhase::Poll).await;
assert_eq!(
waiting.last_tick_started_at(),
running.last_tick_started_at()
);
assert_eq!(waiting.last_tick_finished_at(), Some(clock.now()));
assert_eq!(
waiting.last_tick_outcome(),
Some(WorkerTickOutcome::Success)
);
assert_eq!(waiting.phase_since(), clock.now());
advance(&clock, Duration::from_secs(10)).await;
let running = phase(&mut status, WorkerPhase::Running).await;
assert_eq!(running.last_tick_started_at(), Some(clock.now()));
assert_eq!(
running.last_tick_finished_at(),
waiting.last_tick_finished_at()
);
commands
.send(Ok(WorkerTick::new(0).with_schedule(
WorkerSchedule::WaitForWork {
recheck_after: None,
},
)))
.unwrap();
phase(&mut status, WorkerPhase::QueueIdle).await;
wakeup.notify();
phase(&mut status, WorkerPhase::QueueSpacing).await;
advance(&clock, Duration::from_secs(1)).await;
phase(&mut status, WorkerPhase::Running).await;
commands
.send(Ok(
WorkerTick::new(0).with_schedule(WorkerSchedule::After(Duration::from_secs(4)))
))
.unwrap();
phase(&mut status, WorkerPhase::Delay).await;
advance(&clock, Duration::from_secs(4)).await;
phase(&mut status, WorkerPhase::Running).await;
commands
.send(Ok(
WorkerTick::new(0).with_schedule(WorkerSchedule::Continue)
))
.unwrap();
phase(&mut status, WorkerPhase::QueueSpacing).await;
advance(&clock, Duration::from_secs(1)).await;
phase(&mut status, WorkerPhase::Running).await;
trigger.shutdown();
phase(&mut status, WorkerPhase::Shutdown).await;
commands.send(Ok(WorkerTick::new(0))).unwrap();
assert_eq!(task.await.unwrap().unwrap().ticks(), 5);
let stopped = phase(&mut status, WorkerPhase::Stopped).await;
assert_eq!(stopped.last_tick_finished_at(), Some(clock.now()));
assert_eq!(*second_reader.borrow(), stopped);
assert!(status.changed().await.is_err());
}
#[tokio::test(start_paused = true)]
async fn status_distinguishes_failure_backoff_and_external_cancellation() {
let clock = ManualClock::new();
let wakeup = WorkerWakeup::new();
let mut options = WorkerRunOptions::new()
.with_clock(clock.shared())
.with_wakeup(wakeup.clone());
let mut status = options.subscribe_status();
let config = WorkerSettings::default()
.with_failure_backoff(BackoffSettings::new(
Duration::from_secs(5),
Duration::from_secs(5),
1,
))
.build()
.unwrap();
let (_trigger, shutdown) = shutdown_channel();
let (worker, commands) = controlled();
let task = tokio::spawn(run_worker_until_shutdown_with_options(
worker, config, shutdown, options,
));
phase(&mut status, WorkerPhase::Running).await;
commands.send(Err(())).unwrap();
let backoff = phase(&mut status, WorkerPhase::Backoff).await;
assert_eq!(
backoff.last_tick_outcome(),
Some(WorkerTickOutcome::Failure)
);
assert_eq!(backoff.last_tick_finished_at(), Some(clock.now()));
wakeup.notify();
tokio::task::yield_now().await;
assert_eq!(status.borrow().phase(), WorkerPhase::Backoff);
advance(&clock, Duration::from_secs(5)).await;
let running = phase(&mut status, WorkerPhase::Running).await;
assert_eq!(running.last_tick_started_at(), Some(clock.now()));
task.abort();
assert!(task.await.unwrap_err().is_cancelled());
let cancelled = phase(&mut status, WorkerPhase::Cancelled).await;
assert_eq!(cancelled.phase_since(), clock.now());
assert_eq!(
cancelled.last_tick_finished_at(),
backoff.last_tick_finished_at()
);
assert_eq!(
cancelled.last_tick_outcome(),
Some(WorkerTickOutcome::Failure)
);
assert!(status.changed().await.is_err());
}
#[tokio::test(start_paused = true)]
async fn status_records_errors_and_disabled_runners_without_starting_a_tick() {
for config in [
WorkerSettings::default()
.with_lease(Some(WorkerLeaseSettings::new(
Duration::from_secs(30),
Duration::from_secs(10),
Duration::from_secs(5),
)))
.build()
.unwrap(),
WorkerConfig::disabled(),
] {
let mut options = WorkerRunOptions::new();
let mut status = options.subscribe_status();
let (_trigger, shutdown) = shutdown_channel();
let (worker, _commands) = controlled();
let enabled = config.enabled();
let result =
run_worker_until_shutdown_with_options(worker, config, shutdown, options).await;
let expected = if enabled {
assert!(matches!(result, Err(WorkerRunError::MissingLeaseStore)));
WorkerPhase::Failed
} else {
assert_eq!(result.unwrap().ticks(), 0);
WorkerPhase::Stopped
};
let final_status = phase(&mut status, expected).await;
assert_eq!(final_status.last_tick_started_at(), None);
assert_eq!(final_status.last_tick_finished_at(), None);
assert!(status.changed().await.is_err());
}
}
#[tokio::test(start_paused = true)]
async fn shutdown_timeout_marks_failure_without_completing_the_interrupted_tick() {
let mut options = WorkerRunOptions::new();
let mut status = options.subscribe_status();
let (trigger, shutdown) = shutdown_channel();
let (worker, _commands) = controlled();
let task = tokio::spawn(run_worker_until_shutdown_with_options(
worker,
WorkerSettings::default()
.with_shutdown_timeout(Duration::from_secs(2))
.build()
.unwrap(),
shutdown,
options,
));
phase(&mut status, WorkerPhase::Running).await;
trigger.shutdown();
phase(&mut status, WorkerPhase::Shutdown).await;
tokio::time::advance(Duration::from_secs(2)).await;
assert!(matches!(
task.await.unwrap(),
Err(WorkerRunError::ShutdownTimedOut)
));
let failed = phase(&mut status, WorkerPhase::Failed).await;
assert!(failed.last_tick_started_at().is_some());
assert_eq!(failed.last_tick_finished_at(), None);
assert_eq!(failed.last_tick_outcome(), None);
assert!(status.changed().await.is_err());
}
#[test]
fn unused_options_and_unpolled_runner_publish_cancellation() {
let clock = ManualClock::new();
let mut options = WorkerRunOptions::new();
let status = options.subscribe_status();
options = options.with_clock(clock.shared());
assert_eq!(status.borrow().phase_since(), clock.now());
clock.advance(Duration::from_secs(2)).unwrap();
drop(options);
assert_eq!(status.borrow().phase(), WorkerPhase::Cancelled);
assert_eq!(status.borrow().phase_since(), clock.now());
assert!(status.has_changed().is_err());
let mut options = WorkerRunOptions::new();
let status = options.subscribe_status();
let (_trigger, shutdown) = shutdown_channel();
let (worker, _commands) = controlled();
let future =
run_worker_until_shutdown_with_options(worker, WorkerConfig::default(), shutdown, options);
drop(future);
assert_eq!(status.borrow().phase(), WorkerPhase::Cancelled);
assert_eq!(status.borrow().last_tick_started_at(), None);
assert!(status.has_changed().is_err());
}
struct BusyLeaseStore;
#[async_trait]
impl WorkerLeaseStore for BusyLeaseStore {
async fn try_acquire(
&self,
_: &str,
_: &str,
_: WorkerLeaseConfig,
) -> Result<Option<WorkerLease>, WorkerLeaseError> {
Ok(None)
}
async fn renew(
&self,
_: &WorkerLease,
_: WorkerLeaseConfig,
) -> Result<Option<WorkerLease>, WorkerLeaseError> {
Ok(None)
}
async fn release(&self, _: &WorkerLease) -> Result<(), WorkerLeaseError> {
Ok(())
}
}
#[tokio::test(start_paused = true)]
async fn lease_contention_is_observable_without_reporting_an_execution() {
let clock = ManualClock::new();
let mut options = WorkerRunOptions::new()
.with_clock(clock.shared())
.with_lease_store("owner", Arc::new(BusyLeaseStore));
let mut status = options.subscribe_status();
let config = WorkerSettings::default()
.with_lease(Some(WorkerLeaseSettings::new(
Duration::from_secs(30),
Duration::from_secs(10),
Duration::from_secs(5),
)))
.build()
.unwrap();
let (trigger, shutdown) = shutdown_channel();
let (worker, _commands) = controlled();
let task = tokio::spawn(run_worker_until_shutdown_with_options(
worker, config, shutdown, options,
));
let waiting = phase(&mut status, WorkerPhase::LeaseWait).await;
assert_eq!(waiting.last_tick_started_at(), None);
advance(&clock, Duration::from_secs(10)).await;
tokio::task::yield_now().await;
assert_eq!(status.borrow().phase(), WorkerPhase::LeaseWait);
assert_eq!(status.borrow().phase_since(), waiting.phase_since());
trigger.shutdown();
assert_eq!(task.await.unwrap().unwrap().ticks(), 0);
phase(&mut status, WorkerPhase::Stopped).await;
assert!(status.changed().await.is_err());
}