use std::{
collections::VecDeque,
mem::DropGuard,
sync::{
Arc, Mutex,
atomic::{AtomicUsize, Ordering},
},
time::Duration,
};
use async_trait::async_trait;
use tokio::{
sync::watch,
time::{self, Instant},
};
use crate::{
clock::{Clock, SharedClock},
observability::{MetricRegistry, metric_names},
result::{ClassifyError, ErrorKind},
};
use super::*;
struct TokioClock;
impl Clock for TokioClock {
fn now(&self) -> std::time::Instant {
Instant::now().into_std()
}
}
struct TimedWorker {
steps: Mutex<VecDeque<Duration>>,
starts: Mutex<Vec<Instant>>,
cancellations: AtomicUsize,
shutdown: ShutdownTrigger,
}
impl TimedWorker {
fn new(steps: impl IntoIterator<Item = Duration>, shutdown: ShutdownTrigger) -> Self {
Self {
steps: Mutex::new(steps.into_iter().collect()),
starts: Mutex::new(Vec::new()),
cancellations: AtomicUsize::new(0),
shutdown,
}
}
}
#[async_trait]
impl Worker for TimedWorker {
fn name(&self) -> &str {
"timed"
}
async fn run_once(&self, _: WorkerContext) -> Result<WorkerTick, BoxWorkerError> {
self.starts.lock().unwrap().push(Instant::now());
let (delay, last) = {
let mut steps = self.steps.lock().unwrap();
(
steps.pop_front().expect("unexpected tick"),
steps.is_empty(),
)
};
let cancellation = DropGuard::new(&self.cancellations, |cancellations| {
cancellations.fetch_add(1, Ordering::SeqCst);
});
time::sleep(delay).await;
DropGuard::dismiss(cancellation);
if last {
self.shutdown.shutdown();
}
Ok(WorkerTick::new(1))
}
}
async fn phase(status: &mut watch::Receiver<WorkerStatus>, expected: WorkerPhase) -> WorkerStatus {
loop {
let snapshot = *status.borrow_and_update();
if snapshot.phase() == expected {
return snapshot;
}
status.changed().await.expect("runner stopped unexpectedly");
}
}
#[test]
fn tick_timeout_configuration_is_optional_validated_and_round_trips() {
for json in ["{}", r#"{"tick_timeout":null}"#] {
let settings: WorkerSettings = serde_json::from_str(json).unwrap();
assert_eq!(settings.build().unwrap().tick_timeout(), None);
}
let settings: WorkerSettings = serde_json::from_str(r#"{"tick_timeout":"250ms"}"#).unwrap();
let config = settings.build().unwrap();
assert_eq!(config.tick_timeout(), Some(Duration::from_millis(250)));
let wire = serde_json::to_value(WorkerSettings::from(config)).unwrap();
assert_eq!(wire["tick_timeout"], "250ms");
for timeout in [Duration::ZERO, Duration::MAX] {
let error = WorkerSettings::default()
.with_tick_timeout(Some(timeout))
.build()
.unwrap_err();
assert!(
error
.issues()
.iter()
.any(|issue| issue.path() == "worker.tick_timeout")
);
}
assert_eq!(WorkerTickTimeout.error_kind(), ErrorKind::Timeout);
assert_eq!(WorkerTickTimeout.error_code(), "worker_tick_timed_out");
}
#[tokio::test(start_paused = true)]
async fn timed_out_tick_records_failure_and_retries_only_after_backoff() {
let (trigger, shutdown) = shutdown_channel();
let worker = Arc::new(TimedWorker::new(
[Duration::from_secs(60), Duration::ZERO],
trigger,
));
let metrics = Arc::new(MetricRegistry::new());
let wakeup = WorkerWakeup::new();
let mut options = WorkerRunOptions::new()
.with_clock(Arc::new(TokioClock))
.with_metrics(Some(metrics.clone()))
.with_wakeup(wakeup.clone());
let mut status = options.subscribe_status();
let started_at = Instant::now();
let task = tokio::spawn(run_worker_until_shutdown_with_options(
worker.clone(),
WorkerSettings::default()
.with_tick_timeout(Some(Duration::from_secs(2)))
.with_failure_backoff(BackoffSettings::new(
Duration::from_secs(5),
Duration::from_secs(5),
1,
))
.build()
.unwrap(),
shutdown,
options,
));
let failed = phase(&mut status, WorkerPhase::Backoff).await;
assert_eq!(Instant::now() - started_at, Duration::from_secs(2));
assert_eq!(
failed.last_tick_finished_at(),
Some(Instant::now().into_std())
);
assert_eq!(failed.last_tick_outcome(), Some(WorkerTickOutcome::Failure));
assert_eq!(worker.cancellations.load(Ordering::SeqCst), 1);
wakeup.notify();
time::advance(Duration::from_secs(4)).await;
tokio::task::yield_now().await;
assert_eq!(worker.starts.lock().unwrap().len(), 1);
assert_eq!(status.borrow().phase(), WorkerPhase::Backoff);
time::advance(Duration::from_secs(1)).await;
let summary = task.await.unwrap().unwrap();
assert_eq!(summary.successes(), 1);
assert_eq!(summary.failures(), 1);
let starts = worker.starts.lock().unwrap();
assert_eq!(starts[1] - starts[0], Duration::from_secs(7));
let snapshot = metrics.snapshot().unwrap();
let failures = snapshot
.samples()
.iter()
.find(|sample| {
sample.name() == metric_names::WORKER_TICKS_TOTAL
&& sample
.labels()
.iter()
.any(|(key, value)| key == "outcome" && value == "failure")
})
.unwrap();
assert_eq!(failures.value(), 1.0);
}
#[tokio::test(start_paused = true)]
async fn default_keeps_long_ticks_unbounded() {
let (trigger, shutdown) = shutdown_channel();
let worker = Arc::new(TimedWorker::new([Duration::from_secs(60)], trigger));
let started_at = Instant::now();
let summary = run_worker_until_shutdown(worker.clone(), WorkerConfig::default(), shutdown)
.await
.unwrap();
assert_eq!(Instant::now() - started_at, Duration::from_secs(60));
assert_eq!(summary.successes(), 1);
assert_eq!(worker.cancellations.load(Ordering::SeqCst), 0);
}
#[tokio::test(start_paused = true)]
async fn shutdown_and_tick_deadlines_use_the_earlier_deadline() {
for tick_seconds in [2, 10] {
let (trigger, shutdown) = shutdown_channel();
let worker = Arc::new(TimedWorker::new([Duration::from_secs(60)], trigger.clone()));
let mut options = WorkerRunOptions::new();
let mut status = options.subscribe_status();
let started_at = Instant::now();
let task = tokio::spawn(run_worker_until_shutdown_with_options(
worker.clone(),
WorkerSettings::default()
.with_tick_timeout(Some(Duration::from_secs(tick_seconds)))
.with_shutdown_timeout(Duration::from_secs(5))
.build()
.unwrap(),
shutdown,
options,
));
phase(&mut status, WorkerPhase::Running).await;
trigger.shutdown();
let result = task.await.unwrap();
assert_eq!(
Instant::now() - started_at,
Duration::from_secs(tick_seconds.min(5))
);
if tick_seconds < 5 {
assert_eq!(result.unwrap().failures(), 1);
assert_eq!(status.borrow().phase(), WorkerPhase::Stopped);
assert_eq!(
status.borrow().last_tick_outcome(),
Some(WorkerTickOutcome::Failure)
);
} else {
assert!(matches!(result, Err(WorkerRunError::ShutdownTimedOut)));
assert_eq!(status.borrow().phase(), WorkerPhase::Failed);
assert_eq!(status.borrow().last_tick_outcome(), None);
}
assert_eq!(worker.starts.lock().unwrap().len(), 1);
assert_eq!(worker.cancellations.load(Ordering::SeqCst), 1);
}
}
#[derive(Clone, Copy)]
enum Renewal {
Succeed,
Hang,
FailAfter(Duration),
LoseAfter(Duration),
}
struct LeaseStore {
local: LocalWorkerLeaseStore,
renewal: Renewal,
acquire_delay: Duration,
release_delay: Duration,
renewals: AtomicUsize,
cancelled_renewals: AtomicUsize,
releases: AtomicUsize,
}
impl LeaseStore {
fn new(clock: SharedClock, renewal: Renewal) -> Self {
Self {
local: LocalWorkerLeaseStore::with_clock(clock),
renewal,
acquire_delay: Duration::ZERO,
release_delay: Duration::ZERO,
renewals: AtomicUsize::new(0),
cancelled_renewals: AtomicUsize::new(0),
releases: AtomicUsize::new(0),
}
}
}
#[async_trait]
impl WorkerLeaseStore for LeaseStore {
async fn try_acquire(
&self,
worker: &str,
owner: &str,
config: WorkerLeaseConfig,
) -> Result<Option<WorkerLease>, WorkerLeaseError> {
time::sleep(self.acquire_delay).await;
self.local.try_acquire(worker, owner, config).await
}
async fn renew(
&self,
lease: &WorkerLease,
config: WorkerLeaseConfig,
) -> Result<Option<WorkerLease>, WorkerLeaseError> {
self.renewals.fetch_add(1, Ordering::SeqCst);
let cancellation = DropGuard::new(&self.cancelled_renewals, |cancellations| {
cancellations.fetch_add(1, Ordering::SeqCst);
});
let result = match self.renewal {
Renewal::Succeed => self.local.renew(lease, config).await,
Renewal::Hang => std::future::pending().await,
Renewal::FailAfter(delay) => {
time::sleep(delay).await;
Err(WorkerLeaseError::Store(
std::io::Error::other("renewal failed").into(),
))
}
Renewal::LoseAfter(delay) => {
time::sleep(delay).await;
Ok(None)
}
};
DropGuard::dismiss(cancellation);
result
}
async fn release(&self, lease: &WorkerLease) -> Result<(), WorkerLeaseError> {
self.releases.fetch_add(1, Ordering::SeqCst);
time::sleep(self.release_delay).await;
self.local.release(lease).await
}
}
#[tokio::test(start_paused = true)]
async fn shutdown_budget_is_shared_by_tick_drain_and_lease_release() {
for release_seconds in [1, 4] {
let clock = Arc::new(TokioClock);
let mut store = LeaseStore::new(clock.clone(), Renewal::Succeed);
store.release_delay = Duration::from_secs(release_seconds);
let store = Arc::new(store);
let (trigger, shutdown) = shutdown_channel();
let worker = Arc::new(TimedWorker::new([Duration::from_secs(4)], trigger.clone()));
let mut options = WorkerRunOptions::new()
.with_clock(clock)
.with_lease_store("owner", store.clone());
let mut status = options.subscribe_status();
let config = WorkerSettings::default()
.with_shutdown_timeout(Duration::from_secs(5))
.with_lease(Some(WorkerLeaseSettings::new(
Duration::from_secs(30),
Duration::from_secs(10),
Duration::from_secs(5),
)))
.build()
.unwrap();
let task = tokio::spawn(run_worker_until_shutdown_with_options(
worker.clone(),
config,
shutdown,
options,
));
phase(&mut status, WorkerPhase::Running).await;
time::advance(Duration::from_secs(1)).await;
let stopping_at = Instant::now();
trigger.shutdown();
let result = task.await.unwrap();
assert_eq!(
stopping_at.elapsed(),
Duration::from_secs((3 + release_seconds).min(5))
);
assert_eq!(store.releases.load(Ordering::SeqCst), 1);
assert_eq!(worker.cancellations.load(Ordering::SeqCst), 0);
assert_eq!(
status.borrow().last_tick_outcome(),
Some(WorkerTickOutcome::Success)
);
if release_seconds == 1 {
assert_eq!(result.unwrap().successes(), 1);
assert_eq!(status.borrow().phase(), WorkerPhase::Stopped);
} else {
assert!(matches!(result, Err(WorkerRunError::ShutdownTimedOut)));
assert_eq!(status.borrow().phase(), WorkerPhase::Failed);
}
}
}
fn leased_timeout(timeout: u64) -> WorkerConfig {
WorkerSettings::default()
.with_tick_timeout(Some(Duration::from_secs(timeout)))
.with_lease(Some(WorkerLeaseSettings::new(
Duration::from_secs(10),
Duration::from_secs(6),
Duration::from_secs(5),
)))
.build()
.unwrap()
}
#[tokio::test(start_paused = true)]
async fn tick_deadline_keeps_renewals_live_and_cancels_pending_renewal_before_shutdown_release() {
for (renewal, timeout) in [(Renewal::Succeed, 11), (Renewal::Hang, 5)] {
let clock = Arc::new(TokioClock);
let store = Arc::new(LeaseStore::new(clock.clone(), renewal));
let (trigger, shutdown) = shutdown_channel();
let worker = Arc::new(TimedWorker::new([Duration::from_secs(60)], trigger.clone()));
let mut options = WorkerRunOptions::new()
.with_clock(clock)
.with_lease_store("owner", store.clone());
let mut status = options.subscribe_status();
let started_at = Instant::now();
let task = tokio::spawn(run_worker_until_shutdown_with_options(
worker.clone(),
leased_timeout(timeout),
shutdown,
options,
));
phase(&mut status, WorkerPhase::Backoff).await;
assert_eq!(Instant::now() - started_at, Duration::from_secs(timeout));
assert_eq!(worker.cancellations.load(Ordering::SeqCst), 1);
if matches!(renewal, Renewal::Hang) {
assert_eq!(store.renewals.load(Ordering::SeqCst), 1);
assert_eq!(store.cancelled_renewals.load(Ordering::SeqCst), 1);
} else {
assert_eq!(store.renewals.load(Ordering::SeqCst), 2);
}
assert_eq!(store.releases.load(Ordering::SeqCst), 0);
trigger.shutdown();
assert_eq!(task.await.unwrap().unwrap().failures(), 1);
assert_eq!(store.releases.load(Ordering::SeqCst), 1);
assert_eq!(worker.starts.lock().unwrap().len(), 1);
}
}
#[tokio::test(start_paused = true)]
async fn lease_failures_keep_priority_and_do_not_release_an_uncertain_active_lease() {
for (renewal, timeout, elapsed) in [
(Renewal::FailAfter(Duration::from_secs(1)), 5, 5),
(Renewal::LoseAfter(Duration::from_secs(1)), 5, 5),
(Renewal::Hang, 12, 9),
] {
let clock = Arc::new(TokioClock);
let store = Arc::new(LeaseStore::new(clock.clone(), renewal));
let (trigger, shutdown) = shutdown_channel();
let worker = Arc::new(TimedWorker::new([Duration::from_secs(60)], trigger));
let mut options = WorkerRunOptions::new()
.with_clock(clock)
.with_lease_store("owner", store.clone());
let status = options.subscribe_status();
let started_at = Instant::now();
let result = run_worker_until_shutdown_with_options(
worker.clone(),
leased_timeout(timeout),
shutdown,
options,
)
.await;
match renewal {
Renewal::FailAfter(_) => assert!(matches!(result, Err(WorkerRunError::Lease(_)))),
Renewal::LoseAfter(_) => assert!(matches!(result, Err(WorkerRunError::LeaseLost))),
Renewal::Hang => assert!(matches!(
result,
Err(WorkerRunError::LeaseOperationTimedOut)
)),
Renewal::Succeed => unreachable!(),
}
assert_eq!(Instant::now() - started_at, Duration::from_secs(elapsed));
assert_eq!(store.releases.load(Ordering::SeqCst), 0);
assert_eq!(worker.cancellations.load(Ordering::SeqCst), 1);
assert_eq!(status.borrow().phase(), WorkerPhase::Failed);
assert_eq!(status.borrow().last_tick_outcome(), None);
}
}
#[tokio::test(start_paused = true)]
async fn delayed_renewal_cannot_keep_a_tick_running_past_lease_expiry() {
let clock = Arc::new(TokioClock);
let store = Arc::new(LeaseStore::new(clock.clone(), Renewal::Hang));
let (trigger, shutdown) = shutdown_channel();
let worker = Arc::new(TimedWorker::new([Duration::from_secs(60)], trigger));
let mut options = WorkerRunOptions::new()
.with_clock(clock)
.with_lease_store("owner", store.clone());
let mut status = options.subscribe_status();
let task = tokio::spawn(run_worker_until_shutdown_with_options(
worker.clone(),
leased_timeout(30),
shutdown,
options,
));
phase(&mut status, WorkerPhase::Running).await;
time::advance(Duration::from_secs(8)).await;
tokio::task::yield_now().await;
assert_eq!(store.renewals.load(Ordering::SeqCst), 1);
time::advance(Duration::from_secs(2)).await;
tokio::task::yield_now().await;
assert!(task.is_finished(), "the lease expiry must bound renewal");
assert!(matches!(
task.await.unwrap(),
Err(WorkerRunError::LeaseLost)
));
assert_eq!(worker.cancellations.load(Ordering::SeqCst), 1);
assert_eq!(store.cancelled_renewals.load(Ordering::SeqCst), 1);
assert_eq!(store.releases.load(Ordering::SeqCst), 0);
assert_eq!(status.borrow().phase(), WorkerPhase::Failed);
}
#[tokio::test(start_paused = true)]
async fn expired_lease_takes_priority_over_a_ready_tick() {
let clock = Arc::new(TokioClock);
let store = Arc::new(LeaseStore::new(clock.clone(), Renewal::Hang));
let (trigger, shutdown) = shutdown_channel();
let worker = Arc::new(TimedWorker::new([Duration::from_secs(10)], trigger));
let mut options = WorkerRunOptions::new()
.with_clock(clock)
.with_lease_store("owner", store.clone());
let mut status = options.subscribe_status();
let task = tokio::spawn(run_worker_until_shutdown_with_options(
worker.clone(),
leased_timeout(30),
shutdown,
options,
));
phase(&mut status, WorkerPhase::Running).await;
time::advance(Duration::from_secs(10)).await;
assert!(matches!(
task.await.unwrap(),
Err(WorkerRunError::LeaseLost)
));
assert_eq!(worker.cancellations.load(Ordering::SeqCst), 1);
assert_eq!(store.renewals.load(Ordering::SeqCst), 0);
assert_eq!(store.releases.load(Ordering::SeqCst), 0);
assert_eq!(status.borrow().last_tick_outcome(), None);
}
#[tokio::test(start_paused = true)]
async fn startup_and_lease_acquisition_do_not_consume_the_tick_budget() {
let clock = Arc::new(TokioClock);
let mut store = LeaseStore::new(clock.clone(), Renewal::Succeed);
store.acquire_delay = Duration::from_secs(3);
let (trigger, shutdown) = shutdown_channel();
let worker = Arc::new(TimedWorker::new([Duration::from_secs(1)], trigger));
let started_at = Instant::now();
let config = WorkerSettings::from(leased_timeout(2))
.with_startup_delay(Duration::from_secs(4))
.build()
.unwrap();
let summary = run_worker_until_shutdown_with_options(
worker.clone(),
config,
shutdown,
WorkerRunOptions::new()
.with_clock(clock)
.with_lease_store("owner", Arc::new(store)),
)
.await
.unwrap();
assert_eq!(Instant::now() - started_at, Duration::from_secs(8));
assert_eq!(summary.successes(), 1);
assert_eq!(worker.cancellations.load(Ordering::SeqCst), 0);
}