use std::{
collections::VecDeque,
sync::{
Arc, Mutex,
atomic::{AtomicUsize, Ordering},
},
time::Duration,
};
use async_trait::async_trait;
use tokio::time::{self, Instant};
use super::*;
type TickResult = Result<WorkerTick, ()>;
type Starts = Arc<Mutex<Vec<Instant>>>;
struct ScriptedWorker {
steps: Mutex<VecDeque<TickResult>>,
starts: Starts,
shutdown: ShutdownTrigger,
}
#[async_trait]
impl Worker for ScriptedWorker {
fn name(&self) -> &str {
"scheduled"
}
async fn run_once(&self) -> Result<WorkerTick, BoxWorkerError> {
self.starts.lock().unwrap().push(Instant::now());
let mut steps = self.steps.lock().unwrap();
let result = steps.pop_front().expect("unexpected additional tick");
if steps.is_empty() {
self.shutdown.shutdown();
}
result.map_err(|()| std::io::Error::other("tick failed").into())
}
}
fn scripted(
steps: impl IntoIterator<Item = TickResult>,
) -> (ScriptedWorker, Starts, ShutdownTrigger, ShutdownToken) {
let (trigger, token) = shutdown_channel();
let starts = Arc::new(Mutex::new(Vec::new()));
let worker = ScriptedWorker {
steps: Mutex::new(steps.into_iter().collect()),
starts: starts.clone(),
shutdown: trigger.clone(),
};
(worker, starts, trigger, token)
}
fn tick(schedule: WorkerSchedule) -> TickResult {
Ok(WorkerTick::new(0).with_schedule(schedule))
}
const IDLE: WorkerSchedule = WorkerSchedule::WaitForWork {
recheck_after: None,
};
#[tokio::test(start_paused = true)]
async fn wakeup_reacquires_an_expired_idle_lease_without_an_extra_poll() {
struct TokioClock;
impl crate::clock::Clock for TokioClock {
fn now(&self) -> std::time::Instant {
Instant::now().into_std()
}
}
let clock = Arc::new(TokioClock);
let lease = WorkerLeaseConfig::new(
Duration::from_secs(1),
Duration::from_millis(200),
Duration::from_millis(100),
)
.unwrap();
let config = WorkerSettings::default()
.with_poll_interval(Duration::from_secs(10))
.with_lease(Some(WorkerLeaseSettings::from(lease)))
.build()
.unwrap();
let wakeup = WorkerWakeup::new();
let options = WorkerRunOptions::new()
.with_clock(clock.clone())
.with_lease_store("owner", Arc::new(LocalWorkerLeaseStore::with_clock(clock)))
.with_wakeup(wakeup.clone());
let (worker, starts, _trigger, token) = scripted([tick(IDLE), tick(IDLE)]);
let task = tokio::spawn(run_worker_until_shutdown_with_options(
worker, config, token, options,
));
for _ in 0..8 {
tokio::task::yield_now().await;
}
assert_eq!(starts.lock().unwrap().len(), 1);
time::advance(Duration::from_secs(2)).await;
wakeup.notify();
for _ in 0..8 {
tokio::task::yield_now().await;
}
assert_eq!(starts.lock().unwrap().len(), 2);
task.await.unwrap().unwrap();
}
#[test]
fn external_lease_errors_preserve_the_cause_without_exposing_it_in_diagnostics() {
use crate::result::{ClassifyError, ErrorKind};
use std::error::Error;
let error = WorkerLeaseError::Store(std::io::Error::other("backend credential=secret").into());
assert_eq!(error.error_kind(), ErrorKind::Unavailable);
assert_eq!(
error.source().unwrap().to_string(),
"backend credential=secret"
);
assert!(!format!("{error} {error:?}").contains("secret"));
}
async fn run(
steps: impl IntoIterator<Item = TickResult>,
config: WorkerConfig,
options: WorkerRunOptions,
) -> Vec<Duration> {
let (worker, starts, _trigger, token) = scripted(steps);
run_worker_until_shutdown_with_options(worker, config, token, options)
.await
.unwrap();
starts
.lock()
.unwrap()
.windows(2)
.map(|pair| pair[1] - pair[0])
.collect()
}
#[tokio::test(start_paused = true)]
async fn polling_and_explicit_delays_ignore_notifications_and_queue_spacing() {
let wakeup = WorkerWakeup::new();
wakeup.notify();
let gaps = run(
[
tick(WorkerSchedule::Poll),
Ok(WorkerTick::new(20).with_schedule(WorkerSchedule::After(Duration::from_secs(2)))),
tick(WorkerSchedule::After(Duration::ZERO)),
tick(WorkerSchedule::Poll),
],
WorkerConfig::default(),
WorkerRunOptions::new().with_wakeup(wakeup),
)
.await;
assert_eq!(
gaps,
[
Duration::from_secs(10),
Duration::from_secs(2),
Duration::ZERO
]
);
}
#[tokio::test(start_paused = true)]
async fn empty_continuations_and_zero_recheck_hints_are_rate_limited() {
let gaps = run(
[
tick(WorkerSchedule::Continue),
tick(WorkerSchedule::WaitForWork {
recheck_after: Some(Duration::ZERO),
}),
tick(WorkerSchedule::Continue),
tick(WorkerSchedule::Poll),
],
WorkerConfig::default(),
WorkerRunOptions::new(),
)
.await;
assert_eq!(gaps, [Duration::from_millis(10); 3]);
}
#[tokio::test(start_paused = true)]
async fn idle_polling_survives_missing_notifications_and_dropped_producers() {
let producer = WorkerWakeup::new();
let options = WorkerRunOptions::new().with_wakeup(producer.clone());
drop(producer);
for options in [WorkerRunOptions::new(), options] {
let gaps = run(
[tick(IDLE), tick(IDLE), tick(WorkerSchedule::Poll)],
WorkerConfig::default(),
options,
)
.await;
assert_eq!(gaps, [Duration::from_secs(10); 2]);
}
}
#[tokio::test(start_paused = true)]
async fn early_notifications_are_retained_and_coalesced() {
let wakeup = WorkerWakeup::new();
for _ in 0..1000 {
wakeup.notify();
}
let gaps = run(
[tick(IDLE), tick(IDLE), tick(WorkerSchedule::Poll)],
WorkerConfig::default(),
WorkerRunOptions::new().with_wakeup(wakeup),
)
.await;
assert_eq!(gaps, [Duration::from_millis(10), Duration::from_secs(10)]);
}
#[tokio::test(start_paused = true)]
async fn notification_interrupts_idle_wait_and_earlier_rechecks_still_work() {
let wakeup = WorkerWakeup::new();
let producer = wakeup.clone();
let notify = tokio::spawn(async move {
time::sleep(Duration::from_secs(2)).await;
producer.notify();
});
let gaps = run(
[
tick(IDLE),
tick(WorkerSchedule::WaitForWork {
recheck_after: Some(Duration::from_secs(3)),
}),
tick(WorkerSchedule::WaitForWork {
recheck_after: Some(Duration::from_secs(60)),
}),
tick(WorkerSchedule::Poll),
],
WorkerConfig::default(),
WorkerRunOptions::new().with_wakeup(wakeup),
)
.await;
notify.await.unwrap();
assert_eq!(
gaps,
[
Duration::from_secs(2),
Duration::from_secs(3),
Duration::from_secs(10)
]
);
}
#[tokio::test(start_paused = true)]
async fn notification_storm_cannot_create_unbounded_empty_ticks() {
let wakeup = WorkerWakeup::new();
let producer = wakeup.clone();
let notifications = tokio::spawn(async move {
loop {
producer.notify();
time::sleep(Duration::from_millis(1)).await;
}
});
let gaps = run(
[tick(IDLE); 20],
WorkerConfig::default(),
WorkerRunOptions::new().with_wakeup(wakeup),
)
.await;
notifications.abort();
assert_eq!(gaps, [Duration::from_millis(10); 19]);
}
#[tokio::test(start_paused = true)]
async fn failure_backoff_ignores_notifications_and_resets_only_after_success() {
let wakeup = WorkerWakeup::new();
wakeup.notify();
let gaps = run(
[
Err(()),
Err(()),
tick(WorkerSchedule::Continue),
Err(()),
tick(WorkerSchedule::Poll),
],
WorkerConfig::default(),
WorkerRunOptions::new().with_wakeup(wakeup),
)
.await;
assert_eq!(
gaps,
[
Duration::from_secs(1),
Duration::from_secs(2),
Duration::from_millis(10),
Duration::from_secs(1)
]
);
}
#[tokio::test(start_paused = true)]
async fn notification_does_not_shorten_startup_delay() {
let wakeup = WorkerWakeup::new();
wakeup.notify();
let (worker, starts, _trigger, token) = scripted([tick(IDLE)]);
let start = Instant::now();
let config = WorkerSettings::default()
.with_startup_delay(Duration::from_secs(3))
.build()
.unwrap();
run_worker_until_shutdown_with_options(
worker,
config,
token,
WorkerRunOptions::new().with_wakeup(wakeup),
)
.await
.unwrap();
assert_eq!(starts.lock().unwrap()[0] - start, Duration::from_secs(3));
}
#[tokio::test(start_paused = true)]
async fn shutdown_interrupts_every_wait_without_starting_another_tick() {
for result in [
tick(WorkerSchedule::Poll),
tick(IDLE),
tick(WorkerSchedule::Continue),
tick(WorkerSchedule::After(Duration::MAX)),
Err(()),
] {
let wakeup = WorkerWakeup::new();
let (worker, starts, trigger, token) = scripted([result, tick(WorkerSchedule::Poll)]);
let producer = wakeup.clone();
let stop = tokio::spawn(async move {
time::sleep(Duration::from_millis(5)).await;
producer.notify();
trigger.shutdown();
});
let summary = run_worker_until_shutdown_with_options(
worker,
WorkerConfig::default(),
token,
WorkerRunOptions::new().with_wakeup(wakeup),
)
.await
.unwrap();
stop.await.unwrap();
assert_eq!(starts.lock().unwrap().len(), 1);
assert_eq!(summary.ticks(), 1);
}
}
#[tokio::test(start_paused = true)]
async fn slow_batches_do_not_pay_an_additional_queue_delay() {
struct SlowBatch {
starts: Starts,
shutdown: ShutdownTrigger,
}
#[async_trait]
impl Worker for SlowBatch {
fn name(&self) -> &str {
"slow-batch"
}
async fn run_once(&self) -> Result<WorkerTick, BoxWorkerError> {
let count = {
let mut starts = self.starts.lock().unwrap();
starts.push(Instant::now());
starts.len()
};
if count == 1 {
time::sleep(Duration::from_millis(50)).await;
} else {
self.shutdown.shutdown();
}
Ok(WorkerTick::new(20).with_schedule(WorkerSchedule::Continue))
}
}
let (trigger, token) = shutdown_channel();
let starts = Arc::new(Mutex::new(Vec::new()));
run_worker_until_shutdown(
SlowBatch {
starts: starts.clone(),
shutdown: trigger,
},
WorkerConfig::default(),
token,
)
.await
.unwrap();
let starts = starts.lock().unwrap();
assert_eq!(starts[1] - starts[0], Duration::from_millis(50));
}
#[test]
fn queue_settings_default_validate_and_round_trip() {
let settings: WorkerSettings = serde_json::from_str("{}").unwrap();
assert_eq!(
settings.build().unwrap().min_queue_interval(),
Duration::from_millis(10)
);
let settings = WorkerSettings::default().with_min_queue_interval(Duration::from_millis(25));
let wire = serde_json::to_value(&settings).unwrap();
let config = serde_json::from_value::<WorkerSettings>(wire)
.unwrap()
.build()
.unwrap();
assert_eq!(config.min_queue_interval(), Duration::from_millis(25));
for delay in [Duration::ZERO, Duration::MAX] {
let error = WorkerSettings::default()
.with_min_queue_interval(delay)
.build()
.unwrap_err();
assert!(
error
.issues()
.iter()
.any(|issue| issue.path() == "worker.min_queue_interval")
);
}
}
#[tokio::test(start_paused = true)]
async fn queue_spacing_never_extends_the_polling_fallback() {
let config = WorkerSettings::default()
.with_poll_interval(Duration::from_millis(1))
.build()
.unwrap();
let gaps = run(
[
tick(IDLE),
tick(WorkerSchedule::Continue),
tick(WorkerSchedule::Poll),
],
config,
WorkerRunOptions::new(),
)
.await;
assert_eq!(gaps, [Duration::from_millis(1); 2]);
}
struct ShutdownOnAcquire {
shutdown: ShutdownTrigger,
released: Arc<AtomicUsize>,
}
#[async_trait]
impl WorkerLeaseStore for ShutdownOnAcquire {
async fn try_acquire(
&self,
worker: &str,
owner: &str,
config: WorkerLeaseConfig,
) -> Result<Option<WorkerLease>, WorkerLeaseError> {
self.shutdown.shutdown();
Ok(Some(WorkerLease::new(
worker,
owner,
1,
std::time::Instant::now() + config.lease_duration(),
)?))
}
async fn renew(
&self,
lease: &WorkerLease,
_: WorkerLeaseConfig,
) -> Result<Option<WorkerLease>, WorkerLeaseError> {
Ok(Some(lease.clone()))
}
async fn release(&self, _: &WorkerLease) -> Result<(), WorkerLeaseError> {
self.released.fetch_add(1, Ordering::SeqCst);
Ok(())
}
}
#[tokio::test]
async fn shutdown_during_lease_acquisition_releases_lease_without_starting_work() {
let (worker, starts, trigger, token) = scripted([tick(WorkerSchedule::Continue)]);
let released = Arc::new(AtomicUsize::new(0));
let lease = WorkerLeaseConfig::new(
Duration::from_secs(60),
Duration::from_secs(10),
Duration::from_secs(1),
)
.unwrap();
let config = WorkerSettings::default()
.with_lease(Some(WorkerLeaseSettings::from(lease)))
.build()
.unwrap();
let store = ShutdownOnAcquire {
shutdown: trigger,
released: released.clone(),
};
let summary = run_worker_until_shutdown_with_options(
worker,
config,
token,
WorkerRunOptions::new().with_lease_store("owner", Arc::new(store)),
)
.await
.unwrap();
assert!(starts.lock().unwrap().is_empty());
assert_eq!(summary.ticks(), 0);
assert_eq!(released.load(Ordering::SeqCst), 1);
}
struct ScheduledBatch(WorkerSchedule);
#[async_trait]
impl BatchWorkerJob for ScheduledBatch {
fn name(&self) -> &str {
"scheduled-batch"
}
async fn run_batch(&self) -> Result<BatchRunSummary, BoxWorkerError> {
Ok(BatchRunSummary::new()
.with_skipped(2)
.with_retrying(1)
.with_failed(1)
.with_schedule(self.0))
}
}
#[tokio::test]
async fn batch_adapter_preserves_schedule_independently_of_outcome_counts() {
for schedule in [
WorkerSchedule::Poll,
WorkerSchedule::Continue,
WorkerSchedule::After(Duration::from_secs(2)),
IDLE,
] {
let tick = BatchWorker::new(ScheduledBatch(schedule))
.run_once()
.await
.unwrap();
assert_eq!(tick.processed(), 4);
assert_eq!(tick.schedule(), schedule);
}
let tick = WorkerTick::new(0)
.with_schedule(WorkerSchedule::After(Duration::from_secs(2)))
.with_schedule(IDLE);
assert_eq!(tick.schedule(), IDLE);
let summary = BatchRunSummary::new()
.with_schedule(WorkerSchedule::Continue)
.with_schedule(WorkerSchedule::After(Duration::from_secs(3)));
assert_eq!(
summary.schedule(),
WorkerSchedule::After(Duration::from_secs(3))
);
}
#[derive(Clone)]
struct QueueItem {
id: u32,
attempts: u32,
due: Instant,
}
impl OutboxItem for QueueItem {
fn outbox_id(&self) -> String {
self.id.to_string()
}
fn outbox_attempts(&self) -> u32 {
self.attempts
}
}
#[derive(Default)]
struct QueueState {
pending: VecDeque<QueueItem>,
completed: Vec<(Instant, u32)>,
claims: Vec<(Instant, usize)>,
}
#[derive(Clone, Default)]
struct DueQueue(Arc<Mutex<QueueState>>);
impl DueQueue {
fn enqueue(&self, ids: impl IntoIterator<Item = u32>) {
self.0
.lock()
.unwrap()
.pending
.extend(ids.into_iter().map(|id| QueueItem {
id,
attempts: 0,
due: Instant::now(),
}));
}
}
#[async_trait]
impl OutboxStore<QueueItem> for DueQueue {
async fn claim_due(&self, claim: OutboxClaim) -> Result<Vec<QueueItem>, BoxWorkerError> {
let mut queue = self.0.lock().unwrap();
let mut items = Vec::new();
for _ in 0..queue.pending.len() {
let item = queue.pending.pop_front().unwrap();
if item.due <= Instant::now() && items.len() < claim.limit() as usize {
items.push(item);
} else {
queue.pending.push_back(item);
}
}
queue.claims.push((Instant::now(), items.len()));
Ok(items)
}
async fn persist_transition(
&self,
item: &QueueItem,
state: OutboxItemState,
) -> Result<(), BoxWorkerError> {
let mut queue = self.0.lock().unwrap();
match state {
OutboxItemState::Retry { delay, .. } => queue.pending.push_back(QueueItem {
id: item.id,
attempts: item.attempts + 1,
due: Instant::now() + delay,
}),
OutboxItemState::Succeeded => queue.completed.push((Instant::now(), item.id)),
_ => panic!("unexpected test item state"),
}
Ok(())
}
}
struct QueueHandler {
retry_first: bool,
}
#[async_trait]
impl OutboxHandler<QueueItem> for QueueHandler {
async fn handle(&self, item: &QueueItem) -> Result<OutboxItemState, BoxWorkerError> {
Ok(if self.retry_first && item.id == 0 && item.attempts == 0 {
OutboxItemState::retry("later", Duration::from_secs(5))
} else {
OutboxItemState::Succeeded
})
}
}
struct LimitedBatches<J> {
job: J,
remaining: AtomicUsize,
shutdown: ShutdownTrigger,
}
#[async_trait]
impl<J: BatchWorkerJob> BatchWorkerJob for LimitedBatches<J> {
fn name(&self) -> &str {
self.job.name()
}
async fn run_batch(&self) -> Result<BatchRunSummary, BoxWorkerError> {
let summary = self.job.run_batch().await?;
if self.remaining.fetch_sub(1, Ordering::SeqCst) == 1 {
self.shutdown.shutdown();
}
Ok(summary)
}
}
#[tokio::test(start_paused = true)]
async fn outbox_drains_thirty_one_items_in_two_bounded_batches() {
let queue = DueQueue::default();
queue.enqueue(0..31);
let (shutdown, token) = shutdown_channel();
let job = OutboxJob::new(
"queue",
queue.clone(),
QueueHandler { retry_first: false },
OutboxWorkerSettings::default()
.with_batch_size(20)
.build()
.unwrap(),
)
.with_queue_scheduling();
let worker = BatchWorker::new(LimitedBatches {
job,
remaining: AtomicUsize::new(2),
shutdown,
});
let summary = run_worker_until_shutdown(worker, WorkerConfig::default(), token)
.await
.unwrap();
assert_eq!(summary.processed(), 31);
let queue = queue.0.lock().unwrap();
assert_eq!(
queue
.claims
.iter()
.map(|(_, count)| *count)
.collect::<Vec<_>>(),
[20, 11]
);
assert_eq!(
queue.claims[1].0 - queue.claims[0].0,
Duration::from_millis(10)
);
assert_eq!(queue.completed.len(), 31);
}
#[tokio::test(start_paused = true)]
async fn wakeup_processes_new_work_without_delivering_future_retries_early() {
let queue = DueQueue::default();
queue.enqueue([0]);
let start = Instant::now();
let wakeup = WorkerWakeup::new();
let producer = wakeup.clone();
let producer_queue = queue.clone();
let enqueue = tokio::spawn(async move {
time::sleep(Duration::from_secs(1)).await;
producer_queue.enqueue([1]);
producer.notify();
});
let (shutdown, token) = shutdown_channel();
let job = OutboxJob::new(
"queue",
queue.clone(),
QueueHandler { retry_first: true },
OutboxWorkerConfig::default(),
)
.with_queue_scheduling();
let worker = BatchWorker::new(LimitedBatches {
job,
remaining: AtomicUsize::new(4),
shutdown,
});
let config = WorkerSettings::default()
.with_poll_interval(Duration::from_secs(3))
.build()
.unwrap();
run_worker_until_shutdown_with_options(
worker,
config,
token,
WorkerRunOptions::new().with_wakeup(wakeup),
)
.await
.unwrap();
enqueue.await.unwrap();
let queue = queue.0.lock().unwrap();
assert_eq!(
queue
.claims
.iter()
.map(|(at, count)| ((*at - start).as_secs(), *count))
.collect::<Vec<_>>(),
[(0, 1), (1, 1), (4, 0), (7, 1)]
);
assert_eq!(
queue.completed,
[
(start + Duration::from_secs(1), 1),
(start + Duration::from_secs(7), 0)
]
);
}
struct InitiallyBusyLease {
acquisitions: Starts,
}
#[async_trait]
impl WorkerLeaseStore for InitiallyBusyLease {
async fn try_acquire(
&self,
worker: &str,
owner: &str,
config: WorkerLeaseConfig,
) -> Result<Option<WorkerLease>, WorkerLeaseError> {
let mut starts = self.acquisitions.lock().unwrap();
starts.push(Instant::now());
if starts.len() == 1 {
return Ok(None);
}
Ok(Some(WorkerLease::new(
worker,
owner,
1,
std::time::Instant::now() + config.lease_duration(),
)?))
}
async fn renew(
&self,
lease: &WorkerLease,
_: WorkerLeaseConfig,
) -> Result<Option<WorkerLease>, WorkerLeaseError> {
Ok(Some(lease.clone()))
}
async fn release(&self, _: &WorkerLease) -> Result<(), WorkerLeaseError> {
Ok(())
}
}
#[tokio::test(start_paused = true)]
async fn notifications_and_continuation_do_not_bypass_lease_acquisition() {
let acquisitions = Arc::new(Mutex::new(Vec::new()));
let store = InitiallyBusyLease {
acquisitions: acquisitions.clone(),
};
let lease = WorkerLeaseConfig::new(
Duration::from_secs(60),
Duration::from_secs(10),
Duration::from_secs(1),
)
.unwrap();
let config = WorkerSettings::default()
.with_lease(Some(WorkerLeaseSettings::from(lease)))
.build()
.unwrap();
let (worker, starts, _trigger, token) = scripted([tick(WorkerSchedule::Continue), tick(IDLE)]);
let wakeup = WorkerWakeup::new();
wakeup.notify();
run_worker_until_shutdown_with_options(
worker,
config,
token,
WorkerRunOptions::new()
.with_lease_store("owner", Arc::new(store))
.with_wakeup(wakeup),
)
.await
.unwrap();
let acquisitions = acquisitions.lock().unwrap();
let starts = starts.lock().unwrap();
assert_eq!(acquisitions.len(), 2);
assert_eq!(starts[0] - acquisitions[0], Duration::from_secs(10));
assert_eq!(starts[1] - starts[0], Duration::from_millis(10));
}