use std::{
collections::VecDeque,
sync::{
Arc, Mutex,
atomic::{AtomicUsize, Ordering},
},
time::{Duration, Instant},
};
use async_trait::async_trait;
use tokio::time;
use crate::{
clock::{Clock, ManualClock},
result::{ClassifyError, ErrorKind},
};
use super::*;
fn lease_config(lease_duration: Duration, renew_after: Duration) -> WorkerLeaseConfig {
WorkerLeaseConfig::new(lease_duration, renew_after)
.expect("test lease configuration should be valid")
}
fn worker_config(poll_interval: Duration, lease: Option<WorkerLeaseConfig>) -> WorkerConfig {
WorkerSettings::default()
.with_poll_interval(poll_interval)
.with_lease(lease.map(WorkerLeaseSettings::from))
.build()
.expect("test worker settings should be valid")
}
fn outbox_config(
update: impl FnOnce(OutboxWorkerSettings) -> OutboxWorkerSettings,
) -> OutboxWorkerConfig {
update(OutboxWorkerSettings::default())
.build()
.expect("test outbox settings should be valid")
}
struct CountingWorker {
calls: Arc<AtomicUsize>,
}
struct NamedWorker(&'static str);
struct SlowWorker {
calls: Arc<AtomicUsize>,
duration: Duration,
}
struct TestBatchJob;
#[derive(Debug, Clone)]
struct TestOutboxItem {
id: String,
attempts: u32,
action: TestOutboxAction,
}
#[derive(Debug, Clone, Copy)]
enum TestOutboxAction {
Succeed,
Skip,
Retry,
HandlerError,
Slow,
}
#[derive(Debug, Clone, Default)]
struct TestOutboxStore {
items: Arc<Mutex<VecDeque<TestOutboxItem>>>,
states: Arc<Mutex<Vec<(String, OutboxItemState)>>>,
dead_letters: Arc<Mutex<Vec<String>>>,
}
#[derive(Debug, Clone, Copy)]
struct TestOutboxHandler;
#[async_trait]
impl Worker for CountingWorker {
fn name(&self) -> &str {
"counting"
}
async fn run_once(&self) -> Result<WorkerTick, BoxWorkerError> {
self.calls.fetch_add(1, Ordering::SeqCst);
Ok(WorkerTick::new(1).with_next_delay(Duration::from_millis(1)))
}
}
#[async_trait]
impl Worker for NamedWorker {
fn name(&self) -> &str {
self.0
}
async fn run_once(&self) -> Result<WorkerTick, BoxWorkerError> {
Ok(WorkerTick::new(1).with_next_delay(Duration::from_millis(1)))
}
}
#[async_trait]
impl Worker for SlowWorker {
fn name(&self) -> &str {
"slow"
}
async fn run_once(&self) -> Result<WorkerTick, BoxWorkerError> {
self.calls.fetch_add(1, Ordering::SeqCst);
time::sleep(self.duration).await;
Ok(WorkerTick::new(1).with_next_delay(Duration::from_secs(1)))
}
}
#[async_trait]
impl BatchWorkerJob for TestBatchJob {
fn name(&self) -> &str {
"test batch"
}
async fn run_batch(&self) -> Result<BatchRunSummary, BoxWorkerError> {
Ok(BatchRunSummary::new()
.with_claimed(5)
.with_succeeded(2)
.with_retrying(1)
.with_failed(1)
.with_skipped(1)
.with_next_delay(Duration::from_millis(5)))
}
}
impl OutboxItem for TestOutboxItem {
fn outbox_id(&self) -> String {
self.id.clone()
}
fn outbox_attempts(&self) -> u32 {
self.attempts
}
}
#[async_trait]
impl OutboxStore<TestOutboxItem> for TestOutboxStore {
async fn claim_due(&self, claim: OutboxClaim) -> Result<Vec<TestOutboxItem>, BoxWorkerError> {
let mut items = self.items.lock().unwrap();
Ok((0..claim.limit())
.filter_map(|_| items.pop_front())
.collect())
}
async fn record_state(
&self,
item: &TestOutboxItem,
state: OutboxItemState,
) -> Result<(), BoxWorkerError> {
self.states.lock().unwrap().push((item.id.clone(), state));
Ok(())
}
async fn record_dead_letter(
&self,
item: &TestOutboxItem,
reason: &str,
) -> Result<(), BoxWorkerError> {
self.dead_letters
.lock()
.unwrap()
.push(format!("{}:{reason}", item.id));
Ok(())
}
}
#[async_trait]
impl OutboxHandler<TestOutboxItem> for TestOutboxHandler {
async fn handle(&self, item: &TestOutboxItem) -> Result<OutboxItemState, BoxWorkerError> {
match item.action {
TestOutboxAction::Succeed => Ok(OutboxItemState::Succeeded),
TestOutboxAction::Skip => Ok(OutboxItemState::skipped("not applicable")),
TestOutboxAction::Retry => {
Ok(OutboxItemState::retry("temporary", Duration::from_secs(3)))
}
TestOutboxAction::HandlerError => Err("delivery failed".into()),
TestOutboxAction::Slow => {
time::sleep(Duration::from_millis(50)).await;
Ok(OutboxItemState::Succeeded)
}
}
}
}
#[test]
fn backoff_grows_until_cap() {
let config = BackoffConfig::new(Duration::from_secs(1), Duration::from_secs(3), 2)
.expect("test backoff should be valid");
let mut backoff = RetryBackoff::new(config);
assert_eq!(backoff.record_failure(), Duration::from_secs(1));
assert_eq!(backoff.record_failure(), Duration::from_secs(2));
assert_eq!(backoff.record_failure(), Duration::from_secs(3));
backoff.reset();
assert_eq!(backoff.record_failure(), Duration::from_secs(1));
}
#[cfg(feature = "settings")]
#[test]
fn worker_settings_adapt_to_runtime_config() {
let settings = WorkerSettings {
poll_interval: Duration::from_secs(5),
failure_backoff: BackoffSettings {
initial_delay: Duration::from_millis(200),
max_delay: Duration::from_secs(2),
multiplier: 3,
},
lease: Some(WorkerLeaseSettings {
lease_duration: Duration::from_secs(30),
renew_after: Duration::from_secs(10),
}),
..WorkerSettings::default()
};
settings.validate().expect("settings should validate");
let config = settings.build().expect("settings should build");
assert!(config.enabled());
assert_eq!(config.poll_interval(), Duration::from_secs(5));
assert_eq!(config.failure_backoff().multiplier(), 3);
assert_eq!(
config
.lease()
.expect("lease should be configured")
.renew_after(),
Duration::from_secs(10)
);
}
#[cfg(feature = "settings")]
#[test]
fn worker_settings_report_config_paths() {
let settings = WorkerSettings {
poll_interval: Duration::ZERO,
failure_backoff: BackoffSettings {
initial_delay: Duration::ZERO,
..BackoffSettings::default()
},
lease: Some(WorkerLeaseSettings::new(
Duration::MAX,
Duration::from_secs(1),
)),
..WorkerSettings::default()
};
let error = settings
.validate_at("email.worker")
.expect_err("invalid worker settings should report paths");
let paths = error
.issues()
.iter()
.map(|issue| issue.path())
.collect::<Vec<_>>();
assert!(paths.contains(&"email.worker.poll_interval"));
assert!(paths.contains(&"email.worker.failure_backoff.initial_delay"));
assert!(paths.contains(&"email.worker.lease.lease_duration"));
}
#[tokio::test]
async fn batch_worker_maps_outbox_summary_to_tick() {
let worker = BatchWorker::new(TestBatchJob);
let tick = worker.run_once().await.expect("batch should run");
assert_eq!(tick.processed(), 5);
assert_eq!(tick.next_delay(), Some(Duration::from_millis(5)));
}
#[test]
fn outbox_state_debug_redacts_handler_reasons() {
let reason = "private-handler-error-detail";
let states = [
OutboxItemState::skipped(reason),
OutboxItemState::retry(reason, Duration::from_secs(1)),
OutboxItemState::failed(reason),
];
let output = format!("{states:?}");
assert!(output.contains("[REDACTED]"));
assert!(!output.contains(reason));
}
#[tokio::test]
async fn outbox_job_claims_handles_and_records_states() {
let store = TestOutboxStore::default();
store.items.lock().unwrap().extend([
TestOutboxItem {
id: "one".to_owned(),
attempts: 0,
action: TestOutboxAction::Succeed,
},
TestOutboxItem {
id: "two".to_owned(),
attempts: 0,
action: TestOutboxAction::Skip,
},
TestOutboxItem {
id: "three".to_owned(),
attempts: 0,
action: TestOutboxAction::Retry,
},
TestOutboxItem {
id: "four".to_owned(),
attempts: 4,
action: TestOutboxAction::HandlerError,
},
]);
let job = OutboxJob::new(
"test outbox",
store.clone(),
TestOutboxHandler,
outbox_config(|settings| settings.with_batch_size(10)),
);
let summary = job.run_batch().await.expect("outbox should run");
assert_eq!(summary.claimed(), 4);
assert_eq!(summary.succeeded(), 1);
assert_eq!(summary.skipped(), 1);
assert_eq!(summary.retrying(), 1);
assert_eq!(summary.failed(), 1);
assert_eq!(summary.dead_lettered(), 1);
assert_eq!(summary.next_delay(), Some(Duration::from_secs(3)));
let states = store.states.lock().unwrap();
assert_eq!(states.len(), 4);
assert!(matches!(states[0].1, OutboxItemState::Succeeded));
assert!(matches!(states[1].1, OutboxItemState::Skipped { .. }));
assert!(matches!(states[2].1, OutboxItemState::Retry { .. }));
assert!(matches!(states[3].1, OutboxItemState::Failed { .. }));
assert_eq!(store.dead_letters.lock().unwrap().len(), 1);
}
#[tokio::test]
async fn outbox_job_applies_item_timeout_and_batch_timeout() {
let store = TestOutboxStore::default();
store.items.lock().unwrap().extend([
TestOutboxItem {
id: "slow".to_owned(),
attempts: 4,
action: TestOutboxAction::Slow,
},
TestOutboxItem {
id: "never-started".to_owned(),
attempts: 0,
action: TestOutboxAction::Succeed,
},
]);
let job = OutboxJob::new(
"test outbox",
store,
TestOutboxHandler,
outbox_config(|settings| {
settings
.with_batch_size(10)
.with_item_timeout(Some(Duration::from_millis(1)))
.with_batch_timeout(Some(Duration::from_millis(1)))
}),
);
let summary = job.run_batch().await.expect("outbox should run");
assert_eq!(summary.failed(), 1);
assert!(summary.batch_timed_out());
assert_eq!(summary.next_delay(), Some(Duration::ZERO));
}
#[cfg(feature = "settings")]
#[test]
fn outbox_worker_settings_adapt_to_runtime_config() {
let settings = OutboxWorkerSettings::default()
.with_batch_size(25)
.with_lease_duration(Duration::from_secs(120))
.with_max_attempts(7)
.with_retry_backoff(BackoffSettings::new(
Duration::from_secs(2),
Duration::from_secs(30),
2,
))
.with_item_timeout(Some(Duration::from_secs(10)))
.with_batch_timeout(Some(Duration::from_secs(60)));
settings.validate().expect("settings should validate");
let config = settings.build().expect("settings should build");
assert_eq!(config.batch_size(), 25);
assert_eq!(config.lease_duration(), Duration::from_secs(120));
assert_eq!(config.retry_policy().max_attempts(), 7);
assert_eq!(
config.retry_policy().delay_for_attempts(3),
Duration::from_secs(8)
);
assert_eq!(config.item_timeout(), Some(Duration::from_secs(10)));
assert_eq!(config.batch_timeout(), Some(Duration::from_secs(60)));
}
#[test]
fn worker_errors_are_classified() {
assert_eq!(
WorkerConfigError::InvalidOutboxItemTimeout.error_kind(),
ErrorKind::Validation
);
assert_eq!(
WorkerConfigError::InvalidOutboxBatchTimeout.error_code(),
"outbox_invalid_batch_timeout"
);
assert_eq!(
WorkerRunError::MissingLeaseStore.error_kind(),
ErrorKind::Validation
);
assert_eq!(
WorkerRunError::LeaseLost.error_kind(),
ErrorKind::Unavailable
);
assert_eq!(
WorkerLeaseError::LockPoisoned.error_kind(),
ErrorKind::Unavailable
);
}
#[tokio::test]
async fn worker_loop_stops_on_shutdown() {
let (trigger, token) = shutdown_channel();
let calls = Arc::new(AtomicUsize::new(0));
let worker = CountingWorker {
calls: Arc::clone(&calls),
};
let handle = tokio::spawn(run_worker_until_shutdown(
worker,
worker_config(Duration::from_millis(1), None),
token,
));
time::sleep(Duration::from_millis(5)).await;
trigger.shutdown();
let summary = handle.await.unwrap().unwrap();
assert!(summary.successes() > 0);
assert_eq!(summary.failures(), 0);
assert_eq!(summary.last_outcome(), Some(WorkerTickOutcome::Success));
assert_eq!(summary.state(), WorkerRunState::Healthy);
assert!(calls.load(Ordering::SeqCst) > 0);
}
#[tokio::test]
async fn worker_shutdown_enforces_the_configured_deadline() {
let (trigger, token) = shutdown_channel();
let calls = Arc::new(AtomicUsize::new(0));
let worker = SlowWorker {
calls: Arc::clone(&calls),
duration: Duration::from_secs(1),
};
let config = WorkerSettings::default()
.with_poll_interval(Duration::from_millis(1))
.with_shutdown_timeout(Duration::from_millis(5))
.build()
.expect("worker settings should be valid");
let handle = tokio::spawn(run_worker_until_shutdown(worker, config, token));
while calls.load(Ordering::SeqCst) == 0 {
time::sleep(Duration::from_millis(1)).await;
}
trigger.shutdown();
assert!(matches!(
handle.await.unwrap(),
Err(WorkerRunError::ShutdownTimedOut)
));
}
#[tokio::test]
async fn local_worker_lease_store_acquires_renews_and_releases() {
let store = LocalWorkerLeaseStore::new();
let config = lease_config(Duration::from_secs(30), Duration::from_secs(10));
let lease = store
.try_acquire("email", "node-a", config)
.await
.expect("lease store should work")
.expect("lease should be acquired");
assert!(
store
.try_acquire("email", "node-b", config)
.await
.expect("lease store should work")
.is_none()
);
let renewed = store
.renew(&lease, config)
.await
.expect("lease store should work")
.expect("lease should renew");
assert_eq!(renewed.worker(), "email");
store
.release(&renewed)
.await
.expect("lease store should work");
assert!(
store
.try_acquire("email", "node-b", config)
.await
.expect("lease store should work")
.is_some()
);
}
#[tokio::test]
async fn local_worker_lease_store_rejects_invalid_inputs() {
let store = LocalWorkerLeaseStore::new();
assert!(matches!(
WorkerLease::new(
" email ",
"node-a",
1,
Instant::now() + Duration::from_secs(30)
),
Err(WorkerLeaseError::InvalidConfig(
WorkerConfigError::InvalidWorkerName { .. }
))
));
assert!(matches!(
store
.try_acquire(
"",
"node-a",
lease_config(Duration::from_secs(30), Duration::from_secs(10)),
)
.await,
Err(WorkerLeaseError::InvalidConfig(
WorkerConfigError::InvalidWorkerName { .. }
))
));
assert!(matches!(
store
.try_acquire(
"email",
"",
lease_config(Duration::from_secs(30), Duration::from_secs(10)),
)
.await,
Err(WorkerLeaseError::InvalidConfig(
WorkerConfigError::InvalidLeaseOwner { .. }
))
));
assert!(matches!(
WorkerLeaseConfig::new(Duration::ZERO, Duration::from_secs(10)),
Err(WorkerConfigError::InvalidLeaseDuration)
));
assert!(matches!(
WorkerLeaseConfig::new(Duration::MAX, Duration::from_secs(1)),
Err(WorkerConfigError::InvalidLeaseDuration)
));
}
#[tokio::test]
async fn worker_lease_renewal_uses_injected_runtime_clock() {
let clock = ManualClock::new();
let store = LocalWorkerLeaseStore::with_clock(clock.shared());
let config = lease_config(Duration::from_secs(30), Duration::from_secs(10));
let lease = store
.try_acquire("email", "node-a", config)
.await
.expect("lease store should work")
.expect("lease should be acquired");
let unchanged = ensure_worker_lease(
&store,
Some(lease.clone()),
"email",
"node-a",
config,
clock.now(),
None,
)
.await
.expect("lease check should work")
.expect("lease should still be held");
assert_eq!(unchanged.expires_at(), lease.expires_at());
clock.advance(Duration::from_secs(25));
let renewed = ensure_worker_lease(
&store,
Some(unchanged),
"email",
"node-a",
config,
clock.now(),
None,
)
.await
.expect("lease check should work")
.expect("lease should renew");
assert!(renewed.expires_at() > lease.expires_at());
}
#[tokio::test]
async fn leased_worker_requires_store() {
let (_, token) = shutdown_channel();
let result = run_worker_until_shutdown(
CountingWorker {
calls: Arc::new(AtomicUsize::new(0)),
},
worker_config(
WorkerConfig::default().poll_interval(),
Some(lease_config(
Duration::from_secs(30),
Duration::from_secs(10),
)),
),
token,
)
.await;
assert!(matches!(result, Err(WorkerRunError::MissingLeaseStore)));
}
#[tokio::test]
async fn worker_loop_rejects_invalid_worker_name_and_lease_owner() {
let (_, token) = shutdown_channel();
let result = run_worker_until_shutdown(NamedWorker(""), WorkerConfig::default(), token).await;
assert!(matches!(
result,
Err(WorkerRunError::InvalidConfig(
WorkerConfigError::InvalidWorkerName { .. }
))
));
let store = Arc::new(LocalWorkerLeaseStore::new());
let (_, token) = shutdown_channel();
let result = run_worker_until_shutdown_with_options(
NamedWorker("email"),
worker_config(
WorkerConfig::default().poll_interval(),
Some(lease_config(
Duration::from_secs(30),
Duration::from_secs(10),
)),
),
token,
WorkerRunOptions::new()
.with_lease_store("", Arc::clone(&store) as Arc<dyn WorkerLeaseStore>),
)
.await;
assert!(matches!(
result,
Err(WorkerRunError::InvalidConfig(
WorkerConfigError::InvalidLeaseOwner { .. }
))
));
}
#[tokio::test]
async fn leased_worker_runs_and_releases_on_shutdown() {
let store = Arc::new(LocalWorkerLeaseStore::new());
let (trigger, token) = shutdown_channel();
let calls = Arc::new(AtomicUsize::new(0));
let worker = CountingWorker {
calls: Arc::clone(&calls),
};
let config = lease_config(Duration::from_secs(30), Duration::from_secs(10));
let options = WorkerRunOptions::new()
.with_lease_store("node-a", Arc::clone(&store) as Arc<dyn WorkerLeaseStore>);
let handle = tokio::spawn(run_worker_until_shutdown_with_options(
worker,
worker_config(Duration::from_millis(1), Some(config)),
token,
options,
));
time::sleep(Duration::from_millis(5)).await;
trigger.shutdown();
let summary = handle.await.unwrap().unwrap();
assert!(summary.successes() > 0);
assert!(calls.load(Ordering::SeqCst) > 0);
assert!(
store
.try_acquire("counting", "node-b", config)
.await
.expect("lease store should work")
.is_some()
);
}
#[tokio::test]
async fn leased_worker_renews_while_tick_is_running() {
let store = Arc::new(LocalWorkerLeaseStore::new());
let (trigger, token) = shutdown_channel();
let calls = Arc::new(AtomicUsize::new(0));
let worker = SlowWorker {
calls: Arc::clone(&calls),
duration: Duration::from_millis(500),
};
let lease_config = lease_config(Duration::from_millis(200), Duration::from_millis(150));
let options = WorkerRunOptions::new()
.with_lease_store("node-a", Arc::clone(&store) as Arc<dyn WorkerLeaseStore>);
let handle = tokio::spawn(run_worker_until_shutdown_with_options(
worker,
worker_config(Duration::from_secs(1), Some(lease_config)),
token,
options,
));
time::sleep(Duration::from_millis(350)).await;
assert_eq!(calls.load(Ordering::SeqCst), 1);
assert!(
store
.try_acquire("slow", "node-b", lease_config)
.await
.expect("lease store should work")
.is_none(),
"another owner must not acquire the lease during a long-running tick"
);
trigger.shutdown();
let summary = handle.await.unwrap().unwrap();
assert_eq!(summary.successes(), 1);
}