use std::{
collections::VecDeque,
sync::{
Arc, Mutex,
atomic::{AtomicUsize, Ordering},
},
time::{Duration, Instant},
};
use async_trait::async_trait;
use tokio::time;
use crate::{
clock::{Clock, ManualClock},
result::{ClassifyError, ErrorKind},
};
use super::*;
#[test]
fn outbox_backoff_handles_extreme_attempt_counts() {
for (initial, maximum, multiplier, expected) in [
(
Duration::from_secs(1),
Duration::from_secs(60),
2,
Duration::from_secs(60),
),
(Duration::from_nanos(1), Duration::MAX, 2, Duration::MAX),
(
Duration::from_secs(3),
Duration::from_secs(60),
1,
Duration::from_secs(3),
),
(
Duration::from_secs(3),
Duration::from_secs(3),
2,
Duration::from_secs(3),
),
] {
let config = BackoffConfig::new(initial, maximum, multiplier).unwrap();
let policy = OutboxRetryPolicy::new(u32::MAX, config).unwrap();
assert_eq!(policy.delay_for_attempts(u32::MAX), expected);
let mut backoff = RetryBackoff::new(config);
assert_eq!(policy.delay_for_attempts(0), initial);
for attempt in 1..=100 {
assert_eq!(policy.delay_for_attempts(attempt), backoff.record_failure());
}
}
}
fn lease_config(lease_duration: Duration, renew_before: Duration) -> WorkerLeaseConfig {
WorkerLeaseConfig::new(lease_duration, renew_before, renew_before / 2)
.expect("test lease configuration should be valid")
}
fn worker_config(poll_interval: Duration, lease: Option<WorkerLeaseConfig>) -> WorkerConfig {
WorkerSettings::default()
.with_poll_interval(poll_interval)
.with_lease(lease.map(WorkerLeaseSettings::from))
.build()
.expect("test worker settings should be valid")
}
fn outbox_config(
update: impl FnOnce(OutboxWorkerSettings) -> OutboxWorkerSettings,
) -> OutboxWorkerConfig {
update(OutboxWorkerSettings::default())
.build()
.expect("test outbox settings should be valid")
}
struct CountingWorker {
calls: Arc<AtomicUsize>,
}
struct NamedWorker(&'static str);
struct SlowWorker {
calls: Arc<AtomicUsize>,
duration: Duration,
}
#[derive(Debug, Clone, Default)]
struct HangingAcquireLeaseStore {
acquire_calls: Arc<AtomicUsize>,
}
#[derive(Debug, Clone, Default)]
struct HangingRenewLeaseStore {
renew_calls: Arc<AtomicUsize>,
}
struct TestBatchJob;
#[derive(Debug, Clone)]
struct TestOutboxItem {
id: String,
attempts: u32,
action: TestOutboxAction,
}
#[derive(Debug, Clone, Copy)]
enum TestOutboxAction {
Succeed,
Skip,
Retry,
HandlerError,
Slow,
}
#[derive(Debug, Clone, Default)]
struct TestOutboxStore {
items: Arc<Mutex<VecDeque<TestOutboxItem>>>,
states: Arc<Mutex<Vec<(String, OutboxItemState)>>>,
dead_letters: Arc<Mutex<Vec<String>>>,
transition_action: Arc<Mutex<TestPersistenceAction>>,
}
#[derive(Debug, Clone, Copy)]
struct TestOutboxHandler;
#[derive(Debug, Clone, Copy, Default)]
enum TestPersistenceAction {
#[default]
Succeed,
Fail,
Slow(Duration),
}
impl TestOutboxStore {
fn with_transition_action(self, action: TestPersistenceAction) -> Self {
*self.transition_action.lock().unwrap() = action;
self
}
}
async fn apply_persistence_action(action: TestPersistenceAction) -> Result<(), BoxWorkerError> {
match action {
TestPersistenceAction::Succeed => Ok(()),
TestPersistenceAction::Fail => {
Err(std::io::Error::other("private persistence failure").into())
}
TestPersistenceAction::Slow(duration) => {
time::sleep(duration).await;
Ok(())
}
}
}
#[async_trait]
impl Worker for CountingWorker {
fn name(&self) -> &str {
"counting"
}
async fn run_once(&self) -> Result<WorkerTick, BoxWorkerError> {
self.calls.fetch_add(1, Ordering::SeqCst);
Ok(WorkerTick::new(1).with_next_delay(Duration::from_millis(1)))
}
}
#[async_trait]
impl Worker for NamedWorker {
fn name(&self) -> &str {
self.0
}
async fn run_once(&self) -> Result<WorkerTick, BoxWorkerError> {
Ok(WorkerTick::new(1).with_next_delay(Duration::from_millis(1)))
}
}
#[async_trait]
impl Worker for SlowWorker {
fn name(&self) -> &str {
"slow"
}
async fn run_once(&self) -> Result<WorkerTick, BoxWorkerError> {
self.calls.fetch_add(1, Ordering::SeqCst);
time::sleep(self.duration).await;
Ok(WorkerTick::new(1).with_next_delay(Duration::from_secs(1)))
}
}
#[async_trait]
impl WorkerLeaseStore for HangingAcquireLeaseStore {
async fn try_acquire(
&self,
_worker: &str,
_owner: &str,
_config: WorkerLeaseConfig,
) -> Result<Option<WorkerLease>, WorkerLeaseError> {
self.acquire_calls.fetch_add(1, Ordering::SeqCst);
std::future::pending().await
}
async fn renew(
&self,
_lease: &WorkerLease,
_config: WorkerLeaseConfig,
) -> Result<Option<WorkerLease>, WorkerLeaseError> {
std::future::pending().await
}
async fn release(&self, _lease: &WorkerLease) -> Result<(), WorkerLeaseError> {
Ok(())
}
}
#[async_trait]
impl WorkerLeaseStore for HangingRenewLeaseStore {
async fn try_acquire(
&self,
worker: &str,
owner: &str,
config: WorkerLeaseConfig,
) -> Result<Option<WorkerLease>, WorkerLeaseError> {
let expires_at = Instant::now()
.checked_add(config.lease_duration())
.expect("test lease duration should fit");
Ok(Some(WorkerLease::new(worker, owner, 1, expires_at)?))
}
async fn renew(
&self,
_lease: &WorkerLease,
_config: WorkerLeaseConfig,
) -> Result<Option<WorkerLease>, WorkerLeaseError> {
self.renew_calls.fetch_add(1, Ordering::SeqCst);
std::future::pending().await
}
async fn release(&self, _lease: &WorkerLease) -> Result<(), WorkerLeaseError> {
Ok(())
}
}
#[async_trait]
impl BatchWorkerJob for TestBatchJob {
fn name(&self) -> &str {
"test batch"
}
async fn run_batch(&self) -> Result<BatchRunSummary, BoxWorkerError> {
Ok(BatchRunSummary::new()
.with_claimed(5)
.with_succeeded(2)
.with_retrying(1)
.with_failed(1)
.with_skipped(1)
.with_next_delay(Duration::from_millis(5)))
}
}
impl OutboxItem for TestOutboxItem {
fn outbox_id(&self) -> String {
self.id.clone()
}
fn outbox_attempts(&self) -> u32 {
self.attempts
}
}
#[async_trait]
impl OutboxStore<TestOutboxItem> for TestOutboxStore {
async fn claim_due(&self, claim: OutboxClaim) -> Result<Vec<TestOutboxItem>, BoxWorkerError> {
let mut items = self.items.lock().unwrap();
Ok((0..claim.limit())
.filter_map(|_| items.pop_front())
.collect())
}
async fn persist_transition(
&self,
item: &TestOutboxItem,
state: OutboxItemState,
) -> Result<(), BoxWorkerError> {
let action = *self.transition_action.lock().unwrap();
apply_persistence_action(action).await?;
let mut states = self.states.lock().unwrap();
let mut dead_letters = self.dead_letters.lock().unwrap();
if let OutboxItemState::Failed { reason } = &state {
dead_letters.push(format!("{}:{reason}", item.id));
}
states.push((item.id.clone(), state));
Ok(())
}
}
#[async_trait]
impl OutboxHandler<TestOutboxItem> for TestOutboxHandler {
async fn handle(&self, item: &TestOutboxItem) -> Result<OutboxItemState, BoxWorkerError> {
match item.action {
TestOutboxAction::Succeed => Ok(OutboxItemState::Succeeded),
TestOutboxAction::Skip => Ok(OutboxItemState::skipped("not applicable")),
TestOutboxAction::Retry => {
Ok(OutboxItemState::retry("temporary", Duration::from_secs(3)))
}
TestOutboxAction::HandlerError => Err("delivery failed".into()),
TestOutboxAction::Slow => {
time::sleep(Duration::from_millis(50)).await;
Ok(OutboxItemState::Succeeded)
}
}
}
}
#[test]
fn outbox_reasons_truncate_on_unicode_boundaries() {
for (input, expected) in [
("ä¸".repeat(400), "ä¸".repeat(341)),
("🙂".repeat(400), "🙂".repeat(256)),
(format!("{}🙂", "a".repeat(1023)), "a".repeat(1023)),
] {
assert!(
matches!(OutboxItemState::failed(input), OutboxItemState::Failed { reason } if reason == expected)
);
}
}
#[test]
fn outbox_serde_defaults_match_runtime_and_partial_settings() {
let defaults = OutboxWorkerSettings::default();
for value in [
serde_json::json!({}),
serde_json::json!({"retry_backoff": {}}),
] {
let settings: OutboxWorkerSettings = serde_json::from_value(value).unwrap();
assert_eq!(settings, defaults);
assert_eq!(settings.build().unwrap(), defaults.build().unwrap());
}
let settings: OutboxWorkerSettings = serde_json::from_value(serde_json::json!({
"retry_backoff": {"initial_delay": "45s", "multiplier": 3}
}))
.unwrap();
assert_eq!(
settings.retry_backoff.initial_delay,
Duration::from_secs(45)
);
assert_eq!(settings.retry_backoff.max_delay, Duration::from_secs(1800));
assert_eq!(settings.retry_backoff.multiplier, 3);
let wire = serde_json::to_value(settings).unwrap();
assert_eq!(
serde_json::from_value::<OutboxWorkerSettings>(wire).unwrap(),
settings
);
assert!(
serde_json::from_value::<OutboxWorkerSettings>(serde_json::json!({
"retry_backoff": {"unknown": 1}
}))
.is_err()
);
}
#[test]
fn backoff_grows_until_cap() {
let config = BackoffConfig::new(Duration::from_secs(1), Duration::from_secs(3), 2)
.expect("test backoff should be valid");
let mut backoff = RetryBackoff::new(config);
assert_eq!(backoff.record_failure(), Duration::from_secs(1));
assert_eq!(backoff.record_failure(), Duration::from_secs(2));
assert_eq!(backoff.record_failure(), Duration::from_secs(3));
backoff.reset();
assert_eq!(backoff.record_failure(), Duration::from_secs(1));
}
#[cfg(feature = "settings")]
#[test]
fn worker_settings_adapt_to_runtime_config() {
let settings = WorkerSettings {
poll_interval: Duration::from_secs(5),
failure_backoff: BackoffSettings {
initial_delay: Duration::from_millis(200),
max_delay: Duration::from_secs(2),
multiplier: 3,
},
lease: Some(WorkerLeaseSettings {
lease_duration: Duration::from_secs(30),
renew_before: Duration::from_secs(10),
operation_timeout: Duration::from_secs(5),
}),
..WorkerSettings::default()
};
settings.validate().expect("settings should validate");
let config = settings.build().expect("settings should build");
assert!(config.enabled());
assert_eq!(config.poll_interval(), Duration::from_secs(5));
assert_eq!(config.failure_backoff().multiplier(), 3);
assert_eq!(
config
.lease()
.expect("lease should be configured")
.renew_before(),
Duration::from_secs(10)
);
}
#[cfg(feature = "settings")]
#[test]
fn worker_settings_report_config_paths() {
let settings = WorkerSettings {
poll_interval: Duration::ZERO,
failure_backoff: BackoffSettings {
initial_delay: Duration::ZERO,
..BackoffSettings::default()
},
lease: Some(WorkerLeaseSettings::new(
Duration::MAX,
Duration::from_secs(1),
Duration::from_millis(500),
)),
..WorkerSettings::default()
};
let error = settings
.validate_at("email.worker")
.expect_err("invalid worker settings should report paths");
let paths = error
.issues()
.iter()
.map(|issue| issue.path())
.collect::<Vec<_>>();
assert!(paths.contains(&"email.worker.poll_interval"));
assert!(paths.contains(&"email.worker.failure_backoff.initial_delay"));
assert!(paths.contains(&"email.worker.lease.lease_duration"));
}
#[cfg(feature = "settings")]
#[test]
fn worker_lease_settings_require_operation_timeout_below_renewal_margin() {
let settings = WorkerLeaseSettings::new(
Duration::from_secs(30),
Duration::from_secs(10),
Duration::from_secs(10),
);
let error = settings
.validate_at("email.worker.lease")
.expect_err("lease operation must retain expiry headroom");
assert!(
error
.issues()
.iter()
.any(|issue| issue.path() == "email.worker.lease.operation_timeout")
);
}
#[tokio::test]
async fn batch_worker_maps_outbox_summary_to_tick() {
let worker = BatchWorker::new(TestBatchJob);
let tick = worker.run_once().await.expect("batch should run");
assert_eq!(tick.processed(), 5);
assert_eq!(tick.next_delay(), Some(Duration::from_millis(5)));
}
#[test]
fn outbox_state_debug_redacts_handler_reasons() {
let reason = "private-handler-error-detail";
let states = [
OutboxItemState::skipped(reason),
OutboxItemState::retry(reason, Duration::from_secs(1)),
OutboxItemState::failed(reason),
];
let output = format!("{states:?}");
assert!(output.contains("[REDACTED]"));
assert!(!output.contains(reason));
}
#[tokio::test]
async fn outbox_explicit_retries_obey_attempt_limits_without_changing_other_outcomes() {
for (max_attempts, attempts, action, expected) in [
(
5,
3,
TestOutboxAction::Retry,
OutboxItemState::retry("temporary", Duration::from_secs(3)),
),
(
5,
4,
TestOutboxAction::Retry,
OutboxItemState::failed("temporary"),
),
(
5,
u32::MAX,
TestOutboxAction::Retry,
OutboxItemState::failed("temporary"),
),
(
1,
0,
TestOutboxAction::Retry,
OutboxItemState::failed("temporary"),
),
(5, 4, TestOutboxAction::Succeed, OutboxItemState::Succeeded),
(
5,
4,
TestOutboxAction::Skip,
OutboxItemState::skipped("not applicable"),
),
] {
let store = TestOutboxStore::default();
store.items.lock().unwrap().push_back(TestOutboxItem {
id: "one".to_owned(),
attempts,
action,
});
let job = OutboxJob::new(
"retry limit",
store.clone(),
TestOutboxHandler,
outbox_config(|settings| settings.with_max_attempts(max_attempts)),
);
let summary = job.run_batch().await.unwrap();
let failed = u64::from(matches!(&expected, OutboxItemState::Failed { .. }));
assert_eq!(summary.dead_lettered(), failed);
assert_eq!(
summary.retrying(),
u64::from(matches!(&expected, OutboxItemState::Retry { .. }))
);
assert_eq!(
*store.states.lock().unwrap(),
vec![("one".to_owned(), expected)]
);
assert_eq!(store.dead_letters.lock().unwrap().len() as u64, failed);
}
}
#[tokio::test]
async fn outbox_job_claims_handles_and_records_states() {
let store = TestOutboxStore::default();
store.items.lock().unwrap().extend([
TestOutboxItem {
id: "one".to_owned(),
attempts: 0,
action: TestOutboxAction::Succeed,
},
TestOutboxItem {
id: "two".to_owned(),
attempts: 0,
action: TestOutboxAction::Skip,
},
TestOutboxItem {
id: "three".to_owned(),
attempts: 0,
action: TestOutboxAction::Retry,
},
TestOutboxItem {
id: "four".to_owned(),
attempts: 4,
action: TestOutboxAction::HandlerError,
},
]);
let job = OutboxJob::new(
"test outbox",
store.clone(),
TestOutboxHandler,
outbox_config(|settings| settings.with_batch_size(10)),
);
let summary = job.run_batch().await.expect("outbox should run");
assert_eq!(summary.claimed(), 4);
assert_eq!(summary.succeeded(), 1);
assert_eq!(summary.skipped(), 1);
assert_eq!(summary.retrying(), 1);
assert_eq!(summary.failed(), 1);
assert_eq!(summary.dead_lettered(), 1);
assert_eq!(summary.next_delay(), Some(Duration::from_secs(3)));
let states = store.states.lock().unwrap();
assert_eq!(states.len(), 4);
assert!(matches!(states[0].1, OutboxItemState::Succeeded));
assert!(matches!(states[1].1, OutboxItemState::Skipped { .. }));
assert!(matches!(states[2].1, OutboxItemState::Retry { .. }));
assert!(matches!(
&states[3].1,
OutboxItemState::Failed { reason } if reason == "outbox_handler_failed"
));
assert_eq!(store.dead_letters.lock().unwrap().len(), 1);
}
#[tokio::test]
async fn outbox_job_applies_item_timeout_with_stable_redacted_reason() {
let store = TestOutboxStore::default();
store.items.lock().unwrap().extend([
TestOutboxItem {
id: "slow".to_owned(),
attempts: 4,
action: TestOutboxAction::Slow,
},
TestOutboxItem {
id: "never-started".to_owned(),
attempts: 0,
action: TestOutboxAction::Succeed,
},
]);
let job = OutboxJob::new(
"test outbox",
store.clone(),
TestOutboxHandler,
outbox_config(|settings| {
settings
.with_batch_size(10)
.with_item_timeout(Duration::from_millis(5))
.with_batch_timeout(Duration::from_millis(100))
}),
);
let summary = job.run_batch().await.expect("outbox should run");
assert_eq!(summary.failed(), 1);
assert_eq!(summary.succeeded(), 1);
assert!(!summary.batch_timed_out());
let dead_letters = store.dead_letters.lock().unwrap();
assert_eq!(dead_letters.len(), 1);
assert!(dead_letters[0].ends_with("outbox_item_timed_out"));
}
#[test]
fn outbox_defaults_are_bounded_inside_the_claim_lease() {
let config = OutboxWorkerConfig::default();
assert!(!config.item_timeout().is_zero());
assert!(config.item_timeout() < config.batch_timeout());
assert!(config.batch_timeout() < config.lease_duration());
}
#[test]
fn outbox_runtime_config_rejects_deadlines_without_lease_headroom() {
assert!(matches!(
OutboxWorkerConfig::new(
1,
Duration::from_secs(30),
OutboxRetryPolicy::default(),
Duration::from_secs(5),
Duration::from_secs(30),
),
Err(WorkerConfigError::InvalidOutboxBatchTimeout)
));
}
#[tokio::test]
async fn outbox_batch_budget_bounds_state_persistence_and_fails_the_tick() {
let store = TestOutboxStore::default()
.with_transition_action(TestPersistenceAction::Slow(Duration::from_millis(50)));
store.items.lock().unwrap().push_back(TestOutboxItem {
id: "one".to_owned(),
attempts: 0,
action: TestOutboxAction::Succeed,
});
let worker = BatchWorker::new(OutboxJob::new(
"test outbox",
store.clone(),
TestOutboxHandler,
outbox_config(|settings| {
settings
.with_batch_size(1)
.with_item_timeout(Duration::from_millis(1))
.with_batch_timeout(Duration::from_millis(5))
}),
));
let error = time::timeout(Duration::from_millis(100), worker.run_once())
.await
.expect("batch deadline should bound persistence")
.expect_err("persistence timeout must fail the worker tick");
let error = error
.downcast_ref::<OutboxBatchError>()
.expect("tick should retain stable outbox classification");
assert_eq!(error.persistence_errors(), 1);
assert!(error.timed_out());
assert_eq!(
error.error_code(),
"outbox_transition_persistence_timed_out"
);
}
#[tokio::test]
async fn outbox_transition_persistence_failure_fails_tick_without_source_leakage() {
let store = TestOutboxStore::default().with_transition_action(TestPersistenceAction::Fail);
store.items.lock().unwrap().push_back(TestOutboxItem {
id: "private-item-id".to_owned(),
attempts: 0,
action: TestOutboxAction::Succeed,
});
let worker = BatchWorker::new(OutboxJob::new(
"test outbox",
store,
TestOutboxHandler,
outbox_config(|settings| settings.with_batch_size(1)),
));
let error = worker
.run_once()
.await
.expect_err("state persistence failure must fail the tick");
let output = format!("{error:?}");
let classified = error
.downcast_ref::<OutboxBatchError>()
.expect("tick should expose a stable outbox error");
assert_eq!(classified.persistence_errors(), 1);
assert_eq!(classified.error_kind(), ErrorKind::Dependency);
assert!(!output.contains("private persistence failure"));
assert!(!output.contains("private-item-id"));
}
#[tokio::test]
async fn outbox_terminal_transition_is_atomic_with_dead_letter() {
let store = TestOutboxStore::default().with_transition_action(TestPersistenceAction::Fail);
store.items.lock().unwrap().push_back(TestOutboxItem {
id: "one".to_owned(),
attempts: 4,
action: TestOutboxAction::HandlerError,
});
let worker = BatchWorker::new(OutboxJob::new(
"test outbox",
store.clone(),
TestOutboxHandler,
outbox_config(|settings| settings.with_batch_size(1)),
));
let error = worker
.run_once()
.await
.expect_err("terminal transition persistence failure must fail the tick");
let error = error
.downcast_ref::<OutboxBatchError>()
.expect("tick should expose a stable outbox error");
assert_eq!(error.persistence_errors(), 1);
assert_eq!(error.error_code(), "outbox_transition_persistence_failed");
assert!(store.states.lock().unwrap().is_empty());
assert!(store.dead_letters.lock().unwrap().is_empty());
}
#[cfg(feature = "settings")]
#[test]
fn outbox_worker_settings_require_item_timeout_below_batch_timeout() {
let settings = OutboxWorkerSettings::default()
.with_item_timeout(Duration::from_secs(5))
.with_batch_timeout(Duration::from_secs(5));
let error = settings
.validate_at("email.outbox")
.expect_err("equal timeouts leave no persistence budget");
assert!(
error
.issues()
.iter()
.any(|issue| issue.path() == "email.outbox.item_timeout")
);
}
#[cfg(feature = "settings")]
#[test]
fn outbox_worker_settings_require_batch_timeout_below_lease_duration() {
let settings = OutboxWorkerSettings::default()
.with_lease_duration(Duration::from_secs(30))
.with_item_timeout(Duration::from_secs(5))
.with_batch_timeout(Duration::from_secs(30));
let error = settings
.validate_at("email.outbox")
.expect_err("batch deadline must retain claim-lease headroom");
assert!(
error
.issues()
.iter()
.any(|issue| issue.path() == "email.outbox.batch_timeout")
);
}
#[cfg(feature = "settings")]
#[test]
fn outbox_worker_settings_adapt_to_runtime_config() {
let settings = OutboxWorkerSettings::default()
.with_batch_size(25)
.with_lease_duration(Duration::from_secs(120))
.with_max_attempts(7)
.with_retry_backoff(BackoffSettings::new(
Duration::from_secs(2),
Duration::from_secs(30),
2,
))
.with_item_timeout(Duration::from_secs(10))
.with_batch_timeout(Duration::from_secs(60));
settings.validate().expect("settings should validate");
let config = settings.build().expect("settings should build");
assert_eq!(config.batch_size(), 25);
assert_eq!(config.lease_duration(), Duration::from_secs(120));
assert_eq!(config.retry_policy().max_attempts(), 7);
assert_eq!(
config.retry_policy().delay_for_attempts(3),
Duration::from_secs(8)
);
assert_eq!(config.item_timeout(), Duration::from_secs(10));
assert_eq!(config.batch_timeout(), Duration::from_secs(60));
}
#[test]
fn worker_errors_are_classified() {
assert_eq!(
WorkerConfigError::InvalidOutboxItemTimeout.error_kind(),
ErrorKind::Validation
);
assert_eq!(
WorkerConfigError::InvalidOutboxBatchTimeout.error_code(),
"outbox_invalid_batch_timeout"
);
assert_eq!(
WorkerRunError::MissingLeaseStore.error_kind(),
ErrorKind::Validation
);
assert_eq!(
WorkerRunError::LeaseLost.error_kind(),
ErrorKind::Unavailable
);
assert_eq!(
WorkerRunError::LeaseOperationTimedOut.error_kind(),
ErrorKind::Timeout
);
assert_eq!(
WorkerLeaseError::LockPoisoned.error_kind(),
ErrorKind::Unavailable
);
}
#[tokio::test]
async fn worker_loop_stops_on_shutdown() {
let (trigger, token) = shutdown_channel();
let calls = Arc::new(AtomicUsize::new(0));
let worker = CountingWorker {
calls: Arc::clone(&calls),
};
let handle = tokio::spawn(run_worker_until_shutdown(
worker,
worker_config(Duration::from_millis(1), None),
token,
));
time::sleep(Duration::from_millis(5)).await;
trigger.shutdown();
let summary = handle.await.unwrap().unwrap();
assert!(summary.successes() > 0);
assert_eq!(summary.failures(), 0);
assert_eq!(summary.last_outcome(), Some(WorkerTickOutcome::Success));
assert_eq!(summary.state(), WorkerRunState::Healthy);
assert!(calls.load(Ordering::SeqCst) > 0);
}
#[tokio::test]
async fn worker_shutdown_enforces_the_configured_deadline() {
let (trigger, token) = shutdown_channel();
let calls = Arc::new(AtomicUsize::new(0));
let worker = SlowWorker {
calls: Arc::clone(&calls),
duration: Duration::from_secs(1),
};
let config = WorkerSettings::default()
.with_poll_interval(Duration::from_millis(1))
.with_shutdown_timeout(Duration::from_millis(5))
.build()
.expect("worker settings should be valid");
let handle = tokio::spawn(run_worker_until_shutdown(worker, config, token));
while calls.load(Ordering::SeqCst) == 0 {
time::sleep(Duration::from_millis(1)).await;
}
trigger.shutdown();
assert!(matches!(
handle.await.unwrap(),
Err(WorkerRunError::ShutdownTimedOut)
));
}
#[tokio::test]
async fn local_worker_lease_store_acquires_renews_and_releases() {
let store = LocalWorkerLeaseStore::new();
let config = lease_config(Duration::from_secs(30), Duration::from_secs(10));
let lease = store
.try_acquire("email", "node-a", config)
.await
.expect("lease store should work")
.expect("lease should be acquired");
assert!(
store
.try_acquire("email", "node-b", config)
.await
.expect("lease store should work")
.is_none()
);
let renewed = store
.renew(&lease, config)
.await
.expect("lease store should work")
.expect("lease should renew");
assert_eq!(renewed.worker(), "email");
assert_eq!(renewed.token(), lease.token());
store
.release(&renewed)
.await
.expect("lease store should work");
assert!(
store
.try_acquire("email", "node-b", config)
.await
.expect("lease store should work")
.is_some()
);
}
#[tokio::test]
async fn local_worker_lease_store_fences_concurrent_same_owner_acquires() {
let store = LocalWorkerLeaseStore::new();
let config = lease_config(Duration::from_secs(30), Duration::from_secs(10));
let (first, second) = tokio::join!(
store.try_acquire("email", "node-a", config),
store.try_acquire("email", "node-a", config),
);
let first = first.expect("first acquire should reach the store");
let second = second.expect("second acquire should reach the store");
assert_ne!(first.is_some(), second.is_some());
let active = first.or(second).expect("exactly one acquire should win");
let stale_same_owner = WorkerLease::new(
active.worker(),
active.owner(),
active.token().saturating_add(1),
active.expires_at(),
)
.expect("synthetic stale lease should be valid");
assert!(
store
.renew(&stale_same_owner, config)
.await
.expect("renew should reach the store")
.is_none(),
"only the exact fencing token may renew"
);
assert!(
store
.renew(&active, config)
.await
.expect("active token should reach the store")
.is_some()
);
}
#[tokio::test]
async fn local_worker_lease_store_rejects_invalid_inputs() {
let store = LocalWorkerLeaseStore::new();
assert!(matches!(
WorkerLease::new(
" email ",
"node-a",
1,
Instant::now() + Duration::from_secs(30)
),
Err(WorkerLeaseError::InvalidConfig(
WorkerConfigError::InvalidWorkerName { .. }
))
));
assert!(matches!(
store
.try_acquire(
"",
"node-a",
lease_config(Duration::from_secs(30), Duration::from_secs(10)),
)
.await,
Err(WorkerLeaseError::InvalidConfig(
WorkerConfigError::InvalidWorkerName { .. }
))
));
assert!(matches!(
store
.try_acquire(
"email",
"",
lease_config(Duration::from_secs(30), Duration::from_secs(10)),
)
.await,
Err(WorkerLeaseError::InvalidConfig(
WorkerConfigError::InvalidLeaseOwner { .. }
))
));
assert!(matches!(
WorkerLeaseConfig::new(
Duration::ZERO,
Duration::from_secs(10),
Duration::from_secs(5)
),
Err(WorkerConfigError::InvalidLeaseDuration)
));
assert!(matches!(
WorkerLeaseConfig::new(
Duration::MAX,
Duration::from_secs(1),
Duration::from_millis(500)
),
Err(WorkerConfigError::InvalidLeaseDuration)
));
assert!(matches!(
WorkerLeaseConfig::new(
Duration::from_secs(30),
Duration::from_secs(10),
Duration::from_secs(10)
),
Err(WorkerConfigError::InvalidLeaseOperationTimeout)
));
}
#[tokio::test]
async fn lease_acquire_is_bounded_by_operation_timeout() {
let store = Arc::new(HangingAcquireLeaseStore::default());
let lease = WorkerLeaseConfig::new(
Duration::from_millis(200),
Duration::from_millis(100),
Duration::from_millis(20),
)
.expect("lease config should be valid");
let (_shutdown, token) = shutdown_channel();
let result = time::timeout(
Duration::from_millis(250),
run_worker_until_shutdown_with_options(
CountingWorker {
calls: Arc::new(AtomicUsize::new(0)),
},
worker_config(Duration::from_millis(1), Some(lease)),
token,
WorkerRunOptions::new()
.with_lease_store("node-a", Arc::clone(&store) as Arc<dyn WorkerLeaseStore>),
),
)
.await
.expect("operation deadline should bound a hanging store");
assert!(matches!(
result,
Err(WorkerRunError::LeaseOperationTimedOut)
));
assert_eq!(store.acquire_calls.load(Ordering::SeqCst), 1);
}
#[tokio::test]
async fn lease_renewal_is_bounded_by_operation_timeout() {
let store = Arc::new(HangingRenewLeaseStore::default());
let lease = WorkerLeaseConfig::new(
Duration::from_millis(100),
Duration::from_millis(80),
Duration::from_millis(20),
)
.expect("lease config should be valid");
let (_shutdown, token) = shutdown_channel();
let result = time::timeout(
Duration::from_millis(250),
run_worker_until_shutdown_with_options(
SlowWorker {
calls: Arc::new(AtomicUsize::new(0)),
duration: Duration::from_secs(1),
},
worker_config(Duration::from_secs(1), Some(lease)),
token,
WorkerRunOptions::new()
.with_lease_store("node-a", Arc::clone(&store) as Arc<dyn WorkerLeaseStore>),
),
)
.await
.expect("operation deadline should bound a hanging renewal");
assert!(matches!(
result,
Err(WorkerRunError::LeaseOperationTimedOut)
));
assert_eq!(store.renew_calls.load(Ordering::SeqCst), 1);
}
#[tokio::test]
async fn shutdown_interrupts_a_hanging_lease_acquire() {
let store = Arc::new(HangingAcquireLeaseStore::default());
let lease = WorkerLeaseConfig::new(
Duration::from_secs(10),
Duration::from_secs(6),
Duration::from_secs(5),
)
.expect("lease config should be valid");
let (trigger, token) = shutdown_channel();
let handle = tokio::spawn(run_worker_until_shutdown_with_options(
CountingWorker {
calls: Arc::new(AtomicUsize::new(0)),
},
worker_config(Duration::from_millis(1), Some(lease)),
token,
WorkerRunOptions::new()
.with_lease_store("node-a", Arc::clone(&store) as Arc<dyn WorkerLeaseStore>),
));
while store.acquire_calls.load(Ordering::SeqCst) == 0 {
time::sleep(Duration::from_millis(1)).await;
}
trigger.shutdown();
let summary = time::timeout(Duration::from_millis(250), handle)
.await
.expect("shutdown should interrupt lease acquisition")
.expect("worker task should join")
.expect("interrupted acquisition is a graceful shutdown");
assert_eq!(summary.ticks(), 0);
}
#[tokio::test]
async fn shutdown_interrupts_a_hanging_lease_renewal() {
let store = Arc::new(HangingRenewLeaseStore::default());
let calls = Arc::new(AtomicUsize::new(0));
let lease = WorkerLeaseConfig::new(
Duration::from_millis(200),
Duration::from_millis(150),
Duration::from_millis(100),
)
.expect("lease config should be valid");
let (trigger, token) = shutdown_channel();
let handle = tokio::spawn(run_worker_until_shutdown_with_options(
SlowWorker {
calls: Arc::clone(&calls),
duration: Duration::from_secs(1),
},
worker_config(Duration::from_secs(1), Some(lease)),
token,
WorkerRunOptions::new()
.with_lease_store("node-a", Arc::clone(&store) as Arc<dyn WorkerLeaseStore>),
));
while store.renew_calls.load(Ordering::SeqCst) == 0 {
time::sleep(Duration::from_millis(1)).await;
}
trigger.shutdown();
let summary = time::timeout(Duration::from_millis(250), handle)
.await
.expect("shutdown should interrupt lease renewal")
.expect("worker task should join")
.expect("interrupted renewal is a graceful shutdown");
assert_eq!(calls.load(Ordering::SeqCst), 1);
assert_eq!(summary.successes(), 0);
}
#[tokio::test]
async fn worker_lease_renewal_uses_injected_runtime_clock() {
let clock = ManualClock::new();
let store = LocalWorkerLeaseStore::with_clock(clock.shared());
let config = lease_config(Duration::from_secs(30), Duration::from_secs(10));
let lease = store
.try_acquire("email", "node-a", config)
.await
.expect("lease store should work")
.expect("lease should be acquired");
let unchanged = ensure_worker_lease(
&store,
Some(&lease),
"email",
"node-a",
config,
clock.now(),
None,
)
.await
.expect("lease check should work")
.expect("lease should still be held");
assert_eq!(unchanged.expires_at(), lease.expires_at());
clock.advance(Duration::from_secs(25));
let renewed = ensure_worker_lease(
&store,
Some(&unchanged),
"email",
"node-a",
config,
clock.now(),
None,
)
.await
.expect("lease check should work")
.expect("lease should renew");
assert!(renewed.expires_at() > lease.expires_at());
}
#[tokio::test]
async fn leased_worker_requires_store() {
let (_, token) = shutdown_channel();
let result = run_worker_until_shutdown(
CountingWorker {
calls: Arc::new(AtomicUsize::new(0)),
},
worker_config(
WorkerConfig::default().poll_interval(),
Some(lease_config(
Duration::from_secs(30),
Duration::from_secs(10),
)),
),
token,
)
.await;
assert!(matches!(result, Err(WorkerRunError::MissingLeaseStore)));
}
#[tokio::test]
async fn worker_loop_rejects_invalid_worker_name_and_lease_owner() {
let (_, token) = shutdown_channel();
let result = run_worker_until_shutdown(NamedWorker(""), WorkerConfig::default(), token).await;
assert!(matches!(
result,
Err(WorkerRunError::InvalidConfig(
WorkerConfigError::InvalidWorkerName { .. }
))
));
let store = Arc::new(LocalWorkerLeaseStore::new());
let (_, token) = shutdown_channel();
let result = run_worker_until_shutdown_with_options(
NamedWorker("email"),
worker_config(
WorkerConfig::default().poll_interval(),
Some(lease_config(
Duration::from_secs(30),
Duration::from_secs(10),
)),
),
token,
WorkerRunOptions::new()
.with_lease_store("", Arc::clone(&store) as Arc<dyn WorkerLeaseStore>),
)
.await;
assert!(matches!(
result,
Err(WorkerRunError::InvalidConfig(
WorkerConfigError::InvalidLeaseOwner { .. }
))
));
}
#[tokio::test]
async fn leased_worker_runs_and_releases_on_shutdown() {
let store = Arc::new(LocalWorkerLeaseStore::new());
let (trigger, token) = shutdown_channel();
let calls = Arc::new(AtomicUsize::new(0));
let worker = CountingWorker {
calls: Arc::clone(&calls),
};
let config = lease_config(Duration::from_secs(30), Duration::from_secs(10));
let options = WorkerRunOptions::new()
.with_lease_store("node-a", Arc::clone(&store) as Arc<dyn WorkerLeaseStore>);
let handle = tokio::spawn(run_worker_until_shutdown_with_options(
worker,
worker_config(Duration::from_millis(1), Some(config)),
token,
options,
));
time::sleep(Duration::from_millis(5)).await;
trigger.shutdown();
let summary = handle.await.unwrap().unwrap();
assert!(summary.successes() > 0);
assert!(calls.load(Ordering::SeqCst) > 0);
assert!(
store
.try_acquire("counting", "node-b", config)
.await
.expect("lease store should work")
.is_some()
);
}
#[tokio::test]
async fn leased_worker_renews_while_tick_is_running() {
let store = Arc::new(LocalWorkerLeaseStore::new());
let (trigger, token) = shutdown_channel();
let calls = Arc::new(AtomicUsize::new(0));
let worker = SlowWorker {
calls: Arc::clone(&calls),
duration: Duration::from_millis(500),
};
let lease_config = lease_config(Duration::from_millis(200), Duration::from_millis(150));
let options = WorkerRunOptions::new()
.with_lease_store("node-a", Arc::clone(&store) as Arc<dyn WorkerLeaseStore>);
let handle = tokio::spawn(run_worker_until_shutdown_with_options(
worker,
worker_config(Duration::from_secs(1), Some(lease_config)),
token,
options,
));
time::sleep(Duration::from_millis(350)).await;
assert_eq!(calls.load(Ordering::SeqCst), 1);
assert!(
store
.try_acquire("slow", "node-b", lease_config)
.await
.expect("lease store should work")
.is_none(),
"another owner must not acquire the lease during a long-running tick"
);
trigger.shutdown();
let summary = handle.await.unwrap().unwrap();
assert_eq!(summary.successes(), 1);
}