use std::{
collections::VecDeque,
sync::{
Arc, Mutex,
atomic::{AtomicUsize, Ordering},
},
time::{Duration, Instant},
};
use async_trait::async_trait;
use tokio::time;
use crate::{
clock::{Clock, ManualClock},
result::{ClassifyError, ErrorKind},
};
use super::*;
#[test]
fn outbox_backoff_handles_extreme_attempt_counts() {
for (initial, maximum, multiplier, expected) in [
(
Duration::from_secs(1),
Duration::from_secs(60),
2,
Duration::from_secs(60),
),
(
Duration::from_nanos(1),
Duration::from_secs(86_400),
2,
Duration::from_secs(86_400),
),
(
Duration::from_secs(3),
Duration::from_secs(60),
1,
Duration::from_secs(3),
),
(
Duration::from_secs(3),
Duration::from_secs(3),
2,
Duration::from_secs(3),
),
] {
let config = BackoffConfig::new(initial, maximum, multiplier).unwrap();
let policy = OutboxRetryPolicy::new(u32::MAX, config).unwrap();
assert_eq!(policy.delay_for_attempts(u32::MAX), expected);
let mut backoff = RetryBackoff::new(config);
assert_eq!(policy.delay_for_attempts(0), initial);
for attempt in 1..=100 {
assert_eq!(policy.delay_for_attempts(attempt), backoff.record_failure());
}
}
}
fn lease_config(lease_duration: Duration, renew_before: Duration) -> WorkerLeaseConfig {
WorkerLeaseConfig::new(lease_duration, renew_before, renew_before / 2)
.expect("test lease configuration should be valid")
}
fn worker_config(poll_interval: Duration, lease: Option<WorkerLeaseConfig>) -> WorkerConfig {
WorkerSettings::default()
.with_poll_interval(poll_interval)
.with_lease(lease.map(WorkerLeaseSettings::from))
.build()
.expect("test worker settings should be valid")
}
fn outbox_config(
update: impl FnOnce(OutboxWorkerSettings) -> OutboxWorkerSettings,
) -> OutboxWorkerConfig {
update(OutboxWorkerSettings::default())
.build()
.expect("test outbox settings should be valid")
}
struct CountingWorker {
calls: Arc<AtomicUsize>,
}
struct NamedWorker(&'static str);
struct SlowWorker {
calls: Arc<AtomicUsize>,
duration: Duration,
}
#[derive(Debug, Clone, Default)]
struct HangingAcquireLeaseStore {
acquire_calls: Arc<AtomicUsize>,
}
#[derive(Debug, Clone, Default)]
struct HangingRenewLeaseStore {
renew_calls: Arc<AtomicUsize>,
}
struct TestBatchJob;
#[derive(Debug, Clone)]
struct TestOutboxItem {
id: String,
attempts: u32,
action: TestOutboxAction,
}
#[derive(Debug, Clone, Copy)]
enum TestOutboxAction {
Succeed,
Skip,
Retry,
HandlerError,
Slow,
}
#[derive(Debug, Clone, Default)]
struct TestOutboxStore {
items: Arc<Mutex<VecDeque<TestOutboxItem>>>,
states: Arc<Mutex<Vec<(String, OutboxItemState)>>>,
dead_letters: Arc<Mutex<Vec<String>>>,
transition_action: Arc<Mutex<TestPersistenceAction>>,
}
#[derive(Debug, Clone, Copy)]
struct TestOutboxHandler;
#[derive(Debug, Clone, Copy, Default)]
enum TestPersistenceAction {
#[default]
Succeed,
Fail,
Slow(Duration),
}
impl TestOutboxStore {
fn with_transition_action(self, action: TestPersistenceAction) -> Self {
*self.transition_action.lock().unwrap() = action;
self
}
}
async fn apply_persistence_action(action: TestPersistenceAction) -> Result<(), BoxWorkerError> {
match action {
TestPersistenceAction::Succeed => Ok(()),
TestPersistenceAction::Fail => {
Err(std::io::Error::other("private persistence failure").into())
}
TestPersistenceAction::Slow(duration) => {
time::sleep(duration).await;
Ok(())
}
}
}
#[async_trait]
impl Worker for CountingWorker {
fn name(&self) -> &str {
"counting"
}
async fn run_once(&self, _context: WorkerContext) -> Result<WorkerTick, BoxWorkerError> {
self.calls.fetch_add(1, Ordering::SeqCst);
Ok(WorkerTick::new(1).with_schedule(WorkerSchedule::After(Duration::from_millis(1))))
}
}
#[async_trait]
impl Worker for NamedWorker {
fn name(&self) -> &str {
self.0
}
async fn run_once(&self, _context: WorkerContext) -> Result<WorkerTick, BoxWorkerError> {
Ok(WorkerTick::new(1).with_schedule(WorkerSchedule::After(Duration::from_millis(1))))
}
}
#[async_trait]
impl Worker for SlowWorker {
fn name(&self) -> &str {
"slow"
}
async fn run_once(&self, _context: WorkerContext) -> Result<WorkerTick, BoxWorkerError> {
self.calls.fetch_add(1, Ordering::SeqCst);
time::sleep(self.duration).await;
Ok(WorkerTick::new(1).with_schedule(WorkerSchedule::After(Duration::from_secs(1))))
}
}
#[async_trait]
impl WorkerLeaseStore for HangingAcquireLeaseStore {
async fn try_acquire(
&self,
_worker: &str,
_owner: &str,
_config: WorkerLeaseConfig,
) -> Result<Option<WorkerLease>, WorkerLeaseError> {
self.acquire_calls.fetch_add(1, Ordering::SeqCst);
std::future::pending().await
}
async fn renew(
&self,
_lease: &WorkerLease,
_config: WorkerLeaseConfig,
) -> Result<Option<WorkerLease>, WorkerLeaseError> {
std::future::pending().await
}
async fn release(&self, _lease: &WorkerLease) -> Result<(), WorkerLeaseError> {
Ok(())
}
}
#[async_trait]
impl WorkerLeaseStore for HangingRenewLeaseStore {
async fn try_acquire(
&self,
worker: &str,
owner: &str,
config: WorkerLeaseConfig,
) -> Result<Option<WorkerLease>, WorkerLeaseError> {
let expires_at = Instant::now()
.checked_add(config.lease_duration())
.expect("test lease duration should fit");
Ok(Some(WorkerLease::new(worker, owner, 1, expires_at)?))
}
async fn renew(
&self,
_lease: &WorkerLease,
_config: WorkerLeaseConfig,
) -> Result<Option<WorkerLease>, WorkerLeaseError> {
self.renew_calls.fetch_add(1, Ordering::SeqCst);
std::future::pending().await
}
async fn release(&self, _lease: &WorkerLease) -> Result<(), WorkerLeaseError> {
Ok(())
}
}
#[async_trait]
impl BatchWorkerJob for TestBatchJob {
fn name(&self) -> &str {
"test batch"
}
async fn run_batch(&self, _context: WorkerContext) -> Result<BatchRunSummary, BoxWorkerError> {
Ok(BatchRunSummary::new()
.with_claimed(5)
.with_succeeded(2)
.with_retrying(1)
.with_failed(1)
.with_skipped(1)
.with_schedule(WorkerSchedule::After(Duration::from_millis(5))))
}
}
impl OutboxItem for TestOutboxItem {
fn outbox_attempts(&self) -> u32 {
self.attempts
}
}
#[async_trait]
impl OutboxStore<TestOutboxItem> for TestOutboxStore {
async fn claim_due(
&self,
claim: OutboxClaim,
_context: WorkerContext,
) -> Result<Vec<TestOutboxItem>, BoxWorkerError> {
let mut items = self.items.lock().unwrap();
Ok((0..claim.limit())
.filter_map(|_| items.pop_front())
.collect())
}
async fn persist_transition(
&self,
item: &TestOutboxItem,
state: OutboxItemState,
_context: WorkerContext,
) -> Result<(), BoxWorkerError> {
let action = *self.transition_action.lock().unwrap();
apply_persistence_action(action).await?;
let mut states = self.states.lock().unwrap();
let mut dead_letters = self.dead_letters.lock().unwrap();
if let OutboxItemState::Failed { reason } = &state {
dead_letters.push(format!("{}:{reason}", item.id));
}
states.push((item.id.clone(), state));
Ok(())
}
}
#[async_trait]
impl OutboxHandler<TestOutboxItem> for TestOutboxHandler {
async fn handle(
&self,
item: &TestOutboxItem,
_context: WorkerContext,
) -> Result<OutboxItemState, BoxWorkerError> {
match item.action {
TestOutboxAction::Succeed => Ok(OutboxItemState::Succeeded),
TestOutboxAction::Skip => Ok(OutboxItemState::skipped("not applicable")),
TestOutboxAction::Retry => {
Ok(OutboxItemState::retry("temporary", Duration::from_secs(3)))
}
TestOutboxAction::HandlerError => Err("delivery failed".into()),
TestOutboxAction::Slow => {
time::sleep(Duration::from_millis(50)).await;
Ok(OutboxItemState::Succeeded)
}
}
}
}
#[test]
fn outbox_reasons_truncate_on_unicode_boundaries() {
for (input, expected) in [
("ä¸".repeat(400), "ä¸".repeat(341)),
("🙂".repeat(400), "🙂".repeat(256)),
(format!("{}🙂", "a".repeat(1023)), "a".repeat(1023)),
] {
assert!(
matches!(OutboxItemState::failed(input), OutboxItemState::Failed { reason } if reason == expected)
);
}
}
#[test]
fn outbox_serde_defaults_match_runtime_and_partial_settings() {
let defaults = OutboxWorkerSettings::default();
for value in [
serde_json::json!({}),
serde_json::json!({"retry_backoff": {}}),
] {
let settings: OutboxWorkerSettings = serde_json::from_value(value).unwrap();
assert_eq!(settings, defaults);
assert_eq!(settings.build().unwrap(), defaults.build().unwrap());
}
let settings: OutboxWorkerSettings = serde_json::from_value(serde_json::json!({
"retry_backoff": {"initial_delay": "45s", "multiplier": 3}
}))
.unwrap();
assert_eq!(
settings.retry_backoff.initial_delay,
Duration::from_secs(45)
);
assert_eq!(settings.retry_backoff.max_delay, Duration::from_secs(1800));
assert_eq!(settings.retry_backoff.multiplier, 3);
let wire = serde_json::to_value(settings).unwrap();
assert_eq!(
serde_json::from_value::<OutboxWorkerSettings>(wire).unwrap(),
settings
);
assert!(
serde_json::from_value::<OutboxWorkerSettings>(serde_json::json!({
"retry_backoff": {"unknown": 1}
}))
.is_err()
);
}
#[test]
fn backoff_grows_until_cap() {
let config = BackoffConfig::new(Duration::from_secs(1), Duration::from_secs(3), 2)
.expect("test backoff should be valid");
let mut backoff = RetryBackoff::new(config);
assert_eq!(backoff.record_failure(), Duration::from_secs(1));
assert_eq!(backoff.record_failure(), Duration::from_secs(2));
assert_eq!(backoff.record_failure(), Duration::from_secs(3));
backoff.reset();
assert_eq!(backoff.record_failure(), Duration::from_secs(1));
}
#[test]
fn worker_failure_jitter_is_opt_in_and_round_trips_through_settings() {
let defaults: WorkerSettings = serde_json::from_str("{}").unwrap();
assert!(!defaults.build().unwrap().failure_backoff_jitter());
let config = WorkerSettings::default()
.with_failure_backoff_jitter(true)
.build()
.unwrap();
let wire = serde_json::to_value(WorkerSettings::from(config)).unwrap();
assert_eq!(wire["failure_backoff_jitter"], true);
assert!(
serde_json::from_value::<WorkerSettings>(wire)
.unwrap()
.build()
.unwrap()
.failure_backoff_jitter()
);
}
#[test]
fn worker_settings_adapt_to_runtime_config() {
let settings = WorkerSettings {
poll_interval: Duration::from_secs(5),
failure_backoff: BackoffSettings {
initial_delay: Duration::from_millis(200),
max_delay: Duration::from_secs(2),
multiplier: 3,
},
lease: Some(WorkerLeaseSettings {
lease_duration: Duration::from_secs(30),
renew_before: Duration::from_secs(10),
operation_timeout: Duration::from_secs(5),
}),
..WorkerSettings::default()
};
settings.validate().expect("settings should validate");
let config = settings.build().expect("settings should build");
assert!(config.enabled());
assert_eq!(config.poll_interval(), Duration::from_secs(5));
assert_eq!(config.failure_backoff().multiplier(), 3);
assert_eq!(
config
.lease()
.expect("lease should be configured")
.renew_before(),
Duration::from_secs(10)
);
}
#[test]
fn worker_settings_report_config_paths() {
let settings = WorkerSettings {
poll_interval: Duration::ZERO,
failure_backoff: BackoffSettings {
initial_delay: Duration::ZERO,
..BackoffSettings::default()
},
lease: Some(WorkerLeaseSettings::new(
Duration::MAX,
Duration::from_secs(1),
Duration::from_millis(500),
)),
..WorkerSettings::default()
};
let error = settings
.validate_at("email.worker")
.expect_err("invalid worker settings should report paths");
let paths = error
.issues()
.iter()
.map(|issue| issue.path())
.collect::<Vec<_>>();
assert!(paths.contains(&"email.worker.poll_interval"));
assert!(paths.contains(&"email.worker.failure_backoff.initial_delay"));
assert!(paths.contains(&"email.worker.lease.lease_duration"));
}
#[test]
fn worker_lease_settings_require_operation_timeout_below_renewal_margin() {
let settings = WorkerLeaseSettings::new(
Duration::from_secs(30),
Duration::from_secs(10),
Duration::from_secs(10),
);
let error = settings
.validate_at("email.worker.lease")
.expect_err("lease operation must retain expiry headroom");
assert!(
error
.issues()
.iter()
.any(|issue| issue.path() == "email.worker.lease.operation_timeout")
);
}
#[tokio::test]
async fn batch_worker_maps_outbox_summary_to_tick() {
let worker = BatchWorker::new(TestBatchJob);
let tick = worker
.run_once(WorkerContext::default())
.await
.expect("batch should run");
assert_eq!(tick.processed(), 5);
assert_eq!(
tick.schedule(),
WorkerSchedule::After(Duration::from_millis(5))
);
}
#[tokio::test(start_paused = true)]
async fn shared_outbox_dependencies_compose_through_the_worker_runtime() {
let store = Arc::new(TestOutboxStore::default());
store.items.lock().unwrap().push_back(TestOutboxItem {
id: "shared".to_owned(),
attempts: 0,
action: TestOutboxAction::Succeed,
});
let shared_store: Arc<dyn OutboxStore<TestOutboxItem>> = store.clone();
let shared_handler: Arc<dyn OutboxHandler<TestOutboxItem>> = Arc::new(TestOutboxHandler);
let job: Arc<dyn BatchWorkerJob> = Arc::new(OutboxJob::new(
"shared",
shared_store,
shared_handler,
OutboxWorkerConfig::default(),
));
let worker: Arc<dyn Worker> = Arc::new(BatchWorker::new(job));
let (trigger, token) = shutdown_channel();
let stop = tokio::spawn(async move {
tokio::task::yield_now().await;
trigger.shutdown();
});
let summary = run_worker_until_shutdown(worker, WorkerConfig::default(), token)
.await
.unwrap();
stop.await.unwrap();
assert_eq!(summary.successes(), 1);
assert_eq!(summary.processed(), 1);
assert_eq!(
*store.states.lock().unwrap(),
vec![("shared".to_owned(), OutboxItemState::Succeeded)]
);
}
#[test]
fn outbox_state_debug_redacts_handler_reasons() {
let reason = "private-handler-error-detail";
let states = [
OutboxItemState::skipped(reason),
OutboxItemState::retry(reason, Duration::from_secs(1)),
OutboxItemState::failed(reason),
];
let output = format!("{states:?}");
assert!(output.contains("[REDACTED]"));
assert!(!output.contains(reason));
}
#[tokio::test]
async fn outbox_explicit_retries_obey_attempt_limits_without_changing_other_outcomes() {
for (max_attempts, attempts, action, expected) in [
(
5,
3,
TestOutboxAction::Retry,
OutboxItemState::retry("temporary", Duration::from_secs(3)),
),
(
5,
4,
TestOutboxAction::Retry,
OutboxItemState::failed("temporary"),
),
(
5,
u32::MAX,
TestOutboxAction::Retry,
OutboxItemState::failed("temporary"),
),
(
1,
0,
TestOutboxAction::Retry,
OutboxItemState::failed("temporary"),
),
(5, 4, TestOutboxAction::Succeed, OutboxItemState::Succeeded),
(
5,
4,
TestOutboxAction::Skip,
OutboxItemState::skipped("not applicable"),
),
] {
let store = TestOutboxStore::default();
store.items.lock().unwrap().push_back(TestOutboxItem {
id: "one".to_owned(),
attempts,
action,
});
let job = OutboxJob::new(
"retry limit",
store.clone(),
TestOutboxHandler,
outbox_config(|settings| settings.with_max_attempts(max_attempts)),
);
let summary = job.run_batch(WorkerContext::default()).await.unwrap();
let failed = u64::from(matches!(&expected, OutboxItemState::Failed { .. }));
assert_eq!(summary.dead_lettered(), failed);
assert_eq!(
summary.retrying(),
u64::from(matches!(&expected, OutboxItemState::Retry { .. }))
);
assert_eq!(
*store.states.lock().unwrap(),
vec![("one".to_owned(), expected)]
);
assert_eq!(store.dead_letters.lock().unwrap().len() as u64, failed);
}
}
#[tokio::test(start_paused = true)]
async fn outbox_job_claims_handles_and_records_states() {
let store = TestOutboxStore::default();
store.items.lock().unwrap().extend([
TestOutboxItem {
id: "one".to_owned(),
attempts: 0,
action: TestOutboxAction::Succeed,
},
TestOutboxItem {
id: "two".to_owned(),
attempts: 0,
action: TestOutboxAction::Skip,
},
TestOutboxItem {
id: "three".to_owned(),
attempts: 0,
action: TestOutboxAction::Retry,
},
TestOutboxItem {
id: "four".to_owned(),
attempts: 4,
action: TestOutboxAction::HandlerError,
},
]);
let job = OutboxJob::new(
"test outbox",
store.clone(),
TestOutboxHandler,
outbox_config(|settings| settings.with_batch_size(10)),
);
let summary = job
.run_batch(WorkerContext::default())
.await
.expect("outbox should run");
assert_eq!(summary.claimed(), 4);
assert_eq!(summary.succeeded(), 1);
assert_eq!(summary.skipped(), 1);
assert_eq!(summary.retrying(), 1);
assert_eq!(summary.failed(), 1);
assert_eq!(summary.dead_lettered(), 1);
assert_eq!(summary.schedule(), WorkerSchedule::Poll);
let states = store.states.lock().unwrap();
assert_eq!(states.len(), 4);
assert!(matches!(states[0].1, OutboxItemState::Succeeded));
assert!(matches!(states[1].1, OutboxItemState::Skipped { .. }));
assert!(matches!(states[2].1, OutboxItemState::Retry { .. }));
assert!(matches!(
&states[3].1,
OutboxItemState::Failed { reason } if reason == "outbox_handler_failed"
));
assert_eq!(store.dead_letters.lock().unwrap().len(), 1);
}
#[tokio::test]
async fn outbox_job_applies_item_timeout_with_stable_redacted_reason() {
let store = TestOutboxStore::default();
store.items.lock().unwrap().extend([
TestOutboxItem {
id: "slow".to_owned(),
attempts: 4,
action: TestOutboxAction::Slow,
},
TestOutboxItem {
id: "never-started".to_owned(),
attempts: 0,
action: TestOutboxAction::Succeed,
},
]);
let job = OutboxJob::new(
"test outbox",
store.clone(),
TestOutboxHandler,
outbox_config(|settings| {
settings
.with_batch_size(10)
.with_item_timeout(Duration::from_millis(5))
.with_batch_timeout(Duration::from_millis(100))
}),
);
let summary = job
.run_batch(WorkerContext::default())
.await
.expect("outbox should run");
assert_eq!(summary.failed(), 1);
assert_eq!(summary.succeeded(), 1);
assert!(!summary.batch_timed_out());
let dead_letters = store.dead_letters.lock().unwrap();
assert_eq!(dead_letters.len(), 1);
assert!(dead_letters[0].ends_with("outbox_item_timed_out"));
}
#[tokio::test(start_paused = true)]
async fn outbox_queue_scheduling_uses_claim_size_not_item_outcomes() {
let store = TestOutboxStore::default();
for (id, action) in [
TestOutboxAction::Skip,
TestOutboxAction::Retry,
TestOutboxAction::HandlerError,
]
.into_iter()
.enumerate()
{
store.items.lock().unwrap().push_back(TestOutboxItem {
id: id.to_string(),
attempts: 4,
action,
});
}
let job = OutboxJob::new(
"queue",
store,
TestOutboxHandler,
outbox_config(|settings| settings.with_batch_size(2)),
)
.with_queue_scheduling();
let full = job.run_batch(WorkerContext::default()).await.unwrap();
assert_eq!(full.claimed(), 2);
assert_eq!(full.succeeded(), 0);
assert_eq!(full.schedule(), WorkerSchedule::Continue);
let short = job.run_batch(WorkerContext::default()).await.unwrap();
assert_eq!(short.claimed(), 1);
assert_eq!(short.failed(), 1);
assert_eq!(
short.schedule(),
WorkerSchedule::WaitForWork {
recheck_after: None
}
);
let empty = job.run_batch(WorkerContext::default()).await.unwrap();
assert!(empty.is_empty());
assert_eq!(
empty.schedule(),
WorkerSchedule::WaitForWork {
recheck_after: None
}
);
}
#[tokio::test(start_paused = true)]
async fn outbox_retry_hint_accounts_for_time_spent_finishing_the_batch() {
for queue_scheduling in [false, true] {
let store = TestOutboxStore::default();
store.items.lock().unwrap().extend([
TestOutboxItem {
id: "retry".to_owned(),
attempts: 0,
action: TestOutboxAction::Retry,
},
TestOutboxItem {
id: "slow".to_owned(),
attempts: 0,
action: TestOutboxAction::Slow,
},
]);
let job = OutboxJob::new(
"queue",
store.clone(),
TestOutboxHandler,
OutboxWorkerConfig::default(),
);
let job = if queue_scheduling {
job.with_queue_scheduling()
} else {
job
};
let summary = job.run_batch(WorkerContext::default()).await.unwrap();
let delay = Duration::from_millis(2950);
assert_eq!(
summary.schedule(),
if queue_scheduling {
WorkerSchedule::WaitForWork {
recheck_after: Some(delay),
}
} else {
WorkerSchedule::Poll
}
);
assert!(
matches!(store.states.lock().unwrap()[0].1, OutboxItemState::Retry { delay, .. } if delay == Duration::from_secs(3))
);
}
}
#[tokio::test(start_paused = true)]
async fn outbox_retry_hint_starts_after_persistence_completes() {
for queue_scheduling in [false, true] {
let store = TestOutboxStore::default()
.with_transition_action(TestPersistenceAction::Slow(Duration::from_secs(1)));
store.items.lock().unwrap().push_back(TestOutboxItem {
id: "retry".to_owned(),
attempts: 0,
action: TestOutboxAction::Retry,
});
let job = OutboxJob::new(
"retry",
store,
TestOutboxHandler,
OutboxWorkerConfig::default(),
);
let job = if queue_scheduling {
job.with_queue_scheduling()
} else {
job
};
let summary = job.run_batch(WorkerContext::default()).await.unwrap();
assert_eq!(
summary.schedule(),
if queue_scheduling {
WorkerSchedule::WaitForWork {
recheck_after: Some(Duration::from_secs(3)),
}
} else {
WorkerSchedule::Poll
}
);
}
}
#[tokio::test(start_paused = true)]
async fn outbox_queue_operational_errors_override_continuation() {
for action in [
TestPersistenceAction::Fail,
TestPersistenceAction::Slow(Duration::from_secs(60)),
] {
let store = TestOutboxStore::default().with_transition_action(action);
store.items.lock().unwrap().push_back(TestOutboxItem {
id: "one".to_owned(),
attempts: 0,
action: TestOutboxAction::Succeed,
});
let job = OutboxJob::new(
"queue",
store,
TestOutboxHandler,
outbox_config(|settings| settings.with_batch_size(1)),
)
.with_queue_scheduling();
let error = BatchWorker::new(job)
.run_once(WorkerContext::default())
.await
.unwrap_err();
assert!(error.downcast_ref::<OutboxBatchError>().is_some());
}
}
#[test]
fn outbox_defaults_are_bounded_inside_the_claim_lease() {
let config = OutboxWorkerConfig::default();
assert!(!config.item_timeout().is_zero());
assert!(config.item_timeout() < config.batch_timeout());
assert!(config.batch_timeout() < config.lease_duration());
}
#[test]
fn outbox_runtime_config_rejects_deadlines_without_lease_headroom() {
let error = OutboxWorkerSettings::default()
.with_batch_size(1)
.with_lease_duration(Duration::from_secs(30))
.with_item_timeout(Duration::from_secs(5))
.with_batch_timeout(Duration::from_secs(30))
.build()
.unwrap_err();
assert_eq!(error.issues()[0].path(), "outbox.batch_timeout");
}
#[tokio::test]
async fn outbox_batch_budget_bounds_state_persistence_and_fails_the_tick() {
let store = TestOutboxStore::default()
.with_transition_action(TestPersistenceAction::Slow(Duration::from_millis(50)));
store.items.lock().unwrap().push_back(TestOutboxItem {
id: "one".to_owned(),
attempts: 0,
action: TestOutboxAction::Succeed,
});
let worker = BatchWorker::new(OutboxJob::new(
"test outbox",
store.clone(),
TestOutboxHandler,
outbox_config(|settings| {
settings
.with_batch_size(1)
.with_item_timeout(Duration::from_millis(1))
.with_batch_timeout(Duration::from_millis(5))
}),
));
let error = time::timeout(
Duration::from_millis(100),
worker.run_once(WorkerContext::default()),
)
.await
.expect("batch deadline should bound persistence")
.expect_err("persistence timeout must fail the worker tick");
let error = error
.downcast_ref::<OutboxBatchError>()
.expect("tick should retain stable outbox classification");
assert_eq!(error.persistence_errors(), 1);
assert!(error.timed_out());
assert_eq!(
error.error_code(),
"outbox_transition_persistence_timed_out"
);
}
#[tokio::test]
async fn outbox_transition_persistence_failure_fails_tick_without_source_leakage() {
let store = TestOutboxStore::default().with_transition_action(TestPersistenceAction::Fail);
store.items.lock().unwrap().push_back(TestOutboxItem {
id: "private-item-id".to_owned(),
attempts: 0,
action: TestOutboxAction::Succeed,
});
let worker = BatchWorker::new(OutboxJob::new(
"test outbox",
store,
TestOutboxHandler,
outbox_config(|settings| settings.with_batch_size(1)),
));
let error = worker
.run_once(WorkerContext::default())
.await
.expect_err("state persistence failure must fail the tick");
let output = format!("{error:?}");
let classified = error
.downcast_ref::<OutboxBatchError>()
.expect("tick should expose a stable outbox error");
assert_eq!(classified.persistence_errors(), 1);
assert_eq!(classified.error_kind(), ErrorKind::Dependency);
assert!(!output.contains("private persistence failure"));
assert!(!output.contains("private-item-id"));
}
#[tokio::test]
async fn outbox_terminal_transition_is_atomic_with_dead_letter() {
let store = TestOutboxStore::default().with_transition_action(TestPersistenceAction::Fail);
store.items.lock().unwrap().push_back(TestOutboxItem {
id: "one".to_owned(),
attempts: 4,
action: TestOutboxAction::HandlerError,
});
let worker = BatchWorker::new(OutboxJob::new(
"test outbox",
store.clone(),
TestOutboxHandler,
outbox_config(|settings| settings.with_batch_size(1)),
));
let error = worker
.run_once(WorkerContext::default())
.await
.expect_err("terminal transition persistence failure must fail the tick");
let error = error
.downcast_ref::<OutboxBatchError>()
.expect("tick should expose a stable outbox error");
assert_eq!(error.persistence_errors(), 1);
assert_eq!(error.error_code(), "outbox_transition_persistence_failed");
assert!(store.states.lock().unwrap().is_empty());
assert!(store.dead_letters.lock().unwrap().is_empty());
}
#[test]
fn outbox_worker_settings_require_item_timeout_below_batch_timeout() {
let settings = OutboxWorkerSettings::default()
.with_item_timeout(Duration::from_secs(5))
.with_batch_timeout(Duration::from_secs(5));
let error = settings
.validate_at("email.outbox")
.expect_err("equal timeouts leave no persistence budget");
assert!(
error
.issues()
.iter()
.any(|issue| issue.path() == "email.outbox.item_timeout")
);
}
#[test]
fn outbox_worker_settings_require_batch_timeout_below_lease_duration() {
let settings = OutboxWorkerSettings::default()
.with_lease_duration(Duration::from_secs(30))
.with_item_timeout(Duration::from_secs(5))
.with_batch_timeout(Duration::from_secs(30));
let error = settings
.validate_at("email.outbox")
.expect_err("batch deadline must retain claim-lease headroom");
assert!(
error
.issues()
.iter()
.any(|issue| issue.path() == "email.outbox.batch_timeout")
);
}
#[test]
fn outbox_worker_settings_adapt_to_runtime_config() {
let settings = OutboxWorkerSettings::default()
.with_batch_size(25)
.with_lease_duration(Duration::from_secs(120))
.with_max_attempts(7)
.with_retry_backoff(BackoffSettings::new(
Duration::from_secs(2),
Duration::from_secs(30),
2,
))
.with_item_timeout(Duration::from_secs(10))
.with_batch_timeout(Duration::from_secs(60));
settings.validate().expect("settings should validate");
let config = settings.build().expect("settings should build");
assert_eq!(config.batch_size(), 25);
assert_eq!(config.lease_duration(), Duration::from_secs(120));
assert_eq!(config.retry_policy().max_attempts(), 7);
assert_eq!(
config.retry_policy().delay_for_attempts(3),
Duration::from_secs(8)
);
assert_eq!(config.item_timeout(), Duration::from_secs(10));
assert_eq!(config.batch_timeout(), Duration::from_secs(60));
}
#[test]
fn worker_errors_are_classified() {
assert_eq!(
WorkerConfigError::InvalidLeaseDuration.error_kind(),
ErrorKind::Validation
);
assert_eq!(
WorkerConfigError::InvalidLeaseDuration.error_code(),
"worker_invalid_lease_duration"
);
assert_eq!(
WorkerRunError::MissingLeaseStore.error_kind(),
ErrorKind::Validation
);
assert_eq!(
WorkerRunError::LeaseLost.error_kind(),
ErrorKind::Unavailable
);
assert_eq!(
WorkerRunError::LeaseOperationTimedOut.error_kind(),
ErrorKind::Timeout
);
assert_eq!(
WorkerLeaseError::LockPoisoned.error_kind(),
ErrorKind::Internal
);
assert!(
!WorkerLeaseError::LockPoisoned
.error_descriptor()
.retryable()
);
let failure = WorkerRunError::from(WorkerLeaseError::LockPoisoned);
assert_eq!(failure.error_kind(), ErrorKind::Internal);
assert!(!failure.error_descriptor().retryable());
assert!(
WorkerLeaseError::Store(std::io::Error::other("unavailable").into())
.error_descriptor()
.retryable()
);
}
#[tokio::test]
async fn worker_loop_stops_on_shutdown() {
let (trigger, token) = shutdown_channel();
let calls = Arc::new(AtomicUsize::new(0));
let worker = CountingWorker {
calls: Arc::clone(&calls),
};
let handle = tokio::spawn(run_worker_until_shutdown(
worker,
worker_config(Duration::from_millis(1), None),
token,
));
time::sleep(Duration::from_millis(5)).await;
trigger.shutdown();
let summary = handle.await.unwrap().unwrap();
assert!(summary.successes() > 0);
assert_eq!(summary.failures(), 0);
assert_eq!(summary.last_outcome(), Some(WorkerTickOutcome::Success));
assert_eq!(summary.state(), WorkerRunState::Healthy);
assert!(calls.load(Ordering::SeqCst) > 0);
}
#[tokio::test]
async fn dropped_shutdown_triggers_prevent_new_ticks() {
let (trigger, token) = shutdown_channel();
let last_trigger = trigger.clone();
drop(trigger);
assert!(!token.is_shutdown_requested());
drop(last_trigger);
assert!(token.is_shutdown_requested());
let calls = Arc::new(AtomicUsize::new(0));
let summary = run_worker_until_shutdown(
CountingWorker {
calls: calls.clone(),
},
worker_config(Duration::from_secs(1), None),
token,
)
.await
.unwrap();
assert_eq!(summary.ticks(), 0);
assert_eq!(calls.load(Ordering::SeqCst), 0);
}
#[tokio::test]
async fn worker_shutdown_enforces_the_configured_deadline() {
let (trigger, token) = shutdown_channel();
let calls = Arc::new(AtomicUsize::new(0));
let worker = SlowWorker {
calls: Arc::clone(&calls),
duration: Duration::from_secs(1),
};
let config = WorkerSettings::default()
.with_poll_interval(Duration::from_millis(1))
.with_shutdown_timeout(Duration::from_millis(5))
.build()
.expect("worker settings should be valid");
let handle = tokio::spawn(run_worker_until_shutdown(worker, config, token));
while calls.load(Ordering::SeqCst) == 0 {
time::sleep(Duration::from_millis(1)).await;
}
trigger.shutdown();
assert!(matches!(
handle.await.unwrap(),
Err(WorkerRunError::ShutdownTimedOut)
));
}
#[tokio::test]
async fn local_worker_lease_store_acquires_renews_and_releases() {
let store = LocalWorkerLeaseStore::new();
let config = lease_config(Duration::from_secs(30), Duration::from_secs(10));
let lease = store
.try_acquire("email", "node-a", config)
.await
.expect("lease store should work")
.expect("lease should be acquired");
assert!(
store
.try_acquire("email", "node-b", config)
.await
.expect("lease store should work")
.is_none()
);
let renewed = store
.renew(&lease, config)
.await
.expect("lease store should work")
.expect("lease should renew");
assert_eq!(renewed.worker(), "email");
assert_eq!(renewed.token(), lease.token());
store
.release(&renewed)
.await
.expect("lease store should work");
let replacement = store
.try_acquire("email", "node-b", config)
.await
.expect("lease store should work")
.expect("released lease should be acquired");
assert!(replacement.token() > renewed.token());
let other_worker = store
.try_acquire("billing", "node-b", config)
.await
.expect("lease store should work")
.expect("independent worker lease should be acquired");
assert!(other_worker.token() > replacement.token());
store
.release(&renewed)
.await
.expect("stale release should be harmless");
assert_eq!(
store
.renew(&replacement, config)
.await
.expect("lease store should work")
.expect("stale release must not remove the replacement")
.token(),
replacement.token()
);
}
#[tokio::test]
async fn local_worker_lease_store_fences_concurrent_same_owner_acquires() {
let store = LocalWorkerLeaseStore::new();
let config = lease_config(Duration::from_secs(30), Duration::from_secs(10));
let (first, second) = tokio::join!(
store.try_acquire("email", "node-a", config),
store.try_acquire("email", "node-a", config),
);
let first = first.expect("first acquire should reach the store");
let second = second.expect("second acquire should reach the store");
assert_ne!(first.is_some(), second.is_some());
let active = first.or(second).expect("exactly one acquire should win");
let stale_same_owner = WorkerLease::new(
active.worker(),
active.owner(),
active.token().saturating_add(1),
active.expires_at(),
)
.expect("synthetic stale lease should be valid");
assert!(
store
.renew(&stale_same_owner, config)
.await
.expect("renew should reach the store")
.is_none(),
"only the exact fencing token may renew"
);
assert!(
store
.renew(&active, config)
.await
.expect("active token should reach the store")
.is_some()
);
}
#[tokio::test]
async fn local_worker_lease_store_rejects_invalid_inputs() {
let store = LocalWorkerLeaseStore::new();
assert!(matches!(
WorkerLease::new(
" email ",
"node-a",
1,
Instant::now() + Duration::from_secs(30)
),
Err(WorkerLeaseError::InvalidConfig(
WorkerConfigError::InvalidWorkerName { .. }
))
));
assert!(matches!(
store
.try_acquire(
"",
"node-a",
lease_config(Duration::from_secs(30), Duration::from_secs(10)),
)
.await,
Err(WorkerLeaseError::InvalidConfig(
WorkerConfigError::InvalidWorkerName { .. }
))
));
assert!(matches!(
store
.try_acquire(
"email",
"",
lease_config(Duration::from_secs(30), Duration::from_secs(10)),
)
.await,
Err(WorkerLeaseError::InvalidConfig(
WorkerConfigError::InvalidLeaseOwner { .. }
))
));
for duration in [Duration::ZERO, Duration::MAX] {
let error =
WorkerLeaseConfig::new(duration, Duration::from_secs(1), Duration::from_millis(500))
.unwrap_err();
assert_eq!(error.issues()[0].path(), "worker.lease.lease_duration");
}
let error = WorkerLeaseConfig::new(
Duration::from_secs(30),
Duration::from_secs(10),
Duration::from_secs(10),
)
.unwrap_err();
assert_eq!(error.issues()[0].path(), "worker.lease.operation_timeout");
}
#[tokio::test]
async fn lease_acquire_is_bounded_by_operation_timeout() {
let store = Arc::new(HangingAcquireLeaseStore::default());
let lease = WorkerLeaseConfig::new(
Duration::from_millis(200),
Duration::from_millis(100),
Duration::from_millis(20),
)
.expect("lease config should be valid");
let (_shutdown, token) = shutdown_channel();
let result = time::timeout(
Duration::from_millis(250),
run_worker_until_shutdown_with_options(
CountingWorker {
calls: Arc::new(AtomicUsize::new(0)),
},
worker_config(Duration::from_millis(1), Some(lease)),
token,
WorkerRunOptions::new()
.with_lease_store("node-a", Arc::clone(&store) as Arc<dyn WorkerLeaseStore>),
),
)
.await
.expect("operation deadline should bound a hanging store");
assert!(matches!(
result,
Err(WorkerRunError::LeaseOperationTimedOut)
));
assert_eq!(store.acquire_calls.load(Ordering::SeqCst), 1);
}
#[tokio::test]
async fn lease_renewal_is_bounded_by_operation_timeout() {
let store = Arc::new(HangingRenewLeaseStore::default());
let lease = WorkerLeaseConfig::new(
Duration::from_millis(100),
Duration::from_millis(80),
Duration::from_millis(20),
)
.expect("lease config should be valid");
let (_shutdown, token) = shutdown_channel();
let result = time::timeout(
Duration::from_millis(250),
run_worker_until_shutdown_with_options(
SlowWorker {
calls: Arc::new(AtomicUsize::new(0)),
duration: Duration::from_secs(1),
},
worker_config(Duration::from_secs(1), Some(lease)),
token,
WorkerRunOptions::new()
.with_lease_store("node-a", Arc::clone(&store) as Arc<dyn WorkerLeaseStore>),
),
)
.await
.expect("operation deadline should bound a hanging renewal");
assert!(matches!(
result,
Err(WorkerRunError::LeaseOperationTimedOut)
));
assert_eq!(store.renew_calls.load(Ordering::SeqCst), 1);
}
#[tokio::test]
async fn shutdown_interrupts_a_hanging_lease_acquire() {
let store = Arc::new(HangingAcquireLeaseStore::default());
let lease = WorkerLeaseConfig::new(
Duration::from_secs(10),
Duration::from_secs(6),
Duration::from_secs(5),
)
.expect("lease config should be valid");
let (trigger, token) = shutdown_channel();
let handle = tokio::spawn(run_worker_until_shutdown_with_options(
CountingWorker {
calls: Arc::new(AtomicUsize::new(0)),
},
worker_config(Duration::from_millis(1), Some(lease)),
token,
WorkerRunOptions::new()
.with_lease_store("node-a", Arc::clone(&store) as Arc<dyn WorkerLeaseStore>),
));
while store.acquire_calls.load(Ordering::SeqCst) == 0 {
time::sleep(Duration::from_millis(1)).await;
}
trigger.shutdown();
let summary = time::timeout(Duration::from_millis(250), handle)
.await
.expect("shutdown should interrupt lease acquisition")
.expect("worker task should join")
.expect("interrupted acquisition is a graceful shutdown");
assert_eq!(summary.ticks(), 0);
}
#[tokio::test(start_paused = true)]
async fn shutdown_drains_a_tick_while_lease_renewal_is_pending() {
let store = Arc::new(HangingRenewLeaseStore::default());
let lease = WorkerLeaseConfig::new(
Duration::from_secs(1),
Duration::from_millis(800),
Duration::from_millis(500),
)
.unwrap();
let (trigger, token) = shutdown_channel();
let stop = tokio::spawn(async move {
time::sleep(Duration::from_millis(250)).await;
trigger.shutdown();
});
let summary = run_worker_until_shutdown_with_options(
SlowWorker {
calls: Arc::new(AtomicUsize::new(0)),
duration: Duration::from_millis(300),
},
worker_config(Duration::from_secs(1), Some(lease)),
token,
WorkerRunOptions::new()
.with_lease_store("node-a", Arc::clone(&store) as Arc<dyn WorkerLeaseStore>),
)
.await
.unwrap();
stop.await.unwrap();
assert_eq!(store.renew_calls.load(Ordering::SeqCst), 1);
assert_eq!(summary.ticks(), 1);
assert_eq!(summary.successes(), 1);
assert_eq!(summary.processed(), 1);
assert!(summary.successful());
}
#[tokio::test]
async fn shutdown_reports_a_failed_renewal_instead_of_a_healthy_run() {
let store = Arc::new(HangingRenewLeaseStore::default());
let calls = Arc::new(AtomicUsize::new(0));
let lease = WorkerLeaseConfig::new(
Duration::from_millis(200),
Duration::from_millis(150),
Duration::from_millis(100),
)
.expect("lease config should be valid");
let (trigger, token) = shutdown_channel();
let handle = tokio::spawn(run_worker_until_shutdown_with_options(
SlowWorker {
calls: Arc::clone(&calls),
duration: Duration::from_secs(1),
},
worker_config(Duration::from_secs(1), Some(lease)),
token,
WorkerRunOptions::new()
.with_lease_store("node-a", Arc::clone(&store) as Arc<dyn WorkerLeaseStore>),
));
while store.renew_calls.load(Ordering::SeqCst) == 0 {
time::sleep(Duration::from_millis(1)).await;
}
trigger.shutdown();
let result = time::timeout(Duration::from_millis(250), handle)
.await
.expect("renewal deadline should bound shutdown")
.expect("worker task should join");
assert_eq!(calls.load(Ordering::SeqCst), 1);
assert!(matches!(
result,
Err(WorkerRunError::LeaseOperationTimedOut)
));
}
#[tokio::test]
async fn worker_lease_renewal_uses_injected_runtime_clock() {
let clock = ManualClock::new();
let store = LocalWorkerLeaseStore::with_clock(clock.shared());
let config = lease_config(Duration::from_secs(30), Duration::from_secs(10));
let lease = store
.try_acquire("email", "node-a", config)
.await
.expect("lease store should work")
.expect("lease should be acquired");
let initial_expiry = lease.expires_at();
let mut active_lease = Some(lease);
ensure_worker_lease(
&store,
&mut active_lease,
"email",
"node-a",
config,
clock.now(),
&runner::WorkerMetrics::default(),
)
.await
.expect("lease check should work");
assert_eq!(
active_lease
.as_ref()
.expect("lease should still be held")
.expires_at(),
initial_expiry
);
clock.advance(Duration::from_secs(25)).unwrap();
ensure_worker_lease(
&store,
&mut active_lease,
"email",
"node-a",
config,
clock.now(),
&runner::WorkerMetrics::default(),
)
.await
.expect("lease check should work");
assert!(active_lease.expect("lease should renew").expires_at() > initial_expiry);
}
#[tokio::test]
async fn leased_worker_requires_store() {
let (_, token) = shutdown_channel();
let result = run_worker_until_shutdown(
CountingWorker {
calls: Arc::new(AtomicUsize::new(0)),
},
worker_config(
WorkerConfig::default().poll_interval(),
Some(lease_config(
Duration::from_secs(30),
Duration::from_secs(10),
)),
),
token,
)
.await;
assert!(matches!(result, Err(WorkerRunError::MissingLeaseStore)));
}
#[tokio::test]
async fn worker_loop_rejects_invalid_worker_name_and_lease_owner() {
let (_, token) = shutdown_channel();
let result = run_worker_until_shutdown(NamedWorker(""), WorkerConfig::default(), token).await;
assert!(matches!(
result,
Err(WorkerRunError::InvalidConfig(
WorkerConfigError::InvalidWorkerName { .. }
))
));
let store = Arc::new(LocalWorkerLeaseStore::new());
let (_, token) = shutdown_channel();
let result = run_worker_until_shutdown_with_options(
NamedWorker("email"),
worker_config(
WorkerConfig::default().poll_interval(),
Some(lease_config(
Duration::from_secs(30),
Duration::from_secs(10),
)),
),
token,
WorkerRunOptions::new()
.with_lease_store("", Arc::clone(&store) as Arc<dyn WorkerLeaseStore>),
)
.await;
assert!(matches!(
result,
Err(WorkerRunError::InvalidConfig(
WorkerConfigError::InvalidLeaseOwner { .. }
))
));
}
#[tokio::test]
async fn leased_worker_runs_and_releases_on_shutdown() {
let store = Arc::new(LocalWorkerLeaseStore::new());
let (trigger, token) = shutdown_channel();
let calls = Arc::new(AtomicUsize::new(0));
let worker = CountingWorker {
calls: Arc::clone(&calls),
};
let config = lease_config(Duration::from_secs(30), Duration::from_secs(10));
let options = WorkerRunOptions::new()
.with_lease_store("node-a", Arc::clone(&store) as Arc<dyn WorkerLeaseStore>);
let handle = tokio::spawn(run_worker_until_shutdown_with_options(
worker,
worker_config(Duration::from_millis(1), Some(config)),
token,
options,
));
time::sleep(Duration::from_millis(5)).await;
trigger.shutdown();
let summary = handle.await.unwrap().unwrap();
assert!(summary.successes() > 0);
assert!(calls.load(Ordering::SeqCst) > 0);
assert!(
store
.try_acquire("counting", "node-b", config)
.await
.expect("lease store should work")
.is_some()
);
}
#[tokio::test(start_paused = true)]
async fn exhausted_metrics_capacity_does_not_interrupt_ticks_or_lease_release() {
use crate::observability::{MetricLabels, MetricRegistry, MetricRegistryLimits};
let metrics = Arc::new(MetricRegistry::with_limits(
MetricRegistryLimits::new(1, 1).unwrap(),
));
metrics
.increment_counter("occupied_total", 1, MetricLabels::default())
.unwrap();
let store = Arc::new(LocalWorkerLeaseStore::new());
let lease_config = lease_config(Duration::from_secs(30), Duration::from_secs(10));
let (trigger, token) = shutdown_channel();
let stop = tokio::spawn(async move {
tokio::task::yield_now().await;
trigger.shutdown();
});
let summary = run_worker_until_shutdown_with_options(
CountingWorker {
calls: Arc::new(AtomicUsize::new(0)),
},
worker_config(Duration::from_secs(1), Some(lease_config)),
token,
WorkerRunOptions::new()
.with_metrics(Some(metrics.clone()))
.with_lease_store("node-a", store.clone()),
)
.await
.unwrap();
stop.await.unwrap();
assert_eq!(summary.successes(), 1);
assert_eq!(summary.failures(), 0);
assert_eq!(summary.processed(), 1);
assert_eq!(metrics.snapshot().unwrap().samples().len(), 1);
assert!(
store
.try_acquire("counting", "node-b", lease_config)
.await
.unwrap()
.is_some()
);
}
#[tokio::test]
async fn leased_worker_renews_while_tick_is_running() {
let store = Arc::new(LocalWorkerLeaseStore::new());
let (trigger, token) = shutdown_channel();
let calls = Arc::new(AtomicUsize::new(0));
let worker = SlowWorker {
calls: Arc::clone(&calls),
duration: Duration::from_millis(500),
};
let lease_config = lease_config(Duration::from_millis(200), Duration::from_millis(150));
let options = WorkerRunOptions::new()
.with_lease_store("node-a", Arc::clone(&store) as Arc<dyn WorkerLeaseStore>);
let handle = tokio::spawn(run_worker_until_shutdown_with_options(
worker,
worker_config(Duration::from_secs(1), Some(lease_config)),
token,
options,
));
time::sleep(Duration::from_millis(350)).await;
assert_eq!(calls.load(Ordering::SeqCst), 1);
assert!(
store
.try_acquire("slow", "node-b", lease_config)
.await
.expect("lease store should work")
.is_none(),
"another owner must not acquire the lease during a long-running tick"
);
trigger.shutdown();
let summary = handle.await.unwrap().unwrap();
assert_eq!(summary.successes(), 1);
}
#[test]
fn worker_settings_reject_unrepresentable_timer_deadlines() {
let error = WorkerSettings::default()
.with_startup_delay(Duration::MAX)
.with_poll_interval(Duration::MAX)
.with_min_queue_interval(Duration::MAX)
.with_shutdown_timeout(Duration::MAX)
.with_failure_backoff(BackoffSettings::new(Duration::MAX, Duration::MAX, 2))
.build()
.unwrap_err();
for path in [
"worker.startup_delay",
"worker.poll_interval",
"worker.min_queue_interval",
"worker.shutdown_timeout",
"worker.failure_backoff.initial_delay",
"worker.failure_backoff.max_delay",
] {
assert!(error.issues().iter().any(|issue| issue.path() == path));
}
assert!(BackoffConfig::new(Duration::from_secs(1), Duration::MAX, 2).is_err());
}
#[tokio::test]
async fn runner_context_reaches_every_outbox_boundary_through_shared_adapters() {
struct FencedOutbox {
calls: Mutex<Vec<(&'static str, Option<u64>)>>,
shutdown: ShutdownTrigger,
}
#[async_trait]
impl OutboxStore<TestOutboxItem> for FencedOutbox {
async fn claim_due(
&self,
_claim: OutboxClaim,
context: WorkerContext,
) -> Result<Vec<TestOutboxItem>, BoxWorkerError> {
self.calls
.lock()
.unwrap()
.push(("claim", context.fencing_token()));
Ok(vec![TestOutboxItem {
id: "one".into(),
attempts: 0,
action: TestOutboxAction::Succeed,
}])
}
async fn persist_transition(
&self,
_item: &TestOutboxItem,
_state: OutboxItemState,
context: WorkerContext,
) -> Result<(), BoxWorkerError> {
self.calls
.lock()
.unwrap()
.push(("persist", context.fencing_token()));
Ok(())
}
}
#[async_trait]
impl OutboxHandler<TestOutboxItem> for FencedOutbox {
async fn handle(
&self,
_item: &TestOutboxItem,
context: WorkerContext,
) -> Result<OutboxItemState, BoxWorkerError> {
self.calls
.lock()
.unwrap()
.push(("handle", context.fencing_token()));
self.shutdown.shutdown();
Ok(OutboxItemState::Succeeded)
}
}
for leased in [false, true] {
let (shutdown, token) = shutdown_channel();
let outbox = Arc::new(FencedOutbox {
calls: Mutex::new(Vec::new()),
shutdown,
});
let store: Arc<dyn OutboxStore<TestOutboxItem>> = outbox.clone();
let handler: Arc<dyn OutboxHandler<TestOutboxItem>> = outbox.clone();
let job: Arc<dyn BatchWorkerJob> = Arc::new(OutboxJob::new(
"fenced",
store,
handler,
OutboxWorkerConfig::default(),
));
let worker: Arc<dyn Worker> = Arc::new(BatchWorker::new(job));
let config = worker_config(
Duration::from_secs(1),
leased.then(|| lease_config(Duration::from_secs(30), Duration::from_secs(10))),
);
let summary = run_worker_until_shutdown_with_options(
worker,
config,
token,
WorkerRunOptions::new()
.with_lease_store("node", Arc::new(LocalWorkerLeaseStore::new())),
)
.await
.unwrap();
let expected = leased.then_some(1);
assert_eq!(
*outbox.calls.lock().unwrap(),
[
("claim", expected),
("handle", expected),
("persist", expected)
]
);
assert_eq!(summary.ticks(), 1);
assert_eq!(summary.processed(), 1);
}
}