use std::{
collections::HashMap,
error::Error,
fmt,
marker::PhantomData,
sync::{
Arc, Mutex,
atomic::{AtomicU64, Ordering},
},
time::{Duration, Instant},
};
use async_trait::async_trait;
#[cfg(feature = "config")]
use serde::{Deserialize, Serialize};
use thiserror::Error;
use tokio::{select, sync::watch, time};
#[cfg(feature = "config")]
use crate::config::{
ConfigIssueCollector, ConfigValidationError, SettingsAdapter, ValidateSettings,
};
use crate::{
clock::{SharedClock, system_clock},
observability::{MetricLabels, MetricRecorder, metric_labels, metric_names, metric_outcomes},
result::{ClassifyError, ErrorKind},
};
pub type BoxWorkerError = Box<dyn Error + Send + Sync + 'static>;
pub const MAX_WORKER_NAME_LENGTH: usize = 128;
pub const MAX_WORKER_LEASE_OWNER_LENGTH: usize = 128;
#[async_trait]
pub trait Worker: Send + Sync + 'static {
fn name(&self) -> &str;
async fn run_once(&self) -> Result<WorkerTick, BoxWorkerError>;
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
#[non_exhaustive]
pub struct BatchRunSummary {
claimed: u64,
succeeded: u64,
retrying: u64,
failed: u64,
skipped: u64,
dead_lettered: u64,
state_errors: u64,
dead_letter_errors: u64,
batch_timed_out: bool,
next_delay: Option<Duration>,
}
impl BatchRunSummary {
#[must_use]
pub const fn new() -> Self {
Self {
claimed: 0,
succeeded: 0,
retrying: 0,
failed: 0,
skipped: 0,
dead_lettered: 0,
state_errors: 0,
dead_letter_errors: 0,
batch_timed_out: false,
next_delay: None,
}
}
#[must_use]
pub const fn with_claimed(mut self, claimed: u64) -> Self {
self.claimed = claimed;
self
}
#[must_use]
pub const fn with_succeeded(mut self, succeeded: u64) -> Self {
self.succeeded = succeeded;
self
}
#[must_use]
pub const fn with_retrying(mut self, retrying: u64) -> Self {
self.retrying = retrying;
self
}
#[must_use]
pub const fn with_failed(mut self, failed: u64) -> Self {
self.failed = failed;
self
}
#[must_use]
pub const fn with_skipped(mut self, skipped: u64) -> Self {
self.skipped = skipped;
self
}
#[must_use]
pub const fn with_dead_lettered(mut self, dead_lettered: u64) -> Self {
self.dead_lettered = dead_lettered;
self
}
#[must_use]
pub const fn with_state_errors(mut self, state_errors: u64) -> Self {
self.state_errors = state_errors;
self
}
#[must_use]
pub const fn with_dead_letter_errors(mut self, dead_letter_errors: u64) -> Self {
self.dead_letter_errors = dead_letter_errors;
self
}
#[must_use]
pub const fn with_batch_timed_out(mut self, batch_timed_out: bool) -> Self {
self.batch_timed_out = batch_timed_out;
self
}
#[must_use]
pub const fn with_next_delay(mut self, next_delay: Duration) -> Self {
self.next_delay = Some(next_delay);
self
}
#[must_use]
pub const fn claimed(&self) -> u64 {
self.claimed
}
#[must_use]
pub const fn succeeded(&self) -> u64 {
self.succeeded
}
#[must_use]
pub const fn retrying(&self) -> u64 {
self.retrying
}
#[must_use]
pub const fn failed(&self) -> u64 {
self.failed
}
#[must_use]
pub const fn skipped(&self) -> u64 {
self.skipped
}
#[must_use]
pub const fn dead_lettered(&self) -> u64 {
self.dead_lettered
}
#[must_use]
pub const fn state_errors(&self) -> u64 {
self.state_errors
}
#[must_use]
pub const fn dead_letter_errors(&self) -> u64 {
self.dead_letter_errors
}
#[must_use]
pub const fn batch_timed_out(&self) -> bool {
self.batch_timed_out
}
#[must_use]
pub const fn next_delay(&self) -> Option<Duration> {
self.next_delay
}
#[must_use]
pub const fn handled(&self) -> u64 {
self.succeeded
.saturating_add(self.retrying)
.saturating_add(self.failed)
.saturating_add(self.skipped)
.saturating_add(self.state_errors)
}
#[must_use]
pub const fn empty(&self) -> bool {
self.claimed == 0 && self.handled() == 0 && !self.batch_timed_out
}
#[must_use]
pub const fn has_failures(&self) -> bool {
self.failed > 0
|| self.state_errors > 0
|| self.dead_letter_errors > 0
|| self.batch_timed_out
}
fn record_item_state(&mut self, state: &OutboxItemState) {
match state {
OutboxItemState::Succeeded => {
self.succeeded = self.succeeded.saturating_add(1);
}
OutboxItemState::Skipped { .. } => {
self.skipped = self.skipped.saturating_add(1);
}
OutboxItemState::Retry { delay, .. } => {
self.retrying = self.retrying.saturating_add(1);
self.record_next_delay(*delay);
}
OutboxItemState::Failed { .. } => {
self.failed = self.failed.saturating_add(1);
}
}
}
fn record_state_error(&mut self) {
self.state_errors = self.state_errors.saturating_add(1);
}
fn record_dead_letter(&mut self) {
self.dead_lettered = self.dead_lettered.saturating_add(1);
}
fn record_dead_letter_error(&mut self) {
self.dead_letter_errors = self.dead_letter_errors.saturating_add(1);
}
fn record_batch_timeout(&mut self) {
self.batch_timed_out = true;
self.record_next_delay(Duration::ZERO);
}
fn record_next_delay(&mut self, delay: Duration) {
self.next_delay = Some(self.next_delay.map_or(delay, |current| current.min(delay)));
}
}
#[async_trait]
pub trait BatchWorkerJob: Send + Sync + 'static {
fn name(&self) -> &str;
async fn run_batch(&self) -> Result<BatchRunSummary, BoxWorkerError>;
}
pub trait OutboxItem: Send + Sync + 'static {
fn outbox_id(&self) -> String;
fn outbox_attempts(&self) -> u32;
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[non_exhaustive]
pub struct OutboxClaim {
limit: u32,
lease_duration: Duration,
}
impl OutboxClaim {
#[must_use]
pub const fn new(limit: u32, lease_duration: Duration) -> Self {
Self {
limit,
lease_duration,
}
}
#[must_use]
pub const fn limit(&self) -> u32 {
self.limit
}
#[must_use]
pub const fn lease_duration(&self) -> Duration {
self.lease_duration
}
}
#[derive(Clone, PartialEq, Eq)]
#[non_exhaustive]
pub enum OutboxItemState {
Succeeded,
Skipped {
#[doc = "The reason for this outcome."]
reason: String,
},
Retry {
#[doc = "The reason for this outcome."]
reason: String,
#[doc = "The delay before the next action."]
delay: Duration,
},
Failed {
#[doc = "The reason for this outcome."]
reason: String,
},
}
impl fmt::Debug for OutboxItemState {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::Succeeded => formatter.write_str("OutboxItemState::Succeeded"),
Self::Skipped { .. } => formatter
.debug_struct("OutboxItemState::Skipped")
.field("reason", &"[REDACTED]")
.finish(),
Self::Retry { delay, .. } => formatter
.debug_struct("OutboxItemState::Retry")
.field("reason", &"[REDACTED]")
.field("delay", delay)
.finish(),
Self::Failed { .. } => formatter
.debug_struct("OutboxItemState::Failed")
.field("reason", &"[REDACTED]")
.finish(),
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[non_exhaustive]
pub enum OutboxFailureClass {
Retryable,
Terminal,
}
impl OutboxItemState {
#[must_use]
pub fn retry(reason: impl Into<String>, delay: Duration) -> Self {
Self::Retry {
reason: bounded_outbox_reason(reason),
delay,
}
}
#[must_use]
pub fn failed(reason: impl Into<String>) -> Self {
Self::Failed {
reason: bounded_outbox_reason(reason),
}
}
#[must_use]
pub fn skipped(reason: impl Into<String>) -> Self {
Self::Skipped {
reason: bounded_outbox_reason(reason),
}
}
#[must_use]
pub const fn as_str(&self) -> &'static str {
match self {
Self::Succeeded => "succeeded",
Self::Skipped { .. } => "skipped",
Self::Retry { .. } => "retry",
Self::Failed { .. } => "failed",
}
}
#[must_use]
pub const fn failure_class(&self) -> Option<OutboxFailureClass> {
match self {
Self::Retry { .. } => Some(OutboxFailureClass::Retryable),
Self::Failed { .. } => Some(OutboxFailureClass::Terminal),
Self::Succeeded | Self::Skipped { .. } => None,
}
}
}
#[async_trait]
pub trait OutboxStore<I>: Send + Sync + 'static
where
I: OutboxItem,
{
async fn claim_due(&self, claim: OutboxClaim) -> Result<Vec<I>, BoxWorkerError>;
async fn record_state(&self, item: &I, state: OutboxItemState) -> Result<(), BoxWorkerError>;
async fn record_dead_letter(&self, _item: &I, _reason: &str) -> Result<(), BoxWorkerError> {
Ok(())
}
}
#[async_trait]
pub trait OutboxHandler<I>: Send + Sync + 'static
where
I: OutboxItem,
{
async fn handle(&self, item: &I) -> Result<OutboxItemState, BoxWorkerError>;
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[non_exhaustive]
pub struct OutboxRetryPolicy {
max_attempts: u32,
backoff: BackoffConfig,
}
impl Default for OutboxRetryPolicy {
fn default() -> Self {
Self {
max_attempts: 5,
backoff: BackoffConfig::new(Duration::from_secs(30), Duration::from_secs(30 * 60), 2),
}
}
}
impl OutboxRetryPolicy {
#[must_use]
pub const fn new(max_attempts: u32, backoff: BackoffConfig) -> Self {
Self {
max_attempts,
backoff,
}
}
#[must_use]
pub const fn max_attempts(&self) -> u32 {
self.max_attempts
}
#[must_use]
pub const fn backoff(&self) -> BackoffConfig {
self.backoff
}
#[must_use]
pub const fn should_retry(&self, attempts_after_current_try: u32) -> bool {
attempts_after_current_try < self.max_attempts
}
#[must_use]
pub fn delay_for_attempts(&self, attempts_after_current_try: u32) -> Duration {
let exponent = attempts_after_current_try.saturating_sub(1);
let mut delay = self.backoff.initial_delay;
for _ in 0..exponent {
delay = multiply_duration(delay, self.backoff.multiplier).min(self.backoff.max_delay);
}
delay
}
pub fn validate(&self) -> Result<(), WorkerConfigError> {
if self.max_attempts == 0 {
return Err(WorkerConfigError::InvalidOutboxMaxAttempts);
}
self.backoff.validate()
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[non_exhaustive]
pub struct OutboxWorkerConfig {
batch_size: u32,
lease_duration: Duration,
retry_policy: OutboxRetryPolicy,
item_timeout: Option<Duration>,
batch_timeout: Option<Duration>,
}
impl Default for OutboxWorkerConfig {
fn default() -> Self {
Self {
batch_size: 50,
lease_duration: Duration::from_secs(60),
retry_policy: OutboxRetryPolicy::default(),
item_timeout: None,
batch_timeout: None,
}
}
}
impl OutboxWorkerConfig {
#[must_use]
pub const fn new(
batch_size: u32,
lease_duration: Duration,
retry_policy: OutboxRetryPolicy,
) -> Self {
Self {
batch_size,
lease_duration,
retry_policy,
item_timeout: None,
batch_timeout: None,
}
}
#[must_use]
pub const fn with_batch_size(mut self, batch_size: u32) -> Self {
self.batch_size = batch_size;
self
}
#[must_use]
pub const fn with_lease_duration(mut self, lease_duration: Duration) -> Self {
self.lease_duration = lease_duration;
self
}
#[must_use]
pub const fn with_retry_policy(mut self, retry_policy: OutboxRetryPolicy) -> Self {
self.retry_policy = retry_policy;
self
}
#[must_use]
pub const fn with_item_timeout(mut self, item_timeout: Option<Duration>) -> Self {
self.item_timeout = item_timeout;
self
}
#[must_use]
pub const fn with_batch_timeout(mut self, batch_timeout: Option<Duration>) -> Self {
self.batch_timeout = batch_timeout;
self
}
#[must_use]
pub const fn batch_size(&self) -> u32 {
self.batch_size
}
#[must_use]
pub const fn lease_duration(&self) -> Duration {
self.lease_duration
}
#[must_use]
pub const fn retry_policy(&self) -> OutboxRetryPolicy {
self.retry_policy
}
#[must_use]
pub const fn item_timeout(&self) -> Option<Duration> {
self.item_timeout
}
#[must_use]
pub const fn batch_timeout(&self) -> Option<Duration> {
self.batch_timeout
}
pub fn validate(&self) -> Result<(), WorkerConfigError> {
if self.batch_size == 0 {
return Err(WorkerConfigError::InvalidOutboxBatchSize);
}
if self.lease_duration.is_zero() {
return Err(WorkerConfigError::InvalidOutboxLeaseDuration);
}
if matches!(self.item_timeout, Some(timeout) if timeout.is_zero()) {
return Err(WorkerConfigError::InvalidOutboxItemTimeout);
}
if matches!(self.batch_timeout, Some(timeout) if timeout.is_zero()) {
return Err(WorkerConfigError::InvalidOutboxBatchTimeout);
}
self.retry_policy.validate()
}
}
#[cfg(feature = "config")]
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
#[serde(deny_unknown_fields)]
#[non_exhaustive]
pub struct OutboxWorkerSettings {
#[serde(default = "default_outbox_batch_size")]
pub batch_size: u32,
#[serde(
default = "default_outbox_lease_duration",
with = "crate::config::human_duration"
)]
pub lease_duration: Duration,
#[serde(default = "default_outbox_max_attempts")]
pub max_attempts: u32,
#[serde(default)]
pub retry_backoff: BackoffSettings,
#[serde(default, with = "crate::config::human_duration::option")]
pub item_timeout: Option<Duration>,
#[serde(default, with = "crate::config::human_duration::option")]
pub batch_timeout: Option<Duration>,
}
#[cfg(feature = "config")]
impl Default for OutboxWorkerSettings {
fn default() -> Self {
Self::from(OutboxWorkerConfig::default())
}
}
#[cfg(feature = "config")]
impl From<OutboxWorkerConfig> for OutboxWorkerSettings {
fn from(value: OutboxWorkerConfig) -> Self {
Self {
batch_size: value.batch_size,
lease_duration: value.lease_duration,
max_attempts: value.retry_policy.max_attempts,
retry_backoff: BackoffSettings::from(value.retry_policy.backoff),
item_timeout: value.item_timeout,
batch_timeout: value.batch_timeout,
}
}
}
#[cfg(feature = "config")]
impl TryFrom<OutboxWorkerSettings> for OutboxWorkerConfig {
type Error = WorkerConfigError;
fn try_from(value: OutboxWorkerSettings) -> Result<Self, Self::Error> {
Self::try_from(&value)
}
}
#[cfg(feature = "config")]
impl TryFrom<&OutboxWorkerSettings> for OutboxWorkerConfig {
type Error = WorkerConfigError;
fn try_from(value: &OutboxWorkerSettings) -> Result<Self, Self::Error> {
let config = OutboxWorkerConfig::new(
value.batch_size,
value.lease_duration,
OutboxRetryPolicy::new(
value.max_attempts,
BackoffConfig::try_from(&value.retry_backoff)?,
),
)
.with_item_timeout(value.item_timeout)
.with_batch_timeout(value.batch_timeout);
config.validate()?;
Ok(config)
}
}
#[cfg(feature = "config")]
impl OutboxWorkerSettings {
#[must_use]
pub const fn with_batch_size(mut self, batch_size: u32) -> Self {
self.batch_size = batch_size;
self
}
#[must_use]
pub const fn with_lease_duration(mut self, lease_duration: Duration) -> Self {
self.lease_duration = lease_duration;
self
}
#[must_use]
pub const fn with_max_attempts(mut self, max_attempts: u32) -> Self {
self.max_attempts = max_attempts;
self
}
#[must_use]
pub const fn with_retry_backoff(mut self, retry_backoff: BackoffSettings) -> Self {
self.retry_backoff = retry_backoff;
self
}
#[must_use]
pub const fn with_item_timeout(mut self, item_timeout: Option<Duration>) -> Self {
self.item_timeout = item_timeout;
self
}
#[must_use]
pub const fn with_batch_timeout(mut self, batch_timeout: Option<Duration>) -> Self {
self.batch_timeout = batch_timeout;
self
}
pub fn into_config(self) -> Result<OutboxWorkerConfig, WorkerConfigError> {
OutboxWorkerConfig::try_from(self)
}
pub fn validate_at(&self, path: &str) -> Result<(), ConfigValidationError> {
let mut issues = ConfigIssueCollector::new();
issues.require(
self.batch_size > 0,
format!("{path}.batch_size"),
"must be greater than zero",
);
issues.require(
!self.lease_duration.is_zero(),
format!("{path}.lease_duration"),
"must be greater than zero",
);
issues.require(
self.max_attempts > 0,
format!("{path}.max_attempts"),
"must be greater than zero",
);
if let Some(timeout) = self.item_timeout {
issues.require(
!timeout.is_zero(),
format!("{path}.item_timeout"),
"must be greater than zero when set",
);
}
if let Some(timeout) = self.batch_timeout {
issues.require(
!timeout.is_zero(),
format!("{path}.batch_timeout"),
"must be greater than zero when set",
);
}
issues.extend_result(
self.retry_backoff
.validate_at(&format!("{path}.retry_backoff")),
);
issues.into_result()
}
}
#[cfg(feature = "config")]
impl SettingsAdapter<OutboxWorkerConfig> for OutboxWorkerSettings {
type Error = WorkerConfigError;
fn adapt(&self) -> Result<OutboxWorkerConfig, Self::Error> {
OutboxWorkerConfig::try_from(self)
}
}
#[cfg(feature = "config")]
impl ValidateSettings for OutboxWorkerSettings {
fn validate_settings(&self) -> Result<(), ConfigValidationError> {
self.validate_at("outbox")
}
}
#[derive(Debug, Clone)]
#[non_exhaustive]
pub struct OutboxJob<S, H, I> {
name: String,
store: S,
handler: H,
config: OutboxWorkerConfig,
_item: PhantomData<I>,
}
impl<S, H, I> OutboxJob<S, H, I>
where
I: OutboxItem,
{
#[must_use]
pub fn new(name: impl Into<String>, store: S, handler: H, config: OutboxWorkerConfig) -> Self {
Self {
name: name.into(),
store,
handler,
config,
_item: PhantomData,
}
}
#[must_use]
pub fn with_config(mut self, config: OutboxWorkerConfig) -> Self {
self.config = config;
self
}
#[must_use]
pub fn store(&self) -> &S {
&self.store
}
#[must_use]
pub fn handler(&self) -> &H {
&self.handler
}
#[must_use]
pub const fn config(&self) -> OutboxWorkerConfig {
self.config
}
}
#[async_trait]
impl<S, H, I> BatchWorkerJob for OutboxJob<S, H, I>
where
S: OutboxStore<I>,
H: OutboxHandler<I>,
I: OutboxItem,
{
fn name(&self) -> &str {
&self.name
}
async fn run_batch(&self) -> Result<BatchRunSummary, BoxWorkerError> {
self.config.validate().map_err(box_worker_error)?;
let claim = OutboxClaim::new(self.config.batch_size, self.config.lease_duration);
let items = self.store.claim_due(claim).await?;
let mut summary = BatchRunSummary::new().with_claimed(items.len() as u64);
let batch_started_at = Instant::now();
for item in items {
if let Some(batch_timeout) = self.config.batch_timeout
&& batch_started_at.elapsed() >= batch_timeout
{
summary.record_batch_timeout();
break;
}
let state = handle_outbox_item(&self.handler, &item, self.config).await;
if self.store.record_state(&item, state.clone()).await.is_err() {
summary.record_state_error();
tracing::warn!(
worker = self.name(),
item_id = item.outbox_id(),
state = state.as_str(),
"outbox state update failed"
);
continue;
}
summary.record_item_state(&state);
if let OutboxItemState::Failed { reason } = &state {
match self.store.record_dead_letter(&item, reason).await {
Ok(()) => {
summary.record_dead_letter();
}
Err(_) => {
summary.record_dead_letter_error();
tracing::warn!(
worker = self.name(),
item_id = item.outbox_id(),
"outbox dead-letter hook failed"
);
}
}
}
}
Ok(summary)
}
}
#[derive(Debug, Clone)]
#[non_exhaustive]
pub struct BatchWorker<J> {
job: J,
log_empty_batches: bool,
}
impl<J> BatchWorker<J> {
#[must_use]
pub const fn new(job: J) -> Self {
Self {
job,
log_empty_batches: false,
}
}
#[must_use]
pub const fn with_empty_batch_logging(mut self, enabled: bool) -> Self {
self.log_empty_batches = enabled;
self
}
#[must_use]
pub const fn job(&self) -> &J {
&self.job
}
}
#[async_trait]
impl<J> Worker for BatchWorker<J>
where
J: BatchWorkerJob,
{
fn name(&self) -> &str {
self.job.name()
}
async fn run_once(&self) -> Result<WorkerTick, BoxWorkerError> {
let summary = self.job.run_batch().await?;
if self.log_empty_batches || !summary.empty() {
tracing::info!(
worker = self.name(),
claimed = summary.claimed(),
succeeded = summary.succeeded(),
retrying = summary.retrying(),
failed = summary.failed(),
skipped = summary.skipped(),
dead_lettered = summary.dead_lettered(),
state_errors = summary.state_errors(),
dead_letter_errors = summary.dead_letter_errors(),
batch_timed_out = summary.batch_timed_out(),
handled = summary.handled(),
"batch worker completed"
);
}
let mut tick = WorkerTick::new(summary.handled());
if let Some(delay) = summary.next_delay() {
tick = tick.with_next_delay(delay);
}
Ok(tick)
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
#[non_exhaustive]
pub struct WorkerConfig {
enabled: bool,
poll_interval: Duration,
startup_delay: Duration,
shutdown_timeout: Duration,
failure_backoff: BackoffConfig,
lease: Option<WorkerLeaseConfig>,
}
impl Default for WorkerConfig {
fn default() -> Self {
Self {
enabled: true,
poll_interval: Duration::from_secs(10),
startup_delay: Duration::ZERO,
shutdown_timeout: Duration::from_secs(30),
failure_backoff: BackoffConfig::default(),
lease: None,
}
}
}
impl WorkerConfig {
#[must_use]
pub fn disabled() -> Self {
Self::default().with_enabled(false)
}
#[must_use]
pub const fn with_enabled(mut self, enabled: bool) -> Self {
self.enabled = enabled;
self
}
#[must_use]
pub const fn with_poll_interval(mut self, poll_interval: Duration) -> Self {
self.poll_interval = poll_interval;
self
}
#[must_use]
pub const fn with_startup_delay(mut self, startup_delay: Duration) -> Self {
self.startup_delay = startup_delay;
self
}
#[must_use]
pub const fn with_shutdown_timeout(mut self, shutdown_timeout: Duration) -> Self {
self.shutdown_timeout = shutdown_timeout;
self
}
#[must_use]
pub const fn with_failure_backoff(mut self, failure_backoff: BackoffConfig) -> Self {
self.failure_backoff = failure_backoff;
self
}
#[must_use]
pub fn with_lease(mut self, lease: Option<WorkerLeaseConfig>) -> Self {
self.lease = lease;
self
}
#[must_use]
pub const fn enabled(&self) -> bool {
self.enabled
}
#[must_use]
pub const fn poll_interval(&self) -> Duration {
self.poll_interval
}
#[must_use]
pub const fn startup_delay(&self) -> Duration {
self.startup_delay
}
#[must_use]
pub const fn shutdown_timeout(&self) -> Duration {
self.shutdown_timeout
}
#[must_use]
pub const fn failure_backoff(&self) -> BackoffConfig {
self.failure_backoff
}
#[must_use]
pub const fn lease(&self) -> Option<WorkerLeaseConfig> {
self.lease
}
pub fn validate(&self) -> Result<(), WorkerConfigError> {
if self.enabled && self.poll_interval.is_zero() {
return Err(WorkerConfigError::InvalidPollInterval);
}
if self.shutdown_timeout.is_zero() {
return Err(WorkerConfigError::InvalidShutdownTimeout);
}
self.failure_backoff.validate()?;
if let Some(lease) = self.lease {
lease.validate()?;
}
Ok(())
}
}
#[cfg(feature = "config")]
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
#[non_exhaustive]
pub struct WorkerSettings {
#[serde(default = "default_true")]
pub enabled: bool,
#[serde(
default = "default_worker_poll_interval",
with = "crate::config::human_duration"
)]
pub poll_interval: Duration,
#[serde(default, with = "crate::config::human_duration")]
pub startup_delay: Duration,
#[serde(
default = "default_worker_shutdown_timeout",
with = "crate::config::human_duration"
)]
pub shutdown_timeout: Duration,
#[serde(default)]
pub failure_backoff: BackoffSettings,
#[serde(default)]
pub lease: Option<WorkerLeaseSettings>,
}
#[cfg(feature = "config")]
impl Default for WorkerSettings {
fn default() -> Self {
Self::from(WorkerConfig::default())
}
}
#[cfg(feature = "config")]
impl From<WorkerConfig> for WorkerSettings {
fn from(value: WorkerConfig) -> Self {
Self {
enabled: value.enabled,
poll_interval: value.poll_interval,
startup_delay: value.startup_delay,
shutdown_timeout: value.shutdown_timeout,
failure_backoff: BackoffSettings::from(value.failure_backoff),
lease: value.lease.map(WorkerLeaseSettings::from),
}
}
}
#[cfg(feature = "config")]
impl TryFrom<WorkerSettings> for WorkerConfig {
type Error = WorkerConfigError;
fn try_from(value: WorkerSettings) -> Result<Self, Self::Error> {
Self::try_from(&value)
}
}
#[cfg(feature = "config")]
impl TryFrom<&WorkerSettings> for WorkerConfig {
type Error = WorkerConfigError;
fn try_from(value: &WorkerSettings) -> Result<Self, Self::Error> {
let config = WorkerConfig::default()
.with_enabled(value.enabled)
.with_poll_interval(value.poll_interval)
.with_startup_delay(value.startup_delay)
.with_shutdown_timeout(value.shutdown_timeout)
.with_failure_backoff(BackoffConfig::try_from(&value.failure_backoff)?)
.with_lease(
value
.lease
.as_ref()
.map(WorkerLeaseConfig::try_from)
.transpose()?,
);
config.validate()?;
Ok(config)
}
}
#[cfg(feature = "config")]
impl WorkerSettings {
#[must_use]
pub const fn with_enabled(mut self, enabled: bool) -> Self {
self.enabled = enabled;
self
}
#[must_use]
pub const fn with_poll_interval(mut self, poll_interval: Duration) -> Self {
self.poll_interval = poll_interval;
self
}
#[must_use]
pub const fn with_startup_delay(mut self, startup_delay: Duration) -> Self {
self.startup_delay = startup_delay;
self
}
#[must_use]
pub const fn with_shutdown_timeout(mut self, shutdown_timeout: Duration) -> Self {
self.shutdown_timeout = shutdown_timeout;
self
}
#[must_use]
pub const fn with_failure_backoff(mut self, failure_backoff: BackoffSettings) -> Self {
self.failure_backoff = failure_backoff;
self
}
#[must_use]
pub const fn with_lease(mut self, lease: Option<WorkerLeaseSettings>) -> Self {
self.lease = lease;
self
}
pub fn into_config(self) -> Result<WorkerConfig, WorkerConfigError> {
WorkerConfig::try_from(self)
}
pub fn validate_at(&self, path: &str) -> Result<(), ConfigValidationError> {
let mut issues = ConfigIssueCollector::new();
issues.require(
!self.enabled || !self.poll_interval.is_zero(),
format!("{path}.poll_interval"),
"must be greater than zero when worker is enabled",
);
issues.require(
!self.shutdown_timeout.is_zero(),
format!("{path}.shutdown_timeout"),
"must be greater than zero",
);
issues.extend_result(
self.failure_backoff
.validate_at(&format!("{path}.failure_backoff")),
);
if let Some(lease) = &self.lease {
issues.extend_result(lease.validate_at(&format!("{path}.lease")));
}
issues.into_result()
}
}
#[cfg(feature = "config")]
impl SettingsAdapter<WorkerConfig> for WorkerSettings {
type Error = WorkerConfigError;
fn adapt(&self) -> Result<WorkerConfig, Self::Error> {
WorkerConfig::try_from(self)
}
}
#[cfg(feature = "config")]
impl ValidateSettings for WorkerSettings {
fn validate_settings(&self) -> Result<(), ConfigValidationError> {
self.validate_at("worker")
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[non_exhaustive]
pub struct BackoffConfig {
initial_delay: Duration,
max_delay: Duration,
multiplier: u32,
}
impl Default for BackoffConfig {
fn default() -> Self {
Self {
initial_delay: Duration::from_secs(1),
max_delay: Duration::from_secs(60),
multiplier: 2,
}
}
}
impl BackoffConfig {
#[must_use]
pub const fn new(initial_delay: Duration, max_delay: Duration, multiplier: u32) -> Self {
Self {
initial_delay,
max_delay,
multiplier,
}
}
#[must_use]
pub const fn initial_delay(&self) -> Duration {
self.initial_delay
}
#[must_use]
pub const fn max_delay(&self) -> Duration {
self.max_delay
}
#[must_use]
pub const fn multiplier(&self) -> u32 {
self.multiplier
}
pub fn validate(&self) -> Result<(), WorkerConfigError> {
if self.initial_delay.is_zero() {
return Err(WorkerConfigError::InvalidBackoffInitialDelay);
}
if self.max_delay < self.initial_delay {
return Err(WorkerConfigError::InvalidBackoffMaxDelay);
}
if self.multiplier < 1 {
return Err(WorkerConfigError::InvalidBackoffMultiplier);
}
Ok(())
}
}
#[cfg(feature = "config")]
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
#[serde(deny_unknown_fields)]
#[non_exhaustive]
pub struct BackoffSettings {
#[serde(
default = "default_backoff_initial_delay",
with = "crate::config::human_duration"
)]
pub initial_delay: Duration,
#[serde(
default = "default_backoff_max_delay",
with = "crate::config::human_duration"
)]
pub max_delay: Duration,
#[serde(default = "default_backoff_multiplier")]
pub multiplier: u32,
}
#[cfg(feature = "config")]
impl Default for BackoffSettings {
fn default() -> Self {
Self::from(BackoffConfig::default())
}
}
#[cfg(feature = "config")]
impl From<BackoffConfig> for BackoffSettings {
fn from(value: BackoffConfig) -> Self {
Self {
initial_delay: value.initial_delay,
max_delay: value.max_delay,
multiplier: value.multiplier,
}
}
}
#[cfg(feature = "config")]
impl TryFrom<BackoffSettings> for BackoffConfig {
type Error = WorkerConfigError;
fn try_from(value: BackoffSettings) -> Result<Self, Self::Error> {
Self::try_from(&value)
}
}
#[cfg(feature = "config")]
impl TryFrom<&BackoffSettings> for BackoffConfig {
type Error = WorkerConfigError;
fn try_from(value: &BackoffSettings) -> Result<Self, Self::Error> {
let config = BackoffConfig::new(value.initial_delay, value.max_delay, value.multiplier);
config.validate()?;
Ok(config)
}
}
#[cfg(feature = "config")]
impl BackoffSettings {
#[must_use]
pub const fn new(initial_delay: Duration, max_delay: Duration, multiplier: u32) -> Self {
Self {
initial_delay,
max_delay,
multiplier,
}
}
#[must_use]
pub const fn with_initial_delay(mut self, initial_delay: Duration) -> Self {
self.initial_delay = initial_delay;
self
}
#[must_use]
pub const fn with_max_delay(mut self, max_delay: Duration) -> Self {
self.max_delay = max_delay;
self
}
#[must_use]
pub const fn with_multiplier(mut self, multiplier: u32) -> Self {
self.multiplier = multiplier;
self
}
pub fn into_config(self) -> Result<BackoffConfig, WorkerConfigError> {
BackoffConfig::try_from(self)
}
pub fn validate_at(&self, path: &str) -> Result<(), ConfigValidationError> {
let mut issues = ConfigIssueCollector::new();
issues.require(
!self.initial_delay.is_zero(),
format!("{path}.initial_delay"),
"must be greater than zero",
);
issues.require(
self.max_delay >= self.initial_delay,
format!("{path}.max_delay"),
"must be greater than or equal to initial_delay",
);
issues.require(
self.multiplier >= 1,
format!("{path}.multiplier"),
"must be greater than or equal to one",
);
issues.into_result()
}
}
#[cfg(feature = "config")]
impl SettingsAdapter<BackoffConfig> for BackoffSettings {
type Error = WorkerConfigError;
fn adapt(&self) -> Result<BackoffConfig, Self::Error> {
BackoffConfig::try_from(self)
}
}
#[cfg(feature = "config")]
impl ValidateSettings for BackoffSettings {
fn validate_settings(&self) -> Result<(), ConfigValidationError> {
self.validate_at("worker.failure_backoff")
}
}
#[derive(Debug, Clone)]
#[non_exhaustive]
pub struct RetryBackoff {
config: BackoffConfig,
next_delay: Duration,
}
impl RetryBackoff {
#[must_use]
pub fn new(config: BackoffConfig) -> Self {
Self {
config,
next_delay: config.initial_delay,
}
}
pub fn record_failure(&mut self) -> Duration {
let current = self.next_delay;
self.next_delay =
multiply_duration(current, self.config.multiplier).min(self.config.max_delay);
current
}
pub fn reset(&mut self) {
self.next_delay = self.config.initial_delay;
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[non_exhaustive]
pub struct WorkerLeaseConfig {
lease_duration: Duration,
renew_after: Duration,
}
impl WorkerLeaseConfig {
#[must_use]
pub const fn new(lease_duration: Duration, renew_after: Duration) -> Self {
Self {
lease_duration,
renew_after,
}
}
#[must_use]
pub const fn lease_duration(&self) -> Duration {
self.lease_duration
}
#[must_use]
pub const fn renew_after(&self) -> Duration {
self.renew_after
}
pub fn validate(&self) -> Result<(), WorkerConfigError> {
if self.lease_duration.is_zero()
|| Instant::now().checked_add(self.lease_duration).is_none()
{
return Err(WorkerConfigError::InvalidLeaseDuration);
}
if self.renew_after.is_zero() || self.renew_after >= self.lease_duration {
return Err(WorkerConfigError::InvalidLeaseRenewAfter);
}
Ok(())
}
}
#[cfg(feature = "config")]
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
#[serde(deny_unknown_fields)]
#[non_exhaustive]
pub struct WorkerLeaseSettings {
#[serde(with = "crate::config::human_duration")]
pub lease_duration: Duration,
#[serde(with = "crate::config::human_duration")]
pub renew_after: Duration,
}
#[cfg(feature = "config")]
impl From<WorkerLeaseConfig> for WorkerLeaseSettings {
fn from(value: WorkerLeaseConfig) -> Self {
Self {
lease_duration: value.lease_duration,
renew_after: value.renew_after,
}
}
}
#[cfg(feature = "config")]
impl TryFrom<WorkerLeaseSettings> for WorkerLeaseConfig {
type Error = WorkerConfigError;
fn try_from(value: WorkerLeaseSettings) -> Result<Self, Self::Error> {
Self::try_from(&value)
}
}
#[cfg(feature = "config")]
impl TryFrom<&WorkerLeaseSettings> for WorkerLeaseConfig {
type Error = WorkerConfigError;
fn try_from(value: &WorkerLeaseSettings) -> Result<Self, Self::Error> {
let config = WorkerLeaseConfig::new(value.lease_duration, value.renew_after);
config.validate()?;
Ok(config)
}
}
#[cfg(feature = "config")]
impl WorkerLeaseSettings {
#[must_use]
pub const fn new(lease_duration: Duration, renew_after: Duration) -> Self {
Self {
lease_duration,
renew_after,
}
}
#[must_use]
pub const fn with_lease_duration(mut self, lease_duration: Duration) -> Self {
self.lease_duration = lease_duration;
self
}
#[must_use]
pub const fn with_renew_after(mut self, renew_after: Duration) -> Self {
self.renew_after = renew_after;
self
}
pub fn into_config(self) -> Result<WorkerLeaseConfig, WorkerConfigError> {
WorkerLeaseConfig::try_from(self)
}
pub fn validate_at(&self, path: &str) -> Result<(), ConfigValidationError> {
let mut issues = ConfigIssueCollector::new();
issues.require(
!self.lease_duration.is_zero()
&& Instant::now().checked_add(self.lease_duration).is_some(),
format!("{path}.lease_duration"),
"must be greater than zero and fit the monotonic clock range",
);
issues.require(
!self.renew_after.is_zero() && self.renew_after < self.lease_duration,
format!("{path}.renew_after"),
"must be greater than zero and less than lease_duration",
);
issues.into_result()
}
}
#[cfg(feature = "config")]
impl SettingsAdapter<WorkerLeaseConfig> for WorkerLeaseSettings {
type Error = WorkerConfigError;
fn adapt(&self) -> Result<WorkerLeaseConfig, Self::Error> {
WorkerLeaseConfig::try_from(self)
}
}
#[cfg(feature = "config")]
impl ValidateSettings for WorkerLeaseSettings {
fn validate_settings(&self) -> Result<(), ConfigValidationError> {
self.validate_at("worker.lease")
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
#[non_exhaustive]
pub struct WorkerLease {
worker: String,
owner: String,
token: u64,
expires_at: Instant,
}
impl WorkerLease {
pub fn new(
worker: impl Into<String>,
owner: impl Into<String>,
token: u64,
expires_at: Instant,
) -> Result<Self, WorkerLeaseError> {
let worker = worker.into();
let owner = owner.into();
validate_worker_name(&worker).map_err(WorkerLeaseError::InvalidConfig)?;
validate_lease_owner(&owner).map_err(WorkerLeaseError::InvalidConfig)?;
Ok(Self::from_valid_parts(worker, owner, token, expires_at))
}
fn from_valid_parts(worker: String, owner: String, token: u64, expires_at: Instant) -> Self {
Self {
worker,
owner,
token,
expires_at,
}
}
#[must_use]
pub fn worker(&self) -> &str {
&self.worker
}
#[must_use]
pub fn owner(&self) -> &str {
&self.owner
}
#[must_use]
pub const fn token(&self) -> u64 {
self.token
}
#[must_use]
pub const fn expires_at(&self) -> Instant {
self.expires_at
}
#[must_use]
pub fn should_renew(&self, now: Instant, renew_after: Duration) -> bool {
now.checked_add(renew_after)
.is_none_or(|renew_at| renew_at >= self.expires_at)
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[non_exhaustive]
pub enum WorkerLeaseDecision {
Acquired,
Busy,
Renewed,
Lost,
Released,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Error)]
#[non_exhaustive]
pub enum WorkerLeaseError {
#[error("worker lease store lock is poisoned")]
LockPoisoned,
#[error("invalid worker lease configuration")]
InvalidConfig(#[from] WorkerConfigError),
}
impl ClassifyError for WorkerLeaseError {
fn error_kind(&self) -> ErrorKind {
match self {
Self::LockPoisoned => ErrorKind::Unavailable,
Self::InvalidConfig(source) => source.error_kind(),
}
}
fn error_code(&self) -> std::borrow::Cow<'static, str> {
match self {
Self::LockPoisoned => std::borrow::Cow::Borrowed("worker_lease_unavailable"),
Self::InvalidConfig(source) => source.error_code(),
}
}
}
#[async_trait]
pub trait WorkerLeaseStore: Send + Sync + 'static {
async fn try_acquire(
&self,
worker: &str,
owner: &str,
config: WorkerLeaseConfig,
) -> Result<Option<WorkerLease>, WorkerLeaseError>;
async fn renew(
&self,
lease: &WorkerLease,
config: WorkerLeaseConfig,
) -> Result<Option<WorkerLease>, WorkerLeaseError>;
async fn release(&self, lease: &WorkerLease) -> Result<(), WorkerLeaseError>;
}
#[non_exhaustive]
pub struct LocalWorkerLeaseStore {
leases: Mutex<HashMap<String, WorkerLease>>,
clock: SharedClock,
next_token: AtomicU64,
}
impl LocalWorkerLeaseStore {
#[must_use]
pub fn new() -> Self {
Self::with_clock(system_clock())
}
#[must_use]
pub fn with_clock(clock: SharedClock) -> Self {
Self {
leases: Mutex::new(HashMap::new()),
clock,
next_token: AtomicU64::new(1),
}
}
#[must_use]
pub fn clock(&self) -> &SharedClock {
&self.clock
}
fn next_token(&self) -> u64 {
self.next_token.fetch_add(1, Ordering::Relaxed)
}
}
impl fmt::Debug for LocalWorkerLeaseStore {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
let lease_count = self.leases.lock().map(|leases| leases.len()).ok();
formatter
.debug_struct("LocalWorkerLeaseStore")
.field("lease_count", &lease_count)
.finish_non_exhaustive()
}
}
impl Default for LocalWorkerLeaseStore {
fn default() -> Self {
Self::new()
}
}
#[async_trait]
impl WorkerLeaseStore for LocalWorkerLeaseStore {
async fn try_acquire(
&self,
worker: &str,
owner: &str,
config: WorkerLeaseConfig,
) -> Result<Option<WorkerLease>, WorkerLeaseError> {
config.validate()?;
validate_worker_name(worker)?;
validate_lease_owner(owner)?;
let now = self.clock.now();
let expires_at = worker_lease_expiry_after(now, config.lease_duration)?;
let mut leases = self
.leases
.lock()
.map_err(|_| WorkerLeaseError::LockPoisoned)?;
if let Some(existing) = leases.get(worker)
&& existing.expires_at > now
&& existing.owner != owner
{
return Ok(None);
}
let lease = WorkerLease::from_valid_parts(
worker.trim().to_owned(),
owner.trim().to_owned(),
self.next_token(),
expires_at,
);
leases.insert(worker.to_owned(), lease.clone());
Ok(Some(lease))
}
async fn renew(
&self,
lease: &WorkerLease,
config: WorkerLeaseConfig,
) -> Result<Option<WorkerLease>, WorkerLeaseError> {
config.validate()?;
validate_worker_name(lease.worker())?;
validate_lease_owner(lease.owner())?;
let now = self.clock.now();
let expires_at = worker_lease_expiry_after(now, config.lease_duration)?;
let mut leases = self
.leases
.lock()
.map_err(|_| WorkerLeaseError::LockPoisoned)?;
let Some(existing) = leases.get_mut(lease.worker()) else {
return Ok(None);
};
if existing.owner != lease.owner
|| existing.token != lease.token
|| existing.expires_at <= now
{
return Ok(None);
}
existing.expires_at = expires_at;
Ok(Some(existing.clone()))
}
async fn release(&self, lease: &WorkerLease) -> Result<(), WorkerLeaseError> {
let mut leases = self
.leases
.lock()
.map_err(|_| WorkerLeaseError::LockPoisoned)?;
if let Some(existing) = leases.get(lease.worker())
&& existing.owner == lease.owner
&& existing.token == lease.token
{
leases.remove(lease.worker());
}
Ok(())
}
}
fn worker_lease_expiry_after(
now: Instant,
lease_duration: Duration,
) -> Result<Instant, WorkerLeaseError> {
now.checked_add(lease_duration)
.ok_or(WorkerLeaseError::InvalidConfig(
WorkerConfigError::InvalidLeaseDuration,
))
}
#[derive(Clone)]
#[non_exhaustive]
pub struct WorkerRunOptions {
metrics: Option<Arc<dyn MetricRecorder>>,
clock: SharedClock,
lease_store: Option<Arc<dyn WorkerLeaseStore>>,
lease_owner: String,
}
impl WorkerRunOptions {
#[must_use]
pub fn new() -> Self {
Self {
metrics: None,
clock: system_clock(),
lease_store: None,
lease_owner: "local".to_owned(),
}
}
#[must_use]
pub fn with_metrics(mut self, metrics: Option<Arc<dyn MetricRecorder>>) -> Self {
self.metrics = metrics;
self
}
#[must_use]
pub fn with_clock(mut self, clock: SharedClock) -> Self {
self.clock = clock;
self
}
#[must_use]
pub fn with_lease_store(
mut self,
owner: impl Into<String>,
store: Arc<dyn WorkerLeaseStore>,
) -> Self {
self.lease_owner = normalize_worker_identifier(owner);
self.lease_store = Some(store);
self
}
#[must_use]
pub fn metrics(&self) -> Option<&Arc<dyn MetricRecorder>> {
self.metrics.as_ref()
}
#[must_use]
pub fn clock(&self) -> &SharedClock {
&self.clock
}
#[must_use]
pub fn lease_store(&self) -> Option<&Arc<dyn WorkerLeaseStore>> {
self.lease_store.as_ref()
}
#[must_use]
pub fn lease_owner(&self) -> &str {
&self.lease_owner
}
}
impl Default for WorkerRunOptions {
fn default() -> Self {
Self::new()
}
}
impl fmt::Debug for WorkerRunOptions {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("WorkerRunOptions")
.field("metrics_enabled", &self.metrics.is_some())
.field("lease_enabled", &self.lease_store.is_some())
.field("lease_owner", &self.lease_owner)
.finish_non_exhaustive()
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[non_exhaustive]
pub struct WorkerTick {
processed: u64,
next_delay: Option<Duration>,
}
impl WorkerTick {
#[must_use]
pub const fn new(processed: u64) -> Self {
Self {
processed,
next_delay: None,
}
}
#[must_use]
pub const fn with_next_delay(mut self, next_delay: Duration) -> Self {
self.next_delay = Some(next_delay);
self
}
#[must_use]
pub const fn processed(&self) -> u64 {
self.processed
}
#[must_use]
pub const fn next_delay(&self) -> Option<Duration> {
self.next_delay
}
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
#[non_exhaustive]
pub struct WorkerRunSummary {
ticks: u64,
successes: u64,
failures: u64,
processed: u64,
last_outcome: Option<WorkerTickOutcome>,
}
impl WorkerRunSummary {
#[must_use]
pub const fn ticks(&self) -> u64 {
self.ticks
}
#[must_use]
pub const fn successes(&self) -> u64 {
self.successes
}
#[must_use]
pub const fn failures(&self) -> u64 {
self.failures
}
#[must_use]
pub const fn processed(&self) -> u64 {
self.processed
}
#[must_use]
pub const fn last_outcome(&self) -> Option<WorkerTickOutcome> {
self.last_outcome
}
#[must_use]
pub const fn successful(&self) -> bool {
self.failures == 0
}
#[must_use]
pub const fn state(&self) -> WorkerRunState {
if self.ticks == 0 {
WorkerRunState::Idle
} else if self.failures == 0 {
WorkerRunState::Healthy
} else if self.successes == 0 {
WorkerRunState::Failing
} else {
WorkerRunState::Degraded
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[non_exhaustive]
pub enum WorkerTickOutcome {
Success,
Failure,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[non_exhaustive]
pub enum WorkerRunState {
Idle,
Healthy,
Degraded,
Failing,
}
#[derive(Clone)]
pub struct ShutdownTrigger {
sender: watch::Sender<bool>,
}
impl ShutdownTrigger {
pub fn shutdown(&self) {
let _ = self.sender.send(true);
}
}
impl fmt::Debug for ShutdownTrigger {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("ShutdownTrigger")
.finish_non_exhaustive()
}
}
#[derive(Clone, Debug)]
pub struct ShutdownToken {
receiver: watch::Receiver<bool>,
}
impl ShutdownToken {
#[must_use]
pub fn is_shutdown_requested(&self) -> bool {
*self.receiver.borrow()
}
async fn wait(&mut self) {
if self.is_shutdown_requested() {
return;
}
let _ = self.receiver.changed().await;
}
}
#[must_use]
pub fn shutdown_channel() -> (ShutdownTrigger, ShutdownToken) {
let (sender, receiver) = watch::channel(false);
(ShutdownTrigger { sender }, ShutdownToken { receiver })
}
pub async fn run_worker_until_shutdown<W>(
worker: W,
config: WorkerConfig,
mut shutdown: ShutdownToken,
) -> Result<WorkerRunSummary, WorkerRunError>
where
W: Worker,
{
run_worker_loop(worker, config, &mut shutdown, WorkerRunOptions::new()).await
}
pub async fn run_worker_until_shutdown_with_metrics<W>(
worker: W,
config: WorkerConfig,
mut shutdown: ShutdownToken,
metrics: Arc<dyn MetricRecorder>,
) -> Result<WorkerRunSummary, WorkerRunError>
where
W: Worker,
{
run_worker_loop(
worker,
config,
&mut shutdown,
WorkerRunOptions::new().with_metrics(Some(metrics)),
)
.await
}
pub async fn run_worker_until_shutdown_with_options<W>(
worker: W,
config: WorkerConfig,
mut shutdown: ShutdownToken,
options: WorkerRunOptions,
) -> Result<WorkerRunSummary, WorkerRunError>
where
W: Worker,
{
run_worker_loop(worker, config, &mut shutdown, options).await
}
async fn run_worker_loop<W>(
worker: W,
config: WorkerConfig,
shutdown: &mut ShutdownToken,
options: WorkerRunOptions,
) -> Result<WorkerRunSummary, WorkerRunError>
where
W: Worker,
{
config.validate()?;
let mut summary = WorkerRunSummary::default();
if !config.enabled {
return Ok(summary);
}
validate_worker_name(worker.name()).map_err(WorkerRunError::InvalidConfig)?;
let metrics = options.metrics;
let clock = options.clock;
let lease_owner = options.lease_owner;
if config.lease.is_some() {
validate_lease_owner(&lease_owner).map_err(WorkerRunError::InvalidConfig)?;
}
let lease_runtime = match (config.lease, options.lease_store) {
(Some(config), Some(store)) => Some((config, store)),
(Some(_), None) => return Err(WorkerRunError::MissingLeaseStore),
(None, _) => None,
};
let mut active_lease = None;
if !config.startup_delay.is_zero() {
select! {
() = time::sleep(config.startup_delay) => {}
() = shutdown.wait() => return Ok(summary),
}
}
let mut backoff = RetryBackoff::new(config.failure_backoff);
loop {
if shutdown.is_shutdown_requested() {
release_worker_lease(
lease_store_ref(&lease_runtime),
active_lease.take(),
metrics.as_deref(),
)
.await?;
return Ok(summary);
}
if let Some((lease_config, lease_store)) = &lease_runtime {
active_lease = ensure_worker_lease(
lease_store.as_ref(),
active_lease,
worker.name(),
&lease_owner,
*lease_config,
clock.now(),
metrics.as_deref(),
)
.await?;
if active_lease.is_none() {
select! {
() = time::sleep(config.poll_interval) => continue,
() = shutdown.wait() => {
release_worker_lease(
lease_store_ref(&lease_runtime),
active_lease.take(),
metrics.as_deref(),
).await?;
return Ok(summary);
},
}
}
}
summary.ticks += 1;
let tick_started_at = clock.now();
let tick_result = run_worker_once_with_lease(
&worker,
lease_runtime.as_ref(),
&mut active_lease,
&clock,
metrics.as_deref(),
)
.await?;
let delay = match tick_result {
Ok(tick) => {
summary.successes += 1;
summary.processed = summary.processed.saturating_add(tick.processed);
summary.last_outcome = Some(WorkerTickOutcome::Success);
backoff.reset();
record_worker_metrics(
metrics.as_deref(),
worker.name(),
metric_outcomes::SUCCESS,
tick.processed,
clock.now().saturating_duration_since(tick_started_at),
);
tick.next_delay.unwrap_or(config.poll_interval)
}
Err(_) => {
summary.failures += 1;
summary.last_outcome = Some(WorkerTickOutcome::Failure);
let delay = backoff.record_failure();
record_worker_metrics(
metrics.as_deref(),
worker.name(),
metric_outcomes::FAILURE,
0,
clock.now().saturating_duration_since(tick_started_at),
);
tracing::warn!(
worker = worker.name(),
retry_after_ms = delay.as_millis(),
"worker tick failed"
);
delay
}
};
select! {
() = time::sleep(delay) => {}
() = shutdown.wait() => {
release_worker_lease(
lease_store_ref(&lease_runtime),
active_lease.take(),
metrics.as_deref(),
).await?;
return Ok(summary);
},
}
}
}
async fn run_worker_once_with_lease<W>(
worker: &W,
lease_runtime: Option<&(WorkerLeaseConfig, Arc<dyn WorkerLeaseStore>)>,
active_lease: &mut Option<WorkerLease>,
clock: &SharedClock,
metrics: Option<&dyn MetricRecorder>,
) -> Result<Result<WorkerTick, BoxWorkerError>, WorkerRunError>
where
W: Worker,
{
let Some((config, store)) = lease_runtime else {
return Ok(worker.run_once().await);
};
let mut tick = Box::pin(worker.run_once());
loop {
let Some(lease) = active_lease.as_ref() else {
return Err(WorkerRunError::LeaseLost);
};
let renew_in = lease
.expires_at()
.saturating_duration_since(clock.now())
.saturating_sub(config.renew_after());
select! {
result = &mut tick => return Ok(result),
() = time::sleep(renew_in) => {}
}
let lease = lease.clone();
match store.renew(&lease, *config).await? {
Some(renewed) => {
record_worker_lease_metric(metrics, worker.name(), metric_outcomes::RENEWED);
*active_lease = Some(renewed);
}
None => {
record_worker_lease_metric(metrics, worker.name(), metric_outcomes::LOST);
*active_lease = None;
return Err(WorkerRunError::LeaseLost);
}
}
}
}
fn lease_store_ref(
runtime: &Option<(WorkerLeaseConfig, Arc<dyn WorkerLeaseStore>)>,
) -> Option<&dyn WorkerLeaseStore> {
runtime.as_ref().map(|(_, store)| store.as_ref())
}
async fn ensure_worker_lease(
store: &dyn WorkerLeaseStore,
active_lease: Option<WorkerLease>,
worker: &str,
owner: &str,
config: WorkerLeaseConfig,
now: Instant,
metrics: Option<&dyn MetricRecorder>,
) -> Result<Option<WorkerLease>, WorkerRunError> {
if let Some(lease) = active_lease {
if !lease.should_renew(now, config.renew_after) {
return Ok(Some(lease));
}
return match store.renew(&lease, config).await? {
Some(renewed) => {
record_worker_lease_metric(metrics, worker, metric_outcomes::RENEWED);
Ok(Some(renewed))
}
None => {
record_worker_lease_metric(metrics, worker, metric_outcomes::LOST);
Ok(None)
}
};
}
match store.try_acquire(worker, owner, config).await? {
Some(lease) => {
record_worker_lease_metric(metrics, worker, metric_outcomes::ACQUIRED);
Ok(Some(lease))
}
None => {
record_worker_lease_metric(metrics, worker, metric_outcomes::BUSY);
Ok(None)
}
}
}
async fn release_worker_lease(
store: Option<&dyn WorkerLeaseStore>,
lease: Option<WorkerLease>,
metrics: Option<&dyn MetricRecorder>,
) -> Result<(), WorkerRunError> {
let (Some(store), Some(lease)) = (store, lease) else {
return Ok(());
};
store.release(&lease).await?;
record_worker_lease_metric(metrics, lease.worker(), metric_outcomes::RELEASED);
Ok(())
}
fn record_worker_metrics(
metrics: Option<&dyn MetricRecorder>,
worker: &str,
outcome: &'static str,
processed: u64,
elapsed: Duration,
) {
let Some(metrics) = metrics else {
return;
};
let labels = MetricLabels::from_lossy([
(metric_labels::WORKER, worker),
(metric_labels::OUTCOME, outcome),
]);
let _ = metrics.increment_counter(metric_names::WORKER_TICKS_TOTAL, 1, labels.clone());
let _ = metrics.observe_histogram(
metric_names::WORKER_TICK_DURATION_SECONDS,
elapsed.as_secs_f64(),
labels,
);
if processed > 0 {
let _ = metrics.increment_counter(
metric_names::WORKER_PROCESSED_TOTAL,
processed,
MetricLabels::from_lossy([(metric_labels::WORKER, worker)]),
);
}
}
fn record_worker_lease_metric(
metrics: Option<&dyn MetricRecorder>,
worker: &str,
outcome: &'static str,
) {
let Some(metrics) = metrics else {
return;
};
let _ = metrics.increment_counter(
metric_names::WORKER_LEASE_DECISIONS_TOTAL,
1,
MetricLabels::from_lossy([
(metric_labels::WORKER, worker),
(metric_labels::OUTCOME, outcome),
]),
);
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Error)]
#[non_exhaustive]
pub enum WorkerConfigError {
#[error("worker poll_interval must be greater than zero")]
InvalidPollInterval,
#[error("worker shutdown_timeout must be greater than zero")]
InvalidShutdownTimeout,
#[error("worker backoff initial_delay must be greater than zero")]
InvalidBackoffInitialDelay,
#[error("worker backoff max_delay must be greater than or equal to initial_delay")]
InvalidBackoffMaxDelay,
#[error("worker backoff multiplier must be greater than zero")]
InvalidBackoffMultiplier,
#[error("worker lease_duration must be greater than zero")]
InvalidLeaseDuration,
#[error("worker lease renew_after must be greater than zero and less than lease_duration")]
InvalidLeaseRenewAfter,
#[error("worker name must be non-empty and less than or equal to {max} bytes")]
InvalidWorkerName {
#[doc = "The configured upper bound."]
max: usize,
},
#[error("worker lease owner must be non-empty and less than or equal to {max} bytes")]
InvalidLeaseOwner {
#[doc = "The configured upper bound."]
max: usize,
},
#[error("outbox batch_size must be greater than zero")]
InvalidOutboxBatchSize,
#[error("outbox lease_duration must be greater than zero")]
InvalidOutboxLeaseDuration,
#[error("outbox max_attempts must be greater than zero")]
InvalidOutboxMaxAttempts,
#[error("outbox item_timeout must be greater than zero when set")]
InvalidOutboxItemTimeout,
#[error("outbox batch_timeout must be greater than zero when set")]
InvalidOutboxBatchTimeout,
}
impl ClassifyError for WorkerConfigError {
fn error_kind(&self) -> ErrorKind {
ErrorKind::Validation
}
fn error_code(&self) -> std::borrow::Cow<'static, str> {
std::borrow::Cow::Borrowed(match self {
Self::InvalidPollInterval => "worker_invalid_poll_interval",
Self::InvalidShutdownTimeout => "worker_invalid_shutdown_timeout",
Self::InvalidBackoffInitialDelay => "worker_invalid_backoff_initial_delay",
Self::InvalidBackoffMaxDelay => "worker_invalid_backoff_max_delay",
Self::InvalidBackoffMultiplier => "worker_invalid_backoff_multiplier",
Self::InvalidLeaseDuration => "worker_invalid_lease_duration",
Self::InvalidLeaseRenewAfter => "worker_invalid_lease_renew_after",
Self::InvalidWorkerName { .. } => "worker_invalid_name",
Self::InvalidLeaseOwner { .. } => "worker_invalid_lease_owner",
Self::InvalidOutboxBatchSize => "outbox_invalid_batch_size",
Self::InvalidOutboxLeaseDuration => "outbox_invalid_lease_duration",
Self::InvalidOutboxMaxAttempts => "outbox_invalid_max_attempts",
Self::InvalidOutboxItemTimeout => "outbox_invalid_item_timeout",
Self::InvalidOutboxBatchTimeout => "outbox_invalid_batch_timeout",
})
}
}
#[derive(Debug, Error)]
#[non_exhaustive]
pub enum WorkerRunError {
#[error("invalid worker configuration")]
InvalidConfig(#[from] WorkerConfigError),
#[error("worker lease config requires a lease store")]
MissingLeaseStore,
#[error("worker lease was lost while a tick was running")]
LeaseLost,
#[error("worker lease operation failed")]
Lease(#[from] WorkerLeaseError),
}
impl ClassifyError for WorkerRunError {
fn error_kind(&self) -> ErrorKind {
match self {
Self::InvalidConfig(source) => source.error_kind(),
Self::MissingLeaseStore => ErrorKind::Validation,
Self::LeaseLost => ErrorKind::Unavailable,
Self::Lease(source) => source.error_kind(),
}
}
fn error_code(&self) -> std::borrow::Cow<'static, str> {
match self {
Self::InvalidConfig(source) => source.error_code(),
Self::MissingLeaseStore => std::borrow::Cow::Borrowed("worker_lease_store_missing"),
Self::LeaseLost => std::borrow::Cow::Borrowed("worker_lease_lost"),
Self::Lease(source) => source.error_code(),
}
}
}
fn multiply_duration(duration: Duration, multiplier: u32) -> Duration {
duration.checked_mul(multiplier).unwrap_or(Duration::MAX)
}
fn retry_or_fail_state(
retry_policy: &OutboxRetryPolicy,
prior_attempts: u32,
reason: String,
) -> OutboxItemState {
let attempts_after_current_try = prior_attempts.saturating_add(1);
if retry_policy.should_retry(attempts_after_current_try) {
OutboxItemState::retry(
reason,
retry_policy.delay_for_attempts(attempts_after_current_try),
)
} else {
OutboxItemState::failed(reason)
}
}
async fn handle_outbox_item<H, I>(
handler: &H,
item: &I,
config: OutboxWorkerConfig,
) -> OutboxItemState
where
H: OutboxHandler<I>,
I: OutboxItem,
{
let result = if let Some(timeout) = config.item_timeout {
match time::timeout(timeout, handler.handle(item)).await {
Ok(result) => result,
Err(_) => Err(format!("outbox item processing timed out after {timeout:?}").into()),
}
} else {
handler.handle(item).await
};
match result {
Ok(state) => state,
Err(error) => retry_or_fail_state(
&config.retry_policy,
item.outbox_attempts(),
error.to_string(),
),
}
}
fn bounded_outbox_reason(reason: impl Into<String>) -> String {
const MAX_REASON_LEN: usize = 1024;
let mut reason = reason.into().trim().to_owned();
if reason.len() <= MAX_REASON_LEN {
return reason;
}
reason.truncate(MAX_REASON_LEN);
while !reason.is_char_boundary(reason.len()) {
reason.pop();
}
reason
}
fn box_worker_error(error: impl Error + Send + Sync + 'static) -> BoxWorkerError {
Box::new(error)
}
fn normalize_worker_identifier(value: impl Into<String>) -> String {
value.into().trim().to_owned()
}
fn validate_worker_name(value: &str) -> Result<(), WorkerConfigError> {
validate_worker_identifier(value, MAX_WORKER_NAME_LENGTH).map_err(|()| {
WorkerConfigError::InvalidWorkerName {
max: MAX_WORKER_NAME_LENGTH,
}
})
}
fn validate_lease_owner(value: &str) -> Result<(), WorkerConfigError> {
validate_worker_identifier(value, MAX_WORKER_LEASE_OWNER_LENGTH).map_err(|()| {
WorkerConfigError::InvalidLeaseOwner {
max: MAX_WORKER_LEASE_OWNER_LENGTH,
}
})
}
fn validate_worker_identifier(value: &str, max_len: usize) -> Result<(), ()> {
if value.is_empty()
|| value.len() > max_len
|| value != value.trim()
|| value.chars().any(char::is_control)
{
return Err(());
}
Ok(())
}
#[cfg(feature = "config")]
fn default_true() -> bool {
true
}
#[cfg(feature = "config")]
fn default_worker_poll_interval() -> Duration {
WorkerConfig::default().poll_interval
}
#[cfg(feature = "config")]
fn default_worker_shutdown_timeout() -> Duration {
WorkerConfig::default().shutdown_timeout
}
#[cfg(feature = "config")]
fn default_backoff_initial_delay() -> Duration {
BackoffConfig::default().initial_delay
}
#[cfg(feature = "config")]
fn default_backoff_max_delay() -> Duration {
BackoffConfig::default().max_delay
}
#[cfg(feature = "config")]
fn default_backoff_multiplier() -> u32 {
BackoffConfig::default().multiplier
}
#[cfg(feature = "config")]
fn default_outbox_batch_size() -> u32 {
OutboxWorkerConfig::default().batch_size
}
#[cfg(feature = "config")]
fn default_outbox_lease_duration() -> Duration {
OutboxWorkerConfig::default().lease_duration
}
#[cfg(feature = "config")]
fn default_outbox_max_attempts() -> u32 {
OutboxWorkerConfig::default().retry_policy.max_attempts
}
#[cfg(test)]
mod tests {
use std::{
collections::VecDeque,
sync::{
Arc, Mutex,
atomic::{AtomicUsize, Ordering},
},
};
use crate::clock::{Clock, ManualClock};
use super::*;
struct CountingWorker {
calls: Arc<AtomicUsize>,
}
struct NamedWorker(&'static str);
struct SlowWorker {
calls: Arc<AtomicUsize>,
duration: Duration,
}
struct TestBatchJob;
#[derive(Debug, Clone)]
struct TestOutboxItem {
id: String,
attempts: u32,
action: TestOutboxAction,
}
#[derive(Debug, Clone, Copy)]
enum TestOutboxAction {
Succeed,
Skip,
Retry,
HandlerError,
Slow,
}
#[derive(Debug, Clone, Default)]
struct TestOutboxStore {
items: Arc<Mutex<VecDeque<TestOutboxItem>>>,
states: Arc<Mutex<Vec<(String, OutboxItemState)>>>,
dead_letters: Arc<Mutex<Vec<String>>>,
}
#[derive(Debug, Clone, Copy)]
struct TestOutboxHandler;
#[async_trait]
impl Worker for CountingWorker {
fn name(&self) -> &str {
"counting"
}
async fn run_once(&self) -> Result<WorkerTick, BoxWorkerError> {
self.calls.fetch_add(1, Ordering::SeqCst);
Ok(WorkerTick::new(1).with_next_delay(Duration::from_millis(1)))
}
}
#[async_trait]
impl Worker for NamedWorker {
fn name(&self) -> &str {
self.0
}
async fn run_once(&self) -> Result<WorkerTick, BoxWorkerError> {
Ok(WorkerTick::new(1).with_next_delay(Duration::from_millis(1)))
}
}
#[async_trait]
impl Worker for SlowWorker {
fn name(&self) -> &str {
"slow"
}
async fn run_once(&self) -> Result<WorkerTick, BoxWorkerError> {
self.calls.fetch_add(1, Ordering::SeqCst);
time::sleep(self.duration).await;
Ok(WorkerTick::new(1).with_next_delay(Duration::from_secs(1)))
}
}
#[async_trait]
impl BatchWorkerJob for TestBatchJob {
fn name(&self) -> &str {
"test batch"
}
async fn run_batch(&self) -> Result<BatchRunSummary, BoxWorkerError> {
Ok(BatchRunSummary::new()
.with_claimed(5)
.with_succeeded(2)
.with_retrying(1)
.with_failed(1)
.with_skipped(1)
.with_next_delay(Duration::from_millis(5)))
}
}
impl OutboxItem for TestOutboxItem {
fn outbox_id(&self) -> String {
self.id.clone()
}
fn outbox_attempts(&self) -> u32 {
self.attempts
}
}
#[async_trait]
impl OutboxStore<TestOutboxItem> for TestOutboxStore {
async fn claim_due(
&self,
claim: OutboxClaim,
) -> Result<Vec<TestOutboxItem>, BoxWorkerError> {
let mut items = self.items.lock().unwrap();
Ok((0..claim.limit())
.filter_map(|_| items.pop_front())
.collect())
}
async fn record_state(
&self,
item: &TestOutboxItem,
state: OutboxItemState,
) -> Result<(), BoxWorkerError> {
self.states.lock().unwrap().push((item.id.clone(), state));
Ok(())
}
async fn record_dead_letter(
&self,
item: &TestOutboxItem,
reason: &str,
) -> Result<(), BoxWorkerError> {
self.dead_letters
.lock()
.unwrap()
.push(format!("{}:{reason}", item.id));
Ok(())
}
}
#[async_trait]
impl OutboxHandler<TestOutboxItem> for TestOutboxHandler {
async fn handle(&self, item: &TestOutboxItem) -> Result<OutboxItemState, BoxWorkerError> {
match item.action {
TestOutboxAction::Succeed => Ok(OutboxItemState::Succeeded),
TestOutboxAction::Skip => Ok(OutboxItemState::skipped("not applicable")),
TestOutboxAction::Retry => {
Ok(OutboxItemState::retry("temporary", Duration::from_secs(3)))
}
TestOutboxAction::HandlerError => Err("delivery failed".into()),
TestOutboxAction::Slow => {
time::sleep(Duration::from_millis(50)).await;
Ok(OutboxItemState::Succeeded)
}
}
}
}
#[test]
fn backoff_grows_until_cap() {
let mut backoff = RetryBackoff::new(BackoffConfig::new(
Duration::from_secs(1),
Duration::from_secs(3),
2,
));
assert_eq!(backoff.record_failure(), Duration::from_secs(1));
assert_eq!(backoff.record_failure(), Duration::from_secs(2));
assert_eq!(backoff.record_failure(), Duration::from_secs(3));
backoff.reset();
assert_eq!(backoff.record_failure(), Duration::from_secs(1));
}
#[cfg(feature = "config")]
#[test]
fn worker_settings_adapt_to_runtime_config() {
use crate::config::{SettingsAdapter, ValidateSettings};
let settings = WorkerSettings {
poll_interval: Duration::from_secs(5),
failure_backoff: BackoffSettings {
initial_delay: Duration::from_millis(200),
max_delay: Duration::from_secs(2),
multiplier: 3,
},
lease: Some(WorkerLeaseSettings {
lease_duration: Duration::from_secs(30),
renew_after: Duration::from_secs(10),
}),
..WorkerSettings::default()
};
settings
.validate_settings()
.expect("settings should validate");
let config = settings.adapt().expect("settings should adapt");
assert!(config.enabled());
assert_eq!(config.poll_interval(), Duration::from_secs(5));
assert_eq!(config.failure_backoff().multiplier(), 3);
assert_eq!(
config
.lease()
.expect("lease should be configured")
.renew_after(),
Duration::from_secs(10)
);
}
#[cfg(feature = "config")]
#[test]
fn worker_settings_report_config_paths() {
let settings = WorkerSettings {
poll_interval: Duration::ZERO,
failure_backoff: BackoffSettings {
initial_delay: Duration::ZERO,
..BackoffSettings::default()
},
lease: Some(WorkerLeaseSettings::new(
Duration::MAX,
Duration::from_secs(1),
)),
..WorkerSettings::default()
};
let error = settings
.validate_at("email.worker")
.expect_err("invalid worker settings should report paths");
let paths = error
.issues()
.iter()
.map(|issue| issue.path())
.collect::<Vec<_>>();
assert!(paths.contains(&"email.worker.poll_interval"));
assert!(paths.contains(&"email.worker.failure_backoff.initial_delay"));
assert!(paths.contains(&"email.worker.lease.lease_duration"));
}
#[tokio::test]
async fn batch_worker_maps_outbox_summary_to_tick() {
let worker = BatchWorker::new(TestBatchJob);
let tick = worker.run_once().await.expect("batch should run");
assert_eq!(tick.processed(), 5);
assert_eq!(tick.next_delay(), Some(Duration::from_millis(5)));
}
#[test]
fn outbox_state_debug_redacts_handler_reasons() {
let reason = "private-handler-error-detail";
let states = [
OutboxItemState::skipped(reason),
OutboxItemState::retry(reason, Duration::from_secs(1)),
OutboxItemState::failed(reason),
];
let output = format!("{states:?}");
assert!(output.contains("[REDACTED]"));
assert!(!output.contains(reason));
}
#[tokio::test]
async fn outbox_job_claims_handles_and_records_states() {
let store = TestOutboxStore::default();
store.items.lock().unwrap().extend([
TestOutboxItem {
id: "one".to_owned(),
attempts: 0,
action: TestOutboxAction::Succeed,
},
TestOutboxItem {
id: "two".to_owned(),
attempts: 0,
action: TestOutboxAction::Skip,
},
TestOutboxItem {
id: "three".to_owned(),
attempts: 0,
action: TestOutboxAction::Retry,
},
TestOutboxItem {
id: "four".to_owned(),
attempts: 4,
action: TestOutboxAction::HandlerError,
},
]);
let job = OutboxJob::new(
"test outbox",
store.clone(),
TestOutboxHandler,
OutboxWorkerConfig::default().with_batch_size(10),
);
let summary = job.run_batch().await.expect("outbox should run");
assert_eq!(summary.claimed(), 4);
assert_eq!(summary.succeeded(), 1);
assert_eq!(summary.skipped(), 1);
assert_eq!(summary.retrying(), 1);
assert_eq!(summary.failed(), 1);
assert_eq!(summary.dead_lettered(), 1);
assert_eq!(summary.next_delay(), Some(Duration::from_secs(3)));
let states = store.states.lock().unwrap();
assert_eq!(states.len(), 4);
assert!(matches!(states[0].1, OutboxItemState::Succeeded));
assert!(matches!(states[1].1, OutboxItemState::Skipped { .. }));
assert!(matches!(states[2].1, OutboxItemState::Retry { .. }));
assert!(matches!(states[3].1, OutboxItemState::Failed { .. }));
assert_eq!(store.dead_letters.lock().unwrap().len(), 1);
}
#[tokio::test]
async fn outbox_job_applies_item_timeout_and_batch_timeout() {
let store = TestOutboxStore::default();
store.items.lock().unwrap().extend([
TestOutboxItem {
id: "slow".to_owned(),
attempts: 4,
action: TestOutboxAction::Slow,
},
TestOutboxItem {
id: "never-started".to_owned(),
attempts: 0,
action: TestOutboxAction::Succeed,
},
]);
let job = OutboxJob::new(
"test outbox",
store,
TestOutboxHandler,
OutboxWorkerConfig::default()
.with_batch_size(10)
.with_item_timeout(Some(Duration::from_millis(1)))
.with_batch_timeout(Some(Duration::from_millis(1))),
);
let summary = job.run_batch().await.expect("outbox should run");
assert_eq!(summary.failed(), 1);
assert!(summary.batch_timed_out());
assert_eq!(summary.next_delay(), Some(Duration::ZERO));
}
#[cfg(feature = "config")]
#[test]
fn outbox_worker_settings_adapt_to_runtime_config() {
use crate::config::{SettingsAdapter, ValidateSettings};
let settings = OutboxWorkerSettings::default()
.with_batch_size(25)
.with_lease_duration(Duration::from_secs(120))
.with_max_attempts(7)
.with_retry_backoff(BackoffSettings::new(
Duration::from_secs(2),
Duration::from_secs(30),
2,
))
.with_item_timeout(Some(Duration::from_secs(10)))
.with_batch_timeout(Some(Duration::from_secs(60)));
settings
.validate_settings()
.expect("settings should validate");
let config = settings.adapt().expect("settings should adapt");
assert_eq!(config.batch_size(), 25);
assert_eq!(config.lease_duration(), Duration::from_secs(120));
assert_eq!(config.retry_policy().max_attempts(), 7);
assert_eq!(
config.retry_policy().delay_for_attempts(3),
Duration::from_secs(8)
);
assert_eq!(config.item_timeout(), Some(Duration::from_secs(10)));
assert_eq!(config.batch_timeout(), Some(Duration::from_secs(60)));
}
#[test]
fn worker_errors_are_classified() {
assert_eq!(
WorkerConfigError::InvalidOutboxItemTimeout.error_kind(),
ErrorKind::Validation
);
assert_eq!(
WorkerConfigError::InvalidOutboxBatchTimeout.error_code(),
"outbox_invalid_batch_timeout"
);
assert_eq!(
WorkerRunError::MissingLeaseStore.error_kind(),
ErrorKind::Validation
);
assert_eq!(
WorkerRunError::LeaseLost.error_kind(),
ErrorKind::Unavailable
);
assert_eq!(
WorkerLeaseError::LockPoisoned.error_kind(),
ErrorKind::Unavailable
);
}
#[tokio::test]
async fn worker_loop_stops_on_shutdown() {
let (trigger, token) = shutdown_channel();
let calls = Arc::new(AtomicUsize::new(0));
let worker = CountingWorker {
calls: Arc::clone(&calls),
};
let handle = tokio::spawn(run_worker_until_shutdown(
worker,
WorkerConfig::default().with_poll_interval(Duration::from_millis(1)),
token,
));
time::sleep(Duration::from_millis(5)).await;
trigger.shutdown();
let summary = handle.await.unwrap().unwrap();
assert!(summary.successes() > 0);
assert_eq!(summary.failures(), 0);
assert_eq!(summary.last_outcome(), Some(WorkerTickOutcome::Success));
assert_eq!(summary.state(), WorkerRunState::Healthy);
assert!(calls.load(Ordering::SeqCst) > 0);
}
#[tokio::test]
async fn local_worker_lease_store_acquires_renews_and_releases() {
let store = LocalWorkerLeaseStore::new();
let config = WorkerLeaseConfig::new(Duration::from_secs(30), Duration::from_secs(10));
let lease = store
.try_acquire("email", "node-a", config)
.await
.expect("lease store should work")
.expect("lease should be acquired");
assert!(
store
.try_acquire("email", "node-b", config)
.await
.expect("lease store should work")
.is_none()
);
let renewed = store
.renew(&lease, config)
.await
.expect("lease store should work")
.expect("lease should renew");
assert_eq!(renewed.worker(), "email");
store
.release(&renewed)
.await
.expect("lease store should work");
assert!(
store
.try_acquire("email", "node-b", config)
.await
.expect("lease store should work")
.is_some()
);
}
#[tokio::test]
async fn local_worker_lease_store_rejects_invalid_inputs() {
let store = LocalWorkerLeaseStore::new();
assert!(matches!(
WorkerLease::new(
" email ",
"node-a",
1,
Instant::now() + Duration::from_secs(30)
),
Err(WorkerLeaseError::InvalidConfig(
WorkerConfigError::InvalidWorkerName { .. }
))
));
assert!(matches!(
store
.try_acquire(
"",
"node-a",
WorkerLeaseConfig::new(Duration::from_secs(30), Duration::from_secs(10)),
)
.await,
Err(WorkerLeaseError::InvalidConfig(
WorkerConfigError::InvalidWorkerName { .. }
))
));
assert!(matches!(
store
.try_acquire(
"email",
"",
WorkerLeaseConfig::new(Duration::from_secs(30), Duration::from_secs(10)),
)
.await,
Err(WorkerLeaseError::InvalidConfig(
WorkerConfigError::InvalidLeaseOwner { .. }
))
));
assert!(matches!(
store
.try_acquire(
"email",
"node-a",
WorkerLeaseConfig::new(Duration::ZERO, Duration::from_secs(10)),
)
.await,
Err(WorkerLeaseError::InvalidConfig(
WorkerConfigError::InvalidLeaseDuration
))
));
assert!(matches!(
store
.try_acquire(
"email",
"node-a",
WorkerLeaseConfig::new(Duration::MAX, Duration::from_secs(1)),
)
.await,
Err(WorkerLeaseError::InvalidConfig(
WorkerConfigError::InvalidLeaseDuration
))
));
}
#[tokio::test]
async fn worker_lease_renewal_uses_injected_runtime_clock() {
let clock = ManualClock::new();
let store = LocalWorkerLeaseStore::with_clock(clock.shared());
let config = WorkerLeaseConfig::new(Duration::from_secs(30), Duration::from_secs(10));
let lease = store
.try_acquire("email", "node-a", config)
.await
.expect("lease store should work")
.expect("lease should be acquired");
let unchanged = ensure_worker_lease(
&store,
Some(lease.clone()),
"email",
"node-a",
config,
clock.now(),
None,
)
.await
.expect("lease check should work")
.expect("lease should still be held");
assert_eq!(unchanged.expires_at(), lease.expires_at());
clock.advance(Duration::from_secs(25));
let renewed = ensure_worker_lease(
&store,
Some(unchanged),
"email",
"node-a",
config,
clock.now(),
None,
)
.await
.expect("lease check should work")
.expect("lease should renew");
assert!(renewed.expires_at() > lease.expires_at());
}
#[tokio::test]
async fn leased_worker_requires_store() {
let (_, token) = shutdown_channel();
let result = run_worker_until_shutdown(
CountingWorker {
calls: Arc::new(AtomicUsize::new(0)),
},
WorkerConfig::default().with_lease(Some(WorkerLeaseConfig::new(
Duration::from_secs(30),
Duration::from_secs(10),
))),
token,
)
.await;
assert!(matches!(result, Err(WorkerRunError::MissingLeaseStore)));
}
#[tokio::test]
async fn worker_loop_rejects_invalid_worker_name_and_lease_owner() {
let (_, token) = shutdown_channel();
let result =
run_worker_until_shutdown(NamedWorker(""), WorkerConfig::default(), token).await;
assert!(matches!(
result,
Err(WorkerRunError::InvalidConfig(
WorkerConfigError::InvalidWorkerName { .. }
))
));
let store = Arc::new(LocalWorkerLeaseStore::new());
let (_, token) = shutdown_channel();
let result = run_worker_until_shutdown_with_options(
NamedWorker("email"),
WorkerConfig::default().with_lease(Some(WorkerLeaseConfig::new(
Duration::from_secs(30),
Duration::from_secs(10),
))),
token,
WorkerRunOptions::new()
.with_lease_store("", Arc::clone(&store) as Arc<dyn WorkerLeaseStore>),
)
.await;
assert!(matches!(
result,
Err(WorkerRunError::InvalidConfig(
WorkerConfigError::InvalidLeaseOwner { .. }
))
));
}
#[tokio::test]
async fn leased_worker_runs_and_releases_on_shutdown() {
let store = Arc::new(LocalWorkerLeaseStore::new());
let (trigger, token) = shutdown_channel();
let calls = Arc::new(AtomicUsize::new(0));
let worker = CountingWorker {
calls: Arc::clone(&calls),
};
let config = WorkerLeaseConfig::new(Duration::from_secs(30), Duration::from_secs(10));
let options = WorkerRunOptions::new()
.with_lease_store("node-a", Arc::clone(&store) as Arc<dyn WorkerLeaseStore>);
let handle = tokio::spawn(run_worker_until_shutdown_with_options(
worker,
WorkerConfig::default()
.with_poll_interval(Duration::from_millis(1))
.with_lease(Some(config)),
token,
options,
));
time::sleep(Duration::from_millis(5)).await;
trigger.shutdown();
let summary = handle.await.unwrap().unwrap();
assert!(summary.successes() > 0);
assert!(calls.load(Ordering::SeqCst) > 0);
assert!(
store
.try_acquire("counting", "node-b", config)
.await
.expect("lease store should work")
.is_some()
);
}
#[tokio::test]
async fn leased_worker_renews_while_tick_is_running() {
let store = Arc::new(LocalWorkerLeaseStore::new());
let (trigger, token) = shutdown_channel();
let calls = Arc::new(AtomicUsize::new(0));
let worker = SlowWorker {
calls: Arc::clone(&calls),
duration: Duration::from_millis(500),
};
let lease_config =
WorkerLeaseConfig::new(Duration::from_millis(200), Duration::from_millis(150));
let options = WorkerRunOptions::new()
.with_lease_store("node-a", Arc::clone(&store) as Arc<dyn WorkerLeaseStore>);
let handle = tokio::spawn(run_worker_until_shutdown_with_options(
worker,
WorkerConfig::default()
.with_poll_interval(Duration::from_secs(1))
.with_lease(Some(lease_config)),
token,
options,
));
time::sleep(Duration::from_millis(350)).await;
assert_eq!(calls.load(Ordering::SeqCst), 1);
assert!(
store
.try_acquire("slow", "node-b", lease_config)
.await
.expect("lease store should work")
.is_none(),
"another owner must not acquire the lease during a long-running tick"
);
trigger.shutdown();
let summary = handle.await.unwrap().unwrap();
assert_eq!(summary.successes(), 1);
}
}