use std::time::{Duration, Instant};
use serde::{Deserialize, Serialize};
use thiserror::Error;
use tokio::time;
use crate::result::{ClassifyError, ErrorKind};
use crate::settings::{SettingsIssues, SettingsValidationError};
use super::{MAX_WORKER_LEASE_OWNER_LENGTH, MAX_WORKER_NAME_LENGTH};
#[derive(Debug, Clone, PartialEq, Eq)]
#[non_exhaustive]
pub struct WorkerConfig {
pub(super) enabled: bool,
pub(super) poll_interval: Duration,
pub(super) min_queue_interval: Duration,
pub(super) startup_delay: Duration,
pub(super) shutdown_timeout: Duration,
pub(super) failure_backoff: BackoffConfig,
pub(super) lease: Option<WorkerLeaseConfig>,
}
impl Default for WorkerConfig {
fn default() -> Self {
Self {
enabled: true,
poll_interval: Duration::from_secs(10),
min_queue_interval: Duration::from_millis(10),
startup_delay: Duration::ZERO,
shutdown_timeout: Duration::from_secs(30),
failure_backoff: BackoffConfig::default(),
lease: None,
}
}
}
impl WorkerConfig {
#[must_use]
pub fn disabled() -> Self {
Self {
enabled: false,
..Self::default()
}
}
#[must_use]
pub const fn enabled(&self) -> bool {
self.enabled
}
#[must_use]
pub const fn poll_interval(&self) -> Duration {
self.poll_interval
}
#[must_use]
pub const fn min_queue_interval(&self) -> Duration {
self.min_queue_interval
}
#[must_use]
pub const fn startup_delay(&self) -> Duration {
self.startup_delay
}
#[must_use]
pub const fn shutdown_timeout(&self) -> Duration {
self.shutdown_timeout
}
#[must_use]
pub const fn failure_backoff(&self) -> BackoffConfig {
self.failure_backoff
}
#[must_use]
pub const fn lease(&self) -> Option<WorkerLeaseConfig> {
self.lease
}
}
#[cfg_attr(feature = "tier", derive(tier::TierConfig))]
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(default, deny_unknown_fields)]
#[non_exhaustive]
pub struct WorkerSettings {
pub enabled: bool,
#[serde(with = "crate::settings::human_duration")]
pub poll_interval: Duration,
#[serde(with = "crate::settings::human_duration")]
pub min_queue_interval: Duration,
#[serde(with = "crate::settings::human_duration")]
pub startup_delay: Duration,
#[serde(with = "crate::settings::human_duration")]
pub shutdown_timeout: Duration,
pub failure_backoff: BackoffSettings,
pub lease: Option<WorkerLeaseSettings>,
}
impl Default for WorkerSettings {
fn default() -> Self {
Self::from(WorkerConfig::default())
}
}
impl From<WorkerConfig> for WorkerSettings {
fn from(value: WorkerConfig) -> Self {
Self {
enabled: value.enabled,
poll_interval: value.poll_interval,
min_queue_interval: value.min_queue_interval,
startup_delay: value.startup_delay,
shutdown_timeout: value.shutdown_timeout,
failure_backoff: BackoffSettings::from(value.failure_backoff),
lease: value.lease.map(WorkerLeaseSettings::from),
}
}
}
impl WorkerSettings {
#[must_use]
pub const fn with_enabled(mut self, enabled: bool) -> Self {
self.enabled = enabled;
self
}
#[must_use]
pub const fn with_poll_interval(mut self, poll_interval: Duration) -> Self {
self.poll_interval = poll_interval;
self
}
#[must_use]
pub const fn with_min_queue_interval(mut self, interval: Duration) -> Self {
self.min_queue_interval = interval;
self
}
#[must_use]
pub const fn with_startup_delay(mut self, startup_delay: Duration) -> Self {
self.startup_delay = startup_delay;
self
}
#[must_use]
pub const fn with_shutdown_timeout(mut self, shutdown_timeout: Duration) -> Self {
self.shutdown_timeout = shutdown_timeout;
self
}
#[must_use]
pub const fn with_failure_backoff(mut self, failure_backoff: BackoffSettings) -> Self {
self.failure_backoff = failure_backoff;
self
}
#[must_use]
pub const fn with_lease(mut self, lease: Option<WorkerLeaseSettings>) -> Self {
self.lease = lease;
self
}
pub fn build(self) -> Result<WorkerConfig, SettingsValidationError> {
self.validate()?;
Ok(WorkerConfig {
enabled: self.enabled,
poll_interval: self.poll_interval,
min_queue_interval: self.min_queue_interval,
startup_delay: self.startup_delay,
shutdown_timeout: self.shutdown_timeout,
failure_backoff: self.failure_backoff.into_config(),
lease: self.lease.map(WorkerLeaseSettings::into_config),
})
}
pub fn validate(&self) -> Result<(), SettingsValidationError> {
self.validate_at("worker")
}
pub(crate) fn validate_at(&self, path: &str) -> Result<(), SettingsValidationError> {
let mut issues = SettingsIssues::new();
issues.require(
!self.enabled || !self.poll_interval.is_zero(),
format!("{path}.poll_interval"),
"must be greater than zero when worker is enabled",
);
issues.require(
!self.min_queue_interval.is_zero()
&& time::Instant::now()
.checked_add(self.min_queue_interval)
.is_some(),
format!("{path}.min_queue_interval"),
"must be greater than zero and fit the runtime timer range",
);
issues.require(
!self.shutdown_timeout.is_zero()
&& time::Instant::now()
.checked_add(self.shutdown_timeout)
.is_some(),
format!("{path}.shutdown_timeout"),
"must be greater than zero and fit the runtime timer range",
);
issues.extend_result(
self.failure_backoff
.validate_at(&format!("{path}.failure_backoff")),
);
if let Some(lease) = &self.lease {
issues.extend_result(lease.validate_at(&format!("{path}.lease")));
}
issues.into_result()
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[non_exhaustive]
pub struct BackoffConfig {
pub(super) initial_delay: Duration,
pub(super) max_delay: Duration,
pub(super) multiplier: u32,
}
impl Default for BackoffConfig {
fn default() -> Self {
Self {
initial_delay: Duration::from_secs(1),
max_delay: Duration::from_secs(60),
multiplier: 2,
}
}
}
impl BackoffConfig {
pub fn new(
initial_delay: Duration,
max_delay: Duration,
multiplier: u32,
) -> Result<Self, SettingsValidationError> {
BackoffSettings::new(initial_delay, max_delay, multiplier).build()
}
#[must_use]
pub const fn initial_delay(&self) -> Duration {
self.initial_delay
}
#[must_use]
pub const fn max_delay(&self) -> Duration {
self.max_delay
}
#[must_use]
pub const fn multiplier(&self) -> u32 {
self.multiplier
}
}
#[cfg_attr(feature = "tier", derive(tier::TierConfig))]
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
#[serde(default, deny_unknown_fields)]
#[non_exhaustive]
pub struct BackoffSettings {
#[serde(with = "crate::settings::human_duration")]
pub initial_delay: Duration,
#[serde(with = "crate::settings::human_duration")]
pub max_delay: Duration,
pub multiplier: u32,
}
impl Default for BackoffSettings {
fn default() -> Self {
Self::from(BackoffConfig::default())
}
}
impl From<BackoffConfig> for BackoffSettings {
fn from(value: BackoffConfig) -> Self {
Self {
initial_delay: value.initial_delay,
max_delay: value.max_delay,
multiplier: value.multiplier,
}
}
}
impl BackoffSettings {
#[must_use]
pub const fn new(initial_delay: Duration, max_delay: Duration, multiplier: u32) -> Self {
Self {
initial_delay,
max_delay,
multiplier,
}
}
#[must_use]
pub const fn with_initial_delay(mut self, initial_delay: Duration) -> Self {
self.initial_delay = initial_delay;
self
}
#[must_use]
pub const fn with_max_delay(mut self, max_delay: Duration) -> Self {
self.max_delay = max_delay;
self
}
#[must_use]
pub const fn with_multiplier(mut self, multiplier: u32) -> Self {
self.multiplier = multiplier;
self
}
pub fn build(self) -> Result<BackoffConfig, SettingsValidationError> {
self.validate()?;
Ok(self.into_config())
}
pub(super) fn into_config(self) -> BackoffConfig {
BackoffConfig {
initial_delay: self.initial_delay,
max_delay: self.max_delay,
multiplier: self.multiplier,
}
}
pub fn validate(&self) -> Result<(), SettingsValidationError> {
self.validate_at("worker.failure_backoff")
}
pub(crate) fn validate_at(&self, path: &str) -> Result<(), SettingsValidationError> {
let mut issues = SettingsIssues::new();
issues.require(
!self.initial_delay.is_zero(),
format!("{path}.initial_delay"),
"must be greater than zero",
);
issues.require(
self.max_delay >= self.initial_delay,
format!("{path}.max_delay"),
"must be greater than or equal to initial_delay",
);
issues.require(
self.multiplier >= 1,
format!("{path}.multiplier"),
"must be greater than or equal to one",
);
issues.into_result()
}
}
#[derive(Debug, Clone)]
#[non_exhaustive]
pub struct RetryBackoff {
config: BackoffConfig,
next_delay: Duration,
}
impl RetryBackoff {
#[must_use]
pub fn new(config: BackoffConfig) -> Self {
Self {
config,
next_delay: config.initial_delay,
}
}
pub fn record_failure(&mut self) -> Duration {
let current = self.next_delay;
self.next_delay = current
.saturating_mul(self.config.multiplier)
.min(self.config.max_delay);
current
}
pub fn reset(&mut self) {
self.next_delay = self.config.initial_delay;
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[non_exhaustive]
pub struct WorkerLeaseConfig {
pub(super) lease_duration: Duration,
pub(super) renew_before: Duration,
pub(super) operation_timeout: Duration,
}
impl WorkerLeaseConfig {
pub fn new(
lease_duration: Duration,
renew_before: Duration,
operation_timeout: Duration,
) -> Result<Self, SettingsValidationError> {
WorkerLeaseSettings::new(lease_duration, renew_before, operation_timeout).build()
}
#[must_use]
pub const fn lease_duration(&self) -> Duration {
self.lease_duration
}
#[must_use]
pub const fn renew_before(&self) -> Duration {
self.renew_before
}
#[must_use]
pub const fn operation_timeout(&self) -> Duration {
self.operation_timeout
}
}
#[cfg_attr(feature = "tier", derive(tier::TierConfig))]
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
#[serde(deny_unknown_fields)]
#[non_exhaustive]
pub struct WorkerLeaseSettings {
#[serde(with = "crate::settings::human_duration")]
pub lease_duration: Duration,
#[serde(with = "crate::settings::human_duration")]
pub renew_before: Duration,
#[serde(with = "crate::settings::human_duration")]
pub operation_timeout: Duration,
}
impl From<WorkerLeaseConfig> for WorkerLeaseSettings {
fn from(value: WorkerLeaseConfig) -> Self {
Self {
lease_duration: value.lease_duration,
renew_before: value.renew_before,
operation_timeout: value.operation_timeout,
}
}
}
impl WorkerLeaseSettings {
#[must_use]
pub const fn new(
lease_duration: Duration,
renew_before: Duration,
operation_timeout: Duration,
) -> Self {
Self {
lease_duration,
renew_before,
operation_timeout,
}
}
#[must_use]
pub const fn with_lease_duration(mut self, lease_duration: Duration) -> Self {
self.lease_duration = lease_duration;
self
}
#[must_use]
pub const fn with_renew_before(mut self, renew_before: Duration) -> Self {
self.renew_before = renew_before;
self
}
#[must_use]
pub const fn with_operation_timeout(mut self, operation_timeout: Duration) -> Self {
self.operation_timeout = operation_timeout;
self
}
pub fn build(self) -> Result<WorkerLeaseConfig, SettingsValidationError> {
self.validate()?;
Ok(self.into_config())
}
fn into_config(self) -> WorkerLeaseConfig {
WorkerLeaseConfig {
lease_duration: self.lease_duration,
renew_before: self.renew_before,
operation_timeout: self.operation_timeout,
}
}
pub fn validate(&self) -> Result<(), SettingsValidationError> {
self.validate_at("worker.lease")
}
pub(crate) fn validate_at(&self, path: &str) -> Result<(), SettingsValidationError> {
let mut issues = SettingsIssues::new();
issues.require(
!self.lease_duration.is_zero()
&& Instant::now().checked_add(self.lease_duration).is_some(),
format!("{path}.lease_duration"),
"must be greater than zero and fit the monotonic clock range",
);
issues.require(
!self.renew_before.is_zero() && self.renew_before < self.lease_duration,
format!("{path}.renew_before"),
"must be greater than zero and less than lease_duration",
);
issues.require(
!self.operation_timeout.is_zero()
&& self.operation_timeout < self.renew_before
&& time::Instant::now()
.checked_add(self.operation_timeout)
.is_some(),
format!("{path}.operation_timeout"),
"must be greater than zero, fit the runtime timer range, and be less than renew_before",
);
issues.into_result()
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Error)]
#[non_exhaustive]
pub enum WorkerConfigError {
#[error("worker lease_duration must be greater than zero")]
InvalidLeaseDuration,
#[error("worker name must be non-empty and less than or equal to {max} bytes")]
InvalidWorkerName {
#[doc = "The configured upper bound."]
max: usize,
},
#[error("worker lease owner must be non-empty and less than or equal to {max} bytes")]
InvalidLeaseOwner {
#[doc = "The configured upper bound."]
max: usize,
},
#[error("outbox max_attempts must be greater than zero")]
InvalidOutboxMaxAttempts,
}
impl ClassifyError for WorkerConfigError {
fn error_kind(&self) -> ErrorKind {
ErrorKind::Validation
}
fn error_code(&self) -> crate::result::ErrorCode {
crate::result::ErrorCode::internal(match self {
Self::InvalidLeaseDuration => "worker_invalid_lease_duration",
Self::InvalidWorkerName { .. } => "worker_invalid_name",
Self::InvalidLeaseOwner { .. } => "worker_invalid_lease_owner",
Self::InvalidOutboxMaxAttempts => "outbox_invalid_max_attempts",
})
}
}
pub(super) fn normalize_worker_identifier(value: impl Into<String>) -> String {
value.into().trim().to_owned()
}
pub(super) fn validate_worker_name(value: &str) -> Result<(), WorkerConfigError> {
validate_worker_identifier(value, MAX_WORKER_NAME_LENGTH).map_err(|()| {
WorkerConfigError::InvalidWorkerName {
max: MAX_WORKER_NAME_LENGTH,
}
})
}
pub(super) fn validate_lease_owner(value: &str) -> Result<(), WorkerConfigError> {
validate_worker_identifier(value, MAX_WORKER_LEASE_OWNER_LENGTH).map_err(|()| {
WorkerConfigError::InvalidLeaseOwner {
max: MAX_WORKER_LEASE_OWNER_LENGTH,
}
})
}
fn validate_worker_identifier(value: &str, max_len: usize) -> Result<(), ()> {
if value.is_empty()
|| value.len() > max_len
|| value != value.trim()
|| value.chars().any(char::is_control)
{
return Err(());
}
Ok(())
}