use std::{
fmt,
future::Future,
pin::pin,
sync::Arc,
time::{Duration, Instant},
};
use thiserror::Error;
use tokio::{select, sync::watch, time};
use crate::{
clock::{SharedClock, system_clock},
observability::{
MetricLabels, MetricRecorder, RecordingFailures, metric_labels, metric_names,
metric_outcomes,
},
result::{ClassifyError, ErrorKind},
};
use super::config::{normalize_worker_identifier, validate_lease_owner, validate_worker_name};
use super::status::StatusPublisher;
use super::{
BoxWorkerError, OutboxBatchError, RetryBackoff, Worker, WorkerConfig, WorkerConfigError,
WorkerContext, WorkerLease, WorkerLeaseConfig, WorkerLeaseError, WorkerLeaseStore, WorkerPhase,
WorkerSchedule, WorkerStatus, WorkerWakeup,
};
#[non_exhaustive]
pub struct WorkerRunOptions {
metrics: Option<Arc<dyn MetricRecorder>>,
clock: SharedClock,
lease_store: Option<Arc<dyn WorkerLeaseStore>>,
lease_owner: String,
wakeup: Option<WorkerWakeup>,
status: Option<StatusPublisher>,
}
impl WorkerRunOptions {
#[must_use]
pub fn new() -> Self {
Self {
metrics: None,
clock: system_clock(),
lease_store: None,
lease_owner: "local".to_owned(),
wakeup: None,
status: None,
}
}
#[must_use]
pub fn with_metrics(mut self, metrics: Option<Arc<dyn MetricRecorder>>) -> Self {
self.metrics = metrics;
self
}
#[must_use]
pub fn with_clock(mut self, clock: SharedClock) -> Self {
if let Some(status) = &mut self.status {
status.set_clock(clock.clone());
}
self.clock = clock;
self
}
#[must_use]
pub fn subscribe_status(&mut self) -> watch::Receiver<WorkerStatus> {
match &self.status {
Some(status) => status.subscribe(),
None => {
let (status, receiver) = StatusPublisher::new(self.clock.clone());
self.status = Some(status);
receiver
}
}
}
#[must_use]
pub fn with_wakeup(mut self, wakeup: WorkerWakeup) -> Self {
self.wakeup = Some(wakeup);
self
}
#[must_use]
pub fn with_lease_store(
mut self,
owner: impl Into<String>,
store: Arc<dyn WorkerLeaseStore>,
) -> Self {
self.lease_owner = normalize_worker_identifier(owner);
self.lease_store = Some(store);
self
}
#[must_use]
pub fn metrics(&self) -> Option<&Arc<dyn MetricRecorder>> {
self.metrics.as_ref()
}
#[must_use]
pub fn clock(&self) -> &SharedClock {
&self.clock
}
#[must_use]
pub fn lease_store(&self) -> Option<&Arc<dyn WorkerLeaseStore>> {
self.lease_store.as_ref()
}
#[must_use]
pub fn lease_owner(&self) -> &str {
&self.lease_owner
}
}
impl Default for WorkerRunOptions {
fn default() -> Self {
Self::new()
}
}
impl fmt::Debug for WorkerRunOptions {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("WorkerRunOptions")
.field("metrics_enabled", &self.metrics.is_some())
.field("lease_enabled", &self.lease_store.is_some())
.field("lease_owner", &self.lease_owner)
.field("wakeup_enabled", &self.wakeup.is_some())
.field("status_enabled", &self.status.is_some())
.finish_non_exhaustive()
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[non_exhaustive]
pub struct WorkerTick {
processed: u64,
schedule: WorkerSchedule,
}
impl WorkerTick {
#[must_use]
pub const fn new(processed: u64) -> Self {
Self {
processed,
schedule: WorkerSchedule::Poll,
}
}
#[must_use]
pub const fn with_schedule(mut self, schedule: WorkerSchedule) -> Self {
self.schedule = schedule;
self
}
#[must_use]
pub const fn schedule(&self) -> WorkerSchedule {
self.schedule
}
#[must_use]
pub const fn processed(&self) -> u64 {
self.processed
}
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
#[non_exhaustive]
pub struct WorkerRunSummary {
successes: u64,
failures: u64,
processed: u64,
last_outcome: Option<WorkerTickOutcome>,
}
impl WorkerRunSummary {
#[must_use]
pub const fn ticks(&self) -> u64 {
self.successes + self.failures
}
#[must_use]
pub const fn successes(&self) -> u64 {
self.successes
}
#[must_use]
pub const fn failures(&self) -> u64 {
self.failures
}
#[must_use]
pub const fn processed(&self) -> u64 {
self.processed
}
#[must_use]
pub const fn last_outcome(&self) -> Option<WorkerTickOutcome> {
self.last_outcome
}
#[must_use]
pub const fn successful(&self) -> bool {
self.failures == 0
}
#[must_use]
pub const fn state(&self) -> WorkerRunState {
match (self.successes, self.failures) {
(0, 0) => WorkerRunState::Idle,
(_, 0) => WorkerRunState::Healthy,
(0, _) => WorkerRunState::Failing,
_ => WorkerRunState::Degraded,
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[non_exhaustive]
pub enum WorkerTickOutcome {
Success,
Failure,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[non_exhaustive]
pub enum WorkerRunState {
Idle,
Healthy,
Degraded,
Failing,
}
#[derive(Clone)]
pub struct ShutdownTrigger {
sender: watch::Sender<bool>,
}
impl ShutdownTrigger {
pub fn shutdown(&self) {
let _ = self.sender.send(true);
}
}
impl fmt::Debug for ShutdownTrigger {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("ShutdownTrigger")
.finish_non_exhaustive()
}
}
#[derive(Clone, Debug)]
pub struct ShutdownToken {
receiver: watch::Receiver<bool>,
}
impl ShutdownToken {
#[must_use]
pub fn is_shutdown_requested(&self) -> bool {
*self.receiver.borrow() || self.receiver.has_changed().is_err()
}
pub async fn wait(&mut self) {
if self.is_shutdown_requested() {
return;
}
let _ = self.receiver.changed().await;
}
}
#[must_use]
pub fn shutdown_channel() -> (ShutdownTrigger, ShutdownToken) {
let (sender, receiver) = watch::channel(false);
(ShutdownTrigger { sender }, ShutdownToken { receiver })
}
pub async fn run_worker_until_shutdown<W>(
worker: W,
config: WorkerConfig,
shutdown: ShutdownToken,
) -> Result<WorkerRunSummary, WorkerRunError>
where
W: Worker,
{
run_worker_until_shutdown_with_options(worker, config, shutdown, WorkerRunOptions::new()).await
}
pub async fn run_worker_until_shutdown_with_options<W>(
worker: W,
config: WorkerConfig,
shutdown: ShutdownToken,
mut options: WorkerRunOptions,
) -> Result<WorkerRunSummary, WorkerRunError>
where
W: Worker,
{
let status = options.status.take();
let result = run_worker(worker, config, shutdown, options, status.as_ref()).await;
if let Some(status) = &status {
status.set_phase(if result.is_ok() {
WorkerPhase::Stopped
} else {
WorkerPhase::Failed
});
}
result
}
async fn run_worker<W>(
worker: W,
config: WorkerConfig,
mut shutdown: ShutdownToken,
options: WorkerRunOptions,
status: Option<&StatusPublisher>,
) -> Result<WorkerRunSummary, WorkerRunError>
where
W: Worker,
{
let mut summary = WorkerRunSummary::default();
if !config.enabled {
return Ok(summary);
}
validate_worker_name(worker.name()).map_err(WorkerRunError::InvalidConfig)?;
let metrics = WorkerMetrics {
recorder: options.metrics.as_deref(),
failures: RecordingFailures::default(),
};
let clock = options.clock;
let lease_owner = options.lease_owner;
let wakeup = options.wakeup;
if config.lease.is_some() {
validate_lease_owner(&lease_owner).map_err(WorkerRunError::InvalidConfig)?;
}
let lease_runtime = match (config.lease, options.lease_store) {
(Some(config), Some(store)) => Some((config, store)),
(Some(_), None) => return Err(WorkerRunError::MissingLeaseStore),
(None, _) => None,
};
let mut active_lease = None;
if !config.startup_delay.is_zero() {
if let Some(status) = status {
status.set_phase(WorkerPhase::StartupDelay);
}
if !delay_elapsed_before_shutdown(config.startup_delay, &mut shutdown).await? {
return Ok(summary);
}
}
let mut backoff = RetryBackoff::new(config.failure_backoff);
let run_result = 'run: loop {
if shutdown.is_shutdown_requested() {
break 'run Ok(());
}
if let Some((lease_config, lease_store)) = &lease_runtime {
if let Some(status) = status {
status.set_phase(WorkerPhase::LeaseWait);
}
{
let mut lease_operation = pin!(ensure_worker_lease(
lease_store.as_ref(),
&mut active_lease,
worker.name(),
&lease_owner,
*lease_config,
clock.now(),
&metrics,
));
select! {
result = &mut lease_operation => match result {
Ok(()) => {},
Err(error) => break 'run Err(error),
},
() = shutdown.wait() => break 'run Ok(()),
};
}
if active_lease.is_none() {
if delay_elapsed_before_shutdown(config.poll_interval, &mut shutdown).await? {
continue;
}
break 'run Ok(());
}
}
if shutdown.is_shutdown_requested() {
break 'run Ok(());
}
let tick_timer_started_at = time::Instant::now();
let tick_started_at = clock.now();
let tick_shutdown = shutdown.clone();
let (tick_result, shutdown_deadline) = {
let mut running_tick = pin!(async {
if tick_shutdown.is_shutdown_requested() {
return Ok(LeaseAwareTickResult::NotStarted);
}
if let Some(status) = status {
status.tick_started();
}
run_worker_once_with_lease(
&worker,
lease_runtime.as_ref(),
&mut active_lease,
&clock,
&metrics,
config.tick_timeout,
)
.await
});
select! {
result = &mut running_tick => match result? {
LeaseAwareTickResult::Completed(result) => (Some(result), None),
LeaseAwareTickResult::NotStarted => {
if let Some(status) = status {
status.set_phase(WorkerPhase::Shutdown);
}
let deadline = time::Instant::now()
.checked_add(config.shutdown_timeout)
.ok_or(WorkerRunError::ShutdownTimedOut)?;
(None, Some(deadline))
}
},
() = shutdown.wait() => {
if let Some(status) = status {
status.set_phase(WorkerPhase::Shutdown);
}
let deadline = time::Instant::now()
.checked_add(config.shutdown_timeout)
.ok_or(WorkerRunError::ShutdownTimedOut)?;
let tick_result = time::timeout_at(deadline, &mut running_tick)
.await
.map_err(|_| WorkerRunError::ShutdownTimedOut)??;
match tick_result {
LeaseAwareTickResult::Completed(result) => (Some(result), Some(deadline)),
LeaseAwareTickResult::NotStarted => (None, Some(deadline)),
}
}
}
};
let schedule = tick_result.map_or(WorkerSchedule::Poll, |tick_result| {
if let Some(status) = status {
status.tick_finished(if tick_result.is_ok() {
WorkerTickOutcome::Success
} else {
WorkerTickOutcome::Failure
});
}
record_worker_tick(
&mut summary,
tick_result,
&mut backoff,
config.failure_backoff_jitter,
&metrics,
worker.name(),
clock.now().saturating_duration_since(tick_started_at),
)
});
if let Some(deadline) = shutdown_deadline {
time::timeout_at(
deadline,
release_worker_lease(
lease_store_ref(&lease_runtime),
active_lease.take(),
&metrics,
),
)
.await
.map_err(|_| WorkerRunError::ShutdownTimedOut)??;
return Ok(summary);
}
if let Some(status) = status {
status.set_phase(match schedule {
WorkerSchedule::Poll => WorkerPhase::Poll,
WorkerSchedule::After(_)
if summary.last_outcome == Some(WorkerTickOutcome::Failure) =>
{
WorkerPhase::Backoff
}
WorkerSchedule::After(_) => WorkerPhase::Delay,
WorkerSchedule::Continue => WorkerPhase::QueueSpacing,
WorkerSchedule::WaitForWork { .. } => WorkerPhase::QueueIdle,
});
}
match wait_for_next_tick(
schedule,
&config,
&mut shutdown,
wakeup.as_ref(),
tick_timer_started_at,
status,
)
.await
{
Ok(true) => {}
Ok(false) => break 'run Ok(()),
Err(error) => break 'run Err(error),
}
};
if let Some(status) = status {
status.set_phase(WorkerPhase::Shutdown);
}
let release_result = shutdown_worker_lease(
lease_store_ref(&lease_runtime),
active_lease,
&metrics,
config.shutdown_timeout,
)
.await;
match (run_result, release_result) {
(Err(error), Err(release_error)) => {
tracing::warn!(
worker = worker.name(),
error_kind = release_error.error_kind().as_str(),
error_code = %release_error.error_code(),
"worker lease release failed after the worker stopped with an error"
);
Err(error)
}
(Err(error), _) | (_, Err(error)) => Err(error),
(Ok(()), Ok(())) => Ok(summary),
}
}
async fn delay_elapsed_before_shutdown(
delay: Duration,
shutdown: &mut ShutdownToken,
) -> Result<bool, WorkerRunError> {
wait_for_delay(delay, shutdown, None).await
}
async fn wait_for_delay(
delay: Duration,
shutdown: &mut ShutdownToken,
wakeup: Option<&WorkerWakeup>,
) -> Result<bool, WorkerRunError> {
if shutdown.is_shutdown_requested() {
return Ok(false);
}
let deadline = time::Instant::now()
.checked_add(delay)
.ok_or(WorkerRunError::DelayOutOfRange)?;
Ok(select! {
biased;
() = shutdown.wait() => false,
() = time::sleep_until(deadline) => true,
() = async {
match wakeup {
Some(wakeup) => wakeup.wait().await,
None => std::future::pending().await,
}
} => true,
})
}
async fn wait_for_next_tick(
schedule: WorkerSchedule,
config: &WorkerConfig,
shutdown: &mut ShutdownToken,
wakeup: Option<&WorkerWakeup>,
tick_started_at: time::Instant,
status: Option<&StatusPublisher>,
) -> Result<bool, WorkerRunError> {
match schedule {
WorkerSchedule::Poll => {
return delay_elapsed_before_shutdown(config.poll_interval, shutdown).await;
}
WorkerSchedule::After(delay) => {
return delay_elapsed_before_shutdown(delay, shutdown).await;
}
WorkerSchedule::Continue => {}
WorkerSchedule::WaitForWork { recheck_after } => {
let delay = recheck_after.map_or(config.poll_interval, |delay| {
delay.min(config.poll_interval)
});
if !wait_for_delay(delay, shutdown, wakeup).await? {
return Ok(false);
}
}
}
if let Some(status) = status {
status.set_phase(WorkerPhase::QueueSpacing);
}
let spacing = config.min_queue_interval.min(config.poll_interval);
delay_elapsed_before_shutdown(spacing.saturating_sub(tick_started_at.elapsed()), shutdown).await
}
async fn run_worker_once_with_lease<W>(
worker: &W,
lease_runtime: Option<&(WorkerLeaseConfig, Arc<dyn WorkerLeaseStore>)>,
active_lease: &mut Option<WorkerLease>,
clock: &SharedClock,
metrics: &WorkerMetrics<'_>,
tick_timeout: Option<Duration>,
) -> Result<LeaseAwareTickResult, WorkerRunError>
where
W: Worker,
{
let context = WorkerContext {
fencing_token: active_lease.as_ref().map(WorkerLease::token),
};
let deadline = tick_timeout
.map(|timeout| {
time::Instant::now()
.checked_add(timeout)
.ok_or(WorkerRunError::DelayOutOfRange)
})
.transpose()?;
let tick = async {
let execution = worker.run_once(context);
match deadline {
Some(deadline) => time::timeout_at(deadline, execution)
.await
.unwrap_or_else(|_| Err(WorkerTickTimeout.into())),
None => execution.await,
}
};
tokio::pin!(tick);
let Some((config, store)) = lease_runtime else {
return Ok(LeaseAwareTickResult::Completed(tick.await));
};
loop {
let Some(lease) = active_lease.as_ref() else {
return Err(WorkerRunError::LeaseLost);
};
let expires_at = lease.expires_at();
let remaining = expires_at.saturating_duration_since(clock.now());
if remaining.is_zero() {
metrics.lease(worker.name(), metric_outcomes::LOST);
*active_lease = None;
return Err(WorkerRunError::LeaseLost);
}
let renew_in = remaining.saturating_sub(config.renew_before());
select! {
biased;
() = time::sleep(renew_in) => {},
result = &mut tick => return Ok(LeaseAwareTickResult::Completed(result)),
}
let lease = lease.clone();
let remaining = expires_at.saturating_duration_since(clock.now());
if remaining.is_zero() {
metrics.lease(worker.name(), metric_outcomes::LOST);
*active_lease = None;
return Err(WorkerRunError::LeaseLost);
}
let mut renew = pin!(bounded_lease_store_operation(
config.operation_timeout(),
store.renew(&lease, *config),
));
let renewed = select! {
biased;
() = time::sleep(remaining) => {
metrics.lease(worker.name(), metric_outcomes::LOST);
*active_lease = None;
return Err(WorkerRunError::LeaseLost);
},
result = &mut renew => result?,
result = &mut tick => return Ok(LeaseAwareTickResult::Completed(result)),
};
match renewed {
Some(renewed) => {
metrics.lease(worker.name(), metric_outcomes::RENEWED);
*active_lease = Some(renewed);
}
None => {
metrics.lease(worker.name(), metric_outcomes::LOST);
*active_lease = None;
return Err(WorkerRunError::LeaseLost);
}
}
}
}
enum LeaseAwareTickResult {
Completed(Result<WorkerTick, BoxWorkerError>),
NotStarted,
}
fn lease_store_ref(
runtime: &Option<(WorkerLeaseConfig, Arc<dyn WorkerLeaseStore>)>,
) -> Option<&dyn WorkerLeaseStore> {
runtime.as_ref().map(|(_, store)| store.as_ref())
}
pub(super) async fn ensure_worker_lease(
store: &dyn WorkerLeaseStore,
active_lease: &mut Option<WorkerLease>,
worker: &str,
owner: &str,
config: WorkerLeaseConfig,
now: Instant,
metrics: &WorkerMetrics<'_>,
) -> Result<(), WorkerRunError> {
if let Some(lease) = active_lease.as_ref() {
if !lease.should_renew(now, config.renew_before()) {
return Ok(());
}
match bounded_lease_store_operation(config.operation_timeout(), store.renew(lease, config))
.await?
{
Some(renewed) => {
metrics.lease(worker, metric_outcomes::RENEWED);
*active_lease = Some(renewed);
return Ok(());
}
None => {
metrics.lease(worker, metric_outcomes::LOST);
}
}
}
match bounded_lease_store_operation(
config.operation_timeout(),
store.try_acquire(worker, owner, config),
)
.await?
{
Some(lease) => {
metrics.lease(worker, metric_outcomes::ACQUIRED);
*active_lease = Some(lease);
}
None => {
metrics.lease(worker, metric_outcomes::BUSY);
*active_lease = None;
}
}
Ok(())
}
async fn bounded_lease_store_operation<F, T>(
timeout: Duration,
operation: F,
) -> Result<T, WorkerRunError>
where
F: Future<Output = Result<T, WorkerLeaseError>>,
{
time::timeout(timeout, operation)
.await
.map_err(|_| WorkerRunError::LeaseOperationTimedOut)?
.map_err(WorkerRunError::Lease)
}
async fn release_worker_lease(
store: Option<&dyn WorkerLeaseStore>,
lease: Option<WorkerLease>,
metrics: &WorkerMetrics<'_>,
) -> Result<(), WorkerRunError> {
let (Some(store), Some(lease)) = (store, lease) else {
return Ok(());
};
store.release(&lease).await?;
metrics.lease(lease.worker(), metric_outcomes::RELEASED);
Ok(())
}
async fn shutdown_worker_lease(
store: Option<&dyn WorkerLeaseStore>,
lease: Option<WorkerLease>,
metrics: &WorkerMetrics<'_>,
timeout: Duration,
) -> Result<(), WorkerRunError> {
time::timeout(timeout, release_worker_lease(store, lease, metrics))
.await
.map_err(|_| WorkerRunError::ShutdownTimedOut)?
}
fn record_worker_tick(
summary: &mut WorkerRunSummary,
result: Result<WorkerTick, BoxWorkerError>,
backoff: &mut RetryBackoff,
jitter: bool,
metrics: &WorkerMetrics<'_>,
worker: &str,
elapsed: Duration,
) -> WorkerSchedule {
match result {
Ok(tick) => {
summary.successes += 1;
summary.processed = summary.processed.saturating_add(tick.processed);
summary.last_outcome = Some(WorkerTickOutcome::Success);
backoff.reset();
metrics.tick(worker, metric_outcomes::SUCCESS, tick.processed, elapsed);
tick.schedule
}
Err(error) => {
summary.failures += 1;
summary.last_outcome = Some(WorkerTickOutcome::Failure);
let delay = backoff.record_failure();
let delay = if jitter {
crate::jitter::retry_delay(delay)
} else {
delay
};
metrics.tick(worker, metric_outcomes::FAILURE, 0, elapsed);
let (error_kind, error_code) = classify_worker_tick_error(error.as_ref());
tracing::warn!(
worker,
error_kind = error_kind.as_str(),
error_code = %error_code,
retry_after_ms = delay.as_millis(),
"worker tick failed"
);
WorkerSchedule::After(delay)
}
}
}
fn classify_worker_tick_error(
error: &(dyn std::error::Error + 'static),
) -> (ErrorKind, crate::result::ErrorCode) {
let mut current = Some(error);
while let Some(source) = current {
if let Some(timeout) = source.downcast_ref::<WorkerTickTimeout>() {
return (timeout.error_kind(), timeout.error_code());
}
if let Some(outbox) = source.downcast_ref::<OutboxBatchError>() {
return (outbox.error_kind(), outbox.error_code());
}
current = source.source();
}
(
ErrorKind::Internal,
crate::result::ErrorCode::internal("worker_tick_failed"),
)
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Error)]
#[error("worker tick exceeded its configured deadline")]
pub struct WorkerTickTimeout;
impl ClassifyError for WorkerTickTimeout {
fn error_kind(&self) -> ErrorKind {
ErrorKind::Timeout
}
fn error_code(&self) -> crate::result::ErrorCode {
crate::result::ErrorCode::internal("worker_tick_timed_out")
}
}
#[derive(Default)]
pub(super) struct WorkerMetrics<'a> {
recorder: Option<&'a dyn MetricRecorder>,
failures: RecordingFailures,
}
impl WorkerMetrics<'_> {
fn tick(&self, worker: &str, outcome: &'static str, processed: u64, elapsed: Duration) {
let Some(metrics) = self.recorder else {
return;
};
let labels = MetricLabels::new([
(metric_labels::WORKER, worker),
(metric_labels::OUTCOME, outcome),
]);
self.failures.record(
labels.clone().and_then(|labels| {
metrics.increment_counter(metric_names::WORKER_TICKS_TOTAL, 1, labels)
}),
"worker",
);
self.failures.record(
labels.and_then(|labels| {
metrics.observe_histogram(
metric_names::WORKER_TICK_DURATION_SECONDS,
elapsed.as_secs_f64(),
labels,
)
}),
"worker",
);
if processed > 0 {
self.failures.record(
MetricLabels::new([(metric_labels::WORKER, worker)]).and_then(|labels| {
metrics.increment_counter(
metric_names::WORKER_PROCESSED_TOTAL,
processed,
labels,
)
}),
"worker",
);
}
}
fn lease(&self, worker: &str, outcome: &'static str) {
let Some(metrics) = self.recorder else {
return;
};
self.failures.record(
MetricLabels::new([
(metric_labels::WORKER, worker),
(metric_labels::OUTCOME, outcome),
])
.and_then(|labels| {
metrics.increment_counter(metric_names::WORKER_LEASE_DECISIONS_TOTAL, 1, labels)
}),
"worker",
);
}
}
#[derive(Debug, Error)]
#[non_exhaustive]
pub enum WorkerRunError {
#[error("invalid worker configuration")]
InvalidConfig(#[from] WorkerConfigError),
#[error("worker lease config requires a lease store")]
MissingLeaseStore,
#[error("worker delay exceeds the runtime timer range")]
DelayOutOfRange,
#[error("worker did not finish before the graceful shutdown deadline")]
ShutdownTimedOut,
#[error("worker lease was lost while a tick was running")]
LeaseLost,
#[error("worker lease acquire or renew operation timed out")]
LeaseOperationTimedOut,
#[error("worker lease operation failed")]
Lease(#[from] WorkerLeaseError),
}
impl ClassifyError for WorkerRunError {
fn error_kind(&self) -> ErrorKind {
match self {
Self::InvalidConfig(source) => source.error_kind(),
Self::MissingLeaseStore | Self::DelayOutOfRange => ErrorKind::Validation,
Self::ShutdownTimedOut => ErrorKind::Timeout,
Self::LeaseLost => ErrorKind::Unavailable,
Self::LeaseOperationTimedOut => ErrorKind::Timeout,
Self::Lease(source) => source.error_kind(),
}
}
fn error_code(&self) -> crate::result::ErrorCode {
match self {
Self::InvalidConfig(source) => source.error_code(),
Self::MissingLeaseStore => {
crate::result::ErrorCode::internal("worker_lease_store_missing")
}
Self::DelayOutOfRange => {
crate::result::ErrorCode::internal("worker_delay_out_of_range")
}
Self::ShutdownTimedOut => {
crate::result::ErrorCode::internal("worker_shutdown_timed_out")
}
Self::LeaseLost => crate::result::ErrorCode::internal("worker_lease_lost"),
Self::LeaseOperationTimedOut => {
crate::result::ErrorCode::internal("worker_lease_operation_timed_out")
}
Self::Lease(source) => source.error_code(),
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::observability::{MetricRegistry, MetricRegistryLimits};
#[test]
fn all_failed_worker_recordings_share_one_failure_counter() {
let registry = MetricRegistry::with_limits(MetricRegistryLimits::new(1, 1).unwrap());
registry
.increment_counter("occupied_total", 1, MetricLabels::default())
.unwrap();
let metrics = WorkerMetrics {
recorder: Some(®istry),
failures: RecordingFailures::default(),
};
metrics.lease("worker", metric_outcomes::ACQUIRED);
metrics.tick(
"worker",
metric_outcomes::SUCCESS,
1,
Duration::from_millis(10),
);
metrics.lease("worker", metric_outcomes::RELEASED);
assert_eq!(metrics.failures.count(), 5);
}
#[test]
fn worker_summary_classifies_every_run_state() {
for ((ticks, successes, failures), expected) in [
((0, 0, 0), WorkerRunState::Idle),
((1, 1, 0), WorkerRunState::Healthy),
((1, 0, 1), WorkerRunState::Failing),
((2, 1, 1), WorkerRunState::Degraded),
] {
let summary = WorkerRunSummary {
successes,
failures,
..WorkerRunSummary::default()
};
assert_eq!(summary.ticks(), ticks);
assert_eq!(summary.state(), expected);
}
}
#[test]
fn worker_tick_error_classification_uses_safe_stable_codes() {
let (kind, code) = classify_worker_tick_error(&WorkerTickTimeout);
assert_eq!(kind, ErrorKind::Timeout);
assert_eq!(code, "worker_tick_timed_out");
let outbox = super::super::BatchRunSummary::new()
.with_persistence_errors(1)
.operational_error()
.expect("transition failure should be operational");
let boxed: BoxWorkerError = outbox.into();
let (kind, code) = classify_worker_tick_error(boxed.as_ref());
assert_eq!(kind, ErrorKind::Dependency);
assert_eq!(code, "outbox_transition_persistence_failed");
let private: BoxWorkerError = std::io::Error::other("private worker detail").into();
let (kind, code) = classify_worker_tick_error(private.as_ref());
assert_eq!(kind, ErrorKind::Internal);
assert_eq!(code, "worker_tick_failed");
assert!(!code.as_str().contains("private worker detail"));
}
}