use std::{
fmt,
future::Future,
sync::Arc,
time::{Duration, Instant},
};
use thiserror::Error;
use tokio::{select, sync::watch, time};
use crate::{
clock::{SharedClock, system_clock},
observability::{MetricLabels, MetricRecorder, metric_labels, metric_names, metric_outcomes},
result::{ClassifyError, ErrorKind},
};
use super::config::{normalize_worker_identifier, validate_lease_owner, validate_worker_name};
use super::{
BoxWorkerError, OutboxBatchError, RetryBackoff, Worker, WorkerConfig, WorkerConfigError,
WorkerLease, WorkerLeaseConfig, WorkerLeaseError, WorkerLeaseStore, WorkerSchedule,
WorkerWakeup,
};
#[derive(Clone)]
#[non_exhaustive]
pub struct WorkerRunOptions {
metrics: Option<Arc<dyn MetricRecorder>>,
clock: SharedClock,
lease_store: Option<Arc<dyn WorkerLeaseStore>>,
lease_owner: String,
wakeup: Option<WorkerWakeup>,
}
impl WorkerRunOptions {
#[must_use]
pub fn new() -> Self {
Self {
metrics: None,
clock: system_clock(),
lease_store: None,
lease_owner: "local".to_owned(),
wakeup: None,
}
}
#[must_use]
pub fn with_metrics(mut self, metrics: Option<Arc<dyn MetricRecorder>>) -> Self {
self.metrics = metrics;
self
}
#[must_use]
pub fn with_clock(mut self, clock: SharedClock) -> Self {
self.clock = clock;
self
}
#[must_use]
pub fn with_wakeup(mut self, wakeup: WorkerWakeup) -> Self {
self.wakeup = Some(wakeup);
self
}
#[must_use]
pub fn with_lease_store(
mut self,
owner: impl Into<String>,
store: Arc<dyn WorkerLeaseStore>,
) -> Self {
self.lease_owner = normalize_worker_identifier(owner);
self.lease_store = Some(store);
self
}
#[must_use]
pub fn metrics(&self) -> Option<&Arc<dyn MetricRecorder>> {
self.metrics.as_ref()
}
#[must_use]
pub fn clock(&self) -> &SharedClock {
&self.clock
}
#[must_use]
pub fn lease_store(&self) -> Option<&Arc<dyn WorkerLeaseStore>> {
self.lease_store.as_ref()
}
#[must_use]
pub fn lease_owner(&self) -> &str {
&self.lease_owner
}
}
impl Default for WorkerRunOptions {
fn default() -> Self {
Self::new()
}
}
impl fmt::Debug for WorkerRunOptions {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("WorkerRunOptions")
.field("metrics_enabled", &self.metrics.is_some())
.field("lease_enabled", &self.lease_store.is_some())
.field("lease_owner", &self.lease_owner)
.field("wakeup_enabled", &self.wakeup.is_some())
.finish_non_exhaustive()
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[non_exhaustive]
pub struct WorkerTick {
processed: u64,
schedule: WorkerSchedule,
}
impl WorkerTick {
#[must_use]
pub const fn new(processed: u64) -> Self {
Self {
processed,
schedule: WorkerSchedule::Poll,
}
}
#[must_use]
pub const fn with_schedule(mut self, schedule: WorkerSchedule) -> Self {
self.schedule = schedule;
self
}
#[must_use]
pub const fn schedule(&self) -> WorkerSchedule {
self.schedule
}
#[must_use]
pub const fn processed(&self) -> u64 {
self.processed
}
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
#[non_exhaustive]
pub struct WorkerRunSummary {
ticks: u64,
successes: u64,
failures: u64,
processed: u64,
last_outcome: Option<WorkerTickOutcome>,
}
impl WorkerRunSummary {
#[must_use]
pub const fn ticks(&self) -> u64 {
self.ticks
}
#[must_use]
pub const fn successes(&self) -> u64 {
self.successes
}
#[must_use]
pub const fn failures(&self) -> u64 {
self.failures
}
#[must_use]
pub const fn processed(&self) -> u64 {
self.processed
}
#[must_use]
pub const fn last_outcome(&self) -> Option<WorkerTickOutcome> {
self.last_outcome
}
#[must_use]
pub const fn successful(&self) -> bool {
self.failures == 0
}
#[must_use]
pub const fn state(&self) -> WorkerRunState {
match (self.ticks, self.successes, self.failures) {
(0, _, _) => WorkerRunState::Idle,
(_, _, 0) => WorkerRunState::Healthy,
(_, 0, _) => WorkerRunState::Failing,
_ => WorkerRunState::Degraded,
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[non_exhaustive]
pub enum WorkerTickOutcome {
Success,
Failure,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[non_exhaustive]
pub enum WorkerRunState {
Idle,
Healthy,
Degraded,
Failing,
}
#[derive(Clone)]
pub struct ShutdownTrigger {
sender: watch::Sender<bool>,
}
impl ShutdownTrigger {
pub fn shutdown(&self) {
let _ = self.sender.send(true);
}
}
impl fmt::Debug for ShutdownTrigger {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("ShutdownTrigger")
.finish_non_exhaustive()
}
}
#[derive(Clone, Debug)]
pub struct ShutdownToken {
receiver: watch::Receiver<bool>,
}
impl ShutdownToken {
#[must_use]
pub fn is_shutdown_requested(&self) -> bool {
*self.receiver.borrow()
}
async fn wait(&mut self) {
if self.is_shutdown_requested() {
return;
}
let _ = self.receiver.changed().await;
}
}
#[must_use]
pub fn shutdown_channel() -> (ShutdownTrigger, ShutdownToken) {
let (sender, receiver) = watch::channel(false);
(ShutdownTrigger { sender }, ShutdownToken { receiver })
}
pub async fn run_worker_until_shutdown<W>(
worker: W,
config: WorkerConfig,
mut shutdown: ShutdownToken,
) -> Result<WorkerRunSummary, WorkerRunError>
where
W: Worker,
{
run_worker_loop(worker, config, &mut shutdown, WorkerRunOptions::new()).await
}
pub async fn run_worker_until_shutdown_with_metrics<W>(
worker: W,
config: WorkerConfig,
mut shutdown: ShutdownToken,
metrics: Arc<dyn MetricRecorder>,
) -> Result<WorkerRunSummary, WorkerRunError>
where
W: Worker,
{
run_worker_loop(
worker,
config,
&mut shutdown,
WorkerRunOptions::new().with_metrics(Some(metrics)),
)
.await
}
pub async fn run_worker_until_shutdown_with_options<W>(
worker: W,
config: WorkerConfig,
mut shutdown: ShutdownToken,
options: WorkerRunOptions,
) -> Result<WorkerRunSummary, WorkerRunError>
where
W: Worker,
{
run_worker_loop(worker, config, &mut shutdown, options).await
}
async fn run_worker_loop<W>(
worker: W,
config: WorkerConfig,
shutdown: &mut ShutdownToken,
options: WorkerRunOptions,
) -> Result<WorkerRunSummary, WorkerRunError>
where
W: Worker,
{
let mut summary = WorkerRunSummary::default();
if !config.enabled {
return Ok(summary);
}
validate_worker_name(worker.name()).map_err(WorkerRunError::InvalidConfig)?;
let metrics = options.metrics;
let clock = options.clock;
let lease_owner = options.lease_owner;
let wakeup = options.wakeup;
if config.lease.is_some() {
validate_lease_owner(&lease_owner).map_err(WorkerRunError::InvalidConfig)?;
}
let lease_runtime = match (config.lease, options.lease_store) {
(Some(config), Some(store)) => Some((config, store)),
(Some(_), None) => return Err(WorkerRunError::MissingLeaseStore),
(None, _) => None,
};
let mut active_lease = None;
if !config.startup_delay.is_zero()
&& !delay_elapsed_before_shutdown(config.startup_delay, shutdown).await
{
return Ok(summary);
}
let mut backoff = RetryBackoff::new(config.failure_backoff);
'run: loop {
if shutdown.is_shutdown_requested() {
break 'run;
}
if let Some((lease_config, lease_store)) = &lease_runtime {
let mut lease_operation = Box::pin(ensure_worker_lease(
lease_store.as_ref(),
active_lease.as_ref(),
worker.name(),
&lease_owner,
*lease_config,
clock.now(),
metrics.as_deref(),
));
let next_lease = select! {
result = &mut lease_operation => result?,
() = shutdown.wait() => break 'run,
};
drop(lease_operation);
active_lease = next_lease;
if active_lease.is_none() {
if delay_elapsed_before_shutdown(config.poll_interval, shutdown).await {
continue;
}
break 'run;
}
}
if shutdown.is_shutdown_requested() {
break 'run;
}
summary.ticks += 1;
let tick_timer_started_at = time::Instant::now();
let tick_started_at = clock.now();
let mut running_tick = Box::pin(run_worker_once_with_lease(
&worker,
lease_runtime.as_ref(),
&mut active_lease,
&clock,
metrics.as_deref(),
shutdown.clone(),
));
let (tick_result, shutdown_deadline) = select! {
result = &mut running_tick => match result? {
LeaseAwareTickResult::Completed(result) => (Some(result), None),
LeaseAwareTickResult::ShutdownInterrupted => {
let deadline = time::Instant::now()
.checked_add(config.shutdown_timeout)
.ok_or(WorkerRunError::ShutdownTimedOut)?;
(None, Some(deadline))
}
},
() = shutdown.wait() => {
let deadline = time::Instant::now()
.checked_add(config.shutdown_timeout)
.ok_or(WorkerRunError::ShutdownTimedOut)?;
let tick_result = time::timeout_at(deadline, &mut running_tick)
.await
.map_err(|_| WorkerRunError::ShutdownTimedOut)??;
match tick_result {
LeaseAwareTickResult::Completed(result) => (Some(result), Some(deadline)),
LeaseAwareTickResult::ShutdownInterrupted => (None, Some(deadline)),
}
}
};
drop(running_tick);
let schedule = tick_result.map_or(WorkerSchedule::Poll, |tick_result| {
record_worker_tick(
&mut summary,
tick_result,
&mut backoff,
metrics.as_deref(),
worker.name(),
clock.now().saturating_duration_since(tick_started_at),
)
});
if let Some(deadline) = shutdown_deadline {
time::timeout_at(
deadline,
release_worker_lease(
lease_store_ref(&lease_runtime),
active_lease.take(),
metrics.as_deref(),
),
)
.await
.map_err(|_| WorkerRunError::ShutdownTimedOut)??;
return Ok(summary);
}
if !wait_for_next_tick(
schedule,
&config,
shutdown,
wakeup.as_ref(),
tick_timer_started_at,
)
.await
{
break 'run;
}
}
shutdown_worker_lease(
lease_store_ref(&lease_runtime),
active_lease,
metrics.as_deref(),
config.shutdown_timeout,
)
.await?;
Ok(summary)
}
async fn delay_elapsed_before_shutdown(delay: Duration, shutdown: &mut ShutdownToken) -> bool {
wait_for_delay(delay, shutdown, None).await
}
async fn wait_for_delay(
delay: Duration,
shutdown: &mut ShutdownToken,
wakeup: Option<&WorkerWakeup>,
) -> bool {
select! {
biased;
() = shutdown.wait() => false,
() = async {
match time::Instant::now().checked_add(delay) {
Some(deadline) => time::sleep_until(deadline).await,
None => std::future::pending().await,
}
} => true,
() = async {
match wakeup {
Some(wakeup) => wakeup.wait().await,
None => std::future::pending().await,
}
} => true,
}
}
async fn wait_for_next_tick(
schedule: WorkerSchedule,
config: &WorkerConfig,
shutdown: &mut ShutdownToken,
wakeup: Option<&WorkerWakeup>,
tick_started_at: time::Instant,
) -> bool {
match schedule {
WorkerSchedule::Poll => {
return delay_elapsed_before_shutdown(config.poll_interval, shutdown).await;
}
WorkerSchedule::After(delay) => {
return delay_elapsed_before_shutdown(delay, shutdown).await;
}
WorkerSchedule::Continue => {}
WorkerSchedule::WaitForWork { recheck_after } => {
let delay = recheck_after.map_or(config.poll_interval, |delay| {
delay.min(config.poll_interval)
});
if !wait_for_delay(delay, shutdown, wakeup).await {
return false;
}
}
}
let spacing = config.min_queue_interval.min(config.poll_interval);
delay_elapsed_before_shutdown(spacing.saturating_sub(tick_started_at.elapsed()), shutdown).await
}
async fn run_worker_once_with_lease<W>(
worker: &W,
lease_runtime: Option<&(WorkerLeaseConfig, Arc<dyn WorkerLeaseStore>)>,
active_lease: &mut Option<WorkerLease>,
clock: &SharedClock,
metrics: Option<&dyn MetricRecorder>,
mut shutdown: ShutdownToken,
) -> Result<LeaseAwareTickResult, WorkerRunError>
where
W: Worker,
{
if shutdown.is_shutdown_requested() {
return Ok(LeaseAwareTickResult::ShutdownInterrupted);
}
let Some((config, store)) = lease_runtime else {
return Ok(LeaseAwareTickResult::Completed(worker.run_once().await));
};
let mut tick = Box::pin(worker.run_once());
loop {
let Some(lease) = active_lease.as_ref() else {
return Err(WorkerRunError::LeaseLost);
};
let renew_in = lease
.expires_at()
.saturating_duration_since(clock.now())
.saturating_sub(config.renew_before());
select! {
result = &mut tick => return Ok(LeaseAwareTickResult::Completed(result)),
() = time::sleep(renew_in) => {}
}
let lease = lease.clone();
let mut renew = Box::pin(bounded_lease_store_operation(
config.operation_timeout(),
store.renew(&lease, *config),
));
let renewed = select! {
biased;
result = &mut renew => result?,
() = shutdown.wait() => return Ok(LeaseAwareTickResult::ShutdownInterrupted),
};
match renewed {
Some(renewed) => {
record_worker_lease_metric(metrics, worker.name(), metric_outcomes::RENEWED);
*active_lease = Some(renewed);
}
None => {
record_worker_lease_metric(metrics, worker.name(), metric_outcomes::LOST);
*active_lease = None;
return Err(WorkerRunError::LeaseLost);
}
}
}
}
enum LeaseAwareTickResult {
Completed(Result<WorkerTick, BoxWorkerError>),
ShutdownInterrupted,
}
fn lease_store_ref(
runtime: &Option<(WorkerLeaseConfig, Arc<dyn WorkerLeaseStore>)>,
) -> Option<&dyn WorkerLeaseStore> {
runtime.as_ref().map(|(_, store)| store.as_ref())
}
pub(super) async fn ensure_worker_lease(
store: &dyn WorkerLeaseStore,
active_lease: Option<&WorkerLease>,
worker: &str,
owner: &str,
config: WorkerLeaseConfig,
now: Instant,
metrics: Option<&dyn MetricRecorder>,
) -> Result<Option<WorkerLease>, WorkerRunError> {
if let Some(lease) = active_lease {
if !lease.should_renew(now, config.renew_before()) {
return Ok(Some(lease.clone()));
}
match bounded_lease_store_operation(config.operation_timeout(), store.renew(lease, config))
.await?
{
Some(renewed) => {
record_worker_lease_metric(metrics, worker, metric_outcomes::RENEWED);
return Ok(Some(renewed));
}
None => {
record_worker_lease_metric(metrics, worker, metric_outcomes::LOST);
}
}
}
match bounded_lease_store_operation(
config.operation_timeout(),
store.try_acquire(worker, owner, config),
)
.await?
{
Some(lease) => {
record_worker_lease_metric(metrics, worker, metric_outcomes::ACQUIRED);
Ok(Some(lease))
}
None => {
record_worker_lease_metric(metrics, worker, metric_outcomes::BUSY);
Ok(None)
}
}
}
async fn bounded_lease_store_operation<F, T>(
timeout: Duration,
operation: F,
) -> Result<T, WorkerRunError>
where
F: Future<Output = Result<T, WorkerLeaseError>>,
{
time::timeout(timeout, operation)
.await
.map_err(|_| WorkerRunError::LeaseOperationTimedOut)?
.map_err(WorkerRunError::Lease)
}
async fn release_worker_lease(
store: Option<&dyn WorkerLeaseStore>,
lease: Option<WorkerLease>,
metrics: Option<&dyn MetricRecorder>,
) -> Result<(), WorkerRunError> {
let (Some(store), Some(lease)) = (store, lease) else {
return Ok(());
};
store.release(&lease).await?;
record_worker_lease_metric(metrics, lease.worker(), metric_outcomes::RELEASED);
Ok(())
}
async fn shutdown_worker_lease(
store: Option<&dyn WorkerLeaseStore>,
lease: Option<WorkerLease>,
metrics: Option<&dyn MetricRecorder>,
timeout: Duration,
) -> Result<(), WorkerRunError> {
time::timeout(timeout, release_worker_lease(store, lease, metrics))
.await
.map_err(|_| WorkerRunError::ShutdownTimedOut)?
}
fn record_worker_tick(
summary: &mut WorkerRunSummary,
result: Result<WorkerTick, BoxWorkerError>,
backoff: &mut RetryBackoff,
metrics: Option<&dyn MetricRecorder>,
worker: &str,
elapsed: Duration,
) -> WorkerSchedule {
match result {
Ok(tick) => {
summary.successes += 1;
summary.processed = summary.processed.saturating_add(tick.processed);
summary.last_outcome = Some(WorkerTickOutcome::Success);
backoff.reset();
record_worker_metrics(
metrics,
worker,
metric_outcomes::SUCCESS,
tick.processed,
elapsed,
);
tick.schedule
}
Err(error) => {
summary.failures += 1;
summary.last_outcome = Some(WorkerTickOutcome::Failure);
let delay = backoff.record_failure();
record_worker_metrics(metrics, worker, metric_outcomes::FAILURE, 0, elapsed);
let (error_kind, error_code) = classify_worker_tick_error(error.as_ref());
tracing::warn!(
worker,
error_kind = error_kind.as_str(),
error_code = %error_code,
retry_after_ms = delay.as_millis(),
"worker tick failed"
);
WorkerSchedule::After(delay)
}
}
}
fn classify_worker_tick_error(
error: &(dyn std::error::Error + 'static),
) -> (ErrorKind, crate::result::ErrorCode) {
let mut current = Some(error);
while let Some(source) = current {
if let Some(outbox) = source.downcast_ref::<OutboxBatchError>() {
return (outbox.error_kind(), outbox.error_code());
}
current = source.source();
}
(
ErrorKind::Internal,
crate::result::ErrorCode::internal("worker_tick_failed"),
)
}
fn record_worker_metrics(
metrics: Option<&dyn MetricRecorder>,
worker: &str,
outcome: &'static str,
processed: u64,
elapsed: Duration,
) {
let Some(metrics) = metrics else {
return;
};
let labels = MetricLabels::from_lossy([
(metric_labels::WORKER, worker),
(metric_labels::OUTCOME, outcome),
]);
let _ = metrics.increment_counter(metric_names::WORKER_TICKS_TOTAL, 1, labels.clone());
let _ = metrics.observe_histogram(
metric_names::WORKER_TICK_DURATION_SECONDS,
elapsed.as_secs_f64(),
labels,
);
if processed > 0 {
let _ = metrics.increment_counter(
metric_names::WORKER_PROCESSED_TOTAL,
processed,
MetricLabels::from_lossy([(metric_labels::WORKER, worker)]),
);
}
}
fn record_worker_lease_metric(
metrics: Option<&dyn MetricRecorder>,
worker: &str,
outcome: &'static str,
) {
let Some(metrics) = metrics else {
return;
};
let _ = metrics.increment_counter(
metric_names::WORKER_LEASE_DECISIONS_TOTAL,
1,
MetricLabels::from_lossy([
(metric_labels::WORKER, worker),
(metric_labels::OUTCOME, outcome),
]),
);
}
#[derive(Debug, Error)]
#[non_exhaustive]
pub enum WorkerRunError {
#[error("invalid worker configuration")]
InvalidConfig(#[from] WorkerConfigError),
#[error("worker lease config requires a lease store")]
MissingLeaseStore,
#[error("worker did not finish before the graceful shutdown deadline")]
ShutdownTimedOut,
#[error("worker lease was lost while a tick was running")]
LeaseLost,
#[error("worker lease acquire or renew operation timed out")]
LeaseOperationTimedOut,
#[error("worker lease operation failed")]
Lease(#[from] WorkerLeaseError),
}
impl ClassifyError for WorkerRunError {
fn error_kind(&self) -> ErrorKind {
match self {
Self::InvalidConfig(source) => source.error_kind(),
Self::MissingLeaseStore => ErrorKind::Validation,
Self::ShutdownTimedOut => ErrorKind::Timeout,
Self::LeaseLost => ErrorKind::Unavailable,
Self::LeaseOperationTimedOut => ErrorKind::Timeout,
Self::Lease(source) => source.error_kind(),
}
}
fn error_code(&self) -> crate::result::ErrorCode {
match self {
Self::InvalidConfig(source) => source.error_code(),
Self::MissingLeaseStore => {
crate::result::ErrorCode::internal("worker_lease_store_missing")
}
Self::ShutdownTimedOut => {
crate::result::ErrorCode::internal("worker_shutdown_timed_out")
}
Self::LeaseLost => crate::result::ErrorCode::internal("worker_lease_lost"),
Self::LeaseOperationTimedOut => {
crate::result::ErrorCode::internal("worker_lease_operation_timed_out")
}
Self::Lease(source) => source.error_code(),
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn worker_summary_classifies_every_run_state() {
for ((ticks, successes, failures), expected) in [
((0, 0, 0), WorkerRunState::Idle),
((1, 1, 0), WorkerRunState::Healthy),
((1, 0, 1), WorkerRunState::Failing),
((2, 1, 1), WorkerRunState::Degraded),
] {
let summary = WorkerRunSummary {
ticks,
successes,
failures,
..WorkerRunSummary::default()
};
assert_eq!(summary.state(), expected);
}
}
#[test]
fn worker_tick_error_classification_uses_safe_stable_codes() {
let outbox = super::super::BatchRunSummary::new()
.with_persistence_errors(1)
.operational_error()
.expect("transition failure should be operational");
let boxed: BoxWorkerError = outbox.into();
let (kind, code) = classify_worker_tick_error(boxed.as_ref());
assert_eq!(kind, ErrorKind::Dependency);
assert_eq!(code, "outbox_transition_persistence_failed");
let private: BoxWorkerError = std::io::Error::other("private worker detail").into();
let (kind, code) = classify_worker_tick_error(private.as_ref());
assert_eq!(kind, ErrorKind::Internal);
assert_eq!(code, "worker_tick_failed");
assert!(!code.as_str().contains("private worker detail"));
}
}