footing 0.1.21

Feature-gated foundations for Rust backend services
Documentation
use std::{fmt, future::Future, marker::PhantomData};

use async_trait::async_trait;
use tokio::time::{self, Instant};

use super::super::{BoxWorkerError, Worker, WorkerContext, WorkerSchedule, WorkerTick};
use super::{
    BatchRunSummary, BatchWorkerJob, OutboxClaim, OutboxHandler, OutboxItem, OutboxItemState,
    OutboxRetryPolicy, OutboxStore, OutboxWorkerConfig,
};

/// Generic outbox batch job that can be wrapped in [`BatchWorker`].
#[non_exhaustive]
pub struct OutboxJob<S, H, I> {
    name: String,
    store: S,
    handler: H,
    config: OutboxWorkerConfig,
    queue_scheduling: bool,
    _item: PhantomData<I>,
}

impl<S: Clone, H: Clone, I> Clone for OutboxJob<S, H, I> {
    fn clone(&self) -> Self {
        Self {
            name: self.name.clone(),
            store: self.store.clone(),
            handler: self.handler.clone(),
            config: self.config,
            queue_scheduling: self.queue_scheduling,
            _item: PhantomData,
        }
    }
}

impl<S: fmt::Debug, H: fmt::Debug, I> fmt::Debug for OutboxJob<S, H, I> {
    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
        formatter
            .debug_struct("OutboxJob")
            .field("name", &self.name)
            .field("store", &self.store)
            .field("handler", &self.handler)
            .field("config", &self.config)
            .field("queue_scheduling", &self.queue_scheduling)
            .finish_non_exhaustive()
    }
}

impl<S, H, I> OutboxJob<S, H, I>
where
    I: OutboxItem,
{
    #[must_use]
    /// Creates a new instance.
    pub fn new(name: impl Into<String>, store: S, handler: H, config: OutboxWorkerConfig) -> Self {
        Self {
            name: name.into(),
            store,
            handler,
            config,
            queue_scheduling: false,
            _item: PhantomData,
        }
    }

    #[must_use]
    /// Configures config and returns the updated value.
    pub fn with_config(mut self, config: OutboxWorkerConfig) -> Self {
        self.config = config;
        self
    }

    /// Enables continuous consumption and wakeable idle waits for this job.
    ///
    /// A full claim requests another batch; a shorter claim waits for work.
    /// This is only a hint, so the store must always claim only due, available
    /// items and persist retries with their due times. Item outcomes do not
    /// decide whether more work exists. Operational errors still fail the tick.
    ///
    /// Retry delays in the current batch can request an earlier idle check;
    /// polling covers other retries and missed notifications. By default, this
    /// job uses `Poll`; a retry never postpones checking for other due items.
    #[must_use]
    pub const fn with_queue_scheduling(mut self) -> Self {
        self.queue_scheduling = true;
        self
    }

    #[must_use]
    /// Returns the store.
    pub fn store(&self) -> &S {
        &self.store
    }

    #[must_use]
    /// Returns the handler.
    pub fn handler(&self) -> &H {
        &self.handler
    }

    #[must_use]
    /// Returns the config.
    pub const fn config(&self) -> OutboxWorkerConfig {
        self.config
    }
}

#[async_trait]
impl<S, H, I> BatchWorkerJob for OutboxJob<S, H, I>
where
    S: OutboxStore<I>,
    H: OutboxHandler<I>,
    I: OutboxItem,
{
    fn name(&self) -> &str {
        &self.name
    }

    async fn run_batch(&self, context: WorkerContext) -> Result<BatchRunSummary, BoxWorkerError> {
        let claim = OutboxClaim::new(self.config.batch_size(), self.config.lease_duration());
        let mut summary = BatchRunSummary::new();
        let batch_deadline = match Instant::now().checked_add(self.config.batch_timeout()) {
            Some(deadline) => deadline,
            None => {
                summary.record_batch_timeout();
                return Ok(summary);
            }
        };
        let items =
            match before_batch_deadline(batch_deadline, self.store.claim_due(claim, context)).await
            {
                Ok(result) => result?,
                Err(()) => {
                    summary.record_batch_timeout();
                    return Ok(summary);
                }
            };
        summary = summary.with_claimed(items.len() as u64);
        let mut next_retry_at: Option<Instant> = None;

        for item in items {
            let state = match handle_outbox_item(
                &self.handler,
                &item,
                context,
                self.config,
                batch_deadline,
            )
            .await
            {
                HandleItemResult::State(state) => state,
                HandleItemResult::BatchTimedOut => {
                    summary.record_batch_timeout();
                    break;
                }
            };

            match before_batch_deadline(
                batch_deadline,
                self.store.persist_transition(&item, state.clone(), context),
            )
            .await
            {
                Ok(Ok(())) => {
                    // Persistence may start the retry delay at commit time.
                    if self.queue_scheduling
                        && let OutboxItemState::Retry { delay, .. } = &state
                        && let Some(deadline) = Instant::now().checked_add(*delay)
                    {
                        next_retry_at =
                            Some(next_retry_at.map_or(deadline, |current| current.min(deadline)));
                    }
                    summary.record_item_state(&state);
                    if matches!(state, OutboxItemState::Failed { .. }) {
                        summary.record_dead_letter();
                    }
                }
                Ok(Err(_)) => {
                    summary.record_persistence_error();
                    tracing::warn!(
                        worker = self.name(),
                        error_code = "outbox_transition_persistence_failed",
                        state = state.as_str(),
                        "outbox transition persistence failed"
                    );
                    continue;
                }
                Err(()) => {
                    summary.record_persistence_error();
                    summary.record_batch_timeout();
                    tracing::warn!(
                        worker = self.name(),
                        error_code = "outbox_transition_persistence_timed_out",
                        state = state.as_str(),
                        "outbox transition persistence exceeded the batch deadline"
                    );
                    break;
                }
            }
        }

        let recheck_after =
            next_retry_at.map(|deadline| deadline.saturating_duration_since(Instant::now()));
        let schedule = if self.queue_scheduling {
            if summary.claimed() >= u64::from(self.config.batch_size()) {
                WorkerSchedule::Continue
            } else {
                WorkerSchedule::WaitForWork { recheck_after }
            }
        } else {
            WorkerSchedule::Poll
        };
        Ok(summary.with_schedule(schedule))
    }
}

/// Worker adapter for outbox-like batch jobs.
#[derive(Debug, Clone)]
#[non_exhaustive]
pub struct BatchWorker<J> {
    job: J,
    log_empty_batches: bool,
}

impl<J> BatchWorker<J> {
    #[must_use]
    /// Creates a new instance.
    pub const fn new(job: J) -> Self {
        Self {
            job,
            log_empty_batches: false,
        }
    }

    #[must_use]
    /// Configures empty batch logging and returns the updated value.
    pub const fn with_empty_batch_logging(mut self, enabled: bool) -> Self {
        self.log_empty_batches = enabled;
        self
    }

    #[must_use]
    /// Returns the job.
    pub const fn job(&self) -> &J {
        &self.job
    }
}

#[async_trait]
impl<J> Worker for BatchWorker<J>
where
    J: BatchWorkerJob,
{
    fn name(&self) -> &str {
        self.job.name()
    }

    async fn run_once(&self, context: WorkerContext) -> Result<WorkerTick, BoxWorkerError> {
        let summary = self.job.run_batch(context).await?;
        if self.log_empty_batches || !summary.is_empty() {
            tracing::info!(
                worker = self.name(),
                claimed = summary.claimed(),
                succeeded = summary.succeeded(),
                retrying = summary.retrying(),
                failed = summary.failed(),
                skipped = summary.skipped(),
                dead_lettered = summary.dead_lettered(),
                persistence_errors = summary.persistence_errors(),
                batch_timed_out = summary.batch_timed_out(),
                handled = summary.handled(),
                "batch worker completed"
            );
        }

        if let Some(error) = summary.operational_error() {
            return Err(error.into());
        }
        Ok(WorkerTick::new(summary.handled()).with_schedule(summary.schedule()))
    }
}

fn retry_or_fail_state(
    retry_policy: &OutboxRetryPolicy,
    prior_attempts: u32,
    reason: String,
) -> OutboxItemState {
    let attempts_after_current_try = prior_attempts.saturating_add(1);
    if retry_policy.should_retry(attempts_after_current_try) {
        OutboxItemState::retry(
            reason,
            retry_policy.delay_for_attempts(attempts_after_current_try),
        )
    } else {
        OutboxItemState::failed(reason)
    }
}

async fn handle_outbox_item<H, I>(
    handler: &H,
    item: &I,
    context: WorkerContext,
    config: OutboxWorkerConfig,
    batch_deadline: Instant,
) -> HandleItemResult
where
    H: OutboxHandler<I>,
    I: OutboxItem,
{
    let now = Instant::now();
    let item_deadline = match now.checked_add(config.item_timeout()) {
        Some(deadline) => deadline,
        None => {
            return HandleItemResult::State(retry_or_fail_state(
                &config.retry_policy(),
                item.outbox_attempts(),
                "outbox_item_timed_out".to_owned(),
            ));
        }
    };
    if batch_deadline <= now {
        return HandleItemResult::BatchTimedOut;
    }
    let deadline = batch_deadline.min(item_deadline);
    let result = match time::timeout_at(deadline, handler.handle(item, context)).await {
        Ok(Ok(state)) => Ok(state),
        Ok(Err(_)) => Err("outbox_handler_failed"),
        Err(_) if batch_deadline <= item_deadline => {
            return HandleItemResult::BatchTimedOut;
        }
        Err(_) => Err("outbox_item_timed_out"),
    };

    HandleItemResult::State(match result {
        Ok(OutboxItemState::Retry { reason, .. })
            if !config
                .retry_policy()
                .should_retry(item.outbox_attempts().saturating_add(1)) =>
        {
            OutboxItemState::failed(reason)
        }
        Ok(state) => state,
        Err(reason) => retry_or_fail_state(
            &config.retry_policy(),
            item.outbox_attempts(),
            reason.to_owned(),
        ),
    })
}

enum HandleItemResult {
    State(OutboxItemState),
    BatchTimedOut,
}

async fn before_batch_deadline<F, T>(deadline: Instant, future: F) -> Result<T, ()>
where
    F: Future<Output = T>,
{
    if deadline <= Instant::now() {
        return Err(());
    }
    time::timeout_at(deadline, future).await.map_err(|_| ())
}