use std::{future::Future, marker::PhantomData};
use async_trait::async_trait;
use tokio::time::{self, Instant};
use super::super::{BoxWorkerError, Worker, WorkerTick};
use super::{
BatchRunSummary, BatchWorkerJob, OutboxClaim, OutboxHandler, OutboxItem, OutboxItemState,
OutboxRetryPolicy, OutboxStore, OutboxWorkerConfig,
};
#[derive(Debug, Clone)]
#[non_exhaustive]
pub struct OutboxJob<S, H, I> {
name: String,
store: S,
handler: H,
config: OutboxWorkerConfig,
_item: PhantomData<I>,
}
impl<S, H, I> OutboxJob<S, H, I>
where
I: OutboxItem,
{
#[must_use]
pub fn new(name: impl Into<String>, store: S, handler: H, config: OutboxWorkerConfig) -> Self {
Self {
name: name.into(),
store,
handler,
config,
_item: PhantomData,
}
}
#[must_use]
pub fn with_config(mut self, config: OutboxWorkerConfig) -> Self {
self.config = config;
self
}
#[must_use]
pub fn store(&self) -> &S {
&self.store
}
#[must_use]
pub fn handler(&self) -> &H {
&self.handler
}
#[must_use]
pub const fn config(&self) -> OutboxWorkerConfig {
self.config
}
}
#[async_trait]
impl<S, H, I> BatchWorkerJob for OutboxJob<S, H, I>
where
S: OutboxStore<I>,
H: OutboxHandler<I>,
I: OutboxItem,
{
fn name(&self) -> &str {
&self.name
}
async fn run_batch(&self) -> Result<BatchRunSummary, BoxWorkerError> {
let claim = OutboxClaim::new(self.config.batch_size(), self.config.lease_duration());
let mut summary = BatchRunSummary::new();
let batch_deadline = match Instant::now().checked_add(self.config.batch_timeout()) {
Some(deadline) => deadline,
None => {
summary.record_batch_timeout();
return Ok(summary);
}
};
let items = match before_batch_deadline(batch_deadline, self.store.claim_due(claim)).await {
Ok(result) => result?,
Err(()) => {
summary.record_batch_timeout();
return Ok(summary);
}
};
summary = summary.with_claimed(items.len() as u64);
for item in items {
let state =
match handle_outbox_item(&self.handler, &item, self.config, batch_deadline).await {
HandleItemResult::State(state) => state,
HandleItemResult::BatchTimedOut => {
summary.record_batch_timeout();
break;
}
};
match before_batch_deadline(
batch_deadline,
self.store.persist_transition(&item, state.clone()),
)
.await
{
Ok(Ok(())) => {
summary.record_item_state(&state);
if matches!(state, OutboxItemState::Failed { .. }) {
summary.record_dead_letter();
}
}
Ok(Err(_)) => {
summary.record_persistence_error();
tracing::warn!(
worker = self.name(),
error_code = "outbox_transition_persistence_failed",
state = state.as_str(),
"outbox transition persistence failed"
);
continue;
}
Err(()) => {
summary.record_persistence_error();
summary.record_batch_timeout();
tracing::warn!(
worker = self.name(),
error_code = "outbox_transition_persistence_timed_out",
state = state.as_str(),
"outbox transition persistence exceeded the batch deadline"
);
break;
}
}
}
Ok(summary)
}
}
#[derive(Debug, Clone)]
#[non_exhaustive]
pub struct BatchWorker<J> {
job: J,
log_empty_batches: bool,
}
impl<J> BatchWorker<J> {
#[must_use]
pub const fn new(job: J) -> Self {
Self {
job,
log_empty_batches: false,
}
}
#[must_use]
pub const fn with_empty_batch_logging(mut self, enabled: bool) -> Self {
self.log_empty_batches = enabled;
self
}
#[must_use]
pub const fn job(&self) -> &J {
&self.job
}
}
#[async_trait]
impl<J> Worker for BatchWorker<J>
where
J: BatchWorkerJob,
{
fn name(&self) -> &str {
self.job.name()
}
async fn run_once(&self) -> Result<WorkerTick, BoxWorkerError> {
let summary = self.job.run_batch().await?;
if self.log_empty_batches || !summary.is_empty() {
tracing::info!(
worker = self.name(),
claimed = summary.claimed(),
succeeded = summary.succeeded(),
retrying = summary.retrying(),
failed = summary.failed(),
skipped = summary.skipped(),
dead_lettered = summary.dead_lettered(),
persistence_errors = summary.persistence_errors(),
batch_timed_out = summary.batch_timed_out(),
handled = summary.handled(),
"batch worker completed"
);
}
let mut tick = WorkerTick::new(summary.handled());
if let Some(delay) = summary.next_delay() {
tick = tick.with_next_delay(delay);
}
if let Some(error) = summary.operational_error() {
return Err(error.into());
}
Ok(tick)
}
}
fn retry_or_fail_state(
retry_policy: &OutboxRetryPolicy,
prior_attempts: u32,
reason: String,
) -> OutboxItemState {
let attempts_after_current_try = prior_attempts.saturating_add(1);
if retry_policy.should_retry(attempts_after_current_try) {
OutboxItemState::retry(
reason,
retry_policy.delay_for_attempts(attempts_after_current_try),
)
} else {
OutboxItemState::failed(reason)
}
}
async fn handle_outbox_item<H, I>(
handler: &H,
item: &I,
config: OutboxWorkerConfig,
batch_deadline: Instant,
) -> HandleItemResult
where
H: OutboxHandler<I>,
I: OutboxItem,
{
let now = Instant::now();
let item_deadline = match now.checked_add(config.item_timeout()) {
Some(deadline) => deadline,
None => {
return HandleItemResult::State(retry_or_fail_state(
&config.retry_policy(),
item.outbox_attempts(),
"outbox_item_timed_out".to_owned(),
));
}
};
let (deadline, kind) = if batch_deadline <= item_deadline {
(batch_deadline, HandlerTimeout::Batch)
} else {
(item_deadline, HandlerTimeout::Item)
};
let result: Result<OutboxItemState, &'static str> = match (deadline, kind) {
(deadline, kind) if deadline <= now => match kind {
HandlerTimeout::Batch => return HandleItemResult::BatchTimedOut,
HandlerTimeout::Item => Err("outbox_item_timed_out"),
},
(deadline, kind) => match time::timeout_at(deadline, handler.handle(item)).await {
Ok(Ok(state)) => Ok(state),
Ok(Err(_)) => Err("outbox_handler_failed"),
Err(_) if kind == HandlerTimeout::Batch => {
return HandleItemResult::BatchTimedOut;
}
Err(_) => Err("outbox_item_timed_out"),
},
};
HandleItemResult::State(match result {
Ok(OutboxItemState::Retry { reason, .. })
if !config
.retry_policy()
.should_retry(item.outbox_attempts().saturating_add(1)) =>
{
OutboxItemState::failed(reason)
}
Ok(state) => state,
Err(reason) => retry_or_fail_state(
&config.retry_policy(),
item.outbox_attempts(),
reason.to_owned(),
),
})
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum HandlerTimeout {
Item,
Batch,
}
enum HandleItemResult {
State(OutboxItemState),
BatchTimedOut,
}
async fn before_batch_deadline<F, T>(deadline: Instant, future: F) -> Result<T, ()>
where
F: Future<Output = T>,
{
if deadline <= Instant::now() {
return Err(());
}
time::timeout_at(deadline, future).await.map_err(|_| ())
}