use std::{fmt, future::Future, marker::PhantomData};
use async_trait::async_trait;
use tokio::time::{self, Instant};
use super::super::{BoxWorkerError, Worker, WorkerContext, WorkerSchedule, WorkerTick};
use super::{
BatchRunSummary, BatchWorkerJob, OutboxClaim, OutboxHandler, OutboxItem, OutboxItemState,
OutboxRetryPolicy, OutboxStore, OutboxWorkerConfig,
};
#[non_exhaustive]
pub struct OutboxJob<S, H, I> {
name: String,
store: S,
handler: H,
config: OutboxWorkerConfig,
queue_scheduling: bool,
_item: PhantomData<I>,
}
impl<S: Clone, H: Clone, I> Clone for OutboxJob<S, H, I> {
fn clone(&self) -> Self {
Self {
name: self.name.clone(),
store: self.store.clone(),
handler: self.handler.clone(),
config: self.config,
queue_scheduling: self.queue_scheduling,
_item: PhantomData,
}
}
}
impl<S: fmt::Debug, H: fmt::Debug, I> fmt::Debug for OutboxJob<S, H, I> {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("OutboxJob")
.field("name", &self.name)
.field("store", &self.store)
.field("handler", &self.handler)
.field("config", &self.config)
.field("queue_scheduling", &self.queue_scheduling)
.finish_non_exhaustive()
}
}
impl<S, H, I> OutboxJob<S, H, I>
where
I: OutboxItem,
{
#[must_use]
pub fn new(name: impl Into<String>, store: S, handler: H, config: OutboxWorkerConfig) -> Self {
Self {
name: name.into(),
store,
handler,
config,
queue_scheduling: false,
_item: PhantomData,
}
}
#[must_use]
pub fn with_config(mut self, config: OutboxWorkerConfig) -> Self {
self.config = config;
self
}
#[must_use]
pub const fn with_queue_scheduling(mut self) -> Self {
self.queue_scheduling = true;
self
}
#[must_use]
pub fn store(&self) -> &S {
&self.store
}
#[must_use]
pub fn handler(&self) -> &H {
&self.handler
}
#[must_use]
pub const fn config(&self) -> OutboxWorkerConfig {
self.config
}
}
#[async_trait]
impl<S, H, I> BatchWorkerJob for OutboxJob<S, H, I>
where
S: OutboxStore<I>,
H: OutboxHandler<I>,
I: OutboxItem,
{
fn name(&self) -> &str {
&self.name
}
async fn run_batch(&self, context: WorkerContext) -> Result<BatchRunSummary, BoxWorkerError> {
let claim = OutboxClaim::new(self.config.batch_size(), self.config.lease_duration());
let mut summary = BatchRunSummary::new();
let batch_deadline = match Instant::now().checked_add(self.config.batch_timeout()) {
Some(deadline) => deadline,
None => {
summary.record_batch_timeout();
return Ok(summary);
}
};
let items =
match before_batch_deadline(batch_deadline, self.store.claim_due(claim, context)).await
{
Ok(result) => result?,
Err(()) => {
summary.record_batch_timeout();
return Ok(summary);
}
};
summary = summary.with_claimed(items.len() as u64);
let mut next_retry_at: Option<Instant> = None;
for item in items {
let state = match handle_outbox_item(
&self.handler,
&item,
context,
self.config,
batch_deadline,
)
.await
{
HandleItemResult::State(state) => state,
HandleItemResult::BatchTimedOut => {
summary.record_batch_timeout();
break;
}
};
match before_batch_deadline(
batch_deadline,
self.store.persist_transition(&item, state.clone(), context),
)
.await
{
Ok(Ok(())) => {
if self.queue_scheduling
&& let OutboxItemState::Retry { delay, .. } = &state
&& let Some(deadline) = Instant::now().checked_add(*delay)
{
next_retry_at =
Some(next_retry_at.map_or(deadline, |current| current.min(deadline)));
}
summary.record_item_state(&state);
if matches!(state, OutboxItemState::Failed { .. }) {
summary.record_dead_letter();
}
}
Ok(Err(_)) => {
summary.record_persistence_error();
tracing::warn!(
worker = self.name(),
error_code = "outbox_transition_persistence_failed",
state = state.as_str(),
"outbox transition persistence failed"
);
continue;
}
Err(()) => {
summary.record_persistence_error();
summary.record_batch_timeout();
tracing::warn!(
worker = self.name(),
error_code = "outbox_transition_persistence_timed_out",
state = state.as_str(),
"outbox transition persistence exceeded the batch deadline"
);
break;
}
}
}
let recheck_after =
next_retry_at.map(|deadline| deadline.saturating_duration_since(Instant::now()));
let schedule = if self.queue_scheduling {
if summary.claimed() >= u64::from(self.config.batch_size()) {
WorkerSchedule::Continue
} else {
WorkerSchedule::WaitForWork { recheck_after }
}
} else {
WorkerSchedule::Poll
};
Ok(summary.with_schedule(schedule))
}
}
#[derive(Debug, Clone)]
#[non_exhaustive]
pub struct BatchWorker<J> {
job: J,
log_empty_batches: bool,
}
impl<J> BatchWorker<J> {
#[must_use]
pub const fn new(job: J) -> Self {
Self {
job,
log_empty_batches: false,
}
}
#[must_use]
pub const fn with_empty_batch_logging(mut self, enabled: bool) -> Self {
self.log_empty_batches = enabled;
self
}
#[must_use]
pub const fn job(&self) -> &J {
&self.job
}
}
#[async_trait]
impl<J> Worker for BatchWorker<J>
where
J: BatchWorkerJob,
{
fn name(&self) -> &str {
self.job.name()
}
async fn run_once(&self, context: WorkerContext) -> Result<WorkerTick, BoxWorkerError> {
let summary = self.job.run_batch(context).await?;
if self.log_empty_batches || !summary.is_empty() {
tracing::info!(
worker = self.name(),
claimed = summary.claimed(),
succeeded = summary.succeeded(),
retrying = summary.retrying(),
failed = summary.failed(),
skipped = summary.skipped(),
dead_lettered = summary.dead_lettered(),
persistence_errors = summary.persistence_errors(),
batch_timed_out = summary.batch_timed_out(),
handled = summary.handled(),
"batch worker completed"
);
}
if let Some(error) = summary.operational_error() {
return Err(error.into());
}
Ok(WorkerTick::new(summary.handled()).with_schedule(summary.schedule()))
}
}
fn retry_or_fail_state(
retry_policy: &OutboxRetryPolicy,
prior_attempts: u32,
reason: String,
) -> OutboxItemState {
let attempts_after_current_try = prior_attempts.saturating_add(1);
if retry_policy.should_retry(attempts_after_current_try) {
OutboxItemState::retry(
reason,
retry_policy.delay_for_attempts(attempts_after_current_try),
)
} else {
OutboxItemState::failed(reason)
}
}
async fn handle_outbox_item<H, I>(
handler: &H,
item: &I,
context: WorkerContext,
config: OutboxWorkerConfig,
batch_deadline: Instant,
) -> HandleItemResult
where
H: OutboxHandler<I>,
I: OutboxItem,
{
let now = Instant::now();
let item_deadline = match now.checked_add(config.item_timeout()) {
Some(deadline) => deadline,
None => {
return HandleItemResult::State(retry_or_fail_state(
&config.retry_policy(),
item.outbox_attempts(),
"outbox_item_timed_out".to_owned(),
));
}
};
if batch_deadline <= now {
return HandleItemResult::BatchTimedOut;
}
let deadline = batch_deadline.min(item_deadline);
let result = match time::timeout_at(deadline, handler.handle(item, context)).await {
Ok(Ok(state)) => Ok(state),
Ok(Err(_)) => Err("outbox_handler_failed"),
Err(_) if batch_deadline <= item_deadline => {
return HandleItemResult::BatchTimedOut;
}
Err(_) => Err("outbox_item_timed_out"),
};
HandleItemResult::State(match result {
Ok(OutboxItemState::Retry { reason, .. })
if !config
.retry_policy()
.should_retry(item.outbox_attempts().saturating_add(1)) =>
{
OutboxItemState::failed(reason)
}
Ok(state) => state,
Err(reason) => retry_or_fail_state(
&config.retry_policy(),
item.outbox_attempts(),
reason.to_owned(),
),
})
}
enum HandleItemResult {
State(OutboxItemState),
BatchTimedOut,
}
async fn before_batch_deadline<F, T>(deadline: Instant, future: F) -> Result<T, ()>
where
F: Future<Output = T>,
{
if deadline <= Instant::now() {
return Err(());
}
time::timeout_at(deadline, future).await.map_err(|_| ())
}