use std::{error::Error as StdError, fmt, time::Duration};
use async_trait::async_trait;
use crate::result::{ClassifyError, ErrorKind};
use super::super::{BoxWorkerError, WorkerSchedule};
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
#[non_exhaustive]
pub struct BatchRunSummary {
claimed: u64,
succeeded: u64,
retrying: u64,
failed: u64,
skipped: u64,
dead_lettered: u64,
persistence_errors: u64,
batch_timed_out: bool,
schedule: WorkerSchedule,
}
impl BatchRunSummary {
#[must_use]
pub const fn new() -> Self {
Self {
claimed: 0,
succeeded: 0,
retrying: 0,
failed: 0,
skipped: 0,
dead_lettered: 0,
persistence_errors: 0,
batch_timed_out: false,
schedule: WorkerSchedule::Poll,
}
}
#[must_use]
pub const fn with_claimed(mut self, claimed: u64) -> Self {
self.claimed = claimed;
self
}
#[must_use]
pub const fn with_succeeded(mut self, succeeded: u64) -> Self {
self.succeeded = succeeded;
self
}
#[must_use]
pub const fn with_retrying(mut self, retrying: u64) -> Self {
self.retrying = retrying;
self
}
#[must_use]
pub const fn with_failed(mut self, failed: u64) -> Self {
self.failed = failed;
self
}
#[must_use]
pub const fn with_skipped(mut self, skipped: u64) -> Self {
self.skipped = skipped;
self
}
#[must_use]
pub const fn with_dead_lettered(mut self, dead_lettered: u64) -> Self {
self.dead_lettered = dead_lettered;
self
}
#[must_use]
pub const fn with_persistence_errors(mut self, persistence_errors: u64) -> Self {
self.persistence_errors = persistence_errors;
self
}
#[must_use]
pub const fn with_batch_timed_out(mut self, batch_timed_out: bool) -> Self {
self.batch_timed_out = batch_timed_out;
self
}
#[must_use]
pub const fn with_schedule(mut self, schedule: WorkerSchedule) -> Self {
self.schedule = schedule;
self
}
#[must_use]
pub const fn schedule(&self) -> WorkerSchedule {
self.schedule
}
#[must_use]
pub const fn claimed(&self) -> u64 {
self.claimed
}
#[must_use]
pub const fn succeeded(&self) -> u64 {
self.succeeded
}
#[must_use]
pub const fn retrying(&self) -> u64 {
self.retrying
}
#[must_use]
pub const fn failed(&self) -> u64 {
self.failed
}
#[must_use]
pub const fn skipped(&self) -> u64 {
self.skipped
}
#[must_use]
pub const fn dead_lettered(&self) -> u64 {
self.dead_lettered
}
#[must_use]
pub const fn persistence_errors(&self) -> u64 {
self.persistence_errors
}
#[must_use]
pub const fn batch_timed_out(&self) -> bool {
self.batch_timed_out
}
#[must_use]
pub const fn handled(&self) -> u64 {
self.succeeded
.saturating_add(self.retrying)
.saturating_add(self.failed)
.saturating_add(self.skipped)
.saturating_add(self.persistence_errors)
}
#[must_use]
pub const fn is_empty(&self) -> bool {
self.claimed == 0 && self.handled() == 0 && !self.batch_timed_out
}
#[must_use]
pub const fn has_failures(&self) -> bool {
self.failed > 0 || self.persistence_errors > 0 || self.batch_timed_out
}
#[must_use]
pub const fn operational_error(&self) -> Option<OutboxBatchError> {
if self.persistence_errors == 0 && !self.batch_timed_out {
return None;
}
Some(OutboxBatchError {
persistence_errors: self.persistence_errors,
timed_out: self.batch_timed_out,
})
}
pub(super) fn record_item_state(&mut self, state: &OutboxItemState) {
match state {
OutboxItemState::Succeeded => {
self.succeeded = self.succeeded.saturating_add(1);
}
OutboxItemState::Skipped { .. } => {
self.skipped = self.skipped.saturating_add(1);
}
OutboxItemState::Retry { delay, .. } => {
self.retrying = self.retrying.saturating_add(1);
self.record_delay(*delay);
}
OutboxItemState::Failed { .. } => {
self.failed = self.failed.saturating_add(1);
}
}
}
pub(super) fn record_persistence_error(&mut self) {
self.persistence_errors = self.persistence_errors.saturating_add(1);
}
pub(super) fn record_dead_letter(&mut self) {
self.dead_lettered = self.dead_lettered.saturating_add(1);
}
pub(super) fn record_batch_timeout(&mut self) {
self.batch_timed_out = true;
self.record_delay(Duration::ZERO);
}
fn record_delay(&mut self, delay: Duration) {
let delay = match self.schedule {
WorkerSchedule::After(current) => current.min(delay),
_ => delay,
};
self.schedule = WorkerSchedule::After(delay);
}
}
#[derive(Clone, Copy, PartialEq, Eq)]
#[non_exhaustive]
pub struct OutboxBatchError {
persistence_errors: u64,
timed_out: bool,
}
impl OutboxBatchError {
#[must_use]
pub const fn persistence_errors(self) -> u64 {
self.persistence_errors
}
#[must_use]
pub const fn timed_out(self) -> bool {
self.timed_out
}
pub(super) const fn code(self) -> &'static str {
match (self.persistence_errors > 0, self.timed_out) {
(true, true) => "outbox_transition_persistence_timed_out",
(true, false) => "outbox_transition_persistence_failed",
(false, true) => "outbox_batch_timed_out",
(false, false) => "outbox_batch_failed",
}
}
}
impl fmt::Display for OutboxBatchError {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter.write_str("outbox batch did not complete reliably")
}
}
impl fmt::Debug for OutboxBatchError {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("OutboxBatchError")
.field("code", &self.code())
.field("persistence_errors", &self.persistence_errors)
.field("timed_out", &self.timed_out)
.finish_non_exhaustive()
}
}
impl StdError for OutboxBatchError {}
impl ClassifyError for OutboxBatchError {
fn error_kind(&self) -> ErrorKind {
if self.persistence_errors > 0 {
ErrorKind::Dependency
} else {
ErrorKind::Timeout
}
}
fn error_code(&self) -> crate::result::ErrorCode {
crate::result::ErrorCode::internal(self.code())
}
}
#[async_trait]
pub trait BatchWorkerJob: Send + Sync + 'static {
fn name(&self) -> &str;
async fn run_batch(&self) -> Result<BatchRunSummary, BoxWorkerError>;
}
pub trait OutboxItem: Send + Sync + 'static {
fn outbox_id(&self) -> String;
fn outbox_attempts(&self) -> u32;
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[non_exhaustive]
pub struct OutboxClaim {
limit: u32,
lease_duration: Duration,
}
impl OutboxClaim {
#[must_use]
pub const fn new(limit: u32, lease_duration: Duration) -> Self {
Self {
limit,
lease_duration,
}
}
#[must_use]
pub const fn limit(&self) -> u32 {
self.limit
}
#[must_use]
pub const fn lease_duration(&self) -> Duration {
self.lease_duration
}
}
#[derive(Clone, PartialEq, Eq)]
#[non_exhaustive]
pub enum OutboxItemState {
Succeeded,
Skipped {
#[doc = "The reason for this outcome."]
reason: String,
},
Retry {
#[doc = "The reason for this outcome."]
reason: String,
#[doc = "The delay before the next action."]
delay: Duration,
},
Failed {
#[doc = "The reason for this outcome."]
reason: String,
},
}
impl fmt::Debug for OutboxItemState {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::Succeeded => formatter.write_str("OutboxItemState::Succeeded"),
Self::Skipped { .. } => formatter
.debug_struct("OutboxItemState::Skipped")
.field("reason", &"[REDACTED]")
.finish(),
Self::Retry { delay, .. } => formatter
.debug_struct("OutboxItemState::Retry")
.field("reason", &"[REDACTED]")
.field("delay", delay)
.finish(),
Self::Failed { .. } => formatter
.debug_struct("OutboxItemState::Failed")
.field("reason", &"[REDACTED]")
.finish(),
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[non_exhaustive]
pub enum OutboxFailureClass {
Retryable,
Terminal,
}
impl OutboxItemState {
#[must_use]
pub fn retry(reason: impl Into<String>, delay: Duration) -> Self {
Self::Retry {
reason: bounded_outbox_reason(reason),
delay,
}
}
#[must_use]
pub fn failed(reason: impl Into<String>) -> Self {
Self::Failed {
reason: bounded_outbox_reason(reason),
}
}
#[must_use]
pub fn skipped(reason: impl Into<String>) -> Self {
Self::Skipped {
reason: bounded_outbox_reason(reason),
}
}
#[must_use]
pub const fn as_str(&self) -> &'static str {
match self {
Self::Succeeded => "succeeded",
Self::Skipped { .. } => "skipped",
Self::Retry { .. } => "retry",
Self::Failed { .. } => "failed",
}
}
#[must_use]
pub const fn failure_class(&self) -> Option<OutboxFailureClass> {
match self {
Self::Retry { .. } => Some(OutboxFailureClass::Retryable),
Self::Failed { .. } => Some(OutboxFailureClass::Terminal),
Self::Succeeded | Self::Skipped { .. } => None,
}
}
}
#[async_trait]
pub trait OutboxStore<I>: Send + Sync + 'static
where
I: OutboxItem,
{
async fn claim_due(&self, claim: OutboxClaim) -> Result<Vec<I>, BoxWorkerError>;
async fn persist_transition(
&self,
item: &I,
state: OutboxItemState,
) -> Result<(), BoxWorkerError>;
}
#[async_trait]
pub trait OutboxHandler<I>: Send + Sync + 'static
where
I: OutboxItem,
{
async fn handle(&self, item: &I) -> Result<OutboxItemState, BoxWorkerError>;
}
fn bounded_outbox_reason(reason: impl Into<String>) -> String {
const MAX_REASON_LEN: usize = 1024;
let mut reason = reason.into().trim().to_owned();
if reason.len() <= MAX_REASON_LEN {
return reason;
}
reason.truncate(reason.floor_char_boundary(MAX_REASON_LEN));
reason
}