use std::{
sync::{
Arc, Mutex,
atomic::{AtomicUsize, Ordering},
},
time::Duration,
};
use async_trait::async_trait;
use tokio::time::{self, Instant};
use super::*;
use crate::worker::{BoxWorkerError, WorkerContext, WorkerSchedule};
#[derive(Clone)]
struct Item {
id: usize,
handle_delay: Duration,
persist_delay: Duration,
state: Result<OutboxItemState, ()>,
attempts: u32,
persistence_fails: bool,
}
fn item(id: usize, handle_seconds: u64, persist_seconds: u64) -> Item {
Item {
id,
handle_delay: Duration::from_secs(handle_seconds),
persist_delay: Duration::from_secs(persist_seconds),
state: Ok(OutboxItemState::Succeeded),
attempts: 0,
persistence_fails: false,
}
}
impl OutboxItem for Item {
fn outbox_attempts(&self) -> u32 {
self.attempts
}
}
#[derive(Default)]
struct Activity {
active: AtomicUsize,
peak: AtomicUsize,
handlers: Mutex<Vec<(usize, Instant)>>,
persistence: Mutex<Vec<(usize, Instant)>>,
persisted: Mutex<Vec<(usize, OutboxItemState)>>,
cancelled_handlers: AtomicUsize,
cancelled_persistence: AtomicUsize,
}
struct Active<'a> {
activity: &'a Activity,
persistence: bool,
completed: bool,
}
impl<'a> Active<'a> {
fn new(activity: &'a Activity, id: usize, persistence: bool) -> Self {
let active = activity.active.fetch_add(1, Ordering::SeqCst) + 1;
activity.peak.fetch_max(active, Ordering::SeqCst);
if persistence {
&activity.persistence
} else {
&activity.handlers
}
.lock()
.unwrap()
.push((id, Instant::now()));
Self {
activity,
persistence,
completed: false,
}
}
}
impl Drop for Active<'_> {
fn drop(&mut self) {
self.activity.active.fetch_sub(1, Ordering::SeqCst);
if !self.completed {
if self.persistence {
&self.activity.cancelled_persistence
} else {
&self.activity.cancelled_handlers
}
.fetch_add(1, Ordering::SeqCst);
}
}
}
struct Store {
items: Mutex<Vec<Item>>,
activity: Arc<Activity>,
claim_delay: Duration,
}
#[async_trait]
impl OutboxStore<Item> for Store {
async fn claim_due(
&self,
claim: OutboxClaim,
_: WorkerContext,
) -> Result<Vec<Item>, BoxWorkerError> {
if !self.claim_delay.is_zero() {
time::sleep(self.claim_delay).await;
}
let mut items = self.items.lock().unwrap();
let count = items.len().min(claim.limit() as usize);
Ok(items.drain(..count).collect())
}
async fn persist_transition(
&self,
item: &Item,
state: OutboxItemState,
_: WorkerContext,
) -> Result<(), BoxWorkerError> {
let mut active = Active::new(&self.activity, item.id, true);
if !item.persist_delay.is_zero() {
time::sleep(item.persist_delay).await;
}
active.completed = true;
if item.persistence_fails {
return Err(std::io::Error::other("private storage detail").into());
}
self.activity
.persisted
.lock()
.unwrap()
.push((item.id, state));
Ok(())
}
}
struct Handler(Arc<Activity>);
#[async_trait]
impl OutboxHandler<Item> for Handler {
async fn handle(
&self,
item: &Item,
_: WorkerContext,
) -> Result<OutboxItemState, BoxWorkerError> {
let mut active = Active::new(&self.0, item.id, false);
if !item.handle_delay.is_zero() {
time::sleep(item.handle_delay).await;
}
active.completed = true;
item.state
.clone()
.map_err(|()| std::io::Error::other("private handler detail").into())
}
}
fn job(
items: Vec<Item>,
settings: OutboxWorkerSettings,
) -> (OutboxJob<Store, Handler, Item>, Arc<Activity>) {
let activity = Arc::new(Activity::default());
(
OutboxJob::new(
"concurrent",
Store {
items: Mutex::new(items),
activity: activity.clone(),
claim_delay: Duration::ZERO,
},
Handler(activity.clone()),
settings.build().unwrap(),
),
activity,
)
}
#[test]
fn concurrency_settings_default_validate_and_round_trip() {
let defaults: OutboxWorkerSettings = serde_json::from_str("{}").unwrap();
assert_eq!(defaults.max_concurrency, 1);
assert_eq!(defaults.build().unwrap().max_concurrency(), 1);
let config = defaults.with_batch_size(2).build().unwrap();
assert_eq!(config.max_concurrency(), 1);
let settings = defaults.with_max_concurrency(4);
let wire = serde_json::to_value(settings).unwrap();
assert_eq!(wire["max_concurrency"], 4);
let config = serde_json::from_value::<OutboxWorkerSettings>(wire)
.unwrap()
.build()
.unwrap();
assert_eq!(config.max_concurrency(), 4);
assert_eq!(OutboxWorkerSettings::from(config), settings);
let error = defaults
.with_max_concurrency(0)
.validate_at("email.outbox")
.unwrap_err();
assert_eq!(error.issues()[0].path(), "email.outbox.max_concurrency");
#[cfg(feature = "tier")]
assert!(
<OutboxWorkerSettings as tier::TierMetadata>::metadata()
.fields()
.iter()
.any(|field| field.path() == "max_concurrency")
);
}
#[tokio::test(start_paused = true)]
async fn slots_cover_persistence_and_refill_without_waiting_for_an_earlier_item() {
for concurrency in [1, 2] {
let (job, activity) = job(
vec![item(0, 1, 5), item(1, 1, 1), item(2, 1, 0)],
OutboxWorkerSettings::default().with_max_concurrency(concurrency),
);
let started_at = Instant::now();
let summary = job.run_batch(WorkerContext::default()).await.unwrap();
assert_eq!(summary.claimed(), 3);
assert_eq!(summary.succeeded(), 3);
assert!(!summary.batch_timed_out());
assert_eq!(activity.peak.load(Ordering::SeqCst), concurrency);
assert_eq!(activity.active.load(Ordering::SeqCst), 0);
let starts: Vec<_> = activity
.handlers
.lock()
.unwrap()
.iter()
.map(|(id, at)| (*id, (*at - started_at).as_secs()))
.collect();
let completed: Vec<_> = activity
.persisted
.lock()
.unwrap()
.iter()
.map(|(id, _)| *id)
.collect();
if concurrency == 1 {
assert_eq!(started_at.elapsed(), Duration::from_secs(9));
assert_eq!(starts, [(0, 0), (1, 6), (2, 8)]);
assert_eq!(completed, [0, 1, 2]);
} else {
assert_eq!(started_at.elapsed(), Duration::from_secs(6));
assert_eq!(starts, [(0, 0), (1, 0), (2, 2)]);
assert_eq!(completed, [1, 2, 0]);
}
}
}
#[tokio::test(start_paused = true)]
async fn deadline_collects_all_pending_persistence_failures_without_starting_queued_items() {
let (job, activity) = job(
vec![item(0, 0, 60), item(1, 0, 60), item(2, 0, 0)],
OutboxWorkerSettings::default()
.with_max_concurrency(2)
.with_item_timeout(Duration::from_secs(1))
.with_batch_timeout(Duration::from_secs(3)),
);
let started_at = Instant::now();
let summary = job.run_batch(WorkerContext::default()).await.unwrap();
assert_eq!(started_at.elapsed(), Duration::from_secs(3));
assert_eq!(summary.claimed(), 3);
assert_eq!(summary.persistence_errors(), 2);
assert_eq!(summary.succeeded(), 0);
assert!(summary.batch_timed_out());
assert!(summary.operational_error().is_some());
assert_eq!(activity.cancelled_persistence.load(Ordering::SeqCst), 2);
assert_eq!(activity.active.load(Ordering::SeqCst), 0);
assert_eq!(
activity
.handlers
.lock()
.unwrap()
.iter()
.map(|(id, _)| *id)
.collect::<Vec<_>>(),
[0, 1]
);
assert!(activity.persisted.lock().unwrap().is_empty());
}
#[tokio::test(start_paused = true)]
async fn cancellation_drops_both_handler_and_persistence_futures() {
let (job, activity) = job(
vec![item(0, 0, 60), item(1, 60, 0), item(2, 0, 0)],
OutboxWorkerSettings::default().with_max_concurrency(2),
);
assert!(
time::timeout(
Duration::from_secs(1),
job.run_batch(WorkerContext::default())
)
.await
.is_err()
);
assert_eq!(activity.cancelled_handlers.load(Ordering::SeqCst), 1);
assert_eq!(activity.cancelled_persistence.load(Ordering::SeqCst), 1);
assert_eq!(activity.active.load(Ordering::SeqCst), 0);
time::advance(Duration::from_secs(60)).await;
assert_eq!(activity.handlers.lock().unwrap().len(), 2);
assert!(activity.persisted.lock().unwrap().is_empty());
}
#[tokio::test(start_paused = true)]
async fn item_timeouts_start_at_execution_and_retry_hints_start_at_persistence() {
let (job, activity) = job(
vec![item(0, 4, 0), item(1, 60, 0), item(2, 4, 0)],
OutboxWorkerSettings::default()
.with_max_concurrency(2)
.with_item_timeout(Duration::from_secs(5))
.with_batch_timeout(Duration::from_secs(15)),
);
let started_at = Instant::now();
let summary = job
.with_queue_scheduling()
.run_batch(WorkerContext::default())
.await
.unwrap();
assert_eq!(started_at.elapsed(), Duration::from_secs(8));
assert_eq!(summary.succeeded(), 2);
assert_eq!(summary.retrying(), 1);
assert_eq!(summary.persistence_errors(), 0);
assert!(!summary.batch_timed_out());
assert_eq!(activity.cancelled_handlers.load(Ordering::SeqCst), 1);
assert_eq!(
summary.schedule(),
WorkerSchedule::WaitForWork {
recheck_after: Some(Duration::from_secs(27))
}
);
}
#[tokio::test(start_paused = true)]
async fn parallel_results_preserve_retry_limits_dead_letters_and_persistence_errors() {
let mut items: Vec<_> = (0..6).map(|id| item(id, 0, 0)).collect();
items[1].state = Ok(OutboxItemState::skipped("not needed"));
items[2].state = Ok(OutboxItemState::retry("retry", Duration::from_secs(3)));
items[3].state = items[2].state.clone();
items[3].attempts = 4;
items[4].state = Err(());
items[4].attempts = 4;
items[5].persistence_fails = true;
let (job, activity) = job(
items,
OutboxWorkerSettings::default()
.with_batch_size(6)
.with_max_concurrency(3),
);
let summary = job
.with_queue_scheduling()
.run_batch(WorkerContext::default())
.await
.unwrap();
assert_eq!((summary.claimed(), summary.handled()), (6, 6));
assert_eq!(
(summary.succeeded(), summary.skipped(), summary.retrying()),
(1, 1, 1)
);
assert_eq!(
(
summary.failed(),
summary.dead_lettered(),
summary.persistence_errors()
),
(2, 2, 1)
);
assert!(!summary.batch_timed_out());
assert_eq!(summary.schedule(), WorkerSchedule::Continue);
assert_eq!(activity.persisted.lock().unwrap().len(), 5);
}
#[tokio::test(start_paused = true)]
async fn ready_persistence_at_deadline_does_not_start_a_queued_handler() {
let (job, activity) = job(
vec![item(0, 0, 3), item(1, 0, 0)],
OutboxWorkerSettings::default()
.with_item_timeout(Duration::from_secs(1))
.with_batch_timeout(Duration::from_secs(3)),
);
let summary = job.run_batch(WorkerContext::default()).await.unwrap();
assert_eq!(summary.succeeded(), 1);
assert!(summary.batch_timed_out());
assert_eq!(summary.persistence_errors(), 0);
assert_eq!(activity.handlers.lock().unwrap().len(), 1);
}
#[tokio::test(start_paused = true)]
async fn claim_time_consumes_the_batch_budget_and_expired_claims_start_no_handlers() {
let activity = Arc::new(Activity::default());
let job = OutboxJob::new(
"concurrent",
Store {
items: Mutex::new(vec![item(0, 0, 0), item(1, 0, 0)]),
activity: activity.clone(),
claim_delay: Duration::from_secs(3),
},
Handler(activity.clone()),
OutboxWorkerSettings::default()
.with_max_concurrency(2)
.with_item_timeout(Duration::from_secs(1))
.with_batch_timeout(Duration::from_secs(3))
.build()
.unwrap(),
);
let started_at = Instant::now();
let summary = job.run_batch(WorkerContext::default()).await.unwrap();
assert_eq!(started_at.elapsed(), Duration::from_secs(3));
assert_eq!(summary.claimed(), 2);
assert!(summary.batch_timed_out());
assert!(activity.handlers.lock().unwrap().is_empty());
assert!(activity.persistence.lock().unwrap().is_empty());
}