use std::time::Duration;
use sqlx::{PgPool, Row};
#[derive(Debug, Clone)]
pub struct OutboxRow {
pub id: i64,
pub entity: String,
pub kind: String,
pub entity_id: String,
pub payload: serde_json::Value,
pub attempts: i32
}
#[async_trait::async_trait]
pub trait OutboxHandler: Send + Sync {
type Error: std::fmt::Display + Send;
async fn handle(&self, row: &OutboxRow) -> Result<(), Self::Error>;
}
pub struct OutboxDrainer<H> {
pool: PgPool,
handler: H,
batch_size: i64,
poll_interval: Duration,
base_backoff: Duration,
max_attempts: i32
}
impl<H: OutboxHandler> OutboxDrainer<H> {
#[must_use]
pub fn new(pool: PgPool, handler: H) -> Self {
Self {
pool,
handler,
batch_size: 32,
poll_interval: Duration::from_secs(1),
base_backoff: Duration::from_secs(5),
max_attempts: 10
}
}
#[must_use]
pub fn batch_size(mut self, batch_size: i64) -> Self {
self.batch_size = batch_size;
self
}
#[must_use]
pub fn poll_interval(mut self, poll_interval: Duration) -> Self {
self.poll_interval = poll_interval;
self
}
#[must_use]
pub fn base_backoff(mut self, base_backoff: Duration) -> Self {
self.base_backoff = base_backoff;
self
}
#[must_use]
pub fn max_attempts(mut self, max_attempts: i32) -> Self {
self.max_attempts = max_attempts;
self
}
pub async fn run(self) {
loop {
match self.drain_once().await {
Ok(0) | Err(_) => tokio::time::sleep(self.poll_interval).await,
Ok(_) => {}
}
}
}
pub async fn drain_once(&self) -> Result<usize, sqlx::Error> {
let mut tx = self.pool.begin().await?;
let rows = sqlx::query(
"SELECT id, entity, kind, entity_id, payload, attempts \
FROM entity_outbox \
WHERE processed_at IS NULL AND next_attempt_at <= NOW() \
ORDER BY id \
LIMIT $1 \
FOR UPDATE SKIP LOCKED"
)
.bind(self.batch_size)
.fetch_all(&mut *tx)
.await?;
let mut claimed = Vec::with_capacity(rows.len());
for row in rows {
claimed.push(OutboxRow {
id: row.try_get("id")?,
entity: row.try_get("entity")?,
kind: row.try_get("kind")?,
entity_id: row.try_get("entity_id")?,
payload: row.try_get("payload")?,
attempts: row.try_get("attempts")?
});
}
let actions = process_rows(
&self.handler,
&claimed,
self.base_backoff,
self.max_attempts
)
.await;
let processed = actions.len();
for action in actions {
match action {
OutboxAction::MarkProcessed {
id
} => {
sqlx::query("UPDATE entity_outbox SET processed_at = NOW() WHERE id = $1")
.bind(id)
.execute(&mut *tx)
.await?;
}
OutboxAction::ScheduleRetry {
id,
delay
} => {
sqlx::query(
"UPDATE entity_outbox \
SET attempts = attempts + 1, \
next_attempt_at = NOW() + $2 * INTERVAL '1 second' \
WHERE id = $1"
)
.bind(id)
.bind(delay.as_secs_f64())
.execute(&mut *tx)
.await?;
}
}
}
tx.commit().await?;
Ok(processed)
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum OutboxAction {
MarkProcessed {
id: i64
},
ScheduleRetry {
id: i64,
delay: Duration
}
}
pub async fn process_rows<H: OutboxHandler>(
handler: &H,
rows: &[OutboxRow],
base_backoff: Duration,
max_attempts: i32
) -> Vec<OutboxAction> {
let mut actions = Vec::with_capacity(rows.len());
for row in rows {
let action = match handler.handle(row).await {
Ok(()) => OutboxAction::MarkProcessed {
id: row.id
},
Err(_) => OutboxAction::ScheduleRetry {
id: row.id,
delay: backoff_delay(base_backoff, row.attempts, max_attempts)
}
};
actions.push(action);
}
actions
}
#[must_use]
pub fn backoff_delay(base: Duration, attempts: i32, max_attempts: i32) -> Duration {
if attempts + 1 >= max_attempts {
return Duration::from_secs(60 * 60 * 24 * 30);
}
let exp = u32::try_from(attempts.clamp(0, 20)).unwrap_or(0);
let delay = base.saturating_mul(2u32.saturating_pow(exp));
delay.min(Duration::from_secs(3600))
}
#[cfg(test)]
mod tests {
use super::*;
struct FlakyHandler;
#[async_trait::async_trait]
impl OutboxHandler for FlakyHandler {
type Error = String;
async fn handle(&self, row: &OutboxRow) -> Result<(), Self::Error> {
if row.kind == "created" {
Ok(())
} else {
Err("boom".to_string())
}
}
}
fn row(id: i64, kind: &str, attempts: i32) -> OutboxRow {
OutboxRow {
id,
entity: "users".to_string(),
kind: kind.to_string(),
entity_id: "x".to_string(),
payload: serde_json::json!({}),
attempts
}
}
#[tokio::test]
async fn process_rows_marks_success_and_schedules_retry() {
let rows = vec![row(1, "created", 0), row(2, "updated", 1)];
let actions = process_rows(&FlakyHandler, &rows, Duration::from_secs(5), 10).await;
assert_eq!(
actions,
vec![
OutboxAction::MarkProcessed {
id: 1
},
OutboxAction::ScheduleRetry {
id: 2,
delay: Duration::from_secs(10)
},
]
);
}
#[tokio::test]
async fn process_rows_parks_exhausted_rows() {
let rows = vec![row(3, "updated", 9)];
let actions = process_rows(&FlakyHandler, &rows, Duration::from_secs(5), 10).await;
assert_eq!(
actions,
vec![OutboxAction::ScheduleRetry {
id: 3,
delay: Duration::from_secs(60 * 60 * 24 * 30)
}]
);
}
#[tokio::test]
async fn drainer_builder_configures_all_knobs() {
let pool = sqlx::PgPool::connect_lazy("postgres://localhost/unused")
.expect("lazy pool never connects");
let drainer = OutboxDrainer::new(pool, FlakyHandler)
.batch_size(7)
.poll_interval(Duration::from_millis(250))
.base_backoff(Duration::from_secs(2))
.max_attempts(3);
assert_eq!(drainer.batch_size, 7);
assert_eq!(drainer.poll_interval, Duration::from_millis(250));
assert_eq!(drainer.base_backoff, Duration::from_secs(2));
assert_eq!(drainer.max_attempts, 3);
}
#[test]
fn backoff_grows_exponentially() {
let base = Duration::from_secs(5);
assert_eq!(backoff_delay(base, 0, 10), Duration::from_secs(5));
assert_eq!(backoff_delay(base, 1, 10), Duration::from_secs(10));
assert_eq!(backoff_delay(base, 3, 10), Duration::from_secs(40));
}
#[test]
fn backoff_caps_at_one_hour() {
let base = Duration::from_secs(5);
assert_eq!(backoff_delay(base, 15, 20), Duration::from_secs(3600));
}
#[test]
fn backoff_parks_after_max_attempts() {
let base = Duration::from_secs(5);
let parked = backoff_delay(base, 9, 10);
assert_eq!(parked, Duration::from_secs(60 * 60 * 24 * 30));
}
#[test]
fn backoff_negative_attempts_treated_as_zero() {
let base = Duration::from_secs(5);
assert_eq!(backoff_delay(base, -3, 10), Duration::from_secs(5));
}
}