use async_trait::async_trait;
use sea_query::{Asterisk, Condition, Expr, ExprTrait, IntoIden, Query, SimpleExpr};
use crate::errors::OrionError;
use crate::storage::models::{TraceDlqEntry, TraceDlqSummary};
use crate::storage::schema::TraceDlq;
use crate::storage::{DbBackend, DbPool, build_sqlx};
use super::helpers::{Page, PaginatedResult, Projection};
#[derive(Debug, Clone, Default, serde::Deserialize, serde::Serialize, utoipa::IntoParams)]
#[into_params(parameter_in = Query)]
pub struct TraceDlqFilter {
pub channel: Option<String>,
pub exhausted: Option<bool>,
pub limit: Option<i64>,
pub offset: Option<i64>,
}
fn exhausted() -> SimpleExpr {
Expr::col(TraceDlq::RetryCount).gte(Expr::col(TraceDlq::MaxRetries))
}
fn not_exhausted() -> SimpleExpr {
Expr::col(TraceDlq::RetryCount).lt(Expr::col(TraceDlq::MaxRetries))
}
fn clear_lease(q: &mut sea_query::UpdateStatement) -> &mut sea_query::UpdateStatement {
q.value(
TraceDlq::ClaimedBy,
super::helpers::optional_string_value(None),
)
.value(
TraceDlq::ClaimedUntil,
sea_query::Value::ChronoDateTime(None),
)
}
fn summary_columns() -> [TraceDlq; 9] {
[
TraceDlq::Id,
TraceDlq::TraceId,
TraceDlq::Channel,
TraceDlq::ErrorMessage,
TraceDlq::RetryCount,
TraceDlq::MaxRetries,
TraceDlq::NextRetryAt,
TraceDlq::CreatedAt,
TraceDlq::UpdatedAt,
]
}
fn list_page(filter: &TraceDlqFilter) -> Page {
let (limit, offset) = super::helpers::clamp_pagination(filter.limit, filter.offset);
Page {
from: TraceDlq::Table.into_iden(),
projection: Projection::Columns(
summary_columns()
.into_iter()
.map(IntoIden::into_iden)
.collect(),
),
cond: filter.condition(),
sort: TraceDlq::CreatedAt.into_iden(),
order: sea_query::Order::Desc,
limit,
offset,
}
}
impl TraceDlqFilter {
fn condition(&self) -> Condition {
let mut cond = Condition::all();
if let Some(ref channel) = self.channel {
cond = cond.add(Expr::col(TraceDlq::Channel).eq(channel.as_str()));
}
match self.exhausted {
Some(true) => cond = cond.add(exhausted()),
Some(false) => cond = cond.add(not_exhausted()),
None => {}
}
cond
}
}
#[async_trait]
pub trait TraceDlqRepository: Send + Sync {
#[allow(clippy::too_many_arguments)]
async fn enqueue(
&self,
trace_id: &str,
channel: &str,
payload_json: &str,
metadata_json: &str,
error_message: &str,
retry_count: i64,
max_retries: i64,
) -> Result<TraceDlqEntry, OrionError>;
async fn claim_pending(
&self,
claimant: &str,
limit: i64,
lease_secs: u64,
) -> Result<Vec<TraceDlqEntry>, OrionError>;
async fn record_retry(
&self,
id: &str,
next_retry_at: chrono::NaiveDateTime,
) -> Result<(), OrionError>;
async fn remove(&self, id: &str) -> Result<(), OrionError>;
async fn mark_exhausted(&self, id: &str) -> Result<(), OrionError>;
async fn list_paginated(
&self,
filter: &TraceDlqFilter,
) -> Result<PaginatedResult<TraceDlqSummary>, OrionError>;
async fn count(&self, filter: &TraceDlqFilter) -> Result<i64, OrionError>;
async fn get_by_id(&self, id: &str) -> Result<TraceDlqEntry, OrionError>;
async fn requeue(&self, id: &str) -> Result<TraceDlqEntry, OrionError>;
async fn purge_exhausted(&self, older_than_hours: u64) -> Result<u64, OrionError>;
}
fn due_condition(now: &'static str) -> Condition {
Condition::all()
.add(Expr::col(TraceDlq::NextRetryAt).lte(Expr::cust(now)))
.add(not_exhausted())
.add(
Condition::any()
.add(Expr::col(TraceDlq::ClaimedUntil).is_null())
.add(Expr::col(TraceDlq::ClaimedUntil).lt(Expr::cust(now))),
)
}
fn claim_update_query(
claimant: &str,
limit: i64,
now: &'static str,
lease_until: &str,
skip_locked: bool,
) -> sea_query::UpdateStatement {
let mut due_ids = Query::select()
.column(TraceDlq::Id)
.from(TraceDlq::Table)
.cond_where(due_condition(now))
.order_by(TraceDlq::NextRetryAt, sea_query::Order::Asc)
.limit(Ord::max(limit, 0) as u64)
.to_owned();
if skip_locked {
due_ids.lock_with_behavior(
sea_query::LockType::Update,
sea_query::LockBehavior::SkipLocked,
);
}
let mut update = Query::update()
.table(TraceDlq::Table)
.value(TraceDlq::ClaimedBy, claimant)
.value(TraceDlq::ClaimedUntil, Expr::cust(lease_until.to_owned()))
.and_where(Expr::col(TraceDlq::Id).in_subquery(due_ids))
.to_owned();
update.returning_all();
update
}
fn claim_select_query(limit: i64, now: &'static str) -> sea_query::SelectStatement {
let mut select = Query::select()
.column(Asterisk)
.from(TraceDlq::Table)
.cond_where(due_condition(now))
.order_by(TraceDlq::NextRetryAt, sea_query::Order::Asc)
.limit(Ord::max(limit, 0) as u64)
.to_owned();
select.lock_with_behavior(
sea_query::LockType::Update,
sea_query::LockBehavior::SkipLocked,
);
select
}
fn lease_claimed_query<'a>(
claimant: &str,
lease_until: &str,
ids: impl IntoIterator<Item = &'a str>,
) -> sea_query::UpdateStatement {
Query::update()
.table(TraceDlq::Table)
.value(TraceDlq::ClaimedBy, claimant)
.value(TraceDlq::ClaimedUntil, Expr::cust(lease_until.to_owned()))
.and_where(Expr::col(TraceDlq::Id).is_in(ids))
.to_owned()
}
fn dlq_select(id: &str) -> sea_query::SelectStatement {
Query::select()
.column(Asterisk)
.from(TraceDlq::Table)
.and_where(Expr::col(TraceDlq::Id).eq(id))
.to_owned()
}
pub struct SqlTraceDlqRepository {
pool: DbPool,
}
impl SqlTraceDlqRepository {
pub fn new(pool: DbPool) -> Self {
Self { pool }
}
}
#[async_trait]
impl TraceDlqRepository for SqlTraceDlqRepository {
async fn enqueue(
&self,
trace_id: &str,
channel: &str,
payload_json: &str,
metadata_json: &str,
error_message: &str,
retry_count: i64,
max_retries: i64,
) -> Result<TraceDlqEntry, OrionError> {
crate::metrics::timed_db_op("trace_dlq.enqueue", async {
let id = uuid::Uuid::new_v4().to_string();
let next_retry = chrono::Utc::now()
.naive_utc()
.checked_add_signed(chrono::Duration::seconds(1))
.unwrap_or(chrono::Utc::now().naive_utc());
let mut insert = Query::insert();
insert
.into_table(TraceDlq::Table)
.columns([
TraceDlq::Id,
TraceDlq::TraceId,
TraceDlq::Channel,
TraceDlq::PayloadJson,
TraceDlq::MetadataJson,
TraceDlq::ErrorMessage,
TraceDlq::RetryCount,
TraceDlq::MaxRetries,
TraceDlq::NextRetryAt,
])
.values_panic([
Expr::val(id.as_str()),
Expr::val(trace_id),
Expr::val(channel),
Expr::val(payload_json),
Expr::val(metadata_json),
Expr::val(error_message),
Expr::val(Ord::max(retry_count, 0)),
Expr::val(max_retries),
Expr::val(next_retry),
]);
super::helpers::write_returning_row(
&self.pool,
super::helpers::WriteStatement::Insert(&mut insert),
&mut dlq_select(&id),
OrionError::Storage,
|| OrionError::Internal {
context: "Failed to fetch inserted DLQ entry".to_string(),
source: None,
},
)
.await
})
.await
}
async fn claim_pending(
&self,
claimant: &str,
limit: i64,
lease_secs: u64,
) -> Result<Vec<TraceDlqEntry>, OrionError> {
crate::metrics::timed_db_op("trace_dlq.claim_pending", async {
let backend = crate::storage::get_backend();
let now = super::helpers::sql_now(backend);
let lease_until = super::helpers::sql_now_plus_secs(backend, lease_secs);
match backend {
DbBackend::Postgres | DbBackend::Sqlite => {
let (sql, values) = build_sqlx(&mut claim_update_query(
claimant,
limit,
now,
&lease_until,
backend == DbBackend::Postgres,
));
Ok(self
.pool
.fetch_all_as::<TraceDlqEntry>(&sql, values)
.await?)
}
DbBackend::Mysql => {
let mut tx = self.pool.begin_tx().await.map_err(OrionError::Storage)?;
let (sql, values) = build_sqlx(&mut claim_select_query(limit, now));
let rows: Vec<TraceDlqEntry> = tx.fetch_all_as(&sql, values).await?;
if rows.is_empty() {
tx.commit().await.map_err(OrionError::Storage)?;
return Ok(rows);
}
let (sql, values) = build_sqlx(&mut lease_claimed_query(
claimant,
&lease_until,
rows.iter().map(|r| r.id.as_str()),
));
tx.execute_query(&sql, values).await?;
tx.commit().await.map_err(OrionError::Storage)?;
Ok(rows)
}
}
})
.await
}
async fn record_retry(
&self,
id: &str,
next_retry_at: chrono::NaiveDateTime,
) -> Result<(), OrionError> {
crate::metrics::timed_db_op("trace_dlq.record_retry", async {
let (sql, values) = build_sqlx(
clear_lease(
Query::update()
.table(TraceDlq::Table)
.value(TraceDlq::RetryCount, Expr::col(TraceDlq::RetryCount).add(1))
.value(TraceDlq::NextRetryAt, next_retry_at),
)
.and_where(Expr::col(TraceDlq::Id).eq(id)),
);
self.pool.execute_query(&sql, values).await?;
Ok(())
})
.await
}
async fn remove(&self, id: &str) -> Result<(), OrionError> {
crate::metrics::timed_db_op("trace_dlq.remove", async {
let (sql, values) = build_sqlx(
Query::delete()
.from_table(TraceDlq::Table)
.and_where(Expr::col(TraceDlq::Id).eq(id)),
);
self.pool.execute_query(&sql, values).await?;
Ok(())
})
.await
}
async fn mark_exhausted(&self, id: &str) -> Result<(), OrionError> {
crate::metrics::timed_db_op("trace_dlq.mark_exhausted", async {
let (sql, values) = build_sqlx(
clear_lease(
Query::update()
.table(TraceDlq::Table)
.value(TraceDlq::RetryCount, Expr::col(TraceDlq::MaxRetries)),
)
.and_where(Expr::col(TraceDlq::Id).eq(id)),
);
self.pool.execute_query(&sql, values).await?;
Ok(())
})
.await
}
async fn list_paginated(
&self,
filter: &TraceDlqFilter,
) -> Result<PaginatedResult<TraceDlqSummary>, OrionError> {
crate::metrics::timed_db_op("trace_dlq.list_paginated", async {
super::helpers::paginate(&self.pool, list_page(filter)).await
})
.await
}
async fn count(&self, filter: &TraceDlqFilter) -> Result<i64, OrionError> {
crate::metrics::timed_db_op("trace_dlq.count", async {
super::helpers::count_where(&self.pool, TraceDlq::Table, filter.condition()).await
})
.await
}
async fn get_by_id(&self, id: &str) -> Result<TraceDlqEntry, OrionError> {
crate::metrics::timed_db_op("trace_dlq.get_by_id", async {
let (sql, values) = build_sqlx(&mut dlq_select(id));
super::helpers::fetch_required::<TraceDlqEntry>(&self.pool, &sql, values, || {
OrionError::NotFound(format!("DLQ entry '{id}' not found"))
})
.await
})
.await
}
async fn requeue(&self, id: &str) -> Result<TraceDlqEntry, OrionError> {
crate::metrics::timed_db_op("trace_dlq.requeue", async {
let now = chrono::Utc::now().naive_utc();
let mut update = clear_lease(
Query::update()
.table(TraceDlq::Table)
.value(TraceDlq::RetryCount, 0i64)
.value(TraceDlq::NextRetryAt, now),
)
.and_where(Expr::col(TraceDlq::Id).eq(id))
.to_owned();
super::helpers::write_returning_row(
&self.pool,
super::helpers::WriteStatement::Update(&mut update),
&mut dlq_select(id),
OrionError::Storage,
|| OrionError::NotFound(format!("DLQ entry '{id}' not found")),
)
.await
})
.await
}
async fn purge_exhausted(&self, older_than_hours: u64) -> Result<u64, OrionError> {
crate::metrics::timed_db_op("trace_dlq.purge_exhausted", async {
let cutoff =
super::helpers::cutoff_hours_ago(chrono::Utc::now().naive_utc(), older_than_hours);
super::helpers::delete_chunked(
&self.pool,
TraceDlq::Table,
TraceDlq::Id,
Condition::all()
.add(exhausted())
.add(Expr::col(TraceDlq::CreatedAt).lt(cutoff)),
)
.await
})
.await
}
}
#[cfg(test)]
mod tests {
use super::*;
async fn test_repo() -> SqlTraceDlqRepository {
SqlTraceDlqRepository::new(crate::storage::test_sqlite_pool().await)
}
#[test]
fn list_projection_never_reads_the_payload_columns() {
crate::storage::set_backend_for_test(crate::storage::DbBackend::Sqlite);
let sql = super::super::helpers::page_select(&list_page(&TraceDlqFilter::default()))
.to_string(sea_query::SqliteQueryBuilder);
for withheld in ["payload_json", "metadata_json"] {
assert!(
!sql.contains(withheld),
"the DLQ listing projection names `{withheld}`: {sql}"
);
}
assert!(!sql.contains('*'), "{sql}");
}
async fn make_due(repo: &SqlTraceDlqRepository, id: &str) {
let DbPool::Sqlite(p) = &repo.pool else {
unreachable!("sqlite expected");
};
sqlx::query(
"UPDATE trace_dlq SET next_retry_at = datetime('now', '-2 seconds') WHERE id = ?",
)
.bind(id)
.execute(p)
.await
.expect("backdate");
}
async fn enqueue_due(repo: &SqlTraceDlqRepository, trace_id: &str) -> String {
let entry = repo
.enqueue(trace_id, "orders", "{}", "{}", "boom", 0, 5)
.await
.expect("enqueue");
make_due(repo, &entry.id).await;
entry.id
}
#[tokio::test]
async fn test_claim_leases_and_blocks_second_claimant() {
let repo = test_repo().await;
let id = enqueue_due(&repo, "t1").await;
let claimed = repo.claim_pending("node-a", 10, 60).await.expect("claim");
assert_eq!(claimed.len(), 1);
assert_eq!(claimed[0].id, id);
let claimed = repo.claim_pending("node-b", 10, 60).await.expect("claim");
assert!(claimed.is_empty());
}
#[tokio::test]
async fn test_expired_lease_is_reclaimable() {
let repo = test_repo().await;
enqueue_due(&repo, "t1").await;
assert_eq!(
repo.claim_pending("node-a", 10, 60)
.await
.expect("claim")
.len(),
1
);
let DbPool::Sqlite(p) = &repo.pool else {
unreachable!("sqlite expected");
};
sqlx::query("UPDATE trace_dlq SET claimed_until = datetime('now', '-1 seconds')")
.execute(p)
.await
.expect("expire");
assert_eq!(
repo.claim_pending("node-b", 10, 60)
.await
.expect("claim")
.len(),
1
);
}
#[tokio::test]
async fn test_record_retry_clears_lease() {
let repo = test_repo().await;
let id = enqueue_due(&repo, "t1").await;
assert_eq!(
repo.claim_pending("node-a", 10, 60)
.await
.expect("claim")
.len(),
1
);
let past = chrono::Utc::now()
.naive_utc()
.checked_sub_signed(chrono::Duration::seconds(2))
.expect("past");
repo.record_retry(&id, past).await.expect("retry");
let claimed = repo.claim_pending("node-b", 10, 60).await.expect("claim");
assert_eq!(claimed.len(), 1);
assert_eq!(claimed[0].retry_count, 1);
}
#[tokio::test]
async fn test_poison_message_converges_on_max_retries() {
let repo = test_repo().await;
let max_retries = 3;
let first = repo
.enqueue("t-poison", "orders", "{}", "{}", "boom", 0, max_retries)
.await
.expect("enqueue");
make_due(&repo, &first.id).await;
let mut cycles = 0;
while let Some(claimed) = repo
.claim_pending("node-a", 10, 60)
.await
.expect("claim")
.into_iter()
.next()
{
cycles += 1;
assert!(
cycles <= max_retries + 1,
"poison message is still claimable after {cycles} cycles"
);
let carried = claimed.retry_count + 1;
repo.remove(&claimed.id).await.expect("remove");
let requeued = repo
.enqueue(
"t-poison",
"orders",
"{}",
"{}",
"boom",
carried,
max_retries,
)
.await
.expect("re-enqueue");
assert_eq!(requeued.retry_count, carried, "carried count must persist");
make_due(&repo, &requeued.id).await;
}
assert_eq!(
cycles, max_retries,
"a poison message must be retried exactly max_retries times"
);
}
#[test]
fn per_backend_sql_shapes() {
use sea_query::{MysqlQueryBuilder, PostgresQueryBuilder, SqliteQueryBuilder};
let (sql, values) = claim_update_query(
"node-a",
25,
"LOCALTIMESTAMP",
"LOCALTIMESTAMP + interval '60 seconds'",
true,
)
.build(PostgresQueryBuilder);
assert!(sql.contains("RETURNING"), "{sql}");
assert!(sql.contains("FOR UPDATE SKIP LOCKED"), "{sql}");
assert!(sql.contains("\"next_retry_at\""), "{sql}");
assert!(sql.contains("\"claimed_until\""), "{sql}");
assert!(
sql.contains("LIMIT $"),
"limit must be a placeholder: {sql}"
);
assert!(!sql.contains("25"), "limit must not be inlined: {sql}");
assert!(
values.iter().any(|v| *v == sea_query::Value::from(25u64)),
"limit must travel as a bound value: {values:?}"
);
let (sql, _) = claim_update_query(
"node-a",
25,
"datetime('now')",
"datetime('now', '+60 seconds')",
false,
)
.build(SqliteQueryBuilder);
assert!(sql.contains("RETURNING"), "{sql}");
assert!(!sql.contains("FOR UPDATE"), "{sql}");
assert!(
sql.contains("LIMIT ?"),
"limit must be a placeholder: {sql}"
);
let (sql, _) = claim_select_query(25, "UTC_TIMESTAMP()").build(MysqlQueryBuilder);
assert!(sql.contains("FOR UPDATE SKIP LOCKED"), "{sql}");
assert!(sql.contains("`next_retry_at`"), "{sql}");
assert!(
sql.contains("LIMIT ?"),
"limit must be a placeholder: {sql}"
);
let (sql, _) = lease_claimed_query(
"node-a",
"DATE_ADD(UTC_TIMESTAMP(), INTERVAL 60 SECOND)",
["id-1", "id-2"],
)
.build(MysqlQueryBuilder);
assert!(sql.contains("`claimed_by`"), "{sql}");
assert!(sql.contains("IN (?, ?)"), "{sql}");
}
#[tokio::test]
async fn test_claim_respects_batch_limit() {
let repo = test_repo().await;
for i in 0..5 {
enqueue_due(&repo, &format!("t{i}")).await;
}
assert_eq!(
repo.claim_pending("node-a", 3, 60)
.await
.expect("claim")
.len(),
3
);
assert_eq!(
repo.claim_pending("node-a", 3, 60)
.await
.expect("claim")
.len(),
2
);
}
}