1use std::sync::Arc;
5
6use bytes::Bytes;
7use reliar_core::{ContentType, Message, MessageId, Serializer};
8use reliar_outbox::{
9 AcquiredBatch, CompletedMessage, DeadLetterPage, DeadQuery, FailedMessage, FailureOutcome,
10 MessageRef, OutboxDeadLetters, OutboxEnqueue, OutboxStats, OutboxStore, PoisonedRow,
11 PurgeReport, PurgeRequest, WorkerId,
12};
13use sqlx::{PgPool, Postgres, Transaction};
14use tracing::Instrument as _;
15
16use crate::error::{
17 EnqueueError, PostgresStoreError, is_undefined_table, map_enqueue_error, map_operational_error,
18};
19use crate::records::{RawRow, decode_row};
20use crate::settings::PostgresOutboxSettings;
21
22#[cfg(feature = "json")]
23use reliar_core::JsonSerializer;
24
25const MAX_LIST_DEAD_LIMIT: u32 = 1000;
29
30#[non_exhaustive]
38pub struct PostgresOutboxStore<
39 #[cfg(feature = "json")] Ser = JsonSerializer,
40 #[cfg(not(feature = "json"))] Ser,
41> {
42 pool: PgPool,
43 settings: PostgresOutboxSettings,
44 serializer: Arc<Ser>,
45}
46
47impl<Ser> Clone for PostgresOutboxStore<Ser> {
51 fn clone(&self) -> Self {
52 Self {
53 pool: self.pool.clone(),
54 settings: self.settings.clone(),
55 serializer: Arc::clone(&self.serializer),
56 }
57 }
58}
59
60impl<Ser> std::fmt::Debug for PostgresOutboxStore<Ser> {
61 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
62 f.debug_struct("PostgresOutboxStore")
63 .field("settings", &self.settings)
64 .finish_non_exhaustive()
65 }
66}
67
68struct SchemaCheck {
70 resolved_schema: Option<String>,
71 configured_exists: bool,
72 search_path: String,
73}
74
75async fn verify_schema(pool: &PgPool, schema: &str) -> Result<SchemaCheck, PostgresStoreError> {
76 let qualified = format!("{schema}.outbox");
77 let row = sqlx::query!(
78 r#"SELECT
79 current_setting('search_path') AS "search_path!",
80 (SELECT n.nspname
81 FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace
82 WHERE c.oid = to_regclass('outbox')) AS resolved_schema,
83 (to_regclass($1) IS NOT NULL) AS "configured_exists!""#,
84 qualified,
85 )
86 .fetch_one(pool)
87 .await
88 .map_err(|err| {
89 if is_undefined_table(&err) {
90 PostgresStoreError::NotMigrated {
91 schema: schema.to_owned(),
92 }
93 } else {
94 PostgresStoreError::from(err)
95 }
96 })?;
97
98 Ok(SchemaCheck {
99 resolved_schema: row.resolved_schema,
100 configured_exists: row.configured_exists,
101 search_path: row.search_path,
102 })
103}
104
105async fn other_outbox_schemas(
108 pool: &PgPool,
109 schema: &str,
110) -> Result<Vec<String>, PostgresStoreError> {
111 let schemas = sqlx::query_scalar!(
112 r#"SELECT n.nspname
113 FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace
114 WHERE c.relname = 'outbox' AND n.nspname <> $1"#,
115 schema,
116 )
117 .fetch_all(pool)
118 .await?;
119 Ok(schemas)
120}
121
122impl<Ser: Serializer + Send + Sync + 'static> PostgresOutboxStore<Ser> {
123 pub async fn connect(
135 pool: PgPool,
136 settings: PostgresOutboxSettings,
137 serializer: Ser,
138 ) -> Result<Self, PostgresStoreError> {
139 if !crate::error::is_valid_schema_name(&settings.schema) {
140 return Err(PostgresStoreError::InvalidSchema {
141 schema: settings.schema,
142 });
143 }
144
145 let check = verify_schema(&pool, &settings.schema).await?;
146
147 let resolved_here = check.resolved_schema.as_deref() == Some(settings.schema.as_str());
148 if !resolved_here {
149 if !check.configured_exists {
150 return Err(PostgresStoreError::NotMigrated {
151 schema: settings.schema,
152 });
153 }
154 return Err(PostgresStoreError::SchemaResolution {
155 configured: settings.schema,
156 observed: check.search_path,
157 });
158 }
159
160 let others = other_outbox_schemas(&pool, &settings.schema).await?;
161 if !others.is_empty() {
162 tracing::warn!(
163 configured_schema = %settings.schema,
164 other_schemas = ?others,
165 "a table named `outbox` also exists outside the configured schema; \
166 an unqualified reference from another session could resolve to it"
167 );
168 }
169
170 Ok(Self {
171 pool,
172 settings,
173 serializer: Arc::new(serializer),
174 })
175 }
176
177 #[must_use]
182 pub fn content_type(&self) -> &ContentType {
183 self.serializer.content_type()
184 }
185
186 fn map_err(&self, err: sqlx::Error) -> PostgresStoreError {
190 map_operational_error(&self.settings.schema, err)
191 }
192
193 async fn set_local_timeout(
196 &self,
197 tx: &mut Transaction<'_, Postgres>,
198 ) -> Result<(), PostgresStoreError> {
199 let timeout_ms = i64::try_from(self.settings.statement_timeout.as_millis())
200 .unwrap_or(i64::MAX)
201 .to_string();
202 sqlx::query_scalar!(
203 "SELECT set_config('statement_timeout', $1, true)",
204 timeout_ms
205 )
206 .fetch_one(&mut **tx)
207 .await
208 .map_err(|e| self.map_err(e))?;
209 Ok(())
210 }
211}
212
213async fn insert_enqueued<T>(
218 tx: &mut Transaction<'_, Postgres>,
219 settings: &PostgresOutboxSettings,
220 envelope: &reliar_core::Envelope<T>,
221 payload: &Bytes,
222 content_type: &ContentType,
223) -> Result<(), sqlx::Error> {
224 let restore = if settings.enqueue_sets_search_path {
225 Some(set_search_path(tx, &settings.schema).await?)
226 } else {
227 None
228 };
229
230 let result = insert_row(tx, envelope, payload, content_type).await;
231
232 if result.is_ok()
237 && let Some(previous) = restore
238 {
239 restore_search_path(tx, &previous).await?;
240 }
241
242 result
243}
244
245async fn set_search_path(
251 tx: &mut Transaction<'_, Postgres>,
252 schema: &str,
253) -> Result<String, sqlx::Error> {
254 let previous: String = sqlx::query_scalar!("SELECT current_setting('search_path')")
255 .fetch_one(&mut **tx)
256 .await?
257 .unwrap_or_default();
258 let wanted = format!("{schema},public");
259 sqlx::query_scalar!("SELECT set_config('search_path', $1, true)", wanted)
260 .fetch_one(&mut **tx)
261 .await?;
262 Ok(previous)
263}
264
265async fn restore_search_path(
266 tx: &mut Transaction<'_, Postgres>,
267 previous: &str,
268) -> Result<(), sqlx::Error> {
269 sqlx::query_scalar!("SELECT set_config('search_path', $1, true)", previous)
270 .fetch_one(&mut **tx)
271 .await?;
272 Ok(())
273}
274
275async fn insert_row<T>(
279 tx: &mut Transaction<'_, Postgres>,
280 envelope: &reliar_core::Envelope<T>,
281 payload: &Bytes,
282 content_type: &ContentType,
283) -> Result<(), sqlx::Error> {
284 let corr = &envelope.metadata.correlation;
285 let sent_at_ms = envelope
286 .metadata
287 .delivery
288 .sent_at
289 .map(crate::records::encode_epoch_millis);
290 let rest = crate::records::MetadataRest {
291 trace: crate::records::TraceRest {
292 traceparent: envelope.metadata.trace.traceparent.clone(),
293 tracestate: envelope.metadata.trace.tracestate.clone(),
294 },
295 routing: crate::records::RoutingRest {
296 source: envelope
297 .metadata
298 .routing
299 .source
300 .as_ref()
301 .map(|v| v.as_str().to_owned()),
302 destination: envelope
303 .metadata
304 .routing
305 .destination
306 .as_ref()
307 .map(|v| v.as_str().to_owned()),
308 reply_to: envelope
309 .metadata
310 .routing
311 .reply_to
312 .as_ref()
313 .map(|v| v.as_str().to_owned()),
314 },
315 delivery: crate::records::DeliveryRest {
316 sent_at_ms,
317 deduplication_id: envelope.metadata.delivery.deduplication_id.clone(),
318 },
319 };
320 let metadata_json = if rest.trace.traceparent.is_none()
322 && rest.trace.tracestate.is_none()
323 && rest.routing.source.is_none()
324 && rest.routing.destination.is_none()
325 && rest.routing.reply_to.is_none()
326 && rest.delivery.sent_at_ms.is_none()
327 && rest.delivery.deduplication_id.is_none()
328 {
329 None
330 } else {
331 serde_json::to_value(&rest).ok()
339 };
340
341 let headers_json = envelope.headers().filter(|h| !h.is_empty()).map(|h| {
342 let map: serde_json::Map<String, serde_json::Value> = h
343 .iter()
344 .map(|(k, v)| (k.to_owned(), serde_json::Value::String(v.to_owned())))
345 .collect();
346 serde_json::Value::Object(map)
347 });
348
349 sqlx::query!(
350 r#"INSERT INTO outbox (
351 id, message_type, message_version,
352 correlation_id, conversation_id, causation_id, request_id,
353 content_type, payload, tenant_id, expires_at, ordering_key,
354 metadata, headers, available_at
355 ) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14, now())"#,
356 envelope.id.as_uuid(),
357 envelope.message_type.name(),
358 i32::from(envelope.message_type.version()),
359 corr.correlation_id
360 .as_ref()
361 .map(reliar_core::CorrelationId::as_str),
362 corr.conversation_id.as_uuid(),
363 corr.causation_id.map(|id| id.as_uuid()),
364 corr.request_id.map(|id| id.as_uuid()),
365 content_type.as_str(),
366 &payload[..],
367 envelope.metadata.tenant_id.as_deref(),
368 envelope.metadata.delivery.expires_at,
369 None::<&str>,
374 metadata_json,
375 headers_json,
376 )
377 .execute(&mut **tx)
378 .await?;
379 Ok(())
380}
381
382impl<'c, Ser> OutboxEnqueue<Transaction<'c, Postgres>> for PostgresOutboxStore<Ser>
407where
408 Ser: Serializer + Send + Sync + 'static,
409{
410 type Error = EnqueueError<Ser::Error>;
411
412 fn enqueue_envelope<T: Message + Sync>(
434 &self,
435 tx: &mut Transaction<'c, Postgres>,
436 typed_envelope: reliar_core::Envelope<T>,
437 ) -> impl Future<Output = Result<MessageId, Self::Error>> + Send {
438 let span = tracing::debug_span!(
439 "reliar.outbox.enqueue",
440 message.id = %typed_envelope.id,
441 message.type = %typed_envelope.message_type,
442 );
443 let payload = {
447 let _guard = span.enter();
448 self.serializer
449 .serialize(&typed_envelope.body)
450 .map_err(|source| EnqueueError::Serialize { source })
451 };
452 let envelope = typed_envelope.map_body(|_| ());
454
455 async move {
456 let payload = payload?;
457 insert_enqueued(tx, &self.settings, &envelope, &payload, self.content_type())
458 .await
459 .map_err(|source| map_enqueue_error(envelope.id, source))?;
460 Ok(envelope.id)
461 }
462 .instrument(span)
463 }
464}
465
466#[cfg(feature = "json")]
467impl PostgresOutboxStore<JsonSerializer> {
468 pub async fn new(pool: PgPool) -> Result<Self, PostgresStoreError> {
474 Self::connect(pool, PostgresOutboxSettings::default(), JsonSerializer).await
475 }
476
477 pub async fn with_settings(
484 pool: PgPool,
485 settings: PostgresOutboxSettings,
486 ) -> Result<Self, PostgresStoreError> {
487 Self::connect(pool, settings, JsonSerializer).await
488 }
489}
490
491async fn poison_sweep_rows<'e>(
496 executor: impl sqlx::PgExecutor<'e>,
497 poisoned_ids: &[uuid::Uuid],
498 poisoned_errors: &[String],
499 worker: &str,
500 undecodable: &str,
501) -> Result<(), sqlx::Error> {
502 sqlx::query!(
503 r#"UPDATE outbox o
504 SET dead_at = now(),
505 dead_reason = $4,
506 last_error = f.err,
507 locked_by = NULL,
508 locked_until = NULL,
509 updated_at = now()
510 FROM UNNEST($1::uuid[], $2::text[]) AS f(id, err)
511 WHERE o.id = f.id AND o.locked_by = $3"#,
512 poisoned_ids,
513 poisoned_errors,
514 worker,
515 undecodable,
516 )
517 .execute(executor)
518 .await?;
519 Ok(())
520}
521
522async fn purge_published_rows<'e>(
530 executor: impl sqlx::PgExecutor<'e>,
531 retention_ms: i64,
532 batch_size: i64,
533) -> Result<u64, sqlx::Error> {
534 let result = sqlx::query!(
535 r#"DELETE FROM outbox WHERE id IN (
536 SELECT id FROM outbox
537 WHERE published_at IS NOT NULL
538 AND published_at < now() - ($1::bigint * interval '1 millisecond')
539 LIMIT $2
540 )
541 AND published_at IS NOT NULL
542 AND published_at < now() - ($1::bigint * interval '1 millisecond')"#,
543 retention_ms,
544 batch_size,
545 )
546 .execute(executor)
547 .await?;
548 Ok(result.rows_affected())
549}
550
551async fn purge_dead_retention_rows<'e>(
557 executor: impl sqlx::PgExecutor<'e>,
558 retention_ms: i64,
559 batch_size: i64,
560) -> Result<u64, sqlx::Error> {
561 let result = sqlx::query!(
562 r#"DELETE FROM outbox WHERE id IN (
563 SELECT id FROM outbox
564 WHERE dead_at IS NOT NULL
565 AND dead_at < now() - ($1::bigint * interval '1 millisecond')
566 LIMIT $2
567 )
568 AND dead_at IS NOT NULL
569 AND dead_at < now() - ($1::bigint * interval '1 millisecond')"#,
570 retention_ms,
571 batch_size,
572 )
573 .execute(executor)
574 .await?;
575 Ok(result.rows_affected())
576}
577
578async fn purge_expired_sweep_rows<'e>(
584 executor: impl sqlx::PgExecutor<'e>,
585 batch_size: i64,
586 expired_reason: &str,
587) -> Result<u64, sqlx::Error> {
588 let result = sqlx::query!(
589 r#"UPDATE outbox
590 SET dead_at = now(),
591 dead_reason = $2,
592 last_error = 'reliar: expired before publication',
593 locked_by = NULL,
594 locked_until = NULL,
595 updated_at = now()
596 WHERE id IN (
597 SELECT id FROM outbox
598 WHERE expires_at IS NOT NULL AND expires_at < now()
599 AND published_at IS NULL AND dead_at IS NULL
600 AND (locked_until IS NULL OR locked_until < now())
601 LIMIT $1
602 )
603 AND published_at IS NULL AND dead_at IS NULL
604 AND (locked_until IS NULL OR locked_until < now())"#,
605 batch_size,
606 expired_reason,
607 )
608 .execute(executor)
609 .await?;
610 Ok(result.rows_affected())
611}
612
613async fn complete_rows<'e>(
617 executor: impl sqlx::PgExecutor<'e>,
618 ids: &[uuid::Uuid],
619 worker: &str,
620) -> Result<u64, sqlx::Error> {
621 let result = sqlx::query!(
622 r#"UPDATE outbox
623 SET published_at = now(),
624 attempts = attempts + 1,
625 locked_by = NULL,
626 locked_until = NULL,
627 updated_at = now()
628 WHERE id = ANY($1) AND locked_by = $2"#,
629 ids,
630 worker,
631 )
632 .execute(executor)
633 .await?;
634 Ok(result.rows_affected())
635}
636
637async fn release_rows<'e>(
638 executor: impl sqlx::PgExecutor<'e>,
639 ids: &[uuid::Uuid],
640 worker: &str,
641) -> Result<u64, sqlx::Error> {
642 let result = sqlx::query!(
643 r#"UPDATE outbox
644 SET locked_by = NULL,
645 locked_until = NULL,
646 updated_at = now()
647 WHERE id = ANY($1) AND locked_by = $2"#,
648 ids,
649 worker,
650 )
651 .execute(executor)
652 .await?;
653 Ok(result.rows_affected())
654}
655
656async fn extend_lease_rows<'e>(
657 executor: impl sqlx::PgExecutor<'e>,
658 ids: &[uuid::Uuid],
659 lease_ms: i64,
660 worker: &str,
661) -> Result<u64, sqlx::Error> {
662 let result = sqlx::query!(
663 r#"UPDATE outbox
664 SET locked_until = now() + ($2::bigint * interval '1 millisecond'),
665 updated_at = now()
666 WHERE id = ANY($1) AND locked_by = $3"#,
667 ids,
668 lease_ms,
669 worker,
670 )
671 .execute(executor)
672 .await?;
673 Ok(result.rows_affected())
674}
675
676async fn fail_retry_rows<'e>(
677 executor: impl sqlx::PgExecutor<'e>,
678 ids: &[uuid::Uuid],
679 errors: &[String],
680 delays_ms: &[i64],
681 worker: &str,
682) -> Result<u64, sqlx::Error> {
683 let result = sqlx::query!(
684 r#"UPDATE outbox o
685 SET attempts = o.attempts + 1,
686 last_error = f.err,
687 locked_by = NULL,
688 locked_until = NULL,
689 available_at = now() + (f.delay_ms * interval '1 millisecond'),
690 updated_at = now()
691 FROM UNNEST($1::uuid[], $2::text[], $3::bigint[]) AS f(id, err, delay_ms)
692 WHERE o.id = f.id AND o.locked_by = $4"#,
693 ids,
694 errors,
695 delays_ms,
696 worker,
697 )
698 .execute(executor)
699 .await?;
700 Ok(result.rows_affected())
701}
702
703async fn fail_dead_rows<'e>(
704 executor: impl sqlx::PgExecutor<'e>,
705 ids: &[uuid::Uuid],
706 errors: &[String],
707 reasons: &[&str],
708 worker: &str,
709) -> Result<u64, sqlx::Error> {
710 let result = sqlx::query!(
711 r#"UPDATE outbox o
712 SET attempts = o.attempts + 1,
713 last_error = f.err,
714 dead_at = now(),
715 dead_reason = f.reason,
716 locked_by = NULL,
717 locked_until = NULL,
718 updated_at = now()
719 FROM UNNEST($1::uuid[], $2::text[], $3::text[]) AS f(id, err, reason)
720 WHERE o.id = f.id AND o.locked_by = $4"#,
721 ids,
722 errors,
723 reasons as &[&str],
724 worker,
725 )
726 .execute(executor)
727 .await?;
728 Ok(result.rows_affected())
729}
730
731async fn claim_rows<'e>(
739 executor: impl sqlx::PgExecutor<'e>,
740 batch_size: i64,
741 worker: &str,
742 lease_ms: i64,
743) -> Result<Vec<RawRow>, sqlx::Error> {
744 sqlx::query_as!(
745 RawRow,
746 r#"WITH claimed AS (
747 SELECT id FROM outbox
748 WHERE published_at IS NULL AND dead_at IS NULL
749 AND available_at <= now()
750 AND (locked_until IS NULL OR locked_until < now())
751 AND (expires_at IS NULL OR expires_at > now())
752 ORDER BY available_at, sequence
753 LIMIT $1
754 FOR UPDATE SKIP LOCKED
755 )
756 UPDATE outbox o
757 SET locked_by = $2,
758 locked_until = now() + ($3::bigint * interval '1 millisecond'),
759 updated_at = now()
760 FROM claimed
761 WHERE o.id = claimed.id
762 RETURNING o.id, o.sequence, o.message_type, o.message_version,
763 o.correlation_id, o.conversation_id, o.causation_id, o.request_id,
764 o.content_type, o.payload, o.tenant_id, o.expires_at, o.ordering_key,
765 o.metadata, o.headers, o.metadata_version,
766 o.created_at, o.available_at,
767 o.attempts, o.locked_by, o.locked_until,
768 o.published_at, o.dead_at, o.dead_reason, o.last_error"#,
769 batch_size,
770 worker,
771 lease_ms,
772 )
773 .fetch_all(executor)
774 .await
775}
776
777async fn list_dead_rows<'e>(
781 executor: impl sqlx::PgExecutor<'e>,
782 query: &DeadQuery,
783 limit: i64,
784) -> Result<Vec<RawRow>, sqlx::Error> {
785 sqlx::query_as!(
786 RawRow,
787 r#"SELECT id, sequence, message_type, message_version,
788 correlation_id, conversation_id, causation_id, request_id,
789 content_type, payload, tenant_id, expires_at, ordering_key,
790 metadata, headers, metadata_version,
791 created_at, available_at,
792 attempts, locked_by, locked_until,
793 published_at, dead_at, dead_reason, last_error
794 FROM outbox
795 WHERE dead_at IS NOT NULL
796 AND ($1::text IS NULL OR message_type = $1)
797 AND ($2::text IS NULL OR tenant_id = $2)
798 AND ($3::timestamptz IS NULL OR dead_at < $3)
799 AND ($4::bigint IS NULL OR sequence > $4)
800 ORDER BY sequence ASC
801 LIMIT $5"#,
802 query.message_type,
803 query.tenant_id,
804 query.dead_before,
805 query.after_sequence,
806 limit,
807 )
808 .fetch_all(executor)
809 .await
810}
811
812async fn retry_dead_rows<'e>(
815 executor: impl sqlx::PgExecutor<'e>,
816 ids: &[uuid::Uuid],
817) -> Result<u64, sqlx::Error> {
818 let result = sqlx::query!(
819 r#"UPDATE outbox
820 SET dead_at = NULL,
821 dead_reason = NULL,
822 available_at = now(),
823 attempts = 0,
824 locked_by = NULL,
825 locked_until = NULL,
826 updated_at = now()
827 WHERE id = ANY($1) AND dead_at IS NOT NULL"#,
828 ids,
829 )
830 .execute(executor)
831 .await?;
832 Ok(result.rows_affected())
833}
834
835async fn purge_dead_rows<'e>(
838 executor: impl sqlx::PgExecutor<'e>,
839 ids: &[uuid::Uuid],
840) -> Result<u64, sqlx::Error> {
841 let result = sqlx::query!(
842 "DELETE FROM outbox WHERE id = ANY($1) AND dead_at IS NOT NULL",
843 ids,
844 )
845 .execute(executor)
846 .await?;
847 Ok(result.rows_affected())
848}
849
850impl<Ser: Serializer + Send + Sync + 'static> OutboxStore for PostgresOutboxStore<Ser> {
851 type Error = PostgresStoreError;
852
853 async fn acquire(
862 &self,
863 request: reliar_outbox::AcquireRequest,
864 ) -> Result<AcquiredBatch, Self::Error> {
865 let batch_size = i64::from(request.batch_size);
866 let lease_ms = i64::try_from(request.lease.as_millis()).unwrap_or(i64::MAX);
867 let worker = request.worker.as_str();
868
869 let rows = if self.settings.statement_timeout.is_zero() {
874 claim_rows(&self.pool, batch_size, worker, lease_ms)
875 .await
876 .map_err(|e| self.map_err(e))?
877 } else {
878 let mut tx = self.pool.begin().await.map_err(|e| self.map_err(e))?;
879 let timeout_ms = i64::try_from(self.settings.statement_timeout.as_millis())
880 .unwrap_or(i64::MAX)
881 .to_string();
882 sqlx::query_scalar!(
883 "SELECT set_config('statement_timeout', $1, true)",
884 timeout_ms
885 )
886 .fetch_one(&mut *tx)
887 .await
888 .map_err(|e| self.map_err(e))?;
889 let rows = claim_rows(&mut *tx, batch_size, worker, lease_ms)
890 .await
891 .map_err(|e| self.map_err(e))?;
892 tx.commit().await.map_err(|e| self.map_err(e))?;
893 rows
894 };
895
896 let mut records = Vec::with_capacity(rows.len());
897 let mut poisoned = Vec::new();
898 let mut poisoned_ids = Vec::new();
899 let mut poisoned_errors = Vec::new();
900
901 for raw in rows {
902 match decode_row(raw) {
903 Ok(record) => records.push(record),
904 Err(err) => {
905 poisoned_ids.push(err.id.as_uuid());
906 poisoned_errors.push(crate::records::truncate_last_error(err.detail.clone()));
907 poisoned.push(PoisonedRow::new(err.id, err.sequence, err.detail));
908 }
909 }
910 }
911
912 if !poisoned_ids.is_empty() {
913 let undecodable =
920 crate::records::encode_dead_reason(reliar_outbox::DeadReason::Undecodable);
921 if self.settings.statement_timeout.is_zero() {
922 poison_sweep_rows(
923 &self.pool,
924 &poisoned_ids,
925 &poisoned_errors,
926 worker,
927 undecodable,
928 )
929 .await
930 .map_err(|e| self.map_err(e))?;
931 } else {
932 let mut tx = self.pool.begin().await.map_err(|e| self.map_err(e))?;
933 self.set_local_timeout(&mut tx).await?;
934 poison_sweep_rows(
935 &mut *tx,
936 &poisoned_ids,
937 &poisoned_errors,
938 worker,
939 undecodable,
940 )
941 .await
942 .map_err(|e| self.map_err(e))?;
943 tx.commit().await.map_err(|e| self.map_err(e))?;
944 }
945 }
946
947 Ok(AcquiredBatch::new(records, poisoned))
948 }
949
950 async fn complete(
954 &self,
955 worker: &WorkerId,
956 items: &[CompletedMessage],
957 ) -> Result<u64, Self::Error> {
958 if items.is_empty() {
959 return Ok(0);
960 }
961 let ids: Vec<uuid::Uuid> = items.iter().map(|i| i.message.id.as_uuid()).collect();
962 let affected = if self.settings.statement_timeout.is_zero() {
963 complete_rows(&self.pool, &ids, worker.as_str())
964 .await
965 .map_err(|e| self.map_err(e))?
966 } else {
967 let mut tx = self.pool.begin().await.map_err(|e| self.map_err(e))?;
968 self.set_local_timeout(&mut tx).await?;
969 let affected = complete_rows(&mut *tx, &ids, worker.as_str())
970 .await
971 .map_err(|e| self.map_err(e))?;
972 tx.commit().await.map_err(|e| self.map_err(e))?;
973 affected
974 };
975 log_shortfall("complete", items.len(), affected);
976 Ok(affected)
977 }
978
979 async fn fail(&self, worker: &WorkerId, items: &[FailedMessage]) -> Result<u64, Self::Error> {
984 if items.is_empty() {
985 return Ok(0);
986 }
987
988 let mut retry_ids = Vec::new();
989 let mut retry_errors = Vec::new();
990 let mut retry_delays = Vec::new();
991 let mut dead_ids = Vec::new();
992 let mut dead_errors = Vec::new();
993 let mut dead_reasons = Vec::new();
994
995 for item in items {
996 match item.outcome {
997 FailureOutcome::Retry { delay } => {
998 retry_ids.push(item.message.id.as_uuid());
999 retry_errors.push(item.error.clone());
1000 retry_delays.push(i64::try_from(delay.as_millis()).unwrap_or(i64::MAX));
1001 }
1002 FailureOutcome::Dead { reason } => {
1003 dead_ids.push(item.message.id.as_uuid());
1004 dead_errors.push(item.error.clone());
1005 dead_reasons.push(crate::records::encode_dead_reason(reason));
1006 }
1007 _ => tracing::error!(
1012 id = %item.message.id,
1013 "unrecognised FailureOutcome variant; row left as-is"
1014 ),
1015 }
1016 }
1017
1018 let affected = if self.settings.statement_timeout.is_zero() {
1019 let mut affected = 0u64;
1020 if !retry_ids.is_empty() {
1021 affected += fail_retry_rows(
1022 &self.pool,
1023 &retry_ids,
1024 &retry_errors,
1025 &retry_delays,
1026 worker.as_str(),
1027 )
1028 .await
1029 .map_err(|e| self.map_err(e))?;
1030 }
1031 if !dead_ids.is_empty() {
1032 affected += fail_dead_rows(
1033 &self.pool,
1034 &dead_ids,
1035 &dead_errors,
1036 &dead_reasons,
1037 worker.as_str(),
1038 )
1039 .await
1040 .map_err(|e| self.map_err(e))?;
1041 }
1042 affected
1043 } else {
1044 let mut tx = self.pool.begin().await.map_err(|e| self.map_err(e))?;
1045 self.set_local_timeout(&mut tx).await?;
1046 let mut affected = 0u64;
1047 if !retry_ids.is_empty() {
1048 affected += fail_retry_rows(
1049 &mut *tx,
1050 &retry_ids,
1051 &retry_errors,
1052 &retry_delays,
1053 worker.as_str(),
1054 )
1055 .await
1056 .map_err(|e| self.map_err(e))?;
1057 }
1058 if !dead_ids.is_empty() {
1059 affected += fail_dead_rows(
1060 &mut *tx,
1061 &dead_ids,
1062 &dead_errors,
1063 &dead_reasons,
1064 worker.as_str(),
1065 )
1066 .await
1067 .map_err(|e| self.map_err(e))?;
1068 }
1069 tx.commit().await.map_err(|e| self.map_err(e))?;
1070 affected
1071 };
1072 log_shortfall("fail", items.len(), affected);
1073 Ok(affected)
1074 }
1075
1076 async fn release(&self, worker: &WorkerId, items: &[MessageRef]) -> Result<u64, Self::Error> {
1079 if items.is_empty() {
1080 return Ok(0);
1081 }
1082 let ids: Vec<uuid::Uuid> = items.iter().map(|i| i.id.as_uuid()).collect();
1083 let affected = if self.settings.statement_timeout.is_zero() {
1084 release_rows(&self.pool, &ids, worker.as_str())
1085 .await
1086 .map_err(|e| self.map_err(e))?
1087 } else {
1088 let mut tx = self.pool.begin().await.map_err(|e| self.map_err(e))?;
1089 self.set_local_timeout(&mut tx).await?;
1090 let affected = release_rows(&mut *tx, &ids, worker.as_str())
1091 .await
1092 .map_err(|e| self.map_err(e))?;
1093 tx.commit().await.map_err(|e| self.map_err(e))?;
1094 affected
1095 };
1096 log_shortfall("release", items.len(), affected);
1097 Ok(affected)
1098 }
1099
1100 async fn extend_lease(
1103 &self,
1104 worker: &WorkerId,
1105 items: &[MessageRef],
1106 lease: std::time::Duration,
1107 ) -> Result<u64, Self::Error> {
1108 if items.is_empty() {
1109 return Ok(0);
1110 }
1111 let ids: Vec<uuid::Uuid> = items.iter().map(|i| i.id.as_uuid()).collect();
1112 let lease_ms = i64::try_from(lease.as_millis()).unwrap_or(i64::MAX);
1113 let affected = if self.settings.statement_timeout.is_zero() {
1114 extend_lease_rows(&self.pool, &ids, lease_ms, worker.as_str())
1115 .await
1116 .map_err(|e| self.map_err(e))?
1117 } else {
1118 let mut tx = self.pool.begin().await.map_err(|e| self.map_err(e))?;
1119 self.set_local_timeout(&mut tx).await?;
1120 let affected = extend_lease_rows(&mut *tx, &ids, lease_ms, worker.as_str())
1121 .await
1122 .map_err(|e| self.map_err(e))?;
1123 tx.commit().await.map_err(|e| self.map_err(e))?;
1124 affected
1125 };
1126 log_shortfall("extend_lease", items.len(), affected);
1127 Ok(affected)
1128 }
1129
1130 async fn purge(&self, request: PurgeRequest) -> Result<PurgeReport, Self::Error> {
1137 let batch_size = i64::from(request.batch_size);
1138 let expired_reason = crate::records::encode_dead_reason(reliar_outbox::DeadReason::Expired);
1139
1140 let (published_deleted, dead_deleted, expired_to_dead) =
1141 if self.settings.statement_timeout.is_zero() {
1142 let published_deleted = if let Some(retention) = request.published_retention {
1143 let retention_ms = i64::try_from(retention.as_millis()).unwrap_or(i64::MAX);
1144 purge_published_rows(&self.pool, retention_ms, batch_size)
1145 .await
1146 .map_err(|e| self.map_err(e))?
1147 } else {
1148 0
1149 };
1150
1151 let dead_deleted = if let Some(retention) = request.dead_retention {
1152 let retention_ms = i64::try_from(retention.as_millis()).unwrap_or(i64::MAX);
1153 purge_dead_retention_rows(&self.pool, retention_ms, batch_size)
1154 .await
1155 .map_err(|e| self.map_err(e))?
1156 } else {
1157 0
1158 };
1159
1160 let expired_to_dead =
1161 purge_expired_sweep_rows(&self.pool, batch_size, expired_reason)
1162 .await
1163 .map_err(|e| self.map_err(e))?;
1164
1165 (published_deleted, dead_deleted, expired_to_dead)
1166 } else {
1167 let mut tx = self.pool.begin().await.map_err(|e| self.map_err(e))?;
1173 self.set_local_timeout(&mut tx).await?;
1174
1175 let published_deleted = if let Some(retention) = request.published_retention {
1176 let retention_ms = i64::try_from(retention.as_millis()).unwrap_or(i64::MAX);
1177 purge_published_rows(&mut *tx, retention_ms, batch_size)
1178 .await
1179 .map_err(|e| self.map_err(e))?
1180 } else {
1181 0
1182 };
1183
1184 let dead_deleted = if let Some(retention) = request.dead_retention {
1185 let retention_ms = i64::try_from(retention.as_millis()).unwrap_or(i64::MAX);
1186 purge_dead_retention_rows(&mut *tx, retention_ms, batch_size)
1187 .await
1188 .map_err(|e| self.map_err(e))?
1189 } else {
1190 0
1191 };
1192
1193 let expired_to_dead =
1194 purge_expired_sweep_rows(&mut *tx, batch_size, expired_reason)
1195 .await
1196 .map_err(|e| self.map_err(e))?;
1197
1198 tx.commit().await.map_err(|e| self.map_err(e))?;
1199 (published_deleted, dead_deleted, expired_to_dead)
1200 };
1201
1202 Ok(PurgeReport::new(
1203 published_deleted,
1204 dead_deleted,
1205 expired_to_dead,
1206 ))
1207 }
1208 async fn stats(&self) -> Result<OutboxStats, Self::Error> {
1222 if self.settings.statement_timeout.is_zero() {
1223 let row = sqlx::query!(
1224 r#"SELECT
1225 count(*) FILTER (
1226 WHERE published_at IS NULL AND dead_at IS NULL
1227 AND available_at <= now()
1228 AND (locked_until IS NULL OR locked_until < now())
1229 AND (expires_at IS NULL OR expires_at > now())
1230 ) AS "pending!",
1231 count(*) FILTER (WHERE dead_at IS NOT NULL) AS "dead!",
1232 count(*) FILTER (
1233 WHERE published_at IS NULL AND dead_at IS NULL
1234 AND expires_at IS NOT NULL AND expires_at < now()
1235 ) AS "expired_pending!",
1236 min(available_at) FILTER (
1237 WHERE published_at IS NULL AND dead_at IS NULL
1238 AND available_at <= now()
1239 AND (locked_until IS NULL OR locked_until < now())
1240 AND (expires_at IS NULL OR expires_at > now())
1241 ) AS oldest_pending_available_at,
1242 now() AS "as_of!"
1243 FROM outbox"#
1244 )
1245 .fetch_one(&self.pool)
1246 .await
1247 .map_err(|e| self.map_err(e))?;
1248
1249 return Ok(OutboxStats::new(
1250 u64::try_from(row.pending).unwrap_or(0),
1251 u64::try_from(row.dead).unwrap_or(0),
1252 u64::try_from(row.expired_pending).unwrap_or(0),
1253 row.oldest_pending_available_at,
1254 row.as_of,
1255 ));
1256 }
1257
1258 let mut tx = self.pool.begin().await.map_err(|e| self.map_err(e))?;
1260 self.set_local_timeout(&mut tx).await?;
1261
1262 let row = sqlx::query!(
1263 r#"SELECT
1264 count(*) FILTER (
1265 WHERE published_at IS NULL AND dead_at IS NULL
1266 AND available_at <= now()
1267 AND (locked_until IS NULL OR locked_until < now())
1268 AND (expires_at IS NULL OR expires_at > now())
1269 ) AS "pending!",
1270 count(*) FILTER (WHERE dead_at IS NOT NULL) AS "dead!",
1271 count(*) FILTER (
1272 WHERE published_at IS NULL AND dead_at IS NULL
1273 AND expires_at IS NOT NULL AND expires_at < now()
1274 ) AS "expired_pending!",
1275 min(available_at) FILTER (
1276 WHERE published_at IS NULL AND dead_at IS NULL
1277 AND available_at <= now()
1278 AND (locked_until IS NULL OR locked_until < now())
1279 AND (expires_at IS NULL OR expires_at > now())
1280 ) AS oldest_pending_available_at,
1281 now() AS "as_of!"
1282 FROM outbox"#
1283 )
1284 .fetch_one(&mut *tx)
1285 .await
1286 .map_err(|e| self.map_err(e))?;
1287
1288 tx.commit().await.map_err(|e| self.map_err(e))?;
1289
1290 Ok(OutboxStats::new(
1291 u64::try_from(row.pending).unwrap_or(0),
1292 u64::try_from(row.dead).unwrap_or(0),
1293 u64::try_from(row.expired_pending).unwrap_or(0),
1294 row.oldest_pending_available_at,
1295 row.as_of,
1296 ))
1297 }
1298}
1299
1300impl<Ser: Serializer + Send + Sync + 'static> OutboxDeadLetters for PostgresOutboxStore<Ser> {
1301 type Error = PostgresStoreError;
1302
1303 async fn list_dead(&self, query: DeadQuery) -> Result<DeadLetterPage, Self::Error> {
1310 let capped_limit = query.limit.min(MAX_LIST_DEAD_LIMIT);
1313 let limit = i64::from(capped_limit);
1314
1315 let rows = if self.settings.statement_timeout.is_zero() {
1316 list_dead_rows(&self.pool, &query, limit)
1317 .await
1318 .map_err(|e| self.map_err(e))?
1319 } else {
1320 let mut tx = self.pool.begin().await.map_err(|e| self.map_err(e))?;
1321 self.set_local_timeout(&mut tx).await?;
1322 let rows = list_dead_rows(&mut *tx, &query, limit)
1323 .await
1324 .map_err(|e| self.map_err(e))?;
1325 tx.commit().await.map_err(|e| self.map_err(e))?;
1326 rows
1327 };
1328
1329 let scanned = rows.len();
1330 let mut records = Vec::with_capacity(scanned);
1331 let mut poisoned = Vec::new();
1332 let mut max_sequence: Option<i64> = None;
1333
1334 for raw in rows {
1335 max_sequence = Some(max_sequence.map_or(raw.sequence, |m| m.max(raw.sequence)));
1336 match decode_row(raw) {
1337 Ok(record) => records.push(record),
1338 Err(err) => poisoned.push(PoisonedRow::new(err.id, err.sequence, err.detail)),
1339 }
1340 }
1341
1342 let next_after_sequence = if scanned == capped_limit as usize {
1345 max_sequence
1346 } else {
1347 None
1348 };
1349
1350 Ok(DeadLetterPage::new(records, poisoned, next_after_sequence))
1351 }
1352
1353 async fn retry_dead(&self, refs: &[MessageRef]) -> Result<u64, Self::Error> {
1357 if refs.is_empty() {
1358 return Ok(0);
1359 }
1360 let ids: Vec<uuid::Uuid> = refs.iter().map(|r| r.id.as_uuid()).collect();
1361 let affected = if self.settings.statement_timeout.is_zero() {
1362 retry_dead_rows(&self.pool, &ids)
1363 .await
1364 .map_err(|e| self.map_err(e))?
1365 } else {
1366 let mut tx = self.pool.begin().await.map_err(|e| self.map_err(e))?;
1367 self.set_local_timeout(&mut tx).await?;
1368 let affected = retry_dead_rows(&mut *tx, &ids)
1369 .await
1370 .map_err(|e| self.map_err(e))?;
1371 tx.commit().await.map_err(|e| self.map_err(e))?;
1372 affected
1373 };
1374 Ok(affected)
1375 }
1376
1377 async fn purge_dead(&self, refs: &[MessageRef]) -> Result<u64, Self::Error> {
1379 if refs.is_empty() {
1380 return Ok(0);
1381 }
1382 let ids: Vec<uuid::Uuid> = refs.iter().map(|r| r.id.as_uuid()).collect();
1383 let affected = if self.settings.statement_timeout.is_zero() {
1384 purge_dead_rows(&self.pool, &ids)
1385 .await
1386 .map_err(|e| self.map_err(e))?
1387 } else {
1388 let mut tx = self.pool.begin().await.map_err(|e| self.map_err(e))?;
1389 self.set_local_timeout(&mut tx).await?;
1390 let affected = purge_dead_rows(&mut *tx, &ids)
1391 .await
1392 .map_err(|e| self.map_err(e))?;
1393 tx.commit().await.map_err(|e| self.map_err(e))?;
1394 affected
1395 };
1396 Ok(affected)
1397 }
1398}
1399
1400fn log_shortfall(operation: &'static str, claimed: usize, affected: u64) {
1403 let claimed = claimed as u64;
1404 if affected < claimed {
1405 tracing::debug!(
1406 operation,
1407 claimed,
1408 affected,
1409 "fewer rows affected than claimed"
1410 );
1411 }
1412}