Skip to main content

everruns_durable/persistence/postgres/
dlq.rs

1//! Implementation of [`crate::DeadLetters`].
2
3use super::*;
4
5#[async_trait]
6impl DeadLetters for PostgresWorkflowEventStore {
7    #[instrument(skip(self, error_history))]
8    async fn move_to_dlq(
9        &self,
10        task_id: Uuid,
11        error_history: Vec<String>,
12    ) -> Result<(), StoreError> {
13        let error_json = serde_json::to_value(&error_history)
14            .map(sanitize_json_null_bytes)
15            .map_err(|e| StoreError::Serialization(e.to_string()))?;
16
17        // Get task details and move to DLQ
18        sqlx::query(
19            r#"
20            INSERT INTO durable_dead_letter_queue (
21                original_task_id, workflow_id, activity_id, activity_type,
22                input, attempts, last_error, error_history, options
23            )
24            SELECT id, workflow_id, activity_id, activity_type,
25                   input, attempt, COALESCE(last_error, 'unknown'), $2, options
26            FROM durable_task_queue
27            WHERE id = $1
28            "#,
29        )
30        .bind(task_id)
31        .bind(&error_json)
32        .execute(&self.pool)
33        .await
34        .map_err(|e| {
35            error!(error = %e, "Failed to move task to DLQ");
36            StoreError::Database(e.to_string())
37        })?;
38
39        debug!(%task_id, "moved task to DLQ");
40        Ok(())
41    }
42
43    #[instrument(skip(self))]
44    async fn requeue_from_dlq(&self, dlq_id: Uuid) -> Result<Uuid, StoreError> {
45        let db_err = |context: &'static str| {
46            move |e: sqlx::Error| {
47                error!(error = %e, "{context}");
48                StoreError::Database(e.to_string())
49            }
50        };
51        let mut tx = begin_transaction(&self.pool)
52            .await
53            .map_err(db_err("Failed to begin DLQ requeue"))?;
54
55        let entry = sqlx::query(
56            r#"
57            SELECT workflow_id, activity_id, activity_type, input, options
58            FROM durable_dead_letter_queue
59            WHERE id = $1
60            FOR UPDATE
61            "#,
62        )
63        .bind(dlq_id)
64        .fetch_optional(&mut *tx)
65        .await
66        .map_err(db_err("Failed to read DLQ entry"))?
67        .ok_or(StoreError::TaskNotFound(dlq_id))?;
68
69        // The task goes back with the options it was enqueued with, so it
70        // keeps its queue and retry settings. Rows dead before the options
71        // column existed requeue with defaults. A requeue runs now: any start
72        // delay applied to the first enqueue only.
73        let mut options = entry
74            .get::<Option<serde_json::Value>, _>("options")
75            .and_then(|value| serde_json::from_value::<ActivityOptions>(value).ok())
76            .unwrap_or_default();
77        options.start_delay = None;
78        let options_json =
79            serde_json::to_value(&options).map_err(|e| StoreError::Serialization(e.to_string()))?;
80
81        let task_id = Uuid::now_v7();
82        sqlx::query(
83            r#"
84            INSERT INTO durable_task_queue (
85                id, workflow_id, activity_id, activity_type, input, options,
86                max_attempts, priority,
87                schedule_to_start_timeout_ms, start_to_close_timeout_ms, heartbeat_timeout_ms,
88                queue
89            )
90            VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)
91            "#,
92        )
93        .bind(task_id)
94        .bind(entry.get::<Option<Uuid>, _>("workflow_id"))
95        .bind(entry.get::<String, _>("activity_id"))
96        .bind(entry.get::<String, _>("activity_type"))
97        .bind(entry.get::<serde_json::Value, _>("input"))
98        .bind(&options_json)
99        .bind(options.retry_policy.max_attempts as i32)
100        .bind(options.priority)
101        .bind(options.schedule_to_start_timeout.as_millis() as i64)
102        .bind(options.start_to_close_timeout.as_millis() as i64)
103        .bind(options.heartbeat_timeout.map(|d| d.as_millis() as i64))
104        .bind(options.queue.as_deref())
105        .execute(&mut *tx)
106        .await
107        .map_err(db_err("Failed to requeue from DLQ"))?;
108
109        sqlx::query(
110            r#"
111            UPDATE durable_dead_letter_queue
112            SET requeued_at = NOW(),
113                requeue_count = requeue_count + 1
114            WHERE id = $1
115            "#,
116        )
117        .bind(dlq_id)
118        .execute(&mut *tx)
119        .await
120        .map_err(db_err("Failed to mark DLQ entry requeued"))?;
121
122        crate::commit_transaction(tx)
123            .await
124            .map_err(db_err("Failed to commit DLQ requeue"))?;
125
126        debug!(%dlq_id, %task_id, "requeued task from DLQ");
127        Ok(task_id)
128    }
129
130    #[instrument(skip(self))]
131    async fn list_dlq(
132        &self,
133        filter: DlqFilter,
134        pagination: Pagination,
135    ) -> Result<Vec<DlqEntry>, StoreError> {
136        let rows = sqlx::query(
137            r#"
138            SELECT id, original_task_id, workflow_id, activity_id, activity_type,
139                   input, attempts, last_error, error_history, dead_at
140            FROM durable_dead_letter_queue
141            WHERE ($1::uuid IS NULL OR workflow_id = $1)
142              AND ($2::text IS NULL OR activity_type = $2)
143            ORDER BY dead_at DESC
144            OFFSET $3
145            LIMIT $4
146            "#,
147        )
148        .bind(filter.workflow_id)
149        .bind(&filter.activity_type)
150        .bind(pagination.offset as i64)
151        .bind(pagination.limit as i64)
152        .fetch_all(&self.pool)
153        .await
154        .map_err(|e| {
155            error!(error = %e, "Failed to list DLQ");
156            StoreError::Database(e.to_string())
157        })?;
158
159        let entries = rows
160            .into_iter()
161            .map(|row| {
162                let error_history_json: serde_json::Value = row.get("error_history");
163                let error_history: Vec<String> =
164                    serde_json::from_value(error_history_json).unwrap_or_default();
165
166                DlqEntry {
167                    id: row.get("id"),
168                    original_task_id: row.get("original_task_id"),
169                    workflow_id: row.get::<Option<Uuid>, _>("workflow_id"),
170                    activity_id: row.get("activity_id"),
171                    activity_type: row.get("activity_type"),
172                    input: row.get("input"),
173                    attempts: row.get::<i32, _>("attempts") as u32,
174                    last_error: row.get("last_error"),
175                    error_history,
176                    dead_at: row.get("dead_at"),
177                }
178            })
179            .collect();
180
181        Ok(entries)
182    }
183}