everruns_durable/persistence/postgres/
dlq.rs1use super::*;
4
5#[async_trait]
6impl DeadLetters for PostgresWorkflowEventStore {
7 #[instrument(skip(self, error_history))]
8 async fn move_to_dlq(
9 &self,
10 task_id: Uuid,
11 error_history: Vec<String>,
12 ) -> Result<(), StoreError> {
13 let error_json = serde_json::to_value(&error_history)
14 .map(sanitize_json_null_bytes)
15 .map_err(|e| StoreError::Serialization(e.to_string()))?;
16
17 sqlx::query(
19 r#"
20 INSERT INTO durable_dead_letter_queue (
21 original_task_id, workflow_id, activity_id, activity_type,
22 input, attempts, last_error, error_history, options
23 )
24 SELECT id, workflow_id, activity_id, activity_type,
25 input, attempt, COALESCE(last_error, 'unknown'), $2, options
26 FROM durable_task_queue
27 WHERE id = $1
28 "#,
29 )
30 .bind(task_id)
31 .bind(&error_json)
32 .execute(&self.pool)
33 .await
34 .map_err(|e| {
35 error!(error = %e, "Failed to move task to DLQ");
36 StoreError::Database(e.to_string())
37 })?;
38
39 debug!(%task_id, "moved task to DLQ");
40 Ok(())
41 }
42
43 #[instrument(skip(self))]
44 async fn requeue_from_dlq(&self, dlq_id: Uuid) -> Result<Uuid, StoreError> {
45 let db_err = |context: &'static str| {
46 move |e: sqlx::Error| {
47 error!(error = %e, "{context}");
48 StoreError::Database(e.to_string())
49 }
50 };
51 let mut tx = begin_transaction(&self.pool)
52 .await
53 .map_err(db_err("Failed to begin DLQ requeue"))?;
54
55 let entry = sqlx::query(
56 r#"
57 SELECT workflow_id, activity_id, activity_type, input, options
58 FROM durable_dead_letter_queue
59 WHERE id = $1
60 FOR UPDATE
61 "#,
62 )
63 .bind(dlq_id)
64 .fetch_optional(&mut *tx)
65 .await
66 .map_err(db_err("Failed to read DLQ entry"))?
67 .ok_or(StoreError::TaskNotFound(dlq_id))?;
68
69 let mut options = entry
74 .get::<Option<serde_json::Value>, _>("options")
75 .and_then(|value| serde_json::from_value::<ActivityOptions>(value).ok())
76 .unwrap_or_default();
77 options.start_delay = None;
78 let options_json =
79 serde_json::to_value(&options).map_err(|e| StoreError::Serialization(e.to_string()))?;
80
81 let task_id = Uuid::now_v7();
82 sqlx::query(
83 r#"
84 INSERT INTO durable_task_queue (
85 id, workflow_id, activity_id, activity_type, input, options,
86 max_attempts, priority,
87 schedule_to_start_timeout_ms, start_to_close_timeout_ms, heartbeat_timeout_ms,
88 queue
89 )
90 VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)
91 "#,
92 )
93 .bind(task_id)
94 .bind(entry.get::<Option<Uuid>, _>("workflow_id"))
95 .bind(entry.get::<String, _>("activity_id"))
96 .bind(entry.get::<String, _>("activity_type"))
97 .bind(entry.get::<serde_json::Value, _>("input"))
98 .bind(&options_json)
99 .bind(options.retry_policy.max_attempts as i32)
100 .bind(options.priority)
101 .bind(options.schedule_to_start_timeout.as_millis() as i64)
102 .bind(options.start_to_close_timeout.as_millis() as i64)
103 .bind(options.heartbeat_timeout.map(|d| d.as_millis() as i64))
104 .bind(options.queue.as_deref())
105 .execute(&mut *tx)
106 .await
107 .map_err(db_err("Failed to requeue from DLQ"))?;
108
109 sqlx::query(
110 r#"
111 UPDATE durable_dead_letter_queue
112 SET requeued_at = NOW(),
113 requeue_count = requeue_count + 1
114 WHERE id = $1
115 "#,
116 )
117 .bind(dlq_id)
118 .execute(&mut *tx)
119 .await
120 .map_err(db_err("Failed to mark DLQ entry requeued"))?;
121
122 crate::commit_transaction(tx)
123 .await
124 .map_err(db_err("Failed to commit DLQ requeue"))?;
125
126 debug!(%dlq_id, %task_id, "requeued task from DLQ");
127 Ok(task_id)
128 }
129
130 #[instrument(skip(self))]
131 async fn list_dlq(
132 &self,
133 filter: DlqFilter,
134 pagination: Pagination,
135 ) -> Result<Vec<DlqEntry>, StoreError> {
136 let rows = sqlx::query(
137 r#"
138 SELECT id, original_task_id, workflow_id, activity_id, activity_type,
139 input, attempts, last_error, error_history, dead_at
140 FROM durable_dead_letter_queue
141 WHERE ($1::uuid IS NULL OR workflow_id = $1)
142 AND ($2::text IS NULL OR activity_type = $2)
143 ORDER BY dead_at DESC
144 OFFSET $3
145 LIMIT $4
146 "#,
147 )
148 .bind(filter.workflow_id)
149 .bind(&filter.activity_type)
150 .bind(pagination.offset as i64)
151 .bind(pagination.limit as i64)
152 .fetch_all(&self.pool)
153 .await
154 .map_err(|e| {
155 error!(error = %e, "Failed to list DLQ");
156 StoreError::Database(e.to_string())
157 })?;
158
159 let entries = rows
160 .into_iter()
161 .map(|row| {
162 let error_history_json: serde_json::Value = row.get("error_history");
163 let error_history: Vec<String> =
164 serde_json::from_value(error_history_json).unwrap_or_default();
165
166 DlqEntry {
167 id: row.get("id"),
168 original_task_id: row.get("original_task_id"),
169 workflow_id: row.get::<Option<Uuid>, _>("workflow_id"),
170 activity_id: row.get("activity_id"),
171 activity_type: row.get("activity_type"),
172 input: row.get("input"),
173 attempts: row.get::<i32, _>("attempts") as u32,
174 last_error: row.get("last_error"),
175 error_history,
176 dead_at: row.get("dead_at"),
177 }
178 })
179 .collect();
180
181 Ok(entries)
182 }
183}