1use super::*;
4
5#[async_trait]
6impl TaskQueue for PostgresWorkflowEventStore {
7 #[instrument(skip(self, task))]
8 async fn enqueue_task(&self, task: TaskDefinition) -> Result<Uuid, StoreError> {
9 if let Some(wf_id) = task.workflow_id {
11 let limit = self.max_pending_tasks_per_workflow;
12 let pending_count: i64 = sqlx::query_scalar(
13 r#"
14 SELECT COUNT(*) FROM durable_task_queue
15 WHERE workflow_id = $1 AND status = 'pending'
16 "#,
17 )
18 .bind(wf_id)
19 .fetch_one(&self.pool)
20 .await
21 .map_err(|e| {
22 error!(error = %e, "Failed to count pending tasks");
23 StoreError::Database(e.to_string())
24 })?;
25 if pending_count >= limit as i64 {
26 return Err(StoreError::TaskQueueLimitExceeded {
27 workflow_id: wf_id,
28 current: pending_count as u32,
29 limit,
30 });
31 }
32 } else {
33 let limit = crate::persistence::store::DEFAULT_MAX_PENDING_STANDALONE_TASKS;
35 let pending_count: i64 = sqlx::query_scalar(
36 r#"
37 SELECT COUNT(*) FROM durable_task_queue
38 WHERE workflow_id IS NULL AND status = 'pending'
39 "#,
40 )
41 .fetch_one(&self.pool)
42 .await
43 .map_err(|e| {
44 error!(error = %e, "Failed to count standalone pending tasks");
45 StoreError::Database(e.to_string())
46 })?;
47
48 if pending_count >= limit as i64 {
49 return Err(StoreError::StandaloneTaskQueueLimitExceeded {
50 current: pending_count as u32,
51 limit,
52 });
53 }
54 }
55 let task_id = Uuid::now_v7();
56 let task_input = sanitize_json_null_bytes(task.input.clone());
57 let options_json = serde_json::to_value(&task.options)
58 .map(sanitize_json_null_bytes)
59 .map_err(|e| StoreError::Serialization(e.to_string()))?;
60
61 sqlx::query(
62 r#"
63 INSERT INTO durable_task_queue (
64 id, workflow_id, activity_id, activity_type, input, options,
65 max_attempts, priority, visible_at,
66 schedule_to_start_timeout_ms, start_to_close_timeout_ms, heartbeat_timeout_ms,
67 queue
68 )
69 VALUES ($1, $2, $3, $4, $5, $6, $7, $8, NOW() + $12::bigint * INTERVAL '1 millisecond', $9, $10, $11, $13)
70 "#,
71 )
72 .bind(task_id)
73 .bind(task.workflow_id)
74 .bind(&task.activity_id)
75 .bind(&task.activity_type)
76 .bind(&task_input)
77 .bind(&options_json)
78 .bind(task.options.retry_policy.max_attempts as i32)
79 .bind(task.options.priority)
80 .bind(task.options.schedule_to_start_timeout.as_millis() as i64)
81 .bind(task.options.start_to_close_timeout.as_millis() as i64)
82 .bind(task.options.heartbeat_timeout.map(|d| d.as_millis() as i64))
83 .bind(task.options.start_delay.map_or(0, |d| d.as_millis() as i64))
84 .bind(task.options.queue.as_deref())
85 .execute(&self.pool)
86 .await
87 .map_err(|e| {
88 error!(error = %e, "Failed to enqueue task");
89 StoreError::Database(e.to_string())
90 })?;
91
92 #[cfg(feature = "failpoints")]
93 fail_point!("postgres_enqueue_task_after_insert", |_| {
94 Err(StoreError::Database(
95 "injected: after enqueue insert".into(),
96 ))
97 });
98
99 debug!(%task_id, activity_type = %task.activity_type, "enqueued task");
100 Ok(task_id)
101 }
102
103 #[instrument(skip(self, activity_types))]
104 async fn claim_queue_tasks(
105 &self,
106 worker_id: &str,
107 queue: Option<&str>,
108 activity_types: &[String],
109 max_tasks: usize,
110 ) -> Result<Vec<ClaimedTask>, StoreError> {
111 if activity_types.is_empty() {
112 return Ok(vec![]);
113 }
114
115 let mut tx = begin_transaction(&self.pool)
139 .await
140 .map_err(|e| StoreError::Database(e.to_string()))?;
141
142 let rows = sqlx::query(
143 r#"
144 WITH worker_check AS (
145 SELECT 1 FROM durable_workers
146 WHERE id = $3 AND status != 'draining'
147 ),
148 claimable AS (
149 SELECT tq.id
150 FROM durable_task_queue tq, worker_check
151 WHERE tq.status = 'pending'
152 AND tq.activity_type = ANY($1)
153 AND tq.queue IS NOT DISTINCT FROM $4
154 AND tq.visible_at <= NOW()
155 AND tq.attempt < tq.max_attempts
156 ORDER BY tq.priority DESC, tq.visible_at
157 LIMIT $2
158 FOR UPDATE SKIP LOCKED
159 ),
160 updated AS (
161 UPDATE durable_task_queue t
162 SET status = 'claimed',
163 claimed_by = $3,
164 claimed_at = NOW(),
165 heartbeat_at = NOW(),
166 attempt = attempt + 1
167 FROM claimable c
168 WHERE t.id = c.id
169 RETURNING t.id, t.workflow_id, t.activity_id, t.activity_type,
170 t.input, t.options, t.attempt, t.max_attempts
171 )
172 SELECT u.id, u.workflow_id, u.activity_id, u.activity_type,
173 u.input, u.options, u.attempt, u.max_attempts,
174 w.status AS workflow_status
175 FROM updated u
176 LEFT JOIN durable_workflow_instances w ON w.id = u.workflow_id
177 "#,
178 )
179 .bind(activity_types)
180 .bind(max_tasks as i32)
181 .bind(worker_id)
182 .bind(queue)
183 .fetch_all(&mut *tx)
184 .await
185 .map_err(|e| store_failure("durable.tasks.claim", "Failed to claim tasks", e))?;
186
187 let mut claimed = Vec::with_capacity(rows.len());
188 let mut started_by_workflow: BTreeMap<Uuid, Vec<(Uuid, String, i32)>> = BTreeMap::new();
195 for row in rows {
196 let options_json: serde_json::Value = row.get("options");
197 let options: ActivityOptions = serde_json::from_value(options_json)
198 .map_err(|e| StoreError::Serialization(e.to_string()))?;
199 let task_id: Uuid = row.get("id");
200 let workflow_id = row.get::<Option<Uuid>, _>("workflow_id");
201 let activity_id: String = row.get("activity_id");
202 let attempt = row.get::<i32, _>("attempt");
203
204 claimed.push(ClaimedTask {
205 id: task_id,
206 workflow_id,
207 activity_id: activity_id.clone(),
208 activity_type: row.get("activity_type"),
209 input: row.get("input"),
210 options,
211 attempt: attempt as u32,
212 max_attempts: row.get::<i32, _>("max_attempts") as u32,
213 workflow_status: row
214 .get::<Option<String>, _>("workflow_status")
215 .as_deref()
216 .map(parse_workflow_status)
217 .transpose()?,
218 });
219
220 if attempt == 1
221 && let Some(workflow_id) = workflow_id
222 {
223 started_by_workflow.entry(workflow_id).or_default().push((
224 task_id,
225 activity_id,
226 attempt,
227 ));
228 }
229 }
230
231 if !started_by_workflow.is_empty() {
232 let workflow_ids: Vec<Uuid> = started_by_workflow.keys().copied().collect();
240
241 sqlx::query(
242 r#"
243 SELECT id FROM durable_workflow_instances
244 WHERE id = ANY($1)
245 ORDER BY id
246 FOR UPDATE
247 "#,
248 )
249 .bind(&workflow_ids)
250 .fetch_all(&mut *tx)
251 .await
252 .map_err(|e| {
253 error!(error = %e, "Failed to lock workflows for activity start events");
254 StoreError::Database(e.to_string())
255 })?;
256
257 let seq_rows = sqlx::query(
258 r#"
259 SELECT workflow_id, COALESCE(MAX(sequence_num) + 1, 0)::INTEGER AS next_seq
260 FROM durable_workflow_events
261 WHERE workflow_id = ANY($1)
262 GROUP BY workflow_id
263 "#,
264 )
265 .bind(&workflow_ids)
266 .fetch_all(&mut *tx)
267 .await
268 .map_err(|e| {
269 error!(error = %e, "Failed to get workflow event sequences");
270 StoreError::Database(e.to_string())
271 })?;
272
273 let mut next_seq_by_workflow: BTreeMap<Uuid, i32> = BTreeMap::new();
274 for row in seq_rows {
275 next_seq_by_workflow.insert(row.get("workflow_id"), row.get::<i32, _>("next_seq"));
276 }
277
278 struct StartedEventRow {
282 workflow_id: Uuid,
283 sequence_num: i32,
284 event_data: serde_json::Value,
285 }
286 let mut insert_rows: Vec<StartedEventRow> = Vec::new();
287 for (workflow_id, mut events) in started_by_workflow {
288 events.sort_by_key(|(task_id, _, _)| *task_id);
289 let base_sequence = next_seq_by_workflow.get(&workflow_id).copied().unwrap_or(0);
292 for (offset, (_, activity_id, attempt)) in events.into_iter().enumerate() {
293 let event_data = serde_json::to_value(WorkflowEvent::ActivityStarted {
294 activity_id,
295 attempt: attempt as u32,
296 worker_id: worker_id.to_string(),
297 })
298 .map(sanitize_json_null_bytes)
299 .map_err(|e| StoreError::Serialization(e.to_string()))?;
300 insert_rows.push(StartedEventRow {
301 workflow_id,
302 sequence_num: base_sequence + offset as i32,
303 event_data,
304 });
305 }
306 }
307
308 if !insert_rows.is_empty() {
309 let mut builder = sqlx::QueryBuilder::new(
310 "INSERT INTO durable_workflow_events (workflow_id, sequence_num, event_type, event_data) ",
311 );
312 builder.push_values(insert_rows.iter(), |mut b, r| {
313 b.push_bind(r.workflow_id)
314 .push_bind(r.sequence_num)
315 .push_bind("activity_started")
316 .push_bind(&r.event_data);
317 });
318 builder.build().execute(&mut *tx).await.map_err(|e| {
319 error!(error = %e, "Failed to write activity start events");
320 StoreError::Database(e.to_string())
321 })?;
322 }
323 }
324
325 crate::commit_transaction(tx).await.map_err(|e| {
326 error!(error = %e, "Failed to commit task claim");
327 StoreError::Database(e.to_string())
328 })?;
329
330 #[cfg(feature = "failpoints")]
331 fail_point!("postgres_claim_task_after_query", |_| {
332 Err(StoreError::Database("injected: after claim query".into()))
333 });
334
335 if !claimed.is_empty() {
336 debug!(worker_id, count = claimed.len(), "claimed tasks");
337 }
338
339 Ok(claimed)
340 }
341
342 #[instrument(skip(self, _details))]
343 async fn heartbeat_task(
344 &self,
345 task_id: Uuid,
346 worker_id: &str,
347 _details: Option<serde_json::Value>,
348 ) -> Result<HeartbeatResponse, StoreError> {
349 let result: Option<(Option<String>,)> = sqlx::query_as(
351 r#"
352 UPDATE durable_task_queue q SET heartbeat_at = NOW()
353 WHERE q.id = $1 AND q.claimed_by = $2 AND q.status = 'claimed'
354 RETURNING (SELECT w.status FROM durable_workflow_instances w WHERE w.id = q.workflow_id)
355 "#,
356 )
357 .bind(task_id)
358 .bind(worker_id)
359 .fetch_optional(&self.pool)
360 .await
361 .map_err(|e| store_failure("durable.tasks.heartbeat", "Failed to heartbeat task", e))?;
362
363 #[cfg(feature = "failpoints")]
364 fail_point!("postgres_heartbeat_update", |_| {
365 Err(StoreError::Database("injected: heartbeat update".into()))
366 });
367
368 match result {
369 Some((workflow_status,)) => Ok(HeartbeatResponse {
371 accepted: true,
372 should_cancel: workflow_status.as_deref() == Some("cancelled"),
373 }),
374 None => {
375 Ok(HeartbeatResponse {
377 accepted: false,
378 should_cancel: true,
379 })
380 }
381 }
382 }
383
384 #[instrument(skip(self, _result))]
385 async fn complete_task(
386 &self,
387 task_id: Uuid,
388 worker_id: &str,
389 _result: serde_json::Value,
390 ) -> Result<(), StoreError> {
391 let result = sqlx::query(
396 r#"
397 UPDATE durable_task_queue
398 SET status = 'completed', heartbeat_at = NOW()
399 WHERE id = $1 AND claimed_by = $2 AND status = 'claimed'
400 RETURNING id
401 "#,
402 )
403 .bind(task_id)
404 .bind(worker_id)
405 .fetch_optional(&self.pool)
406 .await
407 .map_err(|e| {
408 error!(error = %e, "Failed to complete task");
409 StoreError::Database(e.to_string())
410 })?;
411
412 #[cfg(feature = "failpoints")]
413 fail_point!("postgres_complete_task_after_update", |_| {
414 Err(StoreError::Database("injected: complete task".into()))
415 });
416
417 match result {
418 Some(_) => {
419 debug!(%task_id, %worker_id, "completed task");
420 Ok(())
421 }
422 None => {
423 debug!(%task_id, %worker_id, "task not owned - was reclaimed or already completed");
425 Err(StoreError::TaskNotOwned(task_id))
426 }
427 }
428 }
429
430 #[instrument(skip(self))]
431 async fn fail_task_with_retry(
432 &self,
433 task_id: Uuid,
434 error: &str,
435 retryable: bool,
436 ) -> Result<TaskFailureOutcome, StoreError> {
437 let mut tx = begin_transaction(&self.pool)
447 .await
448 .map_err(|e| StoreError::Database(e.to_string()))?;
449
450 let row = sqlx::query(
451 r#"
452 SELECT attempt, max_attempts, options, status
453 FROM durable_task_queue
454 WHERE id = $1
455 FOR UPDATE
456 "#,
457 )
458 .bind(task_id)
459 .fetch_optional(&mut *tx)
460 .await
461 .map_err(|e| StoreError::Database(e.to_string()))?
462 .ok_or(StoreError::TaskNotFound(task_id))?;
463
464 let status: String = row.get("status");
465 if status != "claimed" {
466 debug!(
469 %task_id,
470 status,
471 "fail_task: task no longer claimed, skipping"
472 );
473 return Err(StoreError::TaskNotOwned(task_id));
474 }
475
476 let attempt: i32 = row.get("attempt");
477 let max_attempts: i32 = row.get("max_attempts");
478 let options_json: serde_json::Value = row.get("options");
479 let options: ActivityOptions = serde_json::from_value(options_json)
480 .map_err(|e| StoreError::Serialization(e.to_string()))?;
481
482 let outcome = if retryable && attempt < max_attempts {
483 let delay = options.retry_policy.delay_for_attempt((attempt + 1) as u32);
485 let visible_at = Utc::now() + chrono::Duration::from_std(delay).unwrap_or_default();
486
487 sqlx::query(
489 r#"
490 UPDATE durable_task_queue
491 SET status = 'pending',
492 claimed_by = NULL,
493 claimed_at = NULL,
494 heartbeat_at = NULL,
495 last_error = $2,
496 visible_at = $3
497 WHERE id = $1
498 "#,
499 )
500 .bind(task_id)
501 .bind(error)
502 .bind(visible_at)
503 .execute(&mut *tx)
504 .await
505 .map_err(|e| StoreError::Database(e.to_string()))?;
506
507 debug!(%task_id, next_attempt = attempt + 1, "task will retry");
508 TaskFailureOutcome::WillRetry {
509 next_attempt: (attempt + 1) as u32,
510 delay,
511 }
512 } else {
513 sqlx::query(
515 r#"
516 UPDATE durable_task_queue
517 SET status = 'dead',
518 last_error = $2
519 WHERE id = $1
520 "#,
521 )
522 .bind(task_id)
523 .bind(error)
524 .execute(&mut *tx)
525 .await
526 .map_err(|e| StoreError::Database(e.to_string()))?;
527
528 debug!(%task_id, "task moved to DLQ");
529 TaskFailureOutcome::MovedToDlq
530 };
531
532 crate::commit_transaction(tx)
533 .await
534 .map_err(|e| StoreError::Database(e.to_string()))?;
535
536 Ok(outcome)
537 }
538
539 #[instrument(skip(self))]
540 async fn cancel_pending_tasks_for_workflow(
541 &self,
542 workflow_id: Uuid,
543 ) -> Result<u64, StoreError> {
544 let result = sqlx::query(
545 r#"
546 UPDATE durable_task_queue
547 SET status = 'cancelled'
548 WHERE workflow_id = $1 AND status = 'pending'
549 "#,
550 )
551 .bind(workflow_id)
552 .execute(&self.pool)
553 .await
554 .map_err(|e| {
555 error!(error = %e, "Failed to cancel pending tasks for workflow");
556 StoreError::Database(e.to_string())
557 })?;
558
559 let count = result.rows_affected();
560 if count > 0 {
561 debug!(%workflow_id, count, "cancelled pending tasks for workflow");
562 }
563 Ok(count)
564 }
565
566 #[instrument(skip(self))]
567 async fn get_task(&self, task_id: Uuid) -> Result<TaskInfo, StoreError> {
568 let row = sqlx::query(
569 r#"
570 SELECT id, workflow_id, activity_id, activity_type, status,
571 priority, attempt, max_attempts, claimed_by, last_error,
572 created_at, claimed_at, queue
573 FROM durable_task_queue
574 WHERE id = $1
575 "#,
576 )
577 .bind(task_id)
578 .fetch_optional(&self.pool)
579 .await
580 .map_err(|e| {
581 error!(error = %e, "Failed to get task");
582 StoreError::Database(e.to_string())
583 })?
584 .ok_or(StoreError::TaskNotFound(task_id))?;
585
586 let status_str: String = row.get("status");
587 let status = match status_str.as_str() {
588 "pending" => TaskStatus::Pending,
589 "claimed" => TaskStatus::Claimed,
590 "completed" => TaskStatus::Completed,
591 "failed" => TaskStatus::Failed,
592 "dead" => TaskStatus::Dead,
593 "cancelled" => TaskStatus::Cancelled,
594 _ => TaskStatus::Pending,
595 };
596
597 Ok(TaskInfo {
598 id: row.get("id"),
599 workflow_id: row.get::<Option<Uuid>, _>("workflow_id"),
600 activity_id: row.get("activity_id"),
601 activity_type: row.get("activity_type"),
602 status,
603 priority: row.get("priority"),
604 attempt: row.get::<i32, _>("attempt") as u32,
605 max_attempts: row.get::<i32, _>("max_attempts") as u32,
606 claimed_by: row.get("claimed_by"),
607 last_error: row.get("last_error"),
608 created_at: row.get("created_at"),
609 claimed_at: row.get("claimed_at"),
610 queue: row.get("queue"),
611 })
612 }
613
614 #[instrument(skip(self))]
615 async fn reclaim_stale_tasks(
616 &self,
617 stale_threshold: Duration,
618 ) -> Result<ReclaimResult, StoreError> {
619 let threshold =
620 Utc::now() - chrono::Duration::from_std(stale_threshold).unwrap_or_default();
621
622 let dead_rows = sqlx::query(
625 r#"
626 UPDATE durable_task_queue
627 SET status = 'dead',
628 last_error = COALESCE(last_error, 'Worker became unresponsive after exhausting all retry attempts')
629 WHERE status = 'claimed'
630 AND heartbeat_at < $1
631 AND attempt >= max_attempts
632 RETURNING id, workflow_id, activity_id, activity_type, input, last_error
633 "#,
634 )
635 .bind(threshold)
636 .fetch_all(&self.pool)
637 .await
638 .map_err(|e| store_failure("durable.tasks.mark_exhausted_dead", "Failed to mark exhausted stale tasks as dead", e))?;
639
640 let dead_tasks: Vec<DeadTaskInfo> = dead_rows
641 .iter()
642 .map(|r| DeadTaskInfo {
643 task_id: r.get("id"),
644 workflow_id: r.get("workflow_id"),
645 activity_id: r.get("activity_id"),
646 activity_type: r.get("activity_type"),
647 input: r.get("input"),
648 last_error: r.get("last_error"),
649 })
650 .collect();
651
652 if !dead_tasks.is_empty() {
653 info!(
654 count = dead_tasks.len(),
655 "marked exhausted stale tasks as dead"
656 );
657 }
658
659 let threshold_i64 = no_progress_seal_threshold_from_env() as i64;
677 let rows = sqlx::query(
678 r#"
679 WITH stale AS (
680 SELECT q.id,
681 q.workflow_id,
682 q.progress_token AS prev_token,
683 q.no_progress_count AS prev_no_progress,
684 -- Highest durable event sequence for this workflow encoded
685 -- as a monotonic token (-1 => no progress events yet =>
686 -- token 0).
687 --
688 -- We EXCLUDE 'activity_started' events: claim_task writes
689 -- one on the FIRST attempt of a task (see the EVE-639
690 -- started_by_workflow handling; reclaims no longer emit
691 -- it). Counting that first-attempt marker as progress
692 -- would let a task that crashes immediately after dispatch
693 -- — recording nothing else — appear to advance, defeating
694 -- the seal guard (EVE-534). 'activity_started' is the only
695 -- dispatch/bookkeeping event written to
696 -- durable_workflow_events; every other event_type records
697 -- a real workflow fact (scheduled/completed/failed/timer/
698 -- signal/child), so a denylist of just this one type is
699 -- complete.
700 COALESCE(
701 (SELECT MAX(e.sequence_num)
702 FROM durable_workflow_events e
703 WHERE e.workflow_id = q.workflow_id
704 AND e.event_type <> 'activity_started'),
705 -1
706 )::BIGINT + 1 AS cur_token
707 FROM durable_task_queue q
708 WHERE q.status = 'claimed'
709 AND q.heartbeat_at < $1
710 AND q.attempt < q.max_attempts
711 FOR UPDATE OF q SKIP LOCKED
712 ),
713 decided AS (
714 SELECT id,
715 workflow_id,
716 cur_token,
717 -- Standalone tasks (workflow_id IS NULL) have no workflow
718 -- event stream, so cur_token is always 0 and the seal
719 -- guard would DLQ them after N reclaims purely from
720 -- missing workflow context. Exempt them: always treat as
721 -- advanced and never increment no_progress_count, so they
722 -- only DLQ via the max-attempts path (EVE-534).
723 --
724 -- A missing prior token is treated as the 0 baseline (no
725 -- events recorded yet), so a task that records nothing on
726 -- its first attempt already counts as no-progress. The
727 -- token only "advances" when it strictly grows past the
728 -- baseline, which a non-progressing retry can never do.
729 (workflow_id IS NULL
730 OR cur_token > COALESCE(prev_token, 0)) AS advanced,
731 CASE
732 WHEN workflow_id IS NULL THEN 0
733 WHEN cur_token > COALESCE(prev_token, 0) THEN 0
734 ELSE prev_no_progress + 1
735 END AS new_no_progress
736 FROM stale
737 )
738 UPDATE durable_task_queue q
739 SET status = CASE
740 WHEN d.new_no_progress >= $2 THEN 'dead'
741 ELSE 'pending'
742 END,
743 claimed_by = NULL,
744 claimed_at = NULL,
745 -- Leave standalone tasks' progress_token NULL; only
746 -- workflow-scoped tasks track a monotonic token.
747 progress_token = CASE
748 WHEN d.workflow_id IS NULL THEN NULL
749 ELSE d.cur_token
750 END,
751 no_progress_count = d.new_no_progress,
752 last_error = CASE
753 WHEN d.new_no_progress >= $2
754 THEN 'Task sealed by the no-progress guard: no forward progress across '
755 || d.new_no_progress::TEXT
756 || ' consecutive recoveries (EVE-534)'
757 ELSE q.last_error
758 END
759 FROM decided d
760 WHERE q.id = d.id
761 RETURNING q.id,
762 q.workflow_id,
763 q.activity_id,
764 q.activity_type,
765 q.input,
766 q.status AS new_status,
767 q.no_progress_count
768 "#,
769 )
770 .bind(threshold)
771 .bind(threshold_i64)
772 .fetch_all(&self.pool)
773 .await
774 .map_err(|e| store_failure("durable.tasks.reclaim_stale", "Failed to reclaim stale tasks", e))?;
775
776 let mut reclaimed_ids: Vec<Uuid> = Vec::new();
777 let mut sealed_tasks: Vec<SealedTaskInfo> = Vec::new();
778 for r in &rows {
779 let new_status: String = r.get("new_status");
780 if new_status == "dead" {
781 sealed_tasks.push(SealedTaskInfo {
782 task_id: r.get("id"),
783 workflow_id: r.get("workflow_id"),
784 activity_id: r.get("activity_id"),
785 activity_type: r.get("activity_type"),
786 input: r.get("input"),
787 reason: "no_progress".to_string(),
788 no_progress_count: r.get::<i32, _>("no_progress_count") as u32,
789 });
790 } else {
791 reclaimed_ids.push(r.get("id"));
792 }
793 }
794
795 #[cfg(feature = "failpoints")]
796 fail_point!("postgres_reclaim_stale_after_update", |_| {
797 Err(StoreError::Database(
798 "injected: after reclaim update".into(),
799 ))
800 });
801
802 if !reclaimed_ids.is_empty() || !dead_tasks.is_empty() || !sealed_tasks.is_empty() {
803 debug!(
804 reclaimed = reclaimed_ids.len(),
805 dead = dead_tasks.len(),
806 sealed = sealed_tasks.len(),
807 "processed stale tasks"
808 );
809 }
810 if !sealed_tasks.is_empty() {
811 info!(
812 count = sealed_tasks.len(),
813 "sealed non-progressing tasks during reclaim (EVE-534)"
814 );
815 }
816
817 let worker_heartbeat_threshold =
820 Utc::now() - chrono::Duration::seconds(WORKER_HEARTBEAT_TIMEOUT_SECS);
821 let stale_workers = sqlx::query(
822 r#"
823 UPDATE durable_workers
824 SET status = 'stopped'
825 WHERE status = 'active'
826 AND last_heartbeat_at < $1
827 RETURNING id
828 "#,
829 )
830 .bind(worker_heartbeat_threshold)
831 .fetch_all(&self.pool)
832 .await
833 .map_err(|e| {
834 error!(error = %e, "Failed to mark stale workers as stopped");
835 StoreError::Database(e.to_string())
836 })?;
837
838 if !stale_workers.is_empty() {
839 let ids: Vec<String> = stale_workers.iter().map(|r| r.get("id")).collect();
840 info!(count = stale_workers.len(), worker_ids = ?ids, "marked stale workers as stopped");
841 }
842
843 Ok(ReclaimResult {
844 reclaimed_ids,
845 dead_tasks,
846 sealed_tasks,
847 })
848 }
849
850 #[instrument(skip(self))]
851 async fn list_tasks(
852 &self,
853 filter: TaskFilter,
854 pagination: Pagination,
855 ) -> Result<Vec<TaskInfo>, StoreError> {
856 let status_str = filter.status.map(|s| match s {
857 TaskStatus::Pending => "pending",
858 TaskStatus::Claimed => "claimed",
859 TaskStatus::Completed => "completed",
860 TaskStatus::Failed => "failed",
861 TaskStatus::Dead => "dead",
862 TaskStatus::Cancelled => "cancelled",
863 });
864
865 let rows = sqlx::query(
866 r#"
867 SELECT id, workflow_id, activity_id, activity_type, status,
868 priority, attempt, max_attempts, claimed_by, last_error,
869 created_at, claimed_at, queue
870 FROM durable_task_queue
871 WHERE ($1::text IS NULL OR status = $1)
872 AND ($2::text IS NULL OR activity_type = $2)
873 AND ($3::uuid IS NULL OR workflow_id = $3)
874 AND (NOT $6 OR workflow_id IS NULL)
875 ORDER BY created_at ASC
876 OFFSET $4
877 LIMIT $5
878 "#,
879 )
880 .bind(status_str)
881 .bind(&filter.activity_type)
882 .bind(filter.workflow_id)
883 .bind(pagination.offset as i64)
884 .bind(pagination.limit as i64)
885 .bind(filter.standalone_only)
886 .fetch_all(&self.pool)
887 .await
888 .map_err(|e| {
889 error!(error = %e, "Failed to list tasks");
890 StoreError::Database(e.to_string())
891 })?;
892
893 let tasks = rows
894 .into_iter()
895 .map(|row| {
896 let status_str: String = row.get("status");
897 let status = match status_str.as_str() {
898 "pending" => TaskStatus::Pending,
899 "claimed" => TaskStatus::Claimed,
900 "completed" => TaskStatus::Completed,
901 "failed" => TaskStatus::Failed,
902 "dead" => TaskStatus::Dead,
903 "cancelled" => TaskStatus::Cancelled,
904 _ => TaskStatus::Pending,
905 };
906
907 TaskInfo {
908 id: row.get("id"),
909 workflow_id: row.get::<Option<Uuid>, _>("workflow_id"),
910 activity_id: row.get("activity_id"),
911 activity_type: row.get("activity_type"),
912 status,
913 priority: row.get("priority"),
914 attempt: row.get::<i32, _>("attempt") as u32,
915 max_attempts: row.get::<i32, _>("max_attempts") as u32,
916 claimed_by: row.get("claimed_by"),
917 last_error: row.get("last_error"),
918 created_at: row.get("created_at"),
919 claimed_at: row.get("claimed_at"),
920 queue: row.get("queue"),
921 }
922 })
923 .collect();
924
925 Ok(tasks)
926 }
927}
928
929#[cfg(test)]
930mod tests {
931 use super::*;
932 use crate::persistence::db_failure::capture::capture;
933 use sqlx::postgres::PgPoolOptions;
934
935 #[test]
939 fn heartbeat_failure_is_reported_through_the_shared_classifier() {
940 let emitted = capture(|| {
942 let runtime = tokio::runtime::Builder::new_current_thread()
943 .enable_all()
944 .build()
945 .unwrap();
946 runtime.block_on(async {
947 let pool = PgPoolOptions::new()
948 .max_connections(1)
949 .acquire_timeout(Duration::from_millis(200))
950 .connect_lazy("postgres://durable@127.0.0.1:1/durable")
951 .unwrap();
952 let store = PostgresWorkflowEventStore::new(pool);
953 let result = store.heartbeat_task(Uuid::now_v7(), "worker-1", None).await;
954 assert!(matches!(result, Err(StoreError::Database(_))), "{result:?}");
955 });
956 });
957
958 let reported: Vec<_> = emitted.iter().filter(|e| !e.subsystem.is_empty()).collect();
959 assert_eq!(reported.len(), 1, "{emitted:#?}");
960 assert_eq!(reported[0].subsystem, "durable.tasks.heartbeat");
961 assert!(
962 reported[0].db_failure.starts_with("database."),
963 "{emitted:#?}"
964 );
965 }
966}