Skip to main content

everruns_durable/persistence/postgres/
task_queue.rs

1//! Implementation of [`crate::TaskQueue`].
2
3use super::*;
4
5#[async_trait]
6impl TaskQueue for PostgresWorkflowEventStore {
7    #[instrument(skip(self, task))]
8    async fn enqueue_task(&self, task: TaskDefinition) -> Result<Uuid, StoreError> {
9        // Check pending task limits: per-workflow for workflow tasks, global for standalone
10        if let Some(wf_id) = task.workflow_id {
11            let limit = self.max_pending_tasks_per_workflow;
12            let pending_count: i64 = sqlx::query_scalar(
13                r#"
14                SELECT COUNT(*) FROM durable_task_queue
15                WHERE workflow_id = $1 AND status = 'pending'
16                "#,
17            )
18            .bind(wf_id)
19            .fetch_one(&self.pool)
20            .await
21            .map_err(|e| {
22                error!(error = %e, "Failed to count pending tasks");
23                StoreError::Database(e.to_string())
24            })?;
25            if pending_count >= limit as i64 {
26                return Err(StoreError::TaskQueueLimitExceeded {
27                    workflow_id: wf_id,
28                    current: pending_count as u32,
29                    limit,
30                });
31            }
32        } else {
33            // Standalone task: check global standalone limit
34            let limit = crate::persistence::store::DEFAULT_MAX_PENDING_STANDALONE_TASKS;
35            let pending_count: i64 = sqlx::query_scalar(
36                r#"
37                SELECT COUNT(*) FROM durable_task_queue
38                WHERE workflow_id IS NULL AND status = 'pending'
39                "#,
40            )
41            .fetch_one(&self.pool)
42            .await
43            .map_err(|e| {
44                error!(error = %e, "Failed to count standalone pending tasks");
45                StoreError::Database(e.to_string())
46            })?;
47
48            if pending_count >= limit as i64 {
49                return Err(StoreError::StandaloneTaskQueueLimitExceeded {
50                    current: pending_count as u32,
51                    limit,
52                });
53            }
54        }
55        let task_id = Uuid::now_v7();
56        let task_input = sanitize_json_null_bytes(task.input.clone());
57        let options_json = serde_json::to_value(&task.options)
58            .map(sanitize_json_null_bytes)
59            .map_err(|e| StoreError::Serialization(e.to_string()))?;
60
61        sqlx::query(
62            r#"
63            INSERT INTO durable_task_queue (
64                id, workflow_id, activity_id, activity_type, input, options,
65                max_attempts, priority, visible_at,
66                schedule_to_start_timeout_ms, start_to_close_timeout_ms, heartbeat_timeout_ms,
67                queue
68            )
69            VALUES ($1, $2, $3, $4, $5, $6, $7, $8, NOW() + $12::bigint * INTERVAL '1 millisecond', $9, $10, $11, $13)
70            "#,
71        )
72        .bind(task_id)
73        .bind(task.workflow_id)
74        .bind(&task.activity_id)
75        .bind(&task.activity_type)
76        .bind(&task_input)
77        .bind(&options_json)
78        .bind(task.options.retry_policy.max_attempts as i32)
79        .bind(task.options.priority)
80        .bind(task.options.schedule_to_start_timeout.as_millis() as i64)
81        .bind(task.options.start_to_close_timeout.as_millis() as i64)
82        .bind(task.options.heartbeat_timeout.map(|d| d.as_millis() as i64))
83        .bind(task.options.start_delay.map_or(0, |d| d.as_millis() as i64))
84        .bind(task.options.queue.as_deref())
85        .execute(&self.pool)
86        .await
87        .map_err(|e| {
88            error!(error = %e, "Failed to enqueue task");
89            StoreError::Database(e.to_string())
90        })?;
91
92        #[cfg(feature = "failpoints")]
93        fail_point!("postgres_enqueue_task_after_insert", |_| {
94            Err(StoreError::Database(
95                "injected: after enqueue insert".into(),
96            ))
97        });
98
99        debug!(%task_id, activity_type = %task.activity_type, "enqueued task");
100        Ok(task_id)
101    }
102
103    #[instrument(skip(self, activity_types))]
104    async fn claim_queue_tasks(
105        &self,
106        worker_id: &str,
107        queue: Option<&str>,
108        activity_types: &[String],
109        max_tasks: usize,
110    ) -> Result<Vec<ClaimedTask>, StoreError> {
111        if activity_types.is_empty() {
112            return Ok(vec![]);
113        }
114
115        // Use SKIP LOCKED for efficient concurrent claiming.
116        // This transaction:
117        // 1. Verifies worker is not draining (early exit if draining)
118        // 2. Finds pending tasks of `queue` (NULL: the default queue) matching
119        //    activity types
120        // 3. Orders by priority (desc) then visibility time
121        // 4. Limits to max_tasks
122        // 5. Uses SKIP LOCKED to avoid contention
123        // 6. Updates status and claiming info
124        // 7. Appends ActivityStarted events for FIRST-attempt workflow tasks
125        //    before returning the claim, avoiding a separate pre-execution write
126        //    in the worker. Reclaims (attempt > 1) skip this write (EVE-639).
127        //
128        // The ActivityStarted sequence read is deliberately a separate statement
129        // after locking the workflow rows. Under READ COMMITTED, a single-statement
130        // CTE keeps its original snapshot even after waiting on row locks, which
131        // can duplicate sequence numbers during concurrent claims for one workflow.
132        // EVE-639 makes this set-based: one ANY($) lock, one grouped MAX(seq)
133        // read, and one multi-row INSERT, rather than three statements per
134        // workflow.
135        // NOTE: The `attempt < max_attempts` check is critical for preventing infinite
136        // retries when workers panic. Without fail_task being called, the task becomes
137        // stale, gets reclaimed, but must not be claimed if attempts are exhausted.
138        let mut tx = begin_transaction(&self.pool)
139            .await
140            .map_err(|e| StoreError::Database(e.to_string()))?;
141
142        let rows = sqlx::query(
143            r#"
144            WITH worker_check AS (
145                SELECT 1 FROM durable_workers
146                WHERE id = $3 AND status != 'draining'
147            ),
148            claimable AS (
149                SELECT tq.id
150                FROM durable_task_queue tq, worker_check
151                WHERE tq.status = 'pending'
152                  AND tq.activity_type = ANY($1)
153                  AND tq.queue IS NOT DISTINCT FROM $4
154                  AND tq.visible_at <= NOW()
155                  AND tq.attempt < tq.max_attempts
156                ORDER BY tq.priority DESC, tq.visible_at
157                LIMIT $2
158                FOR UPDATE SKIP LOCKED
159            ),
160            updated AS (
161                UPDATE durable_task_queue t
162                SET status = 'claimed',
163                    claimed_by = $3,
164                    claimed_at = NOW(),
165                    heartbeat_at = NOW(),
166                    attempt = attempt + 1
167                FROM claimable c
168                WHERE t.id = c.id
169                RETURNING t.id, t.workflow_id, t.activity_id, t.activity_type,
170                          t.input, t.options, t.attempt, t.max_attempts
171            )
172            SELECT u.id, u.workflow_id, u.activity_id, u.activity_type,
173                   u.input, u.options, u.attempt, u.max_attempts,
174                   w.status AS workflow_status
175            FROM updated u
176            LEFT JOIN durable_workflow_instances w ON w.id = u.workflow_id
177            "#,
178        )
179        .bind(activity_types)
180        .bind(max_tasks as i32)
181        .bind(worker_id)
182        .bind(queue)
183        .fetch_all(&mut *tx)
184        .await
185        .map_err(|e| store_failure("durable.tasks.claim", "Failed to claim tasks", e))?;
186
187        let mut claimed = Vec::with_capacity(rows.len());
188        // EVE-639: only record ActivityStarted on the FIRST attempt of a task,
189        // not on every reclaim. `attempt` is post-increment in the claim UPDATE
190        // above, so the first claim yields attempt == 1. Reclaims (attempt > 1)
191        // record no ActivityStarted event — this is genuine bookkeeping for the
192        // initial dispatch only. (The seal guard in reclaim_stale_tasks still
193        // excludes activity_started from its progress token; see EVE-534.)
194        let mut started_by_workflow: BTreeMap<Uuid, Vec<(Uuid, String, i32)>> = BTreeMap::new();
195        for row in rows {
196            let options_json: serde_json::Value = row.get("options");
197            let options: ActivityOptions = serde_json::from_value(options_json)
198                .map_err(|e| StoreError::Serialization(e.to_string()))?;
199            let task_id: Uuid = row.get("id");
200            let workflow_id = row.get::<Option<Uuid>, _>("workflow_id");
201            let activity_id: String = row.get("activity_id");
202            let attempt = row.get::<i32, _>("attempt");
203
204            claimed.push(ClaimedTask {
205                id: task_id,
206                workflow_id,
207                activity_id: activity_id.clone(),
208                activity_type: row.get("activity_type"),
209                input: row.get("input"),
210                options,
211                attempt: attempt as u32,
212                max_attempts: row.get::<i32, _>("max_attempts") as u32,
213                workflow_status: row
214                    .get::<Option<String>, _>("workflow_status")
215                    .as_deref()
216                    .map(parse_workflow_status)
217                    .transpose()?,
218            });
219
220            if attempt == 1
221                && let Some(workflow_id) = workflow_id
222            {
223                started_by_workflow.entry(workflow_id).or_default().push((
224                    task_id,
225                    activity_id,
226                    attempt,
227                ));
228            }
229        }
230
231        if !started_by_workflow.is_empty() {
232            // Set-based ActivityStarted persistence: lock all involved workflow
233            // rows in one statement, fetch each workflow's next sequence in one
234            // grouped statement, then write every ActivityStarted event in a
235            // single multi-row INSERT — instead of three statements per workflow.
236            // BTreeMap keeps workflow_ids ordered, which gives a deterministic
237            // FOR UPDATE lock order across concurrent claimers, avoiding
238            // deadlocks.
239            let workflow_ids: Vec<Uuid> = started_by_workflow.keys().copied().collect();
240
241            sqlx::query(
242                r#"
243                SELECT id FROM durable_workflow_instances
244                WHERE id = ANY($1)
245                ORDER BY id
246                FOR UPDATE
247                "#,
248            )
249            .bind(&workflow_ids)
250            .fetch_all(&mut *tx)
251            .await
252            .map_err(|e| {
253                error!(error = %e, "Failed to lock workflows for activity start events");
254                StoreError::Database(e.to_string())
255            })?;
256
257            let seq_rows = sqlx::query(
258                r#"
259                SELECT workflow_id, COALESCE(MAX(sequence_num) + 1, 0)::INTEGER AS next_seq
260                FROM durable_workflow_events
261                WHERE workflow_id = ANY($1)
262                GROUP BY workflow_id
263                "#,
264            )
265            .bind(&workflow_ids)
266            .fetch_all(&mut *tx)
267            .await
268            .map_err(|e| {
269                error!(error = %e, "Failed to get workflow event sequences");
270                StoreError::Database(e.to_string())
271            })?;
272
273            let mut next_seq_by_workflow: BTreeMap<Uuid, i32> = BTreeMap::new();
274            for row in seq_rows {
275                next_seq_by_workflow.insert(row.get("workflow_id"), row.get::<i32, _>("next_seq"));
276            }
277
278            // Build the rows for a single multi-row INSERT, assigning contiguous
279            // sequence numbers per workflow. Tasks within a workflow are ordered
280            // by task_id for deterministic sequencing.
281            struct StartedEventRow {
282                workflow_id: Uuid,
283                sequence_num: i32,
284                event_data: serde_json::Value,
285            }
286            let mut insert_rows: Vec<StartedEventRow> = Vec::new();
287            for (workflow_id, mut events) in started_by_workflow {
288                events.sort_by_key(|(task_id, _, _)| *task_id);
289                // A workflow that has events (it always does — WorkflowStarted is
290                // seq 0) returns a row; default to 0 only as a safety net.
291                let base_sequence = next_seq_by_workflow.get(&workflow_id).copied().unwrap_or(0);
292                for (offset, (_, activity_id, attempt)) in events.into_iter().enumerate() {
293                    let event_data = serde_json::to_value(WorkflowEvent::ActivityStarted {
294                        activity_id,
295                        attempt: attempt as u32,
296                        worker_id: worker_id.to_string(),
297                    })
298                    .map(sanitize_json_null_bytes)
299                    .map_err(|e| StoreError::Serialization(e.to_string()))?;
300                    insert_rows.push(StartedEventRow {
301                        workflow_id,
302                        sequence_num: base_sequence + offset as i32,
303                        event_data,
304                    });
305                }
306            }
307
308            if !insert_rows.is_empty() {
309                let mut builder = sqlx::QueryBuilder::new(
310                    "INSERT INTO durable_workflow_events (workflow_id, sequence_num, event_type, event_data) ",
311                );
312                builder.push_values(insert_rows.iter(), |mut b, r| {
313                    b.push_bind(r.workflow_id)
314                        .push_bind(r.sequence_num)
315                        .push_bind("activity_started")
316                        .push_bind(&r.event_data);
317                });
318                builder.build().execute(&mut *tx).await.map_err(|e| {
319                    error!(error = %e, "Failed to write activity start events");
320                    StoreError::Database(e.to_string())
321                })?;
322            }
323        }
324
325        crate::commit_transaction(tx).await.map_err(|e| {
326            error!(error = %e, "Failed to commit task claim");
327            StoreError::Database(e.to_string())
328        })?;
329
330        #[cfg(feature = "failpoints")]
331        fail_point!("postgres_claim_task_after_query", |_| {
332            Err(StoreError::Database("injected: after claim query".into()))
333        });
334
335        if !claimed.is_empty() {
336            debug!(worker_id, count = claimed.len(), "claimed tasks");
337        }
338
339        Ok(claimed)
340    }
341
342    #[instrument(skip(self, _details))]
343    async fn heartbeat_task(
344        &self,
345        task_id: Uuid,
346        worker_id: &str,
347        _details: Option<serde_json::Value>,
348    ) -> Result<HeartbeatResponse, StoreError> {
349        // Renew the claim and report whether the owning workflow was cancelled
350        let result: Option<(Option<String>,)> = sqlx::query_as(
351            r#"
352            UPDATE durable_task_queue q SET heartbeat_at = NOW()
353            WHERE q.id = $1 AND q.claimed_by = $2 AND q.status = 'claimed'
354            RETURNING (SELECT w.status FROM durable_workflow_instances w WHERE w.id = q.workflow_id)
355            "#,
356        )
357        .bind(task_id)
358        .bind(worker_id)
359        .fetch_optional(&self.pool)
360        .await
361        .map_err(|e| store_failure("durable.tasks.heartbeat", "Failed to heartbeat task", e))?;
362
363        #[cfg(feature = "failpoints")]
364        fail_point!("postgres_heartbeat_update", |_| {
365            Err(StoreError::Database("injected: heartbeat update".into()))
366        });
367
368        match result {
369            // Still ours, but the workflow was cancelled: stop the work (EVE-1134).
370            Some((workflow_status,)) => Ok(HeartbeatResponse {
371                accepted: true,
372                should_cancel: workflow_status.as_deref() == Some("cancelled"),
373            }),
374            None => {
375                // Task no longer claimed by this worker (maybe reclaimed or completed)
376                Ok(HeartbeatResponse {
377                    accepted: false,
378                    should_cancel: true,
379                })
380            }
381        }
382    }
383
384    #[instrument(skip(self, _result))]
385    async fn complete_task(
386        &self,
387        task_id: Uuid,
388        worker_id: &str,
389        _result: serde_json::Value,
390    ) -> Result<(), StoreError> {
391        // Only complete if:
392        // 1. Task is still claimed by this worker, OR
393        // 2. Task is still claimed (status = 'claimed') by this worker
394        // This prevents duplicate scheduling when a task is reclaimed due to heartbeat timeout
395        let result = sqlx::query(
396            r#"
397            UPDATE durable_task_queue
398            SET status = 'completed', heartbeat_at = NOW()
399            WHERE id = $1 AND claimed_by = $2 AND status = 'claimed'
400            RETURNING id
401            "#,
402        )
403        .bind(task_id)
404        .bind(worker_id)
405        .fetch_optional(&self.pool)
406        .await
407        .map_err(|e| {
408            error!(error = %e, "Failed to complete task");
409            StoreError::Database(e.to_string())
410        })?;
411
412        #[cfg(feature = "failpoints")]
413        fail_point!("postgres_complete_task_after_update", |_| {
414            Err(StoreError::Database("injected: complete task".into()))
415        });
416
417        match result {
418            Some(_) => {
419                debug!(%task_id, %worker_id, "completed task");
420                Ok(())
421            }
422            None => {
423                // Task was reclaimed by another worker or already completed
424                debug!(%task_id, %worker_id, "task not owned - was reclaimed or already completed");
425                Err(StoreError::TaskNotOwned(task_id))
426            }
427        }
428    }
429
430    #[instrument(skip(self))]
431    async fn fail_task_with_retry(
432        &self,
433        task_id: Uuid,
434        error: &str,
435        retryable: bool,
436    ) -> Result<TaskFailureOutcome, StoreError> {
437        // EVE-639: SELECT FOR UPDATE and the follow-up UPDATE must share ONE
438        // transaction so the row lock is held across the whole read-modify-write.
439        // Previously both ran on `self.pool`, so the FOR UPDATE lock was released
440        // as soon as the SELECT statement returned and a concurrent fail_task /
441        // reclaim_stale_tasks could interleave and double-increment `attempt` or
442        // double-route the task to the DLQ. We also gate on status = 'claimed':
443        // if a reclaimer has already requeued or sealed this task, there is
444        // nothing for this caller to fail and we return TaskNotOwned rather than
445        // clobbering the reclaimer's decision.
446        let mut tx = begin_transaction(&self.pool)
447            .await
448            .map_err(|e| StoreError::Database(e.to_string()))?;
449
450        let row = sqlx::query(
451            r#"
452            SELECT attempt, max_attempts, options, status
453            FROM durable_task_queue
454            WHERE id = $1
455            FOR UPDATE
456            "#,
457        )
458        .bind(task_id)
459        .fetch_optional(&mut *tx)
460        .await
461        .map_err(|e| StoreError::Database(e.to_string()))?
462        .ok_or(StoreError::TaskNotFound(task_id))?;
463
464        let status: String = row.get("status");
465        if status != "claimed" {
466            // Task was reclaimed, completed, or already dead by a concurrent
467            // actor while we waited on the lock. Do not act on it.
468            debug!(
469                %task_id,
470                status,
471                "fail_task: task no longer claimed, skipping"
472            );
473            return Err(StoreError::TaskNotOwned(task_id));
474        }
475
476        let attempt: i32 = row.get("attempt");
477        let max_attempts: i32 = row.get("max_attempts");
478        let options_json: serde_json::Value = row.get("options");
479        let options: ActivityOptions = serde_json::from_value(options_json)
480            .map_err(|e| StoreError::Serialization(e.to_string()))?;
481
482        let outcome = if retryable && attempt < max_attempts {
483            // Calculate retry delay
484            let delay = options.retry_policy.delay_for_attempt((attempt + 1) as u32);
485            let visible_at = Utc::now() + chrono::Duration::from_std(delay).unwrap_or_default();
486
487            // Requeue for retry
488            sqlx::query(
489                r#"
490                UPDATE durable_task_queue
491                SET status = 'pending',
492                    claimed_by = NULL,
493                    claimed_at = NULL,
494                    heartbeat_at = NULL,
495                    last_error = $2,
496                    visible_at = $3
497                WHERE id = $1
498                "#,
499            )
500            .bind(task_id)
501            .bind(error)
502            .bind(visible_at)
503            .execute(&mut *tx)
504            .await
505            .map_err(|e| StoreError::Database(e.to_string()))?;
506
507            debug!(%task_id, next_attempt = attempt + 1, "task will retry");
508            TaskFailureOutcome::WillRetry {
509                next_attempt: (attempt + 1) as u32,
510                delay,
511            }
512        } else {
513            // Move to DLQ
514            sqlx::query(
515                r#"
516                UPDATE durable_task_queue
517                SET status = 'dead',
518                    last_error = $2
519                WHERE id = $1
520                "#,
521            )
522            .bind(task_id)
523            .bind(error)
524            .execute(&mut *tx)
525            .await
526            .map_err(|e| StoreError::Database(e.to_string()))?;
527
528            debug!(%task_id, "task moved to DLQ");
529            TaskFailureOutcome::MovedToDlq
530        };
531
532        crate::commit_transaction(tx)
533            .await
534            .map_err(|e| StoreError::Database(e.to_string()))?;
535
536        Ok(outcome)
537    }
538
539    #[instrument(skip(self))]
540    async fn cancel_pending_tasks_for_workflow(
541        &self,
542        workflow_id: Uuid,
543    ) -> Result<u64, StoreError> {
544        let result = sqlx::query(
545            r#"
546            UPDATE durable_task_queue
547            SET status = 'cancelled'
548            WHERE workflow_id = $1 AND status = 'pending'
549            "#,
550        )
551        .bind(workflow_id)
552        .execute(&self.pool)
553        .await
554        .map_err(|e| {
555            error!(error = %e, "Failed to cancel pending tasks for workflow");
556            StoreError::Database(e.to_string())
557        })?;
558
559        let count = result.rows_affected();
560        if count > 0 {
561            debug!(%workflow_id, count, "cancelled pending tasks for workflow");
562        }
563        Ok(count)
564    }
565
566    #[instrument(skip(self))]
567    async fn get_task(&self, task_id: Uuid) -> Result<TaskInfo, StoreError> {
568        let row = sqlx::query(
569            r#"
570            SELECT id, workflow_id, activity_id, activity_type, status,
571                   priority, attempt, max_attempts, claimed_by, last_error,
572                   created_at, claimed_at, queue
573            FROM durable_task_queue
574            WHERE id = $1
575            "#,
576        )
577        .bind(task_id)
578        .fetch_optional(&self.pool)
579        .await
580        .map_err(|e| {
581            error!(error = %e, "Failed to get task");
582            StoreError::Database(e.to_string())
583        })?
584        .ok_or(StoreError::TaskNotFound(task_id))?;
585
586        let status_str: String = row.get("status");
587        let status = match status_str.as_str() {
588            "pending" => TaskStatus::Pending,
589            "claimed" => TaskStatus::Claimed,
590            "completed" => TaskStatus::Completed,
591            "failed" => TaskStatus::Failed,
592            "dead" => TaskStatus::Dead,
593            "cancelled" => TaskStatus::Cancelled,
594            _ => TaskStatus::Pending,
595        };
596
597        Ok(TaskInfo {
598            id: row.get("id"),
599            workflow_id: row.get::<Option<Uuid>, _>("workflow_id"),
600            activity_id: row.get("activity_id"),
601            activity_type: row.get("activity_type"),
602            status,
603            priority: row.get("priority"),
604            attempt: row.get::<i32, _>("attempt") as u32,
605            max_attempts: row.get::<i32, _>("max_attempts") as u32,
606            claimed_by: row.get("claimed_by"),
607            last_error: row.get("last_error"),
608            created_at: row.get("created_at"),
609            claimed_at: row.get("claimed_at"),
610            queue: row.get("queue"),
611        })
612    }
613
614    #[instrument(skip(self))]
615    async fn reclaim_stale_tasks(
616        &self,
617        stale_threshold: Duration,
618    ) -> Result<ReclaimResult, StoreError> {
619        let threshold =
620            Utc::now() - chrono::Duration::from_std(stale_threshold).unwrap_or_default();
621
622        // First, mark exhausted tasks as dead (they've used all attempts via stale reclaims)
623        // This handles the case where workers panic without calling fail_task.
624        let dead_rows = sqlx::query(
625            r#"
626            UPDATE durable_task_queue
627            SET status = 'dead',
628                last_error = COALESCE(last_error, 'Worker became unresponsive after exhausting all retry attempts')
629            WHERE status = 'claimed'
630              AND heartbeat_at < $1
631              AND attempt >= max_attempts
632            RETURNING id, workflow_id, activity_id, activity_type, input, last_error
633            "#,
634        )
635        .bind(threshold)
636        .fetch_all(&self.pool)
637        .await
638        .map_err(|e| store_failure("durable.tasks.mark_exhausted_dead", "Failed to mark exhausted stale tasks as dead", e))?;
639
640        let dead_tasks: Vec<DeadTaskInfo> = dead_rows
641            .iter()
642            .map(|r| DeadTaskInfo {
643                task_id: r.get("id"),
644                workflow_id: r.get("workflow_id"),
645                activity_id: r.get("activity_id"),
646                activity_type: r.get("activity_type"),
647                input: r.get("input"),
648                last_error: r.get("last_error"),
649            })
650            .collect();
651
652        if !dead_tasks.is_empty() {
653            info!(
654                count = dead_tasks.len(),
655                "marked exhausted stale tasks as dead"
656            );
657        }
658
659        // Forward-progress guard (EVE-534).
660        //
661        // For each stale task that still has retry attempts, derive its current
662        // progress token from durably-recorded facts — the highest
663        // durable_workflow_events.sequence_num for the task's workflow — and
664        // compare it to the token observed at the previous reclaim:
665        //
666        //   - advanced (or first observation): record the new token, reset the
667        //     no-progress counter, and requeue (status='pending').
668        //   - not advanced: increment the no-progress counter. If it reaches the
669        //     configured threshold, SEAL the task — mark the task 'dead' so it
670        //     routes to the DLQ instead of looping forever — otherwise requeue.
671        //
672        // The token is monotonic per workflow, so a non-progressing retry (an
673        // atom that crashes before recording any event) cannot game it. The
674        // whole transition is a single UPDATE so concurrent reclaimers stay
675        // consistent under the row lock.
676        let threshold_i64 = no_progress_seal_threshold_from_env() as i64;
677        let rows = sqlx::query(
678            r#"
679            WITH stale AS (
680                SELECT q.id,
681                       q.workflow_id,
682                       q.progress_token AS prev_token,
683                       q.no_progress_count AS prev_no_progress,
684                       -- Highest durable event sequence for this workflow encoded
685                       -- as a monotonic token (-1 => no progress events yet =>
686                       -- token 0).
687                       --
688                       -- We EXCLUDE 'activity_started' events: claim_task writes
689                       -- one on the FIRST attempt of a task (see the EVE-639
690                       -- started_by_workflow handling; reclaims no longer emit
691                       -- it). Counting that first-attempt marker as progress
692                       -- would let a task that crashes immediately after dispatch
693                       -- — recording nothing else — appear to advance, defeating
694                       -- the seal guard (EVE-534). 'activity_started' is the only
695                       -- dispatch/bookkeeping event written to
696                       -- durable_workflow_events; every other event_type records
697                       -- a real workflow fact (scheduled/completed/failed/timer/
698                       -- signal/child), so a denylist of just this one type is
699                       -- complete.
700                       COALESCE(
701                           (SELECT MAX(e.sequence_num)
702                              FROM durable_workflow_events e
703                             WHERE e.workflow_id = q.workflow_id
704                               AND e.event_type <> 'activity_started'),
705                           -1
706                       )::BIGINT + 1 AS cur_token
707                  FROM durable_task_queue q
708                 WHERE q.status = 'claimed'
709                   AND q.heartbeat_at < $1
710                   AND q.attempt < q.max_attempts
711                   FOR UPDATE OF q SKIP LOCKED
712            ),
713            decided AS (
714                SELECT id,
715                       workflow_id,
716                       cur_token,
717                       -- Standalone tasks (workflow_id IS NULL) have no workflow
718                       -- event stream, so cur_token is always 0 and the seal
719                       -- guard would DLQ them after N reclaims purely from
720                       -- missing workflow context. Exempt them: always treat as
721                       -- advanced and never increment no_progress_count, so they
722                       -- only DLQ via the max-attempts path (EVE-534).
723                       --
724                       -- A missing prior token is treated as the 0 baseline (no
725                       -- events recorded yet), so a task that records nothing on
726                       -- its first attempt already counts as no-progress. The
727                       -- token only "advances" when it strictly grows past the
728                       -- baseline, which a non-progressing retry can never do.
729                       (workflow_id IS NULL
730                        OR cur_token > COALESCE(prev_token, 0)) AS advanced,
731                       CASE
732                           WHEN workflow_id IS NULL THEN 0
733                           WHEN cur_token > COALESCE(prev_token, 0) THEN 0
734                           ELSE prev_no_progress + 1
735                       END AS new_no_progress
736                  FROM stale
737            )
738            UPDATE durable_task_queue q
739               SET status = CASE
740                                WHEN d.new_no_progress >= $2 THEN 'dead'
741                                ELSE 'pending'
742                            END,
743                   claimed_by = NULL,
744                   claimed_at = NULL,
745                   -- Leave standalone tasks' progress_token NULL; only
746                   -- workflow-scoped tasks track a monotonic token.
747                   progress_token = CASE
748                                        WHEN d.workflow_id IS NULL THEN NULL
749                                        ELSE d.cur_token
750                                    END,
751                   no_progress_count = d.new_no_progress,
752                   last_error = CASE
753                                    WHEN d.new_no_progress >= $2
754                                    THEN 'Task sealed by the no-progress guard: no forward progress across '
755                                         || d.new_no_progress::TEXT
756                                         || ' consecutive recoveries (EVE-534)'
757                                    ELSE q.last_error
758                                END
759              FROM decided d
760             WHERE q.id = d.id
761            RETURNING q.id,
762                      q.workflow_id,
763                      q.activity_id,
764                      q.activity_type,
765                      q.input,
766                      q.status AS new_status,
767                      q.no_progress_count
768            "#,
769        )
770        .bind(threshold)
771        .bind(threshold_i64)
772        .fetch_all(&self.pool)
773        .await
774        .map_err(|e| store_failure("durable.tasks.reclaim_stale", "Failed to reclaim stale tasks", e))?;
775
776        let mut reclaimed_ids: Vec<Uuid> = Vec::new();
777        let mut sealed_tasks: Vec<SealedTaskInfo> = Vec::new();
778        for r in &rows {
779            let new_status: String = r.get("new_status");
780            if new_status == "dead" {
781                sealed_tasks.push(SealedTaskInfo {
782                    task_id: r.get("id"),
783                    workflow_id: r.get("workflow_id"),
784                    activity_id: r.get("activity_id"),
785                    activity_type: r.get("activity_type"),
786                    input: r.get("input"),
787                    reason: "no_progress".to_string(),
788                    no_progress_count: r.get::<i32, _>("no_progress_count") as u32,
789                });
790            } else {
791                reclaimed_ids.push(r.get("id"));
792            }
793        }
794
795        #[cfg(feature = "failpoints")]
796        fail_point!("postgres_reclaim_stale_after_update", |_| {
797            Err(StoreError::Database(
798                "injected: after reclaim update".into(),
799            ))
800        });
801
802        if !reclaimed_ids.is_empty() || !dead_tasks.is_empty() || !sealed_tasks.is_empty() {
803            debug!(
804                reclaimed = reclaimed_ids.len(),
805                dead = dead_tasks.len(),
806                sealed = sealed_tasks.len(),
807                "processed stale tasks"
808            );
809        }
810        if !sealed_tasks.is_empty() {
811            info!(
812                count = sealed_tasks.len(),
813                "sealed non-progressing tasks during reclaim (EVE-534)"
814            );
815        }
816
817        // Also mark workers with stale heartbeats as stopped.
818        // This cleans up workers that crashed without calling deregister_worker.
819        let worker_heartbeat_threshold =
820            Utc::now() - chrono::Duration::seconds(WORKER_HEARTBEAT_TIMEOUT_SECS);
821        let stale_workers = sqlx::query(
822            r#"
823            UPDATE durable_workers
824            SET status = 'stopped'
825            WHERE status = 'active'
826              AND last_heartbeat_at < $1
827            RETURNING id
828            "#,
829        )
830        .bind(worker_heartbeat_threshold)
831        .fetch_all(&self.pool)
832        .await
833        .map_err(|e| {
834            error!(error = %e, "Failed to mark stale workers as stopped");
835            StoreError::Database(e.to_string())
836        })?;
837
838        if !stale_workers.is_empty() {
839            let ids: Vec<String> = stale_workers.iter().map(|r| r.get("id")).collect();
840            info!(count = stale_workers.len(), worker_ids = ?ids, "marked stale workers as stopped");
841        }
842
843        Ok(ReclaimResult {
844            reclaimed_ids,
845            dead_tasks,
846            sealed_tasks,
847        })
848    }
849
850    #[instrument(skip(self))]
851    async fn list_tasks(
852        &self,
853        filter: TaskFilter,
854        pagination: Pagination,
855    ) -> Result<Vec<TaskInfo>, StoreError> {
856        let status_str = filter.status.map(|s| match s {
857            TaskStatus::Pending => "pending",
858            TaskStatus::Claimed => "claimed",
859            TaskStatus::Completed => "completed",
860            TaskStatus::Failed => "failed",
861            TaskStatus::Dead => "dead",
862            TaskStatus::Cancelled => "cancelled",
863        });
864
865        let rows = sqlx::query(
866            r#"
867            SELECT id, workflow_id, activity_id, activity_type, status,
868                   priority, attempt, max_attempts, claimed_by, last_error,
869                   created_at, claimed_at, queue
870            FROM durable_task_queue
871            WHERE ($1::text IS NULL OR status = $1)
872              AND ($2::text IS NULL OR activity_type = $2)
873              AND ($3::uuid IS NULL OR workflow_id = $3)
874              AND (NOT $6 OR workflow_id IS NULL)
875            ORDER BY created_at ASC
876            OFFSET $4
877            LIMIT $5
878            "#,
879        )
880        .bind(status_str)
881        .bind(&filter.activity_type)
882        .bind(filter.workflow_id)
883        .bind(pagination.offset as i64)
884        .bind(pagination.limit as i64)
885        .bind(filter.standalone_only)
886        .fetch_all(&self.pool)
887        .await
888        .map_err(|e| {
889            error!(error = %e, "Failed to list tasks");
890            StoreError::Database(e.to_string())
891        })?;
892
893        let tasks = rows
894            .into_iter()
895            .map(|row| {
896                let status_str: String = row.get("status");
897                let status = match status_str.as_str() {
898                    "pending" => TaskStatus::Pending,
899                    "claimed" => TaskStatus::Claimed,
900                    "completed" => TaskStatus::Completed,
901                    "failed" => TaskStatus::Failed,
902                    "dead" => TaskStatus::Dead,
903                    "cancelled" => TaskStatus::Cancelled,
904                    _ => TaskStatus::Pending,
905                };
906
907                TaskInfo {
908                    id: row.get("id"),
909                    workflow_id: row.get::<Option<Uuid>, _>("workflow_id"),
910                    activity_id: row.get("activity_id"),
911                    activity_type: row.get("activity_type"),
912                    status,
913                    priority: row.get("priority"),
914                    attempt: row.get::<i32, _>("attempt") as u32,
915                    max_attempts: row.get::<i32, _>("max_attempts") as u32,
916                    claimed_by: row.get("claimed_by"),
917                    last_error: row.get("last_error"),
918                    created_at: row.get("created_at"),
919                    claimed_at: row.get("claimed_at"),
920                    queue: row.get("queue"),
921                }
922            })
923            .collect();
924
925        Ok(tasks)
926    }
927}
928
929#[cfg(test)]
930mod tests {
931    use super::*;
932    use crate::persistence::db_failure::capture::capture;
933    use sqlx::postgres::PgPoolOptions;
934
935    /// EVERRUNS-2T: a heartbeat that cannot reach the database must be
936    /// reported through the shared classifier, so a pool exhaustion groups
937    /// with every other starved subsystem instead of opening its own issue.
938    #[test]
939    fn heartbeat_failure_is_reported_through_the_shared_classifier() {
940        // Nothing listens on port 1, so every acquire fails without a database.
941        let emitted = capture(|| {
942            let runtime = tokio::runtime::Builder::new_current_thread()
943                .enable_all()
944                .build()
945                .unwrap();
946            runtime.block_on(async {
947                let pool = PgPoolOptions::new()
948                    .max_connections(1)
949                    .acquire_timeout(Duration::from_millis(200))
950                    .connect_lazy("postgres://durable@127.0.0.1:1/durable")
951                    .unwrap();
952                let store = PostgresWorkflowEventStore::new(pool);
953                let result = store.heartbeat_task(Uuid::now_v7(), "worker-1", None).await;
954                assert!(matches!(result, Err(StoreError::Database(_))), "{result:?}");
955            });
956        });
957
958        let reported: Vec<_> = emitted.iter().filter(|e| !e.subsystem.is_empty()).collect();
959        assert_eq!(reported.len(), 1, "{emitted:#?}");
960        assert_eq!(reported[0].subsystem, "durable.tasks.heartbeat");
961        assert!(
962            reported[0].db_failure.starts_with("database."),
963            "{emitted:#?}"
964        );
965    }
966}