Skip to main content

everruns_durable/persistence/postgres/
event_log.rs

1//! Implementation of [`crate::EventLog`].
2
3use super::*;
4
5#[async_trait]
6impl EventLog for PostgresWorkflowEventStore {
7    #[instrument(skip(self, input, trace_context))]
8    async fn create_workflow(
9        &self,
10        workflow_id: Uuid,
11        workflow_type: &str,
12        input: serde_json::Value,
13        trace_context: Option<&TraceContext>,
14    ) -> Result<(), StoreError> {
15        let (trace_id, span_id) = trace_context
16            .map(|tc| (Some(tc.trace_id.clone()), Some(tc.span_id.clone())))
17            .unwrap_or((None, None));
18
19        let input = sanitize_json_null_bytes(input);
20
21        sqlx::query(
22            r#"
23            INSERT INTO durable_workflow_instances (id, workflow_type, status, input, trace_id, span_id)
24            VALUES ($1, $2, 'pending', $3, $4, $5)
25            "#,
26        )
27        .bind(workflow_id)
28        .bind(workflow_type)
29        .bind(&input)
30        .bind(&trace_id)
31        .bind(&span_id)
32        .execute(&self.pool)
33        .await
34        .map_err(|e| {
35            error!(error = %e, "Failed to create workflow");
36            StoreError::Database(e.to_string())
37        })?;
38
39        debug!(%workflow_id, %workflow_type, "created workflow");
40        Ok(())
41    }
42
43    #[instrument(skip(self))]
44    async fn get_workflow_status(&self, workflow_id: Uuid) -> Result<WorkflowStatus, StoreError> {
45        let row = sqlx::query(
46            r#"
47            SELECT status FROM durable_workflow_instances WHERE id = $1
48            "#,
49        )
50        .bind(workflow_id)
51        .fetch_optional(&self.pool)
52        .await
53        .map_err(|e| {
54            error!(error = %e, "Failed to get workflow status");
55            StoreError::Database(e.to_string())
56        })?
57        .ok_or(StoreError::WorkflowNotFound(workflow_id))?;
58
59        let status: String = row.get("status");
60        parse_workflow_status(&status)
61    }
62
63    #[instrument(skip(self))]
64    async fn get_workflow_info(&self, workflow_id: Uuid) -> Result<WorkflowInfo, StoreError> {
65        let row = sqlx::query(
66            r#"
67            SELECT id, workflow_type, status, input, result, error, continued_as_new_id
68            FROM durable_workflow_instances
69            WHERE id = $1
70            "#,
71        )
72        .bind(workflow_id)
73        .fetch_optional(&self.pool)
74        .await
75        .map_err(|e| {
76            error!(error = %e, "Failed to get workflow info");
77            StoreError::Database(e.to_string())
78        })?
79        .ok_or(StoreError::WorkflowNotFound(workflow_id))?;
80
81        let status_str: String = row.get("status");
82        let error_json: Option<serde_json::Value> = row.get("error");
83
84        Ok(WorkflowInfo {
85            id: row.get("id"),
86            workflow_type: row.get("workflow_type"),
87            status: parse_workflow_status(&status_str)?,
88            input: row.get("input"),
89            result: row.get("result"),
90            error: error_json.and_then(|v| serde_json::from_value(v).ok()),
91            continued_as_new_id: row.try_get("continued_as_new_id").ok().flatten(),
92        })
93    }
94
95    #[instrument(skip(self, events))]
96    async fn append_events(
97        &self,
98        workflow_id: Uuid,
99        expected_sequence: i32,
100        events: Vec<WorkflowEvent>,
101    ) -> Result<i32, StoreError> {
102        if events.is_empty() {
103            return Ok(expected_sequence);
104        }
105
106        // EVE-639: single multi-row INSERT instead of FOR UPDATE + MAX(...) +
107        // one INSERT per event. Sequence numbers are assigned deterministically
108        // from `expected_sequence` (the caller's optimistic-concurrency cursor),
109        // and the UNIQUE(workflow_id, sequence_num) constraint is the source of
110        // truth: if another writer already wrote at any of these sequences, the
111        // INSERT fails with a unique violation, which we surface as a
112        // ConcurrencyConflict. The FK on workflow_id surfaces WorkflowNotFound.
113        let mut new_sequence = expected_sequence;
114
115        let mut builder = sqlx::QueryBuilder::new(
116            "INSERT INTO durable_workflow_events (workflow_id, sequence_num, event_type, event_data) ",
117        );
118        builder.push_values(events.iter(), |mut b, event| {
119            let event_type = event_type_name(event);
120            // serde_json::to_value is infallible for these event types in
121            // practice; on the off chance it fails we bind a null event_data,
122            // which the NOT NULL column rejects, turning into a Database error.
123            let event_data = serde_json::to_value(event)
124                .map(sanitize_json_null_bytes)
125                .unwrap_or(serde_json::Value::Null);
126            b.push_bind(workflow_id)
127                .push_bind(new_sequence)
128                .push_bind(event_type)
129                .push_bind(event_data);
130            new_sequence += 1;
131        });
132
133        #[cfg(feature = "failpoints")]
134        fail_point!("postgres_append_events_after_insert", |_| {
135            Err(StoreError::Database("injected: after insert".into()))
136        });
137
138        #[cfg(feature = "failpoints")]
139        fail_point!("postgres_append_events_before_commit", |_| {
140            Err(StoreError::Database("injected: before commit".into()))
141        });
142
143        builder
144            .build()
145            .execute(&self.pool)
146            .await
147            .map_err(|e| map_append_events_error(e, workflow_id, expected_sequence))?;
148
149        debug!(%workflow_id, new_sequence, "appended events");
150        Ok(new_sequence)
151    }
152
153    #[instrument(skip(self))]
154    async fn load_events(
155        &self,
156        workflow_id: Uuid,
157    ) -> Result<Vec<(i32, WorkflowEvent)>, StoreError> {
158        let rows = sqlx::query(
159            r#"
160            SELECT sequence_num, event_data
161            FROM durable_workflow_events
162            WHERE workflow_id = $1
163            ORDER BY sequence_num
164            "#,
165        )
166        .bind(workflow_id)
167        .fetch_all(&self.pool)
168        .await
169        .map_err(|e| {
170            error!(error = %e, "Failed to load events");
171            StoreError::Database(e.to_string())
172        })?;
173
174        #[cfg(feature = "failpoints")]
175        fail_point!("postgres_load_events_after_query", |_| {
176            Err(StoreError::Database(
177                "injected: after load events query".into(),
178            ))
179        });
180
181        let mut events = Vec::with_capacity(rows.len());
182        for row in rows {
183            let seq: i32 = row.get::<i32, _>("sequence_num");
184            let data: serde_json::Value = row.get("event_data");
185            let event: WorkflowEvent = serde_json::from_value(data)
186                .map_err(|e| StoreError::Serialization(e.to_string()))?;
187            events.push((seq, event));
188        }
189
190        Ok(events)
191    }
192
193    #[instrument(skip(self))]
194    async fn count_events(&self, workflow_id: Uuid) -> Result<usize, StoreError> {
195        let count = sqlx::query_scalar::<_, i64>(
196            r#"
197            SELECT COUNT(*)
198            FROM durable_workflow_events
199            WHERE workflow_id = $1
200            "#,
201        )
202        .bind(workflow_id)
203        .fetch_one(&self.pool)
204        .await
205        .map_err(|e| {
206            error!(error = %e, "Failed to count events");
207            StoreError::Database(e.to_string())
208        })?;
209
210        usize::try_from(count).map_err(|_| {
211            StoreError::Database(format!(
212                "event count overflow for workflow {workflow_id}: {count}"
213            ))
214        })
215    }
216
217    #[instrument(skip(self))]
218    async fn count_events_after(
219        &self,
220        workflow_id: Uuid,
221        after_sequence: i32,
222    ) -> Result<usize, StoreError> {
223        let count = sqlx::query_scalar::<_, i64>(
224            r#"
225            SELECT COUNT(*)
226            FROM durable_workflow_events
227            WHERE workflow_id = $1 AND sequence_num > $2
228            "#,
229        )
230        .bind(workflow_id)
231        .bind(after_sequence)
232        .fetch_one(&self.pool)
233        .await
234        .map_err(|e| {
235            error!(error = %e, after_sequence, "Failed to count events after sequence");
236            StoreError::Database(e.to_string())
237        })?;
238
239        usize::try_from(count).map_err(|_| {
240            StoreError::Database(format!(
241                "event count overflow for workflow {workflow_id}: {count}"
242            ))
243        })
244    }
245
246    #[instrument(skip(self))]
247    async fn load_events_after(
248        &self,
249        workflow_id: Uuid,
250        after_sequence: i32,
251    ) -> Result<Vec<(i32, WorkflowEvent)>, StoreError> {
252        let rows = sqlx::query(
253            r#"
254            SELECT sequence_num, event_data
255            FROM durable_workflow_events
256            WHERE workflow_id = $1 AND sequence_num > $2
257            ORDER BY sequence_num
258            "#,
259        )
260        .bind(workflow_id)
261        .bind(after_sequence)
262        .fetch_all(&self.pool)
263        .await
264        .map_err(|e| {
265            error!(error = %e, after_sequence, "Failed to load events after sequence");
266            StoreError::Database(e.to_string())
267        })?;
268
269        let mut events = Vec::with_capacity(rows.len());
270        for row in rows {
271            let seq: i32 = row.get::<i32, _>("sequence_num");
272            let data: serde_json::Value = row.get("event_data");
273            let event: WorkflowEvent = serde_json::from_value(data)
274                .map_err(|e| StoreError::Serialization(e.to_string()))?;
275            events.push((seq, event));
276        }
277
278        Ok(events)
279    }
280
281    #[instrument(skip(self, snapshot_data))]
282    async fn save_snapshot(
283        &self,
284        workflow_id: Uuid,
285        sequence_num: i32,
286        snapshot_data: Vec<u8>,
287    ) -> Result<(), StoreError> {
288        sqlx::query(
289            r#"
290            INSERT INTO durable_workflow_snapshots (workflow_id, sequence_num, snapshot_data)
291            VALUES ($1, $2, $3)
292            ON CONFLICT (workflow_id, sequence_num)
293            DO UPDATE SET snapshot_data = EXCLUDED.snapshot_data, created_at = NOW()
294            "#,
295        )
296        .bind(workflow_id)
297        .bind(sequence_num)
298        .bind(&snapshot_data)
299        .execute(&self.pool)
300        .await
301        .map_err(|e| {
302            error!(error = %e, "Failed to save snapshot");
303            StoreError::Database(e.to_string())
304        })?;
305
306        debug!(%workflow_id, sequence_num, "saved workflow snapshot");
307        Ok(())
308    }
309
310    #[instrument(skip(self))]
311    async fn load_latest_snapshot(
312        &self,
313        workflow_id: Uuid,
314    ) -> Result<Option<crate::persistence::store::WorkflowSnapshot>, StoreError> {
315        let row = sqlx::query(
316            r#"
317            SELECT sequence_num, snapshot_data, created_at
318            FROM durable_workflow_snapshots
319            WHERE workflow_id = $1
320            ORDER BY sequence_num DESC
321            LIMIT 1
322            "#,
323        )
324        .bind(workflow_id)
325        .fetch_optional(&self.pool)
326        .await
327        .map_err(|e| {
328            error!(error = %e, "Failed to load latest snapshot");
329            StoreError::Database(e.to_string())
330        })?;
331
332        Ok(row.map(|r| crate::persistence::store::WorkflowSnapshot {
333            workflow_id,
334            sequence_num: r.get("sequence_num"),
335            snapshot_data: r.get("snapshot_data"),
336            created_at: r.get("created_at"),
337        }))
338    }
339
340    #[instrument(skip(self))]
341    async fn delete_snapshots(&self, workflow_id: Uuid) -> Result<(), StoreError> {
342        sqlx::query(
343            r#"
344            DELETE FROM durable_workflow_snapshots WHERE workflow_id = $1
345            "#,
346        )
347        .bind(workflow_id)
348        .execute(&self.pool)
349        .await
350        .map_err(|e| {
351            error!(error = %e, "Failed to delete snapshots");
352            StoreError::Database(e.to_string())
353        })?;
354
355        debug!(%workflow_id, "deleted workflow snapshots");
356        Ok(())
357    }
358
359    #[instrument(skip(self, result, error))]
360    async fn update_workflow_status(
361        &self,
362        workflow_id: Uuid,
363        status: WorkflowStatus,
364        result: Option<serde_json::Value>,
365        error: Option<WorkflowError>,
366    ) -> Result<(), StoreError> {
367        let status_str = status.to_string();
368        let error_json = error
369            .map(serde_json::to_value)
370            .transpose()
371            .map_err(|e| StoreError::Serialization(e.to_string()))?;
372        // Pending resets transient fields but retains explicit recovery input.
373        let reset_to_pending = matches!(status, WorkflowStatus::Pending);
374        let (started_at, completed_at): (Option<DateTime<Utc>>, Option<DateTime<Utc>>) =
375            match status {
376                WorkflowStatus::Running => (Some(Utc::now()), None),
377                WorkflowStatus::Completed
378                | WorkflowStatus::Failed
379                | WorkflowStatus::Cancelled
380                | WorkflowStatus::ContinuedAsNew => (None, Some(Utc::now())),
381                WorkflowStatus::Pending => (None, None),
382            };
383        let result = result.map(sanitize_json_null_bytes);
384        let error_json = error_json.map(sanitize_json_null_bytes);
385        if reset_to_pending {
386            sqlx::query(
387                r#"
388                UPDATE durable_workflow_instances
389                SET status = $2,
390                    result = $3,
391                    error = NULL,
392                    started_at = NULL,
393                    completed_at = NULL
394                WHERE id = $1
395                "#,
396            )
397            .bind(workflow_id)
398            .bind(&status_str)
399            .bind(&result)
400            .execute(&self.pool)
401            .await
402            .map_err(|e| {
403                error!(error = %e, "Failed to update workflow status");
404                StoreError::Database(e.to_string())
405            })?;
406        } else {
407            sqlx::query(
408                r#"
409                UPDATE durable_workflow_instances
410                SET status = $2,
411                    result = COALESCE($3, result),
412                    error = COALESCE($4, error),
413                    started_at = COALESCE($5, started_at),
414                    completed_at = COALESCE($6, completed_at)
415                WHERE id = $1
416                "#,
417            )
418            .bind(workflow_id)
419            .bind(&status_str)
420            .bind(&result)
421            .bind(&error_json)
422            .bind(started_at)
423            .bind(completed_at)
424            .execute(&self.pool)
425            .await
426            .map_err(|e| {
427                error!(error = %e, "Failed to update workflow status");
428                StoreError::Database(e.to_string())
429            })?;
430        }
431
432        debug!(%workflow_id, %status_str, "updated workflow status");
433        Ok(())
434    }
435
436    #[instrument(skip(self, error))]
437    async fn try_fail_workflow(
438        &self,
439        workflow_id: Uuid,
440        error: WorkflowError,
441    ) -> Result<bool, StoreError> {
442        let error_json = sanitize_json_null_bytes(
443            serde_json::to_value(error).map_err(|e| StoreError::Serialization(e.to_string()))?,
444        );
445        let result = sqlx::query(
446            r#"
447            UPDATE durable_workflow_instances
448            SET status = 'failed',
449                error = $2,
450                completed_at = COALESCE(completed_at, NOW())
451            WHERE id = $1 AND status = 'running'
452            "#,
453        )
454        .bind(workflow_id)
455        .bind(error_json)
456        .execute(&self.pool)
457        .await
458        .map_err(|e| StoreError::Database(e.to_string()))?;
459
460        Ok(result.rows_affected() == 1)
461    }
462
463    #[instrument(skip(self))]
464    async fn try_start_new_run(&self, workflow_id: Uuid) -> Result<bool, StoreError> {
465        // Atomic CAS: transition from terminal/pending → running.
466        // Also cancels stale pending tasks in the same transaction.
467        // Uses SELECT FOR UPDATE to prevent concurrent claims across replicas.
468        let mut tx = begin_transaction(&self.pool).await.map_err(|e| {
469            error!(error = %e, "Failed to begin transaction");
470            StoreError::Database(e.to_string())
471        })?;
472
473        // Lock the workflow row and check status
474        let row = sqlx::query(
475            r#"
476            SELECT status FROM durable_workflow_instances
477            WHERE id = $1
478            FOR UPDATE
479            "#,
480        )
481        .bind(workflow_id)
482        .fetch_optional(&mut *tx)
483        .await
484        .map_err(|e| {
485            error!(error = %e, "Failed to lock workflow for claim");
486            StoreError::Database(e.to_string())
487        })?;
488
489        let Some(row) = row else {
490            tx.rollback().await.ok();
491            return Err(StoreError::WorkflowNotFound(workflow_id));
492        };
493
494        let status: String = row.get("status");
495        if status == "running" {
496            // Active run: cannot claim
497            tx.rollback().await.ok();
498            return Ok(false);
499        }
500
501        let has_claimed_task = sqlx::query_scalar::<_, bool>(
502            r#"
503            SELECT EXISTS(
504                SELECT 1
505                FROM durable_task_queue
506                WHERE workflow_id = $1
507                  AND status = 'claimed'
508            )
509            "#,
510        )
511        .bind(workflow_id)
512        .fetch_one(&mut *tx)
513        .await
514        .map_err(|e| {
515            error!(error = %e, "Failed to check claimed workflow tasks");
516            StoreError::Database(e.to_string())
517        })?;
518
519        if has_claimed_task {
520            tx.rollback().await.ok();
521            return Ok(false);
522        }
523
524        // Claim: set to running, clear transient fields
525        sqlx::query(
526            r#"
527            UPDATE durable_workflow_instances
528            SET status = 'running',
529                result = NULL,
530                error = NULL,
531                started_at = NOW(),
532                completed_at = NULL
533            WHERE id = $1
534            "#,
535        )
536        .bind(workflow_id)
537        .execute(&mut *tx)
538        .await
539        .map_err(|e| {
540            error!(error = %e, "Failed to update workflow to running");
541            StoreError::Database(e.to_string())
542        })?;
543
544        // Cancel any stale pending tasks from the previous run
545        sqlx::query(
546            r#"
547            UPDATE durable_task_queue
548            SET status = 'cancelled'
549            WHERE workflow_id = $1 AND status = 'pending'
550            "#,
551        )
552        .bind(workflow_id)
553        .execute(&mut *tx)
554        .await
555        .map_err(|e| {
556            error!(error = %e, "Failed to cancel stale tasks during claim");
557            StoreError::Database(e.to_string())
558        })?;
559
560        crate::commit_transaction(tx).await.map_err(|e| {
561            error!(error = %e, "Failed to commit workflow claim");
562            StoreError::Database(e.to_string())
563        })?;
564
565        Ok(true)
566    }
567
568    #[instrument(skip(self))]
569    async fn cancel_workflow(&self, workflow_id: Uuid) -> Result<(), StoreError> {
570        // Update workflow status to cancelled
571        let result = sqlx::query(
572            r#"
573            UPDATE durable_workflow_instances
574            SET status = 'cancelled',
575                completed_at = NOW()
576            WHERE id = $1 AND status IN ('pending', 'running')
577            RETURNING id
578            "#,
579        )
580        .bind(workflow_id)
581        .fetch_optional(&self.pool)
582        .await
583        .map_err(|e| {
584            error!(error = %e, "Failed to cancel workflow");
585            StoreError::Database(e.to_string())
586        })?;
587
588        if result.is_none() {
589            return Err(StoreError::WorkflowNotFound(workflow_id));
590        }
591
592        // Cancel any pending tasks for this workflow
593        sqlx::query(
594            r#"
595            UPDATE durable_task_queue
596            SET status = 'cancelled'
597            WHERE workflow_id = $1 AND status = 'pending'
598            "#,
599        )
600        .bind(workflow_id)
601        .execute(&self.pool)
602        .await
603        .map_err(|e| {
604            error!(error = %e, "Failed to cancel workflow tasks");
605            StoreError::Database(e.to_string())
606        })?;
607
608        debug!(%workflow_id, "cancelled workflow");
609        Ok(())
610    }
611
612    #[instrument(skip(self, snapshot_data))]
613    async fn continue_as_new(
614        &self,
615        old_workflow_id: Uuid,
616        workflow_type: &str,
617        input: serde_json::Value,
618        snapshot_data: Vec<u8>,
619    ) -> Result<Uuid, StoreError> {
620        let new_workflow_id = Uuid::now_v7();
621        let input = sanitize_json_null_bytes(input);
622        let start_event = WorkflowEvent::started(input.clone());
623        let event_data = serde_json::to_value(&start_event)
624            .map(sanitize_json_null_bytes)
625            .map_err(|e| StoreError::Serialization(e.to_string()))?;
626
627        let mut tx = begin_transaction(&self.pool).await.map_err(|e| {
628            error!(error = %e, "Failed to begin transaction");
629            StoreError::Database(e.to_string())
630        })?;
631
632        // 1. Mark old workflow as continued_as_new
633        let result = sqlx::query(
634            r#"
635            UPDATE durable_workflow_instances
636            SET status = 'continued_as_new',
637                completed_at = NOW(),
638                continued_as_new_id = $2
639            WHERE id = $1 AND status IN ('pending', 'running')
640            RETURNING id
641            "#,
642        )
643        .bind(old_workflow_id)
644        .bind(new_workflow_id)
645        .fetch_optional(&mut *tx)
646        .await
647        .map_err(|e| {
648            error!(error = %e, "Failed to mark workflow as continued");
649            StoreError::Database(e.to_string())
650        })?;
651
652        if result.is_none() {
653            return Err(StoreError::WorkflowNotFound(old_workflow_id));
654        }
655
656        // 2. Delete old workflow events (archive)
657        sqlx::query(r#"DELETE FROM durable_workflow_events WHERE workflow_id = $1"#)
658            .bind(old_workflow_id)
659            .execute(&mut *tx)
660            .await
661            .map_err(|e| {
662                error!(error = %e, "Failed to delete old events");
663                StoreError::Database(e.to_string())
664            })?;
665
666        // 3. Delete old snapshots
667        sqlx::query(r#"DELETE FROM durable_workflow_snapshots WHERE workflow_id = $1"#)
668            .bind(old_workflow_id)
669            .execute(&mut *tx)
670            .await
671            .map_err(|e| {
672                error!(error = %e, "Failed to delete old snapshots");
673                StoreError::Database(e.to_string())
674            })?;
675
676        // 4. Create new workflow
677        sqlx::query(
678            r#"
679            INSERT INTO durable_workflow_instances (id, workflow_type, status, input, started_at)
680            VALUES ($1, $2, 'running', $3, NOW())
681            "#,
682        )
683        .bind(new_workflow_id)
684        .bind(workflow_type)
685        .bind(&input)
686        .execute(&mut *tx)
687        .await
688        .map_err(|e| {
689            error!(error = %e, "Failed to create new workflow");
690            StoreError::Database(e.to_string())
691        })?;
692
693        // 5. Write WorkflowStarted event on new workflow
694        sqlx::query(
695            r#"
696            INSERT INTO durable_workflow_events (workflow_id, sequence_num, event_type, event_data)
697            VALUES ($1, 0, 'workflow_started', $2)
698            "#,
699        )
700        .bind(new_workflow_id)
701        .bind(&event_data)
702        .execute(&mut *tx)
703        .await
704        .map_err(|e| {
705            error!(error = %e, "Failed to write start event");
706            StoreError::Database(e.to_string())
707        })?;
708
709        // 6. Save snapshot on new workflow at sequence 0
710        sqlx::query(
711            r#"
712            INSERT INTO durable_workflow_snapshots (workflow_id, sequence_num, snapshot_data)
713            VALUES ($1, 0, $2)
714            "#,
715        )
716        .bind(new_workflow_id)
717        .bind(&snapshot_data)
718        .execute(&mut *tx)
719        .await
720        .map_err(|e| {
721            error!(error = %e, "Failed to save snapshot");
722            StoreError::Database(e.to_string())
723        })?;
724
725        crate::commit_transaction(tx).await.map_err(|e| {
726            error!(error = %e, "Failed to commit continue_as_new");
727            StoreError::Database(e.to_string())
728        })?;
729
730        info!(
731            %old_workflow_id,
732            %new_workflow_id,
733            "continued workflow as new"
734        );
735
736        Ok(new_workflow_id)
737    }
738}