Skip to main content

everruns_durable/persistence/
postgres.rs

1//! PostgreSQL implementation of WorkflowEventStore
2//!
3//! Production-ready persistence using PostgreSQL with:
4//! - Optimistic concurrency control via sequence numbers
5//! - Efficient task claiming with SKIP LOCKED
6//! - Event sourcing for workflow replay
7
8use std::collections::BTreeMap;
9use std::time::Duration;
10
11use async_trait::async_trait;
12use chrono::{DateTime, Utc};
13use sqlx::{PgPool, Row};
14use tracing::{debug, error, info, instrument};
15use uuid::Uuid;
16
17mod admin;
18mod begin;
19mod circuit_breakers;
20mod dlq;
21mod event_log;
22mod schedules;
23mod schema;
24mod task_queue;
25mod workers;
26
27pub use begin::{begin_transaction, commit_in_place, commit_transaction};
28
29use super::db_failure::store_failure;
30use super::store::{
31    CapacitySnapshot, CircuitBreakerState, CircuitBreakers, ClaimedTask, CreateScheduleRow,
32    DeadLetters, DeadTaskInfo, DlqEntry, DlqFilter, DurableAdmin, EventLog, HeartbeatResponse,
33    Pagination, ReclaimResult, ScheduleExecutionFilter, ScheduleExecutionRow,
34    ScheduleExecutionStatus, ScheduleFilter, ScheduleRow, ScheduleStats, ScheduleTargetType,
35    SchedulerInstanceInfo, Schedules, SealedTaskInfo, StoreError, SystemHealth, TaskDefinition,
36    TaskFailureOutcome, TaskFilter, TaskInfo, TaskQueue, TaskStatus, TraceContext, UpdateSchedule,
37    WORKER_HEARTBEAT_TIMEOUT_SECS, WorkerFilter, WorkerHeartbeat, WorkerInfo, WorkerRegistry,
38    WorkflowEventInfo, WorkflowFilter, WorkflowInfo, WorkflowInfoExtended, WorkflowStatus,
39    no_progress_seal_threshold_from_env,
40};
41use crate::reliability::{CircuitBreakerConfig, CircuitState};
42use crate::workflow::{ActivityOptions, WorkflowError, WorkflowEvent};
43
44#[cfg(feature = "failpoints")]
45use fail::fail_point;
46
47/// Recursively strip `\0` (null bytes) from all string values in a JSON tree.
48///
49/// PostgreSQL `jsonb` rejects `\u0000`; LLM tool output or other external inputs
50/// may contain them, so all JSONB writes in this module use this helper before binding.
51fn sanitize_json_null_bytes(mut value: serde_json::Value) -> serde_json::Value {
52    fn sanitize_in_place(v: &mut serde_json::Value) {
53        match v {
54            serde_json::Value::String(s) => {
55                s.retain(|c| c != '\0');
56            }
57            serde_json::Value::Array(arr) => {
58                for item in arr.iter_mut() {
59                    sanitize_in_place(item);
60                }
61            }
62            serde_json::Value::Object(map) => {
63                for (_k, val) in map.iter_mut() {
64                    sanitize_in_place(val);
65                }
66            }
67            _ => {}
68        }
69    }
70
71    sanitize_in_place(&mut value);
72    value
73}
74
75/// PostgreSQL implementation of WorkflowEventStore
76///
77/// Uses a connection pool for efficient database access.
78/// Designed for high-throughput with 1000+ concurrent workers.
79///
80/// # Example
81///
82/// ```no_run
83/// # async fn run() -> Result<(), sqlx::Error> {
84/// let pool = sqlx::PgPool::connect("postgres://localhost/everruns").await?;
85/// let store = everruns_durable::PostgresWorkflowEventStore::new(pool);
86/// # Ok(()) }
87/// ```
88#[derive(Clone)]
89pub struct PostgresWorkflowEventStore {
90    pool: PgPool,
91    max_pending_tasks_per_workflow: u32,
92}
93
94impl PostgresWorkflowEventStore {
95    /// Create a new PostgreSQL store with the given connection pool
96    pub fn new(pool: PgPool) -> Self {
97        Self {
98            pool,
99            max_pending_tasks_per_workflow: super::store::max_pending_tasks_per_workflow_from_env(),
100        }
101    }
102
103    /// Get a reference to the connection pool
104    pub fn pool(&self) -> &PgPool {
105        &self.pool
106    }
107
108    async fn existing_initial_task_id(&self, workflow_id: Uuid) -> Result<Uuid, StoreError> {
109        sqlx::query_scalar(
110            r#"
111            SELECT id
112            FROM durable_task_queue
113            WHERE workflow_id = $1
114            ORDER BY created_at ASC, id ASC
115            LIMIT 1
116            "#,
117        )
118        .bind(workflow_id)
119        .fetch_optional(&self.pool)
120        .await
121        .map_err(|e| {
122            error!("Failed to load existing workflow task: {}", e);
123            StoreError::Database(e.to_string())
124        })?
125        .ok_or_else(|| {
126            StoreError::Database(format!(
127                "workflow {workflow_id} already exists without an initial task"
128            ))
129        })
130    }
131
132    /// Create a workflow, write its initial events, and enqueue the first task in
133    /// one transaction. This is the hot path for starting a new workflow run.
134    pub async fn start_workflow_with_task(
135        &self,
136        workflow_id: Uuid,
137        workflow_type: &str,
138        input: serde_json::Value,
139        task: TaskDefinition,
140    ) -> Result<Uuid, StoreError> {
141        let mut tx = begin_transaction(&self.pool)
142            .await
143            .map_err(|e| StoreError::Database(e.to_string()))?;
144
145        if !insert_started_workflow(&mut tx, workflow_id, workflow_type, input, &task).await? {
146            tx.rollback().await.ok();
147            let existing_task_id = self.existing_initial_task_id(workflow_id).await?;
148            debug!(
149                %workflow_id,
150                %existing_task_id,
151                "reused existing initial workflow task after duplicate start"
152            );
153            return Ok(existing_task_id);
154        }
155
156        let task_id = insert_workflow_task(&mut tx, workflow_id, &task).await?;
157
158        crate::commit_transaction(tx).await.map_err(|e| {
159            error!("Failed to commit initial workflow start: {}", e);
160            StoreError::Database(e.to_string())
161        })?;
162
163        debug!(
164            %workflow_id,
165            %task_id,
166            activity_type = %task.activity_type,
167            "started workflow and enqueued initial task"
168        );
169        Ok(task_id)
170    }
171}
172
173/// Insert `workflow_id` as Running with its `WorkflowStarted` and
174/// `ActivityScheduled` events. Returns false, writing nothing, when the
175/// workflow already exists.
176async fn insert_started_workflow(
177    tx: &mut sqlx::Transaction<'_, sqlx::Postgres>,
178    workflow_id: Uuid,
179    workflow_type: &str,
180    input: serde_json::Value,
181    task: &TaskDefinition,
182) -> Result<bool, StoreError> {
183    let workflow_input = sanitize_json_null_bytes(input);
184    let workflow_started = WorkflowEvent::started(workflow_input.clone());
185    let activity_scheduled = WorkflowEvent::ActivityScheduled {
186        activity_id: task.activity_id.clone(),
187        activity_type: task.activity_type.clone(),
188        input: sanitize_json_null_bytes(task.input.clone()),
189        options: task.options.clone(),
190    };
191    let workflow_started_data = serde_json::to_value(&workflow_started)
192        .map(sanitize_json_null_bytes)
193        .map_err(|e| StoreError::Serialization(e.to_string()))?;
194    let activity_scheduled_data = serde_json::to_value(&activity_scheduled)
195        .map(sanitize_json_null_bytes)
196        .map_err(|e| StoreError::Serialization(e.to_string()))?;
197
198    let inserted_workflow_id = sqlx::query_scalar::<_, Uuid>(
199        r#"
200        INSERT INTO durable_workflow_instances (
201            id, workflow_type, status, input, started_at
202        )
203        VALUES ($1, $2, 'running', $3, NOW())
204        ON CONFLICT (id) DO NOTHING
205        RETURNING id
206        "#,
207    )
208    .bind(workflow_id)
209    .bind(workflow_type)
210    .bind(&workflow_input)
211    .fetch_optional(&mut **tx)
212    .await
213    .map_err(|e| {
214        error!("Failed to create started workflow: {}", e);
215        StoreError::Database(e.to_string())
216    })?;
217
218    if inserted_workflow_id.is_none() {
219        return Ok(false);
220    }
221
222    sqlx::query(
223        r#"
224        INSERT INTO durable_workflow_events (
225            workflow_id, sequence_num, event_type, event_data
226        )
227        VALUES
228            ($1, 0, 'workflow_started', $2),
229            ($1, 1, 'activity_scheduled', $3)
230        "#,
231    )
232    .bind(workflow_id)
233    .bind(&workflow_started_data)
234    .bind(&activity_scheduled_data)
235    .execute(&mut **tx)
236    .await
237    .map_err(|e| {
238        error!("Failed to write initial workflow events: {}", e);
239        StoreError::Database(e.to_string())
240    })?;
241
242    Ok(true)
243}
244
245/// Insert `task` as a pending task of `workflow_id`, with no pending-cap check
246/// (callers use it right after creating or resetting the workflow).
247async fn insert_workflow_task(
248    tx: &mut sqlx::Transaction<'_, sqlx::Postgres>,
249    workflow_id: Uuid,
250    task: &TaskDefinition,
251) -> Result<Uuid, StoreError> {
252    let task_id = Uuid::now_v7();
253    let task_input = sanitize_json_null_bytes(task.input.clone());
254    let options_json = serde_json::to_value(&task.options)
255        .map(sanitize_json_null_bytes)
256        .map_err(|e| StoreError::Serialization(e.to_string()))?;
257
258    sqlx::query(
259        r#"
260        INSERT INTO durable_task_queue (
261            id, workflow_id, activity_id, activity_type, input, options,
262            max_attempts, priority, visible_at,
263            schedule_to_start_timeout_ms, start_to_close_timeout_ms, heartbeat_timeout_ms,
264            queue
265        )
266        VALUES ($1, $2, $3, $4, $5, $6, $7, $8, NOW() + $12::bigint * INTERVAL '1 millisecond', $9, $10, $11, $13)
267        "#,
268    )
269    .bind(task_id)
270    .bind(workflow_id)
271    .bind(&task.activity_id)
272    .bind(&task.activity_type)
273    .bind(&task_input)
274    .bind(&options_json)
275    .bind(task.options.retry_policy.max_attempts as i32)
276    .bind(task.options.priority)
277    .bind(task.options.schedule_to_start_timeout.as_millis() as i64)
278    .bind(task.options.start_to_close_timeout.as_millis() as i64)
279    .bind(task.options.heartbeat_timeout.map(|d| d.as_millis() as i64))
280    .bind(task.options.start_delay.map_or(0, |d| d.as_millis() as i64))
281    .bind(task.options.queue.as_deref())
282    .execute(&mut **tx)
283    .await
284    .map_err(|e| {
285        error!("Failed to enqueue initial workflow task: {}", e);
286        StoreError::Database(e.to_string())
287    })?;
288
289    Ok(task_id)
290}
291
292// Helper functions
293
294fn parse_schedule_row(row: sqlx::postgres::PgRow) -> Result<ScheduleRow, StoreError> {
295    use sqlx::Row;
296    let target_type_str: String = row.get("target_type");
297    let target_type = parse_schedule_target_type(&target_type_str)?;
298
299    Ok(ScheduleRow {
300        id: row.get("id"),
301        name: row.get("name"),
302        description: row.get("description"),
303        cron_expression: row.get("cron_expression"),
304        timezone: row.get("timezone"),
305        target_type,
306        target_name: row.get("target_name"),
307        target_input: row.get("target_input"),
308        enabled: row.get("enabled"),
309        max_concurrent: row
310            .get::<Option<i32>, _>("max_concurrent")
311            .map(|v| v as u32),
312        catch_up_missed: row.get("catch_up_missed"),
313        max_catch_up: row.get::<Option<i32>, _>("max_catch_up").map(|v| v as u32),
314        retry_policy: row.get("retry_policy"),
315        last_triggered_at: row.get("last_triggered_at"),
316        next_trigger_at: row.get("next_trigger_at"),
317        claimed_by: row.get("claimed_by"),
318        claimed_at: row.get("claimed_at"),
319        created_at: row.get("created_at"),
320        updated_at: row.get("updated_at"),
321    })
322}
323
324fn parse_schedule_execution_row(
325    row: sqlx::postgres::PgRow,
326) -> Result<ScheduleExecutionRow, StoreError> {
327    use sqlx::Row;
328    let status_str: String = row.get("status");
329    let status = parse_schedule_execution_status(&status_str)?;
330
331    Ok(ScheduleExecutionRow {
332        id: row.get("id"),
333        schedule_id: row.get("schedule_id"),
334        scheduled_at: row.get("scheduled_at"),
335        started_at: row.get("started_at"),
336        completed_at: row.get("completed_at"),
337        status,
338        workflow_id: row.get("workflow_id"),
339        task_id: row.get("task_id"),
340        error: row.get("error"),
341        duration_ms: row.get("duration_ms"),
342        created_at: row.get("created_at"),
343    })
344}
345
346fn parse_schedule_target_type(target_type: &str) -> Result<ScheduleTargetType, StoreError> {
347    match target_type {
348        "workflow" => Ok(ScheduleTargetType::Workflow),
349        "activity" => Ok(ScheduleTargetType::Activity),
350        _ => Err(StoreError::Database(format!(
351            "Unknown schedule target type: {}",
352            target_type
353        ))),
354    }
355}
356
357fn parse_schedule_execution_status(status: &str) -> Result<ScheduleExecutionStatus, StoreError> {
358    match status {
359        "pending" => Ok(ScheduleExecutionStatus::Pending),
360        "running" => Ok(ScheduleExecutionStatus::Running),
361        "completed" => Ok(ScheduleExecutionStatus::Completed),
362        "failed" => Ok(ScheduleExecutionStatus::Failed),
363        "skipped" => Ok(ScheduleExecutionStatus::Skipped),
364        _ => Err(StoreError::Database(format!(
365            "Unknown schedule execution status: {}",
366            status
367        ))),
368    }
369}
370
371fn parse_circuit_state(state: &str) -> Result<CircuitState, StoreError> {
372    match state {
373        "closed" => Ok(CircuitState::Closed),
374        "open" => Ok(CircuitState::Open),
375        "half_open" => Ok(CircuitState::HalfOpen),
376        _ => Err(StoreError::Database(format!(
377            "Unknown circuit state: {}",
378            state
379        ))),
380    }
381}
382
383fn parse_workflow_status(status: &str) -> Result<WorkflowStatus, StoreError> {
384    match status {
385        "pending" => Ok(WorkflowStatus::Pending),
386        "running" => Ok(WorkflowStatus::Running),
387        "completed" => Ok(WorkflowStatus::Completed),
388        "failed" => Ok(WorkflowStatus::Failed),
389        "cancelled" => Ok(WorkflowStatus::Cancelled),
390        "continued_as_new" => Ok(WorkflowStatus::ContinuedAsNew),
391        _ => Err(StoreError::Database(format!(
392            "Unknown workflow status: {}",
393            status
394        ))),
395    }
396}
397
398/// Map an `append_events` INSERT error to the appropriate `StoreError`.
399///
400/// EVE-639: the append path relies on DB constraints rather than a prior
401/// FOR UPDATE + MAX(...) read, so we translate the two expected violations:
402/// - unique `(workflow_id, sequence_num)` (SQLSTATE 23505) → a concurrency
403///   conflict (a competing writer already occupied one of our sequences). The
404///   actual conflicting sequence is not cheaply known here; callers recompute
405///   it via `count_events` on retry, so `actual` is reported as -1 (unknown).
406/// - foreign-key on `workflow_id` (SQLSTATE 23503) → workflow not found.
407fn map_append_events_error(
408    err: sqlx::Error,
409    workflow_id: Uuid,
410    expected_sequence: i32,
411) -> StoreError {
412    if let Some(db_err) = err.as_database_error() {
413        if db_err.is_unique_violation() {
414            return StoreError::ConcurrencyConflict {
415                expected: expected_sequence,
416                actual: -1,
417            };
418        }
419        if db_err.is_foreign_key_violation() {
420            return StoreError::WorkflowNotFound(workflow_id);
421        }
422    }
423    StoreError::Database(err.to_string())
424}
425
426fn event_type_name(event: &WorkflowEvent) -> &'static str {
427    match event {
428        WorkflowEvent::WorkflowStarted { .. } => "workflow_started",
429        WorkflowEvent::WorkflowCompleted { .. } => "workflow_completed",
430        WorkflowEvent::WorkflowFailed { .. } => "workflow_failed",
431        WorkflowEvent::WorkflowCancelled { .. } => "workflow_cancelled",
432        WorkflowEvent::ActivityScheduled { .. } => "activity_scheduled",
433        WorkflowEvent::ActivityStarted { .. } => "activity_started",
434        WorkflowEvent::ActivityCompleted { .. } => "activity_completed",
435        WorkflowEvent::ActivityFailed { .. } => "activity_failed",
436        WorkflowEvent::ActivityTimedOut { .. } => "activity_timed_out",
437        WorkflowEvent::ActivityCancelled { .. } => "activity_cancelled",
438        WorkflowEvent::TimerStarted { .. } => "timer_started",
439        WorkflowEvent::TimerFired { .. } => "timer_fired",
440        WorkflowEvent::TimerCancelled { .. } => "timer_cancelled",
441        WorkflowEvent::ChildWorkflowStarted { .. } => "child_workflow_started",
442        WorkflowEvent::ChildWorkflowCompleted { .. } => "child_workflow_completed",
443        WorkflowEvent::ChildWorkflowFailed { .. } => "child_workflow_failed",
444    }
445}
446
447#[cfg(test)]
448mod tests {
449    // Unit tests for JSON sanitization that do not require a PostgreSQL database.
450    // PostgreSQL-backed integration tests can be run with:
451    // cargo test -p everruns-durable --test integration_test -- --test-threads=1
452
453    use super::*;
454    use serde_json::json;
455
456    #[test]
457    fn sanitize_handles_nested_structures() {
458        let input = json!({
459            "outer": {
460                "inner": "has\0null",
461                "clean": "no nulls"
462            },
463            "arr": ["a\0b", "clean"]
464        });
465        let result = sanitize_json_null_bytes(input);
466        assert_eq!(
467            result,
468            json!({
469                "outer": {
470                    "inner": "hasnull",
471                    "clean": "no nulls"
472                },
473                "arr": ["ab", "clean"]
474            })
475        );
476    }
477
478    #[test]
479    fn sanitize_preserves_non_string_values() {
480        let input = json!({"num": 42, "bool": true, "null": null});
481        let result = sanitize_json_null_bytes(input.clone());
482        assert_eq!(result, input);
483    }
484}