1use std::collections::BTreeMap;
9use std::time::Duration;
10
11use async_trait::async_trait;
12use chrono::{DateTime, Utc};
13use sqlx::{PgPool, Row};
14use tracing::{debug, error, info, instrument};
15use uuid::Uuid;
16
17mod admin;
18mod begin;
19mod circuit_breakers;
20mod dlq;
21mod event_log;
22mod schedules;
23mod schema;
24mod task_queue;
25mod workers;
26
27pub use begin::{begin_transaction, commit_in_place, commit_transaction};
28
29use super::db_failure::store_failure;
30use super::store::{
31 CapacitySnapshot, CircuitBreakerState, CircuitBreakers, ClaimedTask, CreateScheduleRow,
32 DeadLetters, DeadTaskInfo, DlqEntry, DlqFilter, DurableAdmin, EventLog, HeartbeatResponse,
33 Pagination, ReclaimResult, ScheduleExecutionFilter, ScheduleExecutionRow,
34 ScheduleExecutionStatus, ScheduleFilter, ScheduleRow, ScheduleStats, ScheduleTargetType,
35 SchedulerInstanceInfo, Schedules, SealedTaskInfo, StoreError, SystemHealth, TaskDefinition,
36 TaskFailureOutcome, TaskFilter, TaskInfo, TaskQueue, TaskStatus, TraceContext, UpdateSchedule,
37 WORKER_HEARTBEAT_TIMEOUT_SECS, WorkerFilter, WorkerHeartbeat, WorkerInfo, WorkerRegistry,
38 WorkflowEventInfo, WorkflowFilter, WorkflowInfo, WorkflowInfoExtended, WorkflowStatus,
39 no_progress_seal_threshold_from_env,
40};
41use crate::reliability::{CircuitBreakerConfig, CircuitState};
42use crate::workflow::{ActivityOptions, WorkflowError, WorkflowEvent};
43
44#[cfg(feature = "failpoints")]
45use fail::fail_point;
46
47fn sanitize_json_null_bytes(mut value: serde_json::Value) -> serde_json::Value {
52 fn sanitize_in_place(v: &mut serde_json::Value) {
53 match v {
54 serde_json::Value::String(s) => {
55 s.retain(|c| c != '\0');
56 }
57 serde_json::Value::Array(arr) => {
58 for item in arr.iter_mut() {
59 sanitize_in_place(item);
60 }
61 }
62 serde_json::Value::Object(map) => {
63 for (_k, val) in map.iter_mut() {
64 sanitize_in_place(val);
65 }
66 }
67 _ => {}
68 }
69 }
70
71 sanitize_in_place(&mut value);
72 value
73}
74
75#[derive(Clone)]
89pub struct PostgresWorkflowEventStore {
90 pool: PgPool,
91 max_pending_tasks_per_workflow: u32,
92}
93
94impl PostgresWorkflowEventStore {
95 pub fn new(pool: PgPool) -> Self {
97 Self {
98 pool,
99 max_pending_tasks_per_workflow: super::store::max_pending_tasks_per_workflow_from_env(),
100 }
101 }
102
103 pub fn pool(&self) -> &PgPool {
105 &self.pool
106 }
107
108 async fn existing_initial_task_id(&self, workflow_id: Uuid) -> Result<Uuid, StoreError> {
109 sqlx::query_scalar(
110 r#"
111 SELECT id
112 FROM durable_task_queue
113 WHERE workflow_id = $1
114 ORDER BY created_at ASC, id ASC
115 LIMIT 1
116 "#,
117 )
118 .bind(workflow_id)
119 .fetch_optional(&self.pool)
120 .await
121 .map_err(|e| {
122 error!("Failed to load existing workflow task: {}", e);
123 StoreError::Database(e.to_string())
124 })?
125 .ok_or_else(|| {
126 StoreError::Database(format!(
127 "workflow {workflow_id} already exists without an initial task"
128 ))
129 })
130 }
131
132 pub async fn start_workflow_with_task(
135 &self,
136 workflow_id: Uuid,
137 workflow_type: &str,
138 input: serde_json::Value,
139 task: TaskDefinition,
140 ) -> Result<Uuid, StoreError> {
141 let mut tx = begin_transaction(&self.pool)
142 .await
143 .map_err(|e| StoreError::Database(e.to_string()))?;
144
145 if !insert_started_workflow(&mut tx, workflow_id, workflow_type, input, &task).await? {
146 tx.rollback().await.ok();
147 let existing_task_id = self.existing_initial_task_id(workflow_id).await?;
148 debug!(
149 %workflow_id,
150 %existing_task_id,
151 "reused existing initial workflow task after duplicate start"
152 );
153 return Ok(existing_task_id);
154 }
155
156 let task_id = insert_workflow_task(&mut tx, workflow_id, &task).await?;
157
158 crate::commit_transaction(tx).await.map_err(|e| {
159 error!("Failed to commit initial workflow start: {}", e);
160 StoreError::Database(e.to_string())
161 })?;
162
163 debug!(
164 %workflow_id,
165 %task_id,
166 activity_type = %task.activity_type,
167 "started workflow and enqueued initial task"
168 );
169 Ok(task_id)
170 }
171}
172
173async fn insert_started_workflow(
177 tx: &mut sqlx::Transaction<'_, sqlx::Postgres>,
178 workflow_id: Uuid,
179 workflow_type: &str,
180 input: serde_json::Value,
181 task: &TaskDefinition,
182) -> Result<bool, StoreError> {
183 let workflow_input = sanitize_json_null_bytes(input);
184 let workflow_started = WorkflowEvent::started(workflow_input.clone());
185 let activity_scheduled = WorkflowEvent::ActivityScheduled {
186 activity_id: task.activity_id.clone(),
187 activity_type: task.activity_type.clone(),
188 input: sanitize_json_null_bytes(task.input.clone()),
189 options: task.options.clone(),
190 };
191 let workflow_started_data = serde_json::to_value(&workflow_started)
192 .map(sanitize_json_null_bytes)
193 .map_err(|e| StoreError::Serialization(e.to_string()))?;
194 let activity_scheduled_data = serde_json::to_value(&activity_scheduled)
195 .map(sanitize_json_null_bytes)
196 .map_err(|e| StoreError::Serialization(e.to_string()))?;
197
198 let inserted_workflow_id = sqlx::query_scalar::<_, Uuid>(
199 r#"
200 INSERT INTO durable_workflow_instances (
201 id, workflow_type, status, input, started_at
202 )
203 VALUES ($1, $2, 'running', $3, NOW())
204 ON CONFLICT (id) DO NOTHING
205 RETURNING id
206 "#,
207 )
208 .bind(workflow_id)
209 .bind(workflow_type)
210 .bind(&workflow_input)
211 .fetch_optional(&mut **tx)
212 .await
213 .map_err(|e| {
214 error!("Failed to create started workflow: {}", e);
215 StoreError::Database(e.to_string())
216 })?;
217
218 if inserted_workflow_id.is_none() {
219 return Ok(false);
220 }
221
222 sqlx::query(
223 r#"
224 INSERT INTO durable_workflow_events (
225 workflow_id, sequence_num, event_type, event_data
226 )
227 VALUES
228 ($1, 0, 'workflow_started', $2),
229 ($1, 1, 'activity_scheduled', $3)
230 "#,
231 )
232 .bind(workflow_id)
233 .bind(&workflow_started_data)
234 .bind(&activity_scheduled_data)
235 .execute(&mut **tx)
236 .await
237 .map_err(|e| {
238 error!("Failed to write initial workflow events: {}", e);
239 StoreError::Database(e.to_string())
240 })?;
241
242 Ok(true)
243}
244
245async fn insert_workflow_task(
248 tx: &mut sqlx::Transaction<'_, sqlx::Postgres>,
249 workflow_id: Uuid,
250 task: &TaskDefinition,
251) -> Result<Uuid, StoreError> {
252 let task_id = Uuid::now_v7();
253 let task_input = sanitize_json_null_bytes(task.input.clone());
254 let options_json = serde_json::to_value(&task.options)
255 .map(sanitize_json_null_bytes)
256 .map_err(|e| StoreError::Serialization(e.to_string()))?;
257
258 sqlx::query(
259 r#"
260 INSERT INTO durable_task_queue (
261 id, workflow_id, activity_id, activity_type, input, options,
262 max_attempts, priority, visible_at,
263 schedule_to_start_timeout_ms, start_to_close_timeout_ms, heartbeat_timeout_ms,
264 queue
265 )
266 VALUES ($1, $2, $3, $4, $5, $6, $7, $8, NOW() + $12::bigint * INTERVAL '1 millisecond', $9, $10, $11, $13)
267 "#,
268 )
269 .bind(task_id)
270 .bind(workflow_id)
271 .bind(&task.activity_id)
272 .bind(&task.activity_type)
273 .bind(&task_input)
274 .bind(&options_json)
275 .bind(task.options.retry_policy.max_attempts as i32)
276 .bind(task.options.priority)
277 .bind(task.options.schedule_to_start_timeout.as_millis() as i64)
278 .bind(task.options.start_to_close_timeout.as_millis() as i64)
279 .bind(task.options.heartbeat_timeout.map(|d| d.as_millis() as i64))
280 .bind(task.options.start_delay.map_or(0, |d| d.as_millis() as i64))
281 .bind(task.options.queue.as_deref())
282 .execute(&mut **tx)
283 .await
284 .map_err(|e| {
285 error!("Failed to enqueue initial workflow task: {}", e);
286 StoreError::Database(e.to_string())
287 })?;
288
289 Ok(task_id)
290}
291
292fn parse_schedule_row(row: sqlx::postgres::PgRow) -> Result<ScheduleRow, StoreError> {
295 use sqlx::Row;
296 let target_type_str: String = row.get("target_type");
297 let target_type = parse_schedule_target_type(&target_type_str)?;
298
299 Ok(ScheduleRow {
300 id: row.get("id"),
301 name: row.get("name"),
302 description: row.get("description"),
303 cron_expression: row.get("cron_expression"),
304 timezone: row.get("timezone"),
305 target_type,
306 target_name: row.get("target_name"),
307 target_input: row.get("target_input"),
308 enabled: row.get("enabled"),
309 max_concurrent: row
310 .get::<Option<i32>, _>("max_concurrent")
311 .map(|v| v as u32),
312 catch_up_missed: row.get("catch_up_missed"),
313 max_catch_up: row.get::<Option<i32>, _>("max_catch_up").map(|v| v as u32),
314 retry_policy: row.get("retry_policy"),
315 last_triggered_at: row.get("last_triggered_at"),
316 next_trigger_at: row.get("next_trigger_at"),
317 claimed_by: row.get("claimed_by"),
318 claimed_at: row.get("claimed_at"),
319 created_at: row.get("created_at"),
320 updated_at: row.get("updated_at"),
321 })
322}
323
324fn parse_schedule_execution_row(
325 row: sqlx::postgres::PgRow,
326) -> Result<ScheduleExecutionRow, StoreError> {
327 use sqlx::Row;
328 let status_str: String = row.get("status");
329 let status = parse_schedule_execution_status(&status_str)?;
330
331 Ok(ScheduleExecutionRow {
332 id: row.get("id"),
333 schedule_id: row.get("schedule_id"),
334 scheduled_at: row.get("scheduled_at"),
335 started_at: row.get("started_at"),
336 completed_at: row.get("completed_at"),
337 status,
338 workflow_id: row.get("workflow_id"),
339 task_id: row.get("task_id"),
340 error: row.get("error"),
341 duration_ms: row.get("duration_ms"),
342 created_at: row.get("created_at"),
343 })
344}
345
346fn parse_schedule_target_type(target_type: &str) -> Result<ScheduleTargetType, StoreError> {
347 match target_type {
348 "workflow" => Ok(ScheduleTargetType::Workflow),
349 "activity" => Ok(ScheduleTargetType::Activity),
350 _ => Err(StoreError::Database(format!(
351 "Unknown schedule target type: {}",
352 target_type
353 ))),
354 }
355}
356
357fn parse_schedule_execution_status(status: &str) -> Result<ScheduleExecutionStatus, StoreError> {
358 match status {
359 "pending" => Ok(ScheduleExecutionStatus::Pending),
360 "running" => Ok(ScheduleExecutionStatus::Running),
361 "completed" => Ok(ScheduleExecutionStatus::Completed),
362 "failed" => Ok(ScheduleExecutionStatus::Failed),
363 "skipped" => Ok(ScheduleExecutionStatus::Skipped),
364 _ => Err(StoreError::Database(format!(
365 "Unknown schedule execution status: {}",
366 status
367 ))),
368 }
369}
370
371fn parse_circuit_state(state: &str) -> Result<CircuitState, StoreError> {
372 match state {
373 "closed" => Ok(CircuitState::Closed),
374 "open" => Ok(CircuitState::Open),
375 "half_open" => Ok(CircuitState::HalfOpen),
376 _ => Err(StoreError::Database(format!(
377 "Unknown circuit state: {}",
378 state
379 ))),
380 }
381}
382
383fn parse_workflow_status(status: &str) -> Result<WorkflowStatus, StoreError> {
384 match status {
385 "pending" => Ok(WorkflowStatus::Pending),
386 "running" => Ok(WorkflowStatus::Running),
387 "completed" => Ok(WorkflowStatus::Completed),
388 "failed" => Ok(WorkflowStatus::Failed),
389 "cancelled" => Ok(WorkflowStatus::Cancelled),
390 "continued_as_new" => Ok(WorkflowStatus::ContinuedAsNew),
391 _ => Err(StoreError::Database(format!(
392 "Unknown workflow status: {}",
393 status
394 ))),
395 }
396}
397
398fn map_append_events_error(
408 err: sqlx::Error,
409 workflow_id: Uuid,
410 expected_sequence: i32,
411) -> StoreError {
412 if let Some(db_err) = err.as_database_error() {
413 if db_err.is_unique_violation() {
414 return StoreError::ConcurrencyConflict {
415 expected: expected_sequence,
416 actual: -1,
417 };
418 }
419 if db_err.is_foreign_key_violation() {
420 return StoreError::WorkflowNotFound(workflow_id);
421 }
422 }
423 StoreError::Database(err.to_string())
424}
425
426fn event_type_name(event: &WorkflowEvent) -> &'static str {
427 match event {
428 WorkflowEvent::WorkflowStarted { .. } => "workflow_started",
429 WorkflowEvent::WorkflowCompleted { .. } => "workflow_completed",
430 WorkflowEvent::WorkflowFailed { .. } => "workflow_failed",
431 WorkflowEvent::WorkflowCancelled { .. } => "workflow_cancelled",
432 WorkflowEvent::ActivityScheduled { .. } => "activity_scheduled",
433 WorkflowEvent::ActivityStarted { .. } => "activity_started",
434 WorkflowEvent::ActivityCompleted { .. } => "activity_completed",
435 WorkflowEvent::ActivityFailed { .. } => "activity_failed",
436 WorkflowEvent::ActivityTimedOut { .. } => "activity_timed_out",
437 WorkflowEvent::ActivityCancelled { .. } => "activity_cancelled",
438 WorkflowEvent::TimerStarted { .. } => "timer_started",
439 WorkflowEvent::TimerFired { .. } => "timer_fired",
440 WorkflowEvent::TimerCancelled { .. } => "timer_cancelled",
441 WorkflowEvent::ChildWorkflowStarted { .. } => "child_workflow_started",
442 WorkflowEvent::ChildWorkflowCompleted { .. } => "child_workflow_completed",
443 WorkflowEvent::ChildWorkflowFailed { .. } => "child_workflow_failed",
444 }
445}
446
447#[cfg(test)]
448mod tests {
449 use super::*;
454 use serde_json::json;
455
456 #[test]
457 fn sanitize_handles_nested_structures() {
458 let input = json!({
459 "outer": {
460 "inner": "has\0null",
461 "clean": "no nulls"
462 },
463 "arr": ["a\0b", "clean"]
464 });
465 let result = sanitize_json_null_bytes(input);
466 assert_eq!(
467 result,
468 json!({
469 "outer": {
470 "inner": "hasnull",
471 "clean": "no nulls"
472 },
473 "arr": ["ab", "clean"]
474 })
475 );
476 }
477
478 #[test]
479 fn sanitize_preserves_non_string_values() {
480 let input = json!({"num": 42, "bool": true, "null": null});
481 let result = sanitize_json_null_bytes(input.clone());
482 assert_eq!(result, input);
483 }
484}