1use async_trait::async_trait;
10use chrono::{DateTime, Utc};
11use deadpool_postgres::{Config, ManagerConfig, Pool, RecyclingMethod, Runtime};
12use journio_core::dialect::Dialect;
13use journio_core::error::{JournioError, JournioErrorCode, JournioResult};
14use journio_core::system_db::{ForkWorkflow, InitWorkflow, InitWorkflowResult, SystemDatabase};
15use journio_core::types::{
16 ListWorkflowsFilter, QueueConfig, ScheduleStatus, StepRecord, StreamEntry, VersionInfo,
17 WorkflowSchedule, WorkflowStatus, WorkflowStatusType,
18};
19use journio_core::value::Interchange;
20use std::collections::HashMap;
21use std::sync::Mutex;
22use std::sync::atomic::{AtomicBool, Ordering};
23use std::time::Duration;
24use tokio::sync::Notify;
25use tokio_postgres::{AsyncMessage, NoTls};
26use tokio_util::sync::CancellationToken;
27
28use crate::dialect::PostgresDialect;
29use crate::error::{is_foreign_key_violation, is_unique_violation, sqlstate};
30use crate::migrations::run_migrations;
31
32const NULL_TOPIC: &str = "__null__topic__";
35const NOTIFICATIONS_CHANNEL: &str = "journio_notifications_channel";
36const WORKFLOW_EVENTS_CHANNEL: &str = "journio_workflow_events_channel";
37const STREAMS_CHANNEL: &str = "journio_streams_channel";
38const STREAM_CLOSED_SENTINEL: &str = "__JOURNIO_STREAM_CLOSED__";
39const INTERNAL_QUEUE_NAME: &str = "_journio_internal_queue";
40
41const SERIALIZATION_JSON: &str = "JOURNIO_JSON";
44
45pub struct PostgresSystemDatabase {
51 pool: Pool,
52 connect_config: tokio_postgres::Config,
53 schema: String,
54 dialect: PostgresDialect,
55 listener_cancel: CancellationToken,
56 listener_started: AtomicBool,
57 notification_waiters: std::sync::Arc<Mutex<HashMap<String, std::sync::Arc<Notify>>>>,
58 event_waiters: std::sync::Arc<Mutex<HashMap<String, std::sync::Arc<Notify>>>>,
59 stream_waiters: std::sync::Arc<Mutex<HashMap<String, std::sync::Arc<Notify>>>>,
60}
61
62impl PostgresSystemDatabase {
63 pub fn connect(database_url: &str, schema: &str) -> Result<Self, JournioError> {
65 let mut cfg = Config::new();
66 cfg.url = Some(database_url.to_string());
67 cfg.options = Some(format!("-c search_path={schema},public"));
68 cfg.manager = Some(ManagerConfig {
69 recycling_method: RecyclingMethod::Fast,
70 });
71 let mut pool_cfg = cfg.pool.unwrap_or_default();
72 pool_cfg.max_size = 8;
73 cfg.pool = Some(pool_cfg);
74 let connect_config = cfg.get_pg_config().map_err(init_err)?;
75 let pool = cfg
76 .create_pool(Some(Runtime::Tokio1), NoTls)
77 .map_err(init_err)?;
78 Ok(Self {
79 pool,
80 connect_config,
81 schema: schema.to_string(),
82 dialect: PostgresDialect,
83 listener_cancel: CancellationToken::new(),
84 listener_started: AtomicBool::new(false),
85 notification_waiters: std::sync::Arc::new(Mutex::new(HashMap::new())),
86 event_waiters: std::sync::Arc::new(Mutex::new(HashMap::new())),
87 stream_waiters: std::sync::Arc::new(Mutex::new(HashMap::new())),
88 })
89 }
90
91 pub fn pool(&self) -> &Pool {
92 &self.pool
93 }
94
95 pub fn schema(&self) -> &str {
96 &self.schema
97 }
98
99 fn q(&self, body: &str) -> String {
100 body.to_string()
101 }
102
103 fn spawn_listener(&self) {
104 let connect_config = self.connect_config.clone();
105 let cancel = self.listener_cancel.clone();
106 let notifications = self.notification_waiters.clone();
107 let events = self.event_waiters.clone();
108 let streams = self.stream_waiters.clone();
109
110 tokio::spawn(async move {
111 run_listener_loop(connect_config, cancel, notifications, events, streams).await;
112 });
113 }
114
115 async fn wait_on_map(
116 &self,
117 waiters: &std::sync::Arc<Mutex<HashMap<String, std::sync::Arc<Notify>>>>,
118 key: &str,
119 timeout: Duration,
120 ) -> JournioResult<()> {
121 let notify = {
122 let mut map = waiters.lock().expect("waiters lock");
123 map.entry(key.to_string())
124 .or_insert_with(|| std::sync::Arc::new(Notify::new()))
125 .clone()
126 };
127
128 tokio::select! {
129 _ = notify.notified() => {}
130 _ = tokio::time::sleep(timeout) => {}
131 _ = self.listener_cancel.cancelled() => {}
132 }
133 Ok(())
134 }
135}
136
137async fn run_listener_loop(
138 connect_config: tokio_postgres::Config,
139 cancel: CancellationToken,
140 notification_waiters: std::sync::Arc<Mutex<HashMap<String, std::sync::Arc<Notify>>>>,
141 event_waiters: std::sync::Arc<Mutex<HashMap<String, std::sync::Arc<Notify>>>>,
142 stream_waiters: std::sync::Arc<Mutex<HashMap<String, std::sync::Arc<Notify>>>>,
143) {
144 loop {
145 if cancel.is_cancelled() {
146 return;
147 }
148
149 let connect = connect_config.connect(NoTls).await;
150 let (client, mut connection) = match connect {
151 Ok(parts) => parts,
152 Err(err) => {
153 tracing::warn!(error = %err, "failed to connect postgres listener");
154 tokio::time::sleep(Duration::from_millis(250)).await;
155 continue;
156 }
157 };
158
159 if let Err(err) = client.batch_execute(&format!("LISTEN {NOTIFICATIONS_CHANNEL}; LISTEN {WORKFLOW_EVENTS_CHANNEL}; LISTEN {STREAMS_CHANNEL};")).await {
160 tracing::warn!(error = %err, "failed to register LISTEN channels");
161 tokio::time::sleep(Duration::from_millis(250)).await;
162 continue;
163 }
164
165 loop {
166 tokio::select! {
167 _ = cancel.cancelled() => return,
168 message = std::future::poll_fn(|cx| connection.poll_message(cx)) => {
169 match message {
170 Some(Ok(AsyncMessage::Notification(notification))) => {
171 let payload = notification.payload().to_string();
172 match notification.channel() {
173 NOTIFICATIONS_CHANNEL => notify_waiters(¬ification_waiters, &payload),
174 WORKFLOW_EVENTS_CHANNEL => notify_waiters(&event_waiters, &payload),
175 STREAMS_CHANNEL => notify_waiters(&stream_waiters, &payload),
176 _ => {}
177 }
178 }
179 Some(Ok(_)) => {}
180 Some(Err(err)) => {
181 tracing::warn!(error = %err, "postgres LISTEN/NOTIFY receive failed");
182 break;
183 }
184 None => break,
185 }
186 }
187 }
188 }
189 }
190}
191
192fn notify_waiters(
193 waiters: &std::sync::Arc<Mutex<HashMap<String, std::sync::Arc<Notify>>>>,
194 key: &str,
195) {
196 let notify = waiters.lock().expect("waiters lock").get(key).cloned();
197 if let Some(notify) = notify {
198 notify.notify_waiters();
199 }
200}
201
202#[async_trait]
203impl SystemDatabase for PostgresSystemDatabase {
204 fn dialect(&self) -> &dyn Dialect {
205 &self.dialect
206 }
207
208 async fn migrate(&self) -> JournioResult<()> {
209 run_migrations(&self.pool, &self.schema).await
210 }
211
212 async fn launch(&self) -> JournioResult<()> {
213 if self
214 .listener_started
215 .compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst)
216 .is_ok()
217 {
218 self.spawn_listener();
219 }
220 Ok(())
221 }
222
223 async fn shutdown(&self) -> JournioResult<()> {
224 self.listener_cancel.cancel();
225 Ok(())
226 }
227
228 async fn init_workflow(&self, init: InitWorkflow) -> JournioResult<InitWorkflowResult> {
234 let attempts: i64 = match init.status {
235 WorkflowStatusType::Enqueued | WorkflowStatusType::Delayed => 0,
236 _ => 1,
237 };
238 let now_ms = Utc::now().timestamp_millis();
239 let created_ms = now_ms;
240 let timeout_ms: Option<i64> = init.timeout.map(|d| d.as_millis() as i64);
241 let deadline_ms: Option<i64> = init.deadline.map(|t| t.timestamp_millis());
242 let delay_until_ms: Option<i64> = init.delay_until.map(|t| t.timestamp_millis());
243 let inputs_str: Option<String> = init
244 .input
245 .as_ref()
246 .map(|v| serde_json::to_string(v).unwrap_or_default());
247 let roles_json =
248 serde_json::to_string(&init.authenticated_roles).unwrap_or_else(|_| "null".into());
249 let recovery_increment: i64 = i64::from(i32::from(init.increment_attempts));
250
251 let query = self.q(
252 "INSERT INTO workflow_status (
253 workflow_uuid, status, name, queue_name, authenticated_user,
254 assumed_role, authenticated_roles, executor_id, application_version,
255 application_id, created_at, recovery_attempts, updated_at,
256 workflow_timeout_ms, workflow_deadline_epoch_ms, inputs,
257 deduplication_id, priority, queue_partition_key, owner_xid,
258 parent_workflow_id, class_name, config_name, serialization,
259 delay_until_epoch_ms
260 ) VALUES($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25)
261 ON CONFLICT (workflow_uuid)
262 DO UPDATE SET
263 recovery_attempts = CASE
264 WHEN EXCLUDED.status NOT IN ($26, $27) THEN workflow_status.recovery_attempts + $28
265 ELSE workflow_status.recovery_attempts
266 END,
267 updated_at = EXCLUDED.updated_at,
268 executor_id = CASE
269 WHEN EXCLUDED.status IN ($26, $27) THEN workflow_status.executor_id
270 ELSE EXCLUDED.executor_id
271 END
272 RETURNING recovery_attempts, status, name, queue_name, queue_partition_key, workflow_timeout_ms, workflow_deadline_epoch_ms, owner_xid",
273 );
274
275 let status_str = status_to_str(init.status);
276 let enqueued_str = status_to_str(WorkflowStatusType::Enqueued);
277 let delayed_str = status_to_str(WorkflowStatusType::Delayed);
278
279 let client = self.pool.get().await.map_err(pool_err)?;
280 let row = client
281 .query_one(
282 &query,
283 &[
284 &init.workflow_id, &status_str, &init.name, &init.queue_name,
285 &init.authenticated_user, &init.assumed_role, &roles_json, &init.executor_id,
286 &init.application_version, &init.application_id, &created_ms, &attempts,
287 &now_ms, &timeout_ms, &deadline_ms, &inputs_str, &init.deduplication_id,
288 &init.priority, &init.queue_partition_key, &None::<String>,
289 &init.parent_workflow_id, &init.class_name, &init.config_name,
290 &init.serialization, &delay_until_ms, &enqueued_str, &delayed_str,
291 &recovery_increment,
292 ],
293 )
294 .await
295 .map_err(|e| {
296 if is_unique_violation(&e) {
297 JournioError {
298 code: JournioErrorCode::QueueDeduplicated,
299 message: format!(
300 "Workflow {} was deduplicated due to an existing workflow in queue {:?} with deduplication ID {:?}",
301 init.workflow_id, init.queue_name, init.deduplication_id
302 ),
303 workflow_id: Some(init.workflow_id.clone()),
304 queue_name: init.queue_name.clone(),
305 deduplication_id: init.deduplication_id.clone(),
306 source: Some(Box::new(e)),
307 ..Default::default()
308 }
309 } else {
310 db_err(e)
311 }
312 })?;
313
314 let attempts_out: i64 = row.get(0);
315 let status_out_str: String = row.get(1);
316 let name_out: String = row.get(2);
317 let queue_name_out: Option<String> = row.get(3);
318 let queue_partition_out: Option<String> = row.get(4);
319 let timeout_out: Option<i64> = row.get(5);
320 let deadline_out: Option<i64> = row.get(6);
321
322 if !init.name.is_empty() && name_out != init.name {
323 return Err(JournioError {
324 code: JournioErrorCode::ConflictingWorkflowError,
325 message: format!(
326 "Conflicting workflow invocation with the same ID ({}): Workflow already exists with a different name: {}, but the provided name is: {}",
327 init.workflow_id, name_out, init.name
328 ),
329 workflow_id: Some(init.workflow_id),
330 ..Default::default()
331 });
332 }
333 if let Some(qn) = &init.queue_name {
334 if let Some(existing) = &queue_name_out {
335 if qn != existing {
336 return Err(JournioError {
337 code: JournioErrorCode::ConflictingWorkflowError,
338 message: format!(
339 "Conflicting workflow invocation with the same ID ({}): Workflow already exists in a different queue: {}, but the provided queue is: {}",
340 init.workflow_id, existing, qn
341 ),
342 workflow_id: Some(init.workflow_id),
343 ..Default::default()
344 });
345 }
346 }
347 }
348
349 let status_out = parse_status(&status_out_str);
350 if !matches!(
351 status_out,
352 WorkflowStatusType::Success | WorkflowStatusType::Error
353 ) && init.max_retries > 0
354 && attempts_out > i64::from(init.max_retries) + 1
355 {
356 let dlq = self.q(
357 "UPDATE workflow_status SET status = $1, deduplication_id = NULL, started_at_epoch_ms = NULL, queue_name = NULL WHERE workflow_uuid = $2 AND status = $3",
358 );
359 client
360 .execute(
361 &dlq,
362 &[
363 &status_to_str(WorkflowStatusType::MaxRecoveryAttemptsExceeded),
364 &init.workflow_id,
365 &status_str,
366 ],
367 )
368 .await
369 .map_err(db_err)?;
370 return Err(JournioError {
371 code: JournioErrorCode::DeadLetterQueueError,
372 message: format!(
373 "Workflow {} has been moved to the dead-letter queue after exceeding the maximum of {} retries",
374 init.workflow_id, init.max_retries
375 ),
376 workflow_id: Some(init.workflow_id),
377 ..Default::default()
378 });
379 }
380
381 Ok(InitWorkflowResult {
382 status: status_out,
383 attempts: attempts_out,
384 name: name_out,
385 queue_name: queue_name_out,
386 queue_partition_key: queue_partition_out,
387 timeout: timeout_out
388 .filter(|&m| m > 0)
389 .map(|m| Duration::from_millis(m as u64)),
390 deadline: deadline_out.map(timestamp_ms),
391 })
392 }
393
394 async fn record_workflow_result(
396 &self,
397 workflow_id: &str,
398 status: WorkflowStatusType,
399 output: Option<&Interchange>,
400 error: Option<&str>,
401 ) -> JournioResult<()> {
402 let output_str: Option<String> =
403 output.map(|v| serde_json::to_string(v).unwrap_or_default());
404 let now_ms = Utc::now().timestamp_millis();
405 let query = self.q(
406 "UPDATE workflow_status \
407 SET status = $1, output = $2, error = $3, updated_at = $4, completed_at = $4, deduplication_id = NULL \
408 WHERE workflow_uuid = $5 AND NOT (status = $6 AND CAST($1 AS TEXT) IN ($7, $8))",
409 );
410 let status_str = status_to_str(status);
411 let cancelled_str = status_to_str(WorkflowStatusType::Cancelled);
412 let success_str = status_to_str(WorkflowStatusType::Success);
413 let error_str = status_to_str(WorkflowStatusType::Error);
414 let client = self.pool.get().await.map_err(pool_err)?;
415 client
416 .execute(
417 &query,
418 &[
419 &status_str,
420 &output_str,
421 &error,
422 &now_ms,
423 &workflow_id,
424 &cancelled_str,
425 &success_str,
426 &error_str,
427 ],
428 )
429 .await
430 .map_err(db_err)?;
431 Ok(())
432 }
433
434 async fn get_workflow_status(
435 &self,
436 workflow_id: &str,
437 ) -> JournioResult<Option<WorkflowStatus>> {
438 let query = self.q(WORKFLOW_STATUS_BY_ID);
439 let client = self.pool.get().await.map_err(pool_err)?;
440 let row = client
441 .query_opt(&query, &[&workflow_id])
442 .await
443 .map_err(db_err)?;
444 Ok(row.map(|r| row_to_workflow_status(&r)))
445 }
446
447 async fn list_workflows(&self, limit: i64) -> JournioResult<Vec<WorkflowStatus>> {
448 let query = format!(
449 "{} ORDER BY created_at DESC LIMIT $1",
450 self.q(WORKFLOW_STATUS_SELECT)
451 );
452 let client = self.pool.get().await.map_err(pool_err)?;
453 let rows = client.query(&query, &[&limit]).await.map_err(db_err)?;
454 Ok(rows.iter().map(row_to_workflow_status).collect())
455 }
456
457 async fn cancel_workflows(&self, workflow_ids: &[String]) -> JournioResult<Vec<String>> {
458 if workflow_ids.is_empty() {
459 return Ok(Vec::new());
460 }
461
462 let now_ms = Utc::now().timestamp_millis();
463 let query = self.q("WITH existing AS (
464 SELECT workflow_uuid FROM workflow_status WHERE workflow_uuid = ANY($3)
465 ), updated AS (
466 UPDATE workflow_status
467 SET status = $1, updated_at = $2, completed_at = $2, started_at_epoch_ms = NULL,
468 queue_name = NULL, deduplication_id = NULL
469 WHERE workflow_uuid = ANY($3) AND status NOT IN ($4, $5, $6)
470 RETURNING workflow_uuid
471 )
472 SELECT workflow_uuid FROM existing");
473 let client = self.pool.get().await.map_err(pool_err)?;
474 let rows = client
475 .query(
476 &query,
477 &[
478 &status_to_str(WorkflowStatusType::Cancelled),
479 &now_ms,
480 &workflow_ids,
481 &status_to_str(WorkflowStatusType::Success),
482 &status_to_str(WorkflowStatusType::Error),
483 &status_to_str(WorkflowStatusType::Cancelled),
484 ],
485 )
486 .await
487 .map_err(db_err)?;
488 Ok(rows.into_iter().map(|row| row.get(0)).collect())
489 }
490
491 async fn resume_workflows(
492 &self,
493 workflow_ids: &[String],
494 queue_name: Option<&str>,
495 ) -> JournioResult<Vec<String>> {
496 if workflow_ids.is_empty() {
497 return Ok(Vec::new());
498 }
499
500 let now_ms = Utc::now().timestamp_millis();
501 let queue_name = queue_name.unwrap_or(INTERNAL_QUEUE_NAME).to_string();
502 let query = self.q("WITH existing AS (
503 SELECT workflow_uuid FROM workflow_status WHERE workflow_uuid = ANY($5)
504 ), updated AS (
505 UPDATE workflow_status
506 SET status = $1, queue_name = $2, recovery_attempts = $3,
507 workflow_deadline_epoch_ms = NULL, deduplication_id = NULL,
508 started_at_epoch_ms = NULL, updated_at = $4, completed_at = NULL
509 WHERE workflow_uuid = ANY($5) AND status NOT IN ($6, $7)
510 RETURNING workflow_uuid
511 )
512 SELECT workflow_uuid FROM existing");
513 let client = self.pool.get().await.map_err(pool_err)?;
514 let rows = client
515 .query(
516 &query,
517 &[
518 &status_to_str(WorkflowStatusType::Enqueued),
519 &queue_name,
520 &0_i64,
521 &now_ms,
522 &workflow_ids,
523 &status_to_str(WorkflowStatusType::Success),
524 &status_to_str(WorkflowStatusType::Error),
525 ],
526 )
527 .await
528 .map_err(db_err)?;
529 Ok(rows.into_iter().map(|row| row.get(0)).collect())
530 }
531
532 async fn get_workflow_children(&self, workflow_id: &str) -> JournioResult<Vec<WorkflowStatus>> {
533 let query = self.q(
534 "WITH RECURSIVE descendants AS (
535 SELECT workflow_uuid, status, name, authenticated_user, assumed_role, authenticated_roles,
536 output, error, executor_id, created_at, updated_at, application_version, application_id,
537 recovery_attempts, queue_name, workflow_timeout_ms, workflow_deadline_epoch_ms,
538 started_at_epoch_ms, deduplication_id, inputs, priority, queue_partition_key,
539 forked_from, was_forked_from, parent_workflow_id, completed_at, class_name, config_name,
540 serialization, delay_until_epoch_ms
541 FROM workflow_status
542 WHERE parent_workflow_id = $1
543 UNION ALL
544 SELECT ws.workflow_uuid, ws.status, ws.name, ws.authenticated_user, ws.assumed_role, ws.authenticated_roles,
545 ws.output, ws.error, ws.executor_id, ws.created_at, ws.updated_at, ws.application_version, ws.application_id,
546 ws.recovery_attempts, ws.queue_name, ws.workflow_timeout_ms, ws.workflow_deadline_epoch_ms,
547 ws.started_at_epoch_ms, ws.deduplication_id, ws.inputs, ws.priority, ws.queue_partition_key,
548 ws.forked_from, ws.was_forked_from, ws.parent_workflow_id, ws.completed_at, ws.class_name, ws.config_name,
549 ws.serialization, ws.delay_until_epoch_ms
550 FROM workflow_status ws
551 INNER JOIN descendants d ON ws.parent_workflow_id = d.workflow_uuid
552 )
553 SELECT * FROM descendants
554 ORDER BY created_at ASC",
555 );
556 let client = self.pool.get().await.map_err(pool_err)?;
557 let rows = client
558 .query(&query, &[&workflow_id])
559 .await
560 .map_err(db_err)?;
561 Ok(rows.iter().map(row_to_workflow_status).collect())
562 }
563
564 async fn fork_workflow(&self, input: ForkWorkflow) -> JournioResult<String> {
565 if input.start_step < 0 {
566 return Err(JournioError::new(
567 JournioErrorCode::InitializationError,
568 format!("startStep must be >= 0, got {}", input.start_step),
569 ));
570 }
571
572 let forked_workflow_id = input
573 .forked_workflow_id
574 .unwrap_or_else(|| uuid::Uuid::new_v4().to_string());
575 let queue_name = input
576 .queue_name
577 .unwrap_or_else(|| INTERNAL_QUEUE_NAME.to_string());
578 let now_ms = Utc::now().timestamp_millis();
579
580 let mut client = self.pool.get().await.map_err(pool_err)?;
581 let tx = client.transaction().await.map_err(db_err)?;
582 let original_row = tx
583 .query_opt(
584 &self.q(WORKFLOW_STATUS_BY_ID),
585 &[&input.original_workflow_id],
586 )
587 .await
588 .map_err(db_err)?;
589 let Some(original_row) = original_row else {
590 return Err(JournioError::new(
591 JournioErrorCode::NonExistentWorkflowError,
592 format!("workflow {} does not exist", input.original_workflow_id),
593 ));
594 };
595 let original = row_to_workflow_status(&original_row);
596 let authenticated_roles =
597 serde_json::to_string(&original.authenticated_roles).unwrap_or_else(|_| "null".into());
598 let application_version = input
599 .application_version
600 .unwrap_or_else(|| original.application_version.clone());
601 let inputs = original
602 .input
603 .as_ref()
604 .map(|value| serde_json::to_string(value).unwrap_or_else(|_| "null".into()));
605
606 tx.execute(
607 &self.q(
608 "INSERT INTO workflow_status (
609 workflow_uuid, status, name, authenticated_user, assumed_role, authenticated_roles,
610 executor_id, application_version, application_id, queue_name, queue_partition_key,
611 inputs, created_at, updated_at, recovery_attempts, forked_from, serialization,
612 class_name, config_name
613 ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19)",
614 ),
615 &[
616 &forked_workflow_id,
617 &status_to_str(WorkflowStatusType::Enqueued),
618 &original.name,
619 &original.authenticated_user,
620 &original.assumed_role,
621 &authenticated_roles,
622 &original.executor_id,
623 &application_version,
624 &original.application_id,
625 &queue_name,
626 &input.queue_partition_key,
627 &inputs,
628 &now_ms,
629 &now_ms,
630 &0_i64,
631 &input.original_workflow_id,
632 &original.serialization,
633 &original.class_name,
634 &original.config_name,
635 ],
636 )
637 .await
638 .map_err(db_err)?;
639
640 tx.execute(
641 &self.q("UPDATE workflow_status SET was_forked_from = TRUE WHERE workflow_uuid = $1"),
642 &[&input.original_workflow_id],
643 )
644 .await
645 .map_err(db_err)?;
646
647 if input.start_step > 0 {
648 tx.execute(
649 &self.q(
650 "INSERT INTO operation_outputs
651 (workflow_uuid, function_id, output, error, function_name, child_workflow_id, started_at_epoch_ms, completed_at_epoch_ms, serialization)
652 SELECT $1, function_id, output, error, function_name, child_workflow_id, started_at_epoch_ms, completed_at_epoch_ms, serialization
653 FROM operation_outputs
654 WHERE workflow_uuid = $2 AND function_id < $3",
655 ),
656 &[&forked_workflow_id, &input.original_workflow_id, &input.start_step],
657 )
658 .await
659 .map_err(db_err)?;
660
661 tx.execute(
662 &self.q("INSERT INTO workflow_events_history
663 (workflow_uuid, function_id, key, value, serialization)
664 SELECT $1, function_id, key, value, serialization
665 FROM workflow_events_history
666 WHERE workflow_uuid = $2 AND function_id < $3"),
667 &[
668 &forked_workflow_id,
669 &input.original_workflow_id,
670 &input.start_step,
671 ],
672 )
673 .await
674 .map_err(db_err)?;
675
676 tx.execute(
677 &self.q(
678 "INSERT INTO workflow_events (workflow_uuid, key, value, serialization)
679 SELECT $1, h.key, h.value, h.serialization
680 FROM workflow_events_history h
681 INNER JOIN (
682 SELECT key, MAX(function_id) AS max_fid
683 FROM workflow_events_history
684 WHERE workflow_uuid = $2 AND function_id < $3
685 GROUP BY key
686 ) latest ON h.key = latest.key AND h.function_id = latest.max_fid
687 WHERE h.workflow_uuid = $2 AND h.function_id < $3",
688 ),
689 &[
690 &forked_workflow_id,
691 &input.original_workflow_id,
692 &input.start_step,
693 ],
694 )
695 .await
696 .map_err(db_err)?;
697
698 tx.execute(
699 &self.q(
700 "INSERT INTO streams (workflow_uuid, key, value, \"offset\", function_id, serialization)
701 SELECT $1, key, value, \"offset\", function_id, serialization
702 FROM streams
703 WHERE workflow_uuid = $2 AND function_id < $3",
704 ),
705 &[&forked_workflow_id, &input.original_workflow_id, &input.start_step],
706 )
707 .await
708 .map_err(db_err)?;
709 }
710
711 tx.commit().await.map_err(db_err)?;
712 Ok(forked_workflow_id)
713 }
714
715 async fn get_queue(&self, queue_name: &str) -> JournioResult<Option<QueueConfig>> {
716 let query = self.q(
717 "SELECT queue_id, name, concurrency, worker_concurrency, rate_limit_max, rate_limit_period_sec,
718 priority_enabled, partition_queue, polling_interval_sec
719 FROM queues WHERE name = $1",
720 );
721 let client = self.pool.get().await.map_err(pool_err)?;
722 let row = client
723 .query_opt(&query, &[&queue_name])
724 .await
725 .map_err(db_err)?;
726 Ok(row.as_ref().map(row_to_queue_config))
727 }
728
729 async fn upsert_queue(&self, queue: &QueueConfig) -> JournioResult<()> {
730 let now_ms = Utc::now().timestamp_millis();
731 let query = self.q(
732 "INSERT INTO queues (
733 queue_id, name, concurrency, worker_concurrency, rate_limit_max, rate_limit_period_sec,
734 priority_enabled, partition_queue, polling_interval_sec, created_at, updated_at
735 ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)
736 ON CONFLICT (name)
737 DO UPDATE SET
738 concurrency = excluded.concurrency,
739 worker_concurrency = excluded.worker_concurrency,
740 rate_limit_max = excluded.rate_limit_max,
741 rate_limit_period_sec = excluded.rate_limit_period_sec,
742 priority_enabled = excluded.priority_enabled,
743 partition_queue = excluded.partition_queue,
744 polling_interval_sec = excluded.polling_interval_sec,
745 updated_at = excluded.updated_at",
746 );
747 let client = self.pool.get().await.map_err(pool_err)?;
748 client
749 .execute(
750 &query,
751 &[
752 &queue.queue_id,
753 &queue.name,
754 &queue.concurrency,
755 &queue.worker_concurrency,
756 &queue.rate_limit_max,
757 &queue.rate_limit_period_sec,
758 &queue.priority_enabled,
759 &queue.partition_queue,
760 &queue.polling_interval_sec,
761 &now_ms,
762 &now_ms,
763 ],
764 )
765 .await
766 .map_err(db_err)?;
767 Ok(())
768 }
769
770 async fn record_step_output(&self, step: &StepRecord) -> JournioResult<()> {
776 let now_ms = Utc::now().timestamp_millis();
777 let query = self.q("INSERT INTO operation_outputs \
778 (workflow_uuid, function_id, function_name, output, error, child_workflow_id, \
779 started_at_epoch_ms, completed_at_epoch_ms, serialization) \
780 VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)");
781 let client = self.pool.get().await.map_err(pool_err)?;
782 client
783 .execute(
784 &query,
785 &[
786 &step.workflow_uuid,
787 &step.function_id,
788 &step.function_name,
789 &step.output,
790 &step.error,
791 &step.child_workflow_id,
792 &now_ms,
793 &now_ms,
794 &"JOURNIO_JSON",
795 ],
796 )
797 .await
798 .map_err(|e| {
799 if is_unique_violation(&e) {
800 JournioError {
801 code: JournioErrorCode::ConflictingIDError,
802 message: format!("Conflicting workflow ID {}", step.workflow_uuid),
803 workflow_id: Some(step.workflow_uuid.clone()),
804 source: Some(Box::new(e)),
805 ..Default::default()
806 }
807 } else {
808 db_err(e)
809 }
810 })?;
811 Ok(())
812 }
813
814 async fn get_steps(&self, workflow_id: &str) -> JournioResult<Vec<StepRecord>> {
816 let query = self.q(
817 "SELECT workflow_uuid, function_id, function_name, output, error, child_workflow_id \
818 FROM operation_outputs WHERE workflow_uuid = $1 ORDER BY function_id ASC",
819 );
820 let client = self.pool.get().await.map_err(pool_err)?;
821 let rows = client
822 .query(&query, &[&workflow_id])
823 .await
824 .map_err(db_err)?;
825 Ok(rows
826 .iter()
827 .map(|r| StepRecord {
828 workflow_uuid: r.get(0),
829 function_id: r.get(1),
830 function_name: r.get(2),
831 output: r.get(3),
832 error: r.get(4),
833 child_workflow_id: r.get(5),
834 })
835 .collect())
836 }
837
838 async fn dequeue_workflow(
839 &self,
840 queue_name: &str,
841 executor_id: &str,
842 ) -> JournioResult<Option<WorkflowStatus>> {
843 let now_ms = Utc::now().timestamp_millis();
844 let queue = self.get_queue(queue_name).await?;
845 let candidates_query = self.q("SELECT workflow_uuid, queue_partition_key
846 FROM workflow_status
847 WHERE queue_name = $1
848 AND status IN ($2, $3)
849 AND (
850 status = $2
851 OR delay_until_epoch_ms IS NULL
852 OR delay_until_epoch_ms <= $4
853 )
854 ORDER BY priority ASC, created_at ASC
855 FOR UPDATE SKIP LOCKED");
856 let mut client = self.pool.get().await.map_err(pool_err)?;
857 let tx = client.transaction().await.map_err(db_err)?;
858 let candidates = tx
859 .query(
860 &candidates_query,
861 &[
862 &queue_name,
863 &status_to_str(WorkflowStatusType::Enqueued),
864 &status_to_str(WorkflowStatusType::Delayed),
865 &now_ms,
866 ],
867 )
868 .await
869 .map_err(db_err)?;
870
871 let mut selected_id = None;
872 for candidate in candidates {
873 let workflow_id: String = candidate.get(0);
874 let partition_key: Option<String> = candidate.get(1);
875
876 if queue
877 .as_ref()
878 .is_some_and(|cfg| cfg.partition_queue && partition_key.is_none())
879 {
880 continue;
881 }
882
883 if let Some(cfg) = queue.as_ref() {
884 if let Some(limit) = cfg.concurrency {
885 let query = if cfg.partition_queue {
886 self.q("SELECT COUNT(*) FROM workflow_status
887 WHERE queue_name = $1 AND status = $2 AND queue_partition_key = $3")
888 } else {
889 self.q("SELECT COUNT(*) FROM workflow_status
890 WHERE queue_name = $1 AND status = $2")
891 };
892 let count: i64 = if cfg.partition_queue {
893 tx.query_one(
894 &query,
895 &[
896 &queue_name,
897 &status_to_str(WorkflowStatusType::Pending),
898 &partition_key,
899 ],
900 )
901 .await
902 .map_err(db_err)?
903 .get(0)
904 } else {
905 tx.query_one(
906 &query,
907 &[&queue_name, &status_to_str(WorkflowStatusType::Pending)],
908 )
909 .await
910 .map_err(db_err)?
911 .get(0)
912 };
913 if count >= i64::from(limit) {
914 continue;
915 }
916 }
917
918 if let (Some(limit), Some(period_sec)) =
919 (cfg.rate_limit_max, cfg.rate_limit_period_sec)
920 {
921 let cutoff_ms = now_ms - (period_sec * 1000.0) as i64;
922 let query = if cfg.partition_queue {
923 self.q("SELECT COUNT(*) FROM workflow_status
924 WHERE queue_name = $1
925 AND rate_limited = TRUE
926 AND status NOT IN ($2, $3)
927 AND started_at_epoch_ms > $4
928 AND queue_partition_key = $5")
929 } else {
930 self.q("SELECT COUNT(*) FROM workflow_status
931 WHERE queue_name = $1
932 AND rate_limited = TRUE
933 AND status NOT IN ($2, $3)
934 AND started_at_epoch_ms > $4")
935 };
936 let count: i64 = if cfg.partition_queue {
937 tx.query_one(
938 &query,
939 &[
940 &queue_name,
941 &status_to_str(WorkflowStatusType::Enqueued),
942 &status_to_str(WorkflowStatusType::Delayed),
943 &cutoff_ms,
944 &partition_key,
945 ],
946 )
947 .await
948 .map_err(db_err)?
949 .get(0)
950 } else {
951 tx.query_one(
952 &query,
953 &[
954 &queue_name,
955 &status_to_str(WorkflowStatusType::Enqueued),
956 &status_to_str(WorkflowStatusType::Delayed),
957 &cutoff_ms,
958 ],
959 )
960 .await
961 .map_err(db_err)?
962 .get(0)
963 };
964 if count >= i64::from(limit) {
965 continue;
966 }
967 }
968 }
969
970 selected_id = Some(workflow_id);
971 break;
972 }
973
974 let Some(selected_id) = selected_id else {
975 tx.commit().await.map_err(db_err)?;
976 return Ok(None);
977 };
978
979 let update_query = self.q(
980 "UPDATE workflow_status
981 SET status = $1,
982 executor_id = $2,
983 started_at_epoch_ms = $3,
984 rate_limited = $4,
985 workflow_deadline_epoch_ms = CASE
986 WHEN workflow_timeout_ms IS NOT NULL AND workflow_deadline_epoch_ms IS NULL
987 THEN $3 + workflow_timeout_ms
988 ELSE workflow_deadline_epoch_ms
989 END
990 WHERE workflow_uuid = $5
991 AND status IN ($6, $7)
992 RETURNING workflow_uuid, status, name, authenticated_user, assumed_role, authenticated_roles,
993 output, error, executor_id, created_at, updated_at, application_version, application_id,
994 recovery_attempts, queue_name, workflow_timeout_ms, workflow_deadline_epoch_ms,
995 started_at_epoch_ms, deduplication_id, inputs, priority, queue_partition_key,
996 forked_from, was_forked_from, parent_workflow_id, completed_at, class_name, config_name,
997 serialization, delay_until_epoch_ms",
998 );
999 let row = tx
1000 .query_opt(
1001 &update_query,
1002 &[
1003 &status_to_str(WorkflowStatusType::Pending),
1004 &executor_id,
1005 &now_ms,
1006 &queue
1007 .as_ref()
1008 .is_some_and(|cfg| cfg.rate_limit_max.is_some()),
1009 &selected_id,
1010 &status_to_str(WorkflowStatusType::Enqueued),
1011 &status_to_str(WorkflowStatusType::Delayed),
1012 ],
1013 )
1014 .await
1015 .map_err(db_err)?;
1016 tx.commit().await.map_err(db_err)?;
1017 Ok(row.as_ref().map(row_to_workflow_status))
1018 }
1019
1020 async fn list_runnable_queues(&self) -> JournioResult<Vec<String>> {
1021 let query = self.q("SELECT DISTINCT queue_name
1022 FROM workflow_status
1023 WHERE queue_name IS NOT NULL
1024 AND status IN ($1, $2)
1025 ORDER BY queue_name ASC");
1026 let client = self.pool.get().await.map_err(pool_err)?;
1027 let rows = client
1028 .query(
1029 &query,
1030 &[
1031 &status_to_str(WorkflowStatusType::Enqueued),
1032 &status_to_str(WorkflowStatusType::Delayed),
1033 ],
1034 )
1035 .await
1036 .map_err(db_err)?;
1037 Ok(rows
1038 .into_iter()
1039 .filter_map(|row| row.get::<_, Option<String>>(0))
1040 .collect())
1041 }
1042
1043 async fn list_queues(&self) -> JournioResult<Vec<QueueConfig>> {
1044 let query = self.q(
1045 "SELECT queue_id, name, concurrency, worker_concurrency, rate_limit_max,
1046 rate_limit_period_sec, priority_enabled, partition_queue, polling_interval_sec
1047 FROM queues
1048 ORDER BY name ASC",
1049 );
1050 let client = self.pool.get().await.map_err(pool_err)?;
1051 let rows = client.query(&query, &[]).await.map_err(db_err)?;
1052 Ok(rows.iter().map(row_to_queue_config).collect())
1053 }
1054
1055 async fn send(
1061 &self,
1062 destination_id: &str,
1063 topic: &str,
1064 message: &Interchange,
1065 ) -> JournioResult<()> {
1066 let topic = if topic.is_empty() { NULL_TOPIC } else { topic };
1067 let message_str = serde_json::to_string(message).unwrap_or_else(|_| "null".into());
1068 let message_uuid = uuid::Uuid::new_v4().to_string();
1069 let created_ms = Utc::now().timestamp_millis();
1070 let query = self.q(
1071 "INSERT INTO notifications (destination_uuid, topic, message, serialization, message_uuid, created_at_epoch_ms) \
1072 VALUES ($1, $2, $3, $4, $5, $6)",
1073 );
1074 let client = self.pool.get().await.map_err(pool_err)?;
1075 client
1076 .execute(
1077 &query,
1078 &[
1079 &destination_id,
1080 &topic,
1081 &message_str,
1082 &SERIALIZATION_JSON,
1083 &message_uuid,
1084 &created_ms,
1085 ],
1086 )
1087 .await
1088 .map_err(|e| {
1089 if is_foreign_key_violation(&e) {
1090 JournioError {
1091 code: JournioErrorCode::NonExistentWorkflowError,
1092 message: format!("destination workflow {destination_id} does not exist"),
1093 workflow_id: Some(destination_id.to_string()),
1094 source: Some(Box::new(e)),
1095 ..Default::default()
1096 }
1097 } else {
1098 db_err(e)
1099 }
1100 })?;
1101 notify_waiters(
1104 &self.notification_waiters,
1105 &format!("{destination_id}::{topic}"),
1106 );
1107 Ok(())
1108 }
1109
1110 async fn consume_notification(
1114 &self,
1115 workflow_id: &str,
1116 topic: &str,
1117 ) -> JournioResult<Option<journio_core::Notification>> {
1118 let topic = if topic.is_empty() { NULL_TOPIC } else { topic };
1119 let query = self.q("WITH oldest_entry AS ( \
1120 SELECT message_uuid FROM notifications \
1121 WHERE destination_uuid = $1 AND topic = $2 AND consumed = false \
1122 ORDER BY created_at_epoch_ms ASC LIMIT 1) \
1123 UPDATE notifications SET consumed = true \
1124 WHERE message_uuid = (SELECT message_uuid FROM oldest_entry) \
1125 RETURNING message, serialization");
1126 let client = self.pool.get().await.map_err(pool_err)?;
1127 let row = client
1128 .query_opt(&query, &[&workflow_id, &topic])
1129 .await
1130 .map_err(db_err)?;
1131 let Some(row) = row else { return Ok(None) };
1132 let message_str: Option<String> = row.get(0);
1133 let serialization: Option<String> = row.get(1);
1134 let message = message_str
1135 .and_then(|s| serde_json::from_str::<Interchange>(&s).ok())
1136 .unwrap_or(Interchange::Null);
1137 Ok(Some(journio_core::Notification {
1138 message,
1139 serialization,
1140 }))
1141 }
1142
1143 async fn wait_for_notification(
1144 &self,
1145 workflow_id: &str,
1146 topic: &str,
1147 timeout: Duration,
1148 ) -> JournioResult<()> {
1149 self.wait_on_map(
1150 &self.notification_waiters,
1151 &format!(
1152 "{}::{}",
1153 workflow_id,
1154 if topic.is_empty() { NULL_TOPIC } else { topic }
1155 ),
1156 timeout,
1157 )
1158 .await
1159 }
1160
1161 async fn set_event(
1169 &self,
1170 workflow_id: &str,
1171 key: &str,
1172 value: &Interchange,
1173 function_id: i32,
1174 ) -> JournioResult<()> {
1175 let value_str = serde_json::to_string(value).unwrap_or_else(|_| "null".into());
1176 let mut client = self.pool.get().await.map_err(pool_err)?;
1177 let tx = client.transaction().await.map_err(db_err)?;
1178
1179 tx.execute(
1180 &self.q(
1181 "INSERT INTO workflow_events (workflow_uuid, key, value, serialization) \
1182 VALUES ($1, $2, $3, $4) \
1183 ON CONFLICT (workflow_uuid, key) \
1184 DO UPDATE SET value = EXCLUDED.value, serialization = EXCLUDED.serialization",
1185 ),
1186 &[&workflow_id, &key, &value_str, &SERIALIZATION_JSON],
1187 )
1188 .await
1189 .map_err(db_err)?;
1190
1191 tx.execute(
1192 &self.q(
1193 "INSERT INTO workflow_events_history (workflow_uuid, function_id, key, value, serialization) \
1194 VALUES ($1, $2, $3, $4, $5) \
1195 ON CONFLICT (workflow_uuid, function_id, key) \
1196 DO UPDATE SET value = EXCLUDED.value, serialization = EXCLUDED.serialization",
1197 ),
1198 &[&workflow_id, &function_id, &key, &value_str, &SERIALIZATION_JSON],
1199 )
1200 .await
1201 .map_err(db_err)?;
1202
1203 tx.commit().await.map_err(db_err)?;
1204 notify_waiters(&self.event_waiters, &format!("{workflow_id}::{key}"));
1205 Ok(())
1206 }
1207
1208 async fn get_event_value(
1211 &self,
1212 workflow_id: &str,
1213 key: &str,
1214 ) -> JournioResult<Option<Interchange>> {
1215 let query =
1216 self.q("SELECT value FROM workflow_events WHERE workflow_uuid = $1 AND key = $2");
1217 let client = self.pool.get().await.map_err(pool_err)?;
1218 let row = client
1219 .query_opt(&query, &[&workflow_id, &key])
1220 .await
1221 .map_err(db_err)?;
1222 let Some(row) = row else { return Ok(None) };
1223 let value_str: Option<String> = row.get(0);
1224 Ok(value_str.and_then(|s| serde_json::from_str::<Interchange>(&s).ok()))
1225 }
1226
1227 async fn wait_for_event(
1228 &self,
1229 workflow_id: &str,
1230 key: &str,
1231 timeout: Duration,
1232 ) -> JournioResult<()> {
1233 self.wait_on_map(
1234 &self.event_waiters,
1235 &format!("{workflow_id}::{key}"),
1236 timeout,
1237 )
1238 .await
1239 }
1240
1241 async fn write_stream(
1242 &self,
1243 workflow_id: &str,
1244 key: &str,
1245 value: &str,
1246 function_id: i32,
1247 serialization: Option<&str>,
1248 ) -> JournioResult<()> {
1249 let check_closed = self.q("SELECT 1 FROM streams
1250 WHERE workflow_uuid = $1 AND key = $2 AND value = $3
1251 LIMIT 1");
1252 let insert = self.q(
1253 "INSERT INTO streams (workflow_uuid, key, value, \"offset\", function_id, serialization)
1254 SELECT $1, $2, $3,
1255 COALESCE((SELECT MAX(\"offset\") FROM streams WHERE workflow_uuid = $1 AND key = $2), -1) + 1,
1256 $4, $5",
1257 );
1258 let client = self.pool.get().await.map_err(pool_err)?;
1259 let exists = client
1260 .query_opt(
1261 &check_closed,
1262 &[&workflow_id, &key, &STREAM_CLOSED_SENTINEL],
1263 )
1264 .await
1265 .map_err(db_err)?;
1266 if exists.is_some() {
1267 return Err(JournioError::new(
1268 JournioErrorCode::WorkflowExecutionError,
1269 format!("stream {key:?} is already closed"),
1270 ));
1271 }
1272 client
1273 .execute(
1274 &insert,
1275 &[&workflow_id, &key, &value, &function_id, &serialization],
1276 )
1277 .await
1278 .map_err(db_err)?;
1279 notify_waiters(&self.stream_waiters, &format!("{workflow_id}::{key}"));
1280 Ok(())
1281 }
1282
1283 async fn read_stream(
1284 &self,
1285 workflow_id: &str,
1286 key: &str,
1287 from_offset: i64,
1288 ) -> JournioResult<(Vec<StreamEntry>, bool)> {
1289 let from_offset = i32::try_from(from_offset).map_err(|_| {
1290 JournioError::new(
1291 JournioErrorCode::InitializationError,
1292 format!("stream offset {from_offset} does not fit in postgres INTEGER"),
1293 )
1294 })?;
1295 let query = self.q("SELECT value, \"offset\", serialization
1296 FROM streams
1297 WHERE workflow_uuid = $1 AND key = $2 AND \"offset\" >= $3
1298 ORDER BY \"offset\" ASC");
1299 let client = self.pool.get().await.map_err(pool_err)?;
1300 let rows = client
1301 .query(&query, &[&workflow_id, &key, &from_offset])
1302 .await
1303 .map_err(db_err)?;
1304
1305 let mut entries = Vec::new();
1306 let mut closed = false;
1307 for row in rows {
1308 let value: String = row.get(0);
1309 let offset: i32 = row.get(1);
1310 let serialization: Option<String> = row.get(2);
1311 if value == STREAM_CLOSED_SENTINEL {
1312 closed = true;
1313 break;
1314 }
1315 entries.push(StreamEntry {
1316 value,
1317 offset: i64::from(offset),
1318 serialization,
1319 });
1320 }
1321 Ok((entries, closed))
1322 }
1323
1324 async fn wait_for_stream(
1325 &self,
1326 workflow_id: &str,
1327 key: &str,
1328 timeout: Duration,
1329 ) -> JournioResult<()> {
1330 self.wait_on_map(
1331 &self.stream_waiters,
1332 &format!("{workflow_id}::{key}"),
1333 timeout,
1334 )
1335 .await
1336 }
1337
1338 async fn get_workflows_for_recovery(
1339 &self,
1340 executor_id: &str,
1341 ) -> JournioResult<Vec<WorkflowStatus>> {
1342 let query = format!(
1343 "{} WHERE executor_id = $1 AND status IN ('PENDING', 'ENQUEUED', 'DELAYED') ORDER BY created_at ASC",
1344 self.q(WORKFLOW_STATUS_SELECT)
1345 );
1346 let client = self.pool.get().await.map_err(pool_err)?;
1347 let rows = client
1348 .query(&query, &[&executor_id])
1349 .await
1350 .map_err(db_err)?;
1351 Ok(rows.iter().map(row_to_workflow_status).collect())
1352 }
1353
1354 async fn delete_workflows_before(&self, before: DateTime<Utc>) -> JournioResult<u64> {
1355 let before_ms = before.timestamp_millis();
1356 let query = self.q(
1357 "DELETE FROM workflow_status WHERE status IN ('SUCCESS','ERROR','CANCELLED') AND completed_at < $1",
1358 );
1359 let client = self.pool.get().await.map_err(pool_err)?;
1360 let n = client
1361 .execute(&query, &[&before_ms])
1362 .await
1363 .map_err(db_err)?;
1364 Ok(n)
1365 }
1366
1367 async fn upsert_schedule(&self, schedule: &WorkflowSchedule) -> JournioResult<()> {
1368 let query = self.q(
1369 "INSERT INTO workflow_schedules
1370 (schedule_id, schedule_name, workflow_name, workflow_class_name, schedule, status, context, last_fired_at, automatic_backfill, cron_timezone, queue_name)
1371 VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)
1372 ON CONFLICT (schedule_name)
1373 DO UPDATE SET
1374 schedule_id = excluded.schedule_id,
1375 workflow_name = excluded.workflow_name,
1376 workflow_class_name = excluded.workflow_class_name,
1377 schedule = excluded.schedule,
1378 status = excluded.status,
1379 context = excluded.context,
1380 last_fired_at = excluded.last_fired_at,
1381 automatic_backfill = excluded.automatic_backfill,
1382 cron_timezone = excluded.cron_timezone,
1383 queue_name = excluded.queue_name",
1384 );
1385 let context_json =
1386 serde_json::to_string(&schedule.context).unwrap_or_else(|_| "null".into());
1387 let last_fired_at = schedule.last_fired_at.map(|value| value.to_rfc3339());
1388 let client = self.pool.get().await.map_err(pool_err)?;
1389 client
1390 .execute(
1391 &query,
1392 &[
1393 &schedule.schedule_id,
1394 &schedule.schedule_name,
1395 &schedule.workflow_name,
1396 &schedule.workflow_class_name,
1397 &schedule.schedule,
1398 &schedule_status_to_str(schedule.status),
1399 &context_json,
1400 &last_fired_at,
1401 &schedule.automatic_backfill,
1402 &schedule.cron_timezone,
1403 &schedule.queue_name,
1404 ],
1405 )
1406 .await
1407 .map_err(db_err)?;
1408 Ok(())
1409 }
1410
1411 async fn list_schedules(&self) -> JournioResult<Vec<WorkflowSchedule>> {
1412 let query = self.q(
1413 "SELECT schedule_id, schedule_name, workflow_name, workflow_class_name, schedule, status, context, last_fired_at, automatic_backfill, cron_timezone, queue_name
1414 FROM workflow_schedules
1415 ORDER BY schedule_name ASC",
1416 );
1417 let client = self.pool.get().await.map_err(pool_err)?;
1418 let rows = client.query(&query, &[]).await.map_err(db_err)?;
1419 Ok(rows.iter().map(row_to_workflow_schedule).collect())
1420 }
1421
1422 async fn update_schedule_last_fired_at(
1423 &self,
1424 schedule_name: &str,
1425 fired_at: DateTime<Utc>,
1426 ) -> JournioResult<()> {
1427 let query =
1428 self.q("UPDATE workflow_schedules SET last_fired_at = $1 WHERE schedule_name = $2");
1429 let fired_at = fired_at.to_rfc3339();
1430 let client = self.pool.get().await.map_err(pool_err)?;
1431 client
1432 .execute(&query, &[&fired_at, &schedule_name])
1433 .await
1434 .map_err(db_err)?;
1435 Ok(())
1436 }
1437
1438 async fn get_schedule(&self, schedule_name: &str) -> JournioResult<Option<WorkflowSchedule>> {
1439 let query = self.q(
1440 "SELECT schedule_id, schedule_name, workflow_name, workflow_class_name, schedule, status, context, last_fired_at, automatic_backfill, cron_timezone, queue_name
1441 FROM workflow_schedules WHERE schedule_name = $1",
1442 );
1443 let client = self.pool.get().await.map_err(pool_err)?;
1444 let row = client
1445 .query_opt(&query, &[&schedule_name])
1446 .await
1447 .map_err(db_err)?;
1448 Ok(row.map(|r| row_to_workflow_schedule(&r)))
1449 }
1450
1451 async fn delete_schedule(&self, schedule_name: &str) -> JournioResult<()> {
1452 let query = self.q("DELETE FROM workflow_schedules WHERE schedule_name = $1");
1453 let client = self.pool.get().await.map_err(pool_err)?;
1454 client
1455 .execute(&query, &[&schedule_name])
1456 .await
1457 .map_err(db_err)?;
1458 Ok(())
1459 }
1460
1461 async fn update_schedule_status(
1462 &self,
1463 schedule_name: &str,
1464 status: ScheduleStatus,
1465 ) -> JournioResult<()> {
1466 let query = self.q("UPDATE workflow_schedules SET status = $1 WHERE schedule_name = $2");
1467 let client = self.pool.get().await.map_err(pool_err)?;
1468 client
1469 .execute(&query, &[&schedule_status_to_str(status), &schedule_name])
1470 .await
1471 .map_err(db_err)?;
1472 Ok(())
1473 }
1474
1475 async fn create_application_version(&self, version_name: &str) -> JournioResult<()> {
1476 let query = self.q(
1477 "INSERT INTO application_versions (version_id, version_name, version_timestamp, created_at)
1478 VALUES ($1, $2, $3, $4)
1479 ON CONFLICT (version_name) DO NOTHING",
1480 );
1481 let now_ms = Utc::now().timestamp_millis();
1482 let version_id = uuid::Uuid::new_v4().to_string();
1483 let client = self.pool.get().await.map_err(pool_err)?;
1484 client
1485 .execute(&query, &[&version_id, &version_name, &now_ms, &now_ms])
1486 .await
1487 .map_err(db_err)?;
1488 Ok(())
1489 }
1490
1491 async fn update_application_version_timestamp(
1492 &self,
1493 version_name: &str,
1494 timestamp_ms: i64,
1495 ) -> JournioResult<()> {
1496 let query = self
1497 .q("UPDATE application_versions SET version_timestamp = $1 WHERE version_name = $2");
1498 let client = self.pool.get().await.map_err(pool_err)?;
1499 client
1500 .execute(&query, &[×tamp_ms, &version_name])
1501 .await
1502 .map_err(db_err)?;
1503 Ok(())
1504 }
1505
1506 async fn list_application_versions(&self) -> JournioResult<Vec<VersionInfo>> {
1507 let query =
1508 self.q("SELECT version_id, version_name, version_timestamp, created_at FROM application_versions ORDER BY version_timestamp DESC");
1509 let client = self.pool.get().await.map_err(pool_err)?;
1510 let rows = client.query(&query, &[]).await.map_err(db_err)?;
1511 Ok(rows
1512 .iter()
1513 .map(|r| VersionInfo {
1514 version_id: r.get(0),
1515 version_name: r.get(1),
1516 version_timestamp: r.get(2),
1517 created_at: r.get(3),
1518 })
1519 .collect())
1520 }
1521
1522 async fn get_latest_application_version(&self) -> JournioResult<Option<VersionInfo>> {
1523 let query = self.q(
1524 "SELECT version_id, version_name, version_timestamp, created_at FROM application_versions ORDER BY version_timestamp DESC LIMIT 1",
1525 );
1526 let client = self.pool.get().await.map_err(pool_err)?;
1527 let row = client.query_opt(&query, &[]).await.map_err(db_err)?;
1528 Ok(row.map(|r| VersionInfo {
1529 version_id: r.get(0),
1530 version_name: r.get(1),
1531 version_timestamp: r.get(2),
1532 created_at: r.get(3),
1533 }))
1534 }
1535
1536 async fn set_workflow_delay(
1537 &self,
1538 workflow_id: &str,
1539 delay_until: DateTime<Utc>,
1540 ) -> JournioResult<()> {
1541 let query = self.q(
1542 "UPDATE workflow_status SET delay_until_epoch_ms = $1, updated_at = $2 WHERE workflow_uuid = $3 AND status = $4",
1543 );
1544 let now_ms = Utc::now().timestamp_millis();
1545 let delay_ms = delay_until.timestamp_millis();
1546 let delayed = status_to_str(WorkflowStatusType::Delayed);
1547 let client = self.pool.get().await.map_err(pool_err)?;
1548 client
1549 .execute(&query, &[&delay_ms, &now_ms, &workflow_id, &delayed])
1550 .await
1551 .map_err(db_err)?;
1552 Ok(())
1553 }
1554
1555 async fn delete_workflows(
1556 &self,
1557 workflow_ids: &[String],
1558 delete_children: bool,
1559 ) -> JournioResult<()> {
1560 if workflow_ids.is_empty() {
1561 return Ok(());
1562 }
1563
1564 let mut ids: Vec<String> = workflow_ids.to_vec();
1565 if delete_children {
1566 ids = gather_descendants(&self.pool, &ids).await?;
1567 }
1568
1569 let query = self.q("DELETE FROM workflow_status WHERE workflow_uuid = ANY($1)");
1570 let client = self.pool.get().await.map_err(pool_err)?;
1571 client.execute(&query, &[&ids]).await.map_err(db_err)?;
1572 Ok(())
1573 }
1574
1575 async fn list_workflows_filtered(
1576 &self,
1577 filter: &ListWorkflowsFilter,
1578 ) -> JournioResult<Vec<WorkflowStatus>> {
1579 use tokio_postgres::types::ToSql;
1580
1581 let workflow_ids = filter.workflow_ids.clone();
1588 let workflow_id_prefix_patterns: Vec<String> = filter
1589 .workflow_id_prefixes
1590 .iter()
1591 .map(|prefix| format!("{prefix}%"))
1592 .collect();
1593 let statuses: Vec<String> = filter
1594 .statuses
1595 .iter()
1596 .map(|status| status_to_str(*status))
1597 .collect();
1598 let names = filter.names.clone();
1599 let application_versions = filter.application_versions.clone();
1600 let queue_names = filter.queue_names.clone();
1601 let authenticated_users = filter.authenticated_users.clone();
1602 let executor_ids = filter.executor_ids.clone();
1603 let forked_from = filter.forked_from.clone();
1604 let parent_workflow_ids = filter.parent_workflow_ids.clone();
1605 let deduplication_ids = filter.deduplication_ids.clone();
1606 let start_ms = filter.start_time.map(|t| t.timestamp_millis());
1607 let end_ms = filter.end_time.map(|t| t.timestamp_millis());
1608 let completed_after_ms = filter.completed_after.map(|t| t.timestamp_millis());
1609 let completed_before_ms = filter.completed_before.map(|t| t.timestamp_millis());
1610 let limit = filter.limit.unwrap_or(100);
1611 let offset = filter.offset.unwrap_or(0);
1612
1613 let mut clauses: Vec<String> = Vec::new();
1614 let mut params: Vec<&(dyn ToSql + Sync)> = Vec::new();
1615 let mut idx = 1usize;
1616
1617 macro_rules! bind_array {
1618 ($values:expr, $predicate:literal) => {{
1619 if !$values.is_empty() {
1620 params.push(&$values);
1621 clauses.push(format!(concat!($predicate, " = ANY(${})"), idx));
1622 idx += 1;
1623 }
1624 }};
1625 }
1626 macro_rules! bind_scalar {
1627 ($value:expr, $predicate:literal) => {{
1628 if $value.is_some() {
1629 params.push(&$value);
1630 clauses.push(format!(concat!($predicate, " ${}"), idx));
1631 idx += 1;
1632 }
1633 }};
1634 }
1635
1636 bind_array!(workflow_ids, "workflow_uuid");
1637 if !workflow_id_prefix_patterns.is_empty() {
1638 params.push(&workflow_id_prefix_patterns);
1639 clauses.push(format!("workflow_uuid LIKE ANY(${})", idx));
1640 idx += 1;
1641 }
1642 bind_array!(statuses, "status");
1643 bind_array!(names, "name");
1644 bind_array!(application_versions, "application_version");
1645 if filter.queues_only {
1646 clauses.push("queue_name IS NOT NULL".to_string());
1647 } else {
1648 bind_array!(queue_names, "queue_name");
1649 }
1650 bind_array!(authenticated_users, "authenticated_user");
1651 bind_array!(executor_ids, "executor_id");
1652 bind_array!(forked_from, "forked_from");
1653 bind_array!(parent_workflow_ids, "parent_workflow_id");
1654 bind_array!(deduplication_ids, "deduplication_id");
1655 bind_scalar!(start_ms, "created_at >=");
1656 bind_scalar!(end_ms, "created_at <=");
1657 bind_scalar!(completed_after_ms, "completed_at >=");
1658 bind_scalar!(completed_before_ms, "completed_at <=");
1659
1660 let direction = if filter.sort_desc { "DESC" } else { "ASC" };
1661 let mut query = WORKFLOW_STATUS_SELECT.to_string();
1662 if !clauses.is_empty() {
1663 query.push_str(" WHERE ");
1664 query.push_str(&clauses.join(" AND "));
1665 }
1666 query.push_str(&format!(" ORDER BY created_at {direction}"));
1667 params.push(&limit);
1668 query.push_str(&format!(" LIMIT ${idx}"));
1669 idx += 1;
1670 if offset > 0 {
1671 params.push(&offset);
1672 query.push_str(&format!(" OFFSET ${idx}"));
1673 }
1674
1675 let query = self.q(&query);
1676 let client = self.pool.get().await.map_err(pool_err)?;
1677 let rows = client.query(&query, ¶ms).await.map_err(db_err)?;
1678 Ok(rows.iter().map(row_to_workflow_status).collect())
1679 }
1680}
1681
1682const WORKFLOW_STATUS_SELECT: &str = "SELECT workflow_uuid, status, name, authenticated_user, assumed_role, authenticated_roles, \
1685 output, error, executor_id, created_at, updated_at, application_version, application_id, \
1686 recovery_attempts, queue_name, workflow_timeout_ms, workflow_deadline_epoch_ms, \
1687 started_at_epoch_ms, deduplication_id, inputs, priority, queue_partition_key, \
1688 forked_from, was_forked_from, parent_workflow_id, completed_at, class_name, config_name, \
1689 serialization, delay_until_epoch_ms \
1690 FROM workflow_status";
1691
1692const WORKFLOW_STATUS_BY_ID: &str = "SELECT workflow_uuid, status, name, authenticated_user, assumed_role, authenticated_roles, \
1693 output, error, executor_id, created_at, updated_at, application_version, application_id, \
1694 recovery_attempts, queue_name, workflow_timeout_ms, workflow_deadline_epoch_ms, \
1695 started_at_epoch_ms, deduplication_id, inputs, priority, queue_partition_key, \
1696 forked_from, was_forked_from, parent_workflow_id, completed_at, class_name, config_name, \
1697 serialization, delay_until_epoch_ms \
1698 FROM workflow_status WHERE workflow_uuid = $1";
1699
1700fn init_err(e: impl std::fmt::Display) -> JournioError {
1701 JournioError::new(JournioErrorCode::InitializationError, e.to_string())
1702}
1703
1704pub(crate) fn db_err(e: tokio_postgres::Error) -> JournioError {
1705 JournioError {
1706 code: if matches!(
1707 sqlstate(&e),
1708 Some(crate::error::sqlstate::SERIALIZATION_FAILURE)
1709 | Some(crate::error::sqlstate::DEADLOCK_DETECTED)
1710 ) {
1711 JournioErrorCode::WorkflowExecutionError
1712 } else {
1713 JournioErrorCode::InitializationError
1714 },
1715 message: e.to_string(),
1716 source: Some(Box::new(e)),
1717 ..Default::default()
1718 }
1719}
1720
1721pub(crate) fn pool_err(e: deadpool_postgres::PoolError) -> JournioError {
1722 JournioError::new(JournioErrorCode::InitializationError, e.to_string())
1723}
1724
1725fn status_to_str(s: WorkflowStatusType) -> String {
1726 match s {
1727 WorkflowStatusType::Pending => "PENDING",
1728 WorkflowStatusType::Enqueued => "ENQUEUED",
1729 WorkflowStatusType::Delayed => "DELAYED",
1730 WorkflowStatusType::Success => "SUCCESS",
1731 WorkflowStatusType::Error => "ERROR",
1732 WorkflowStatusType::Cancelled => "CANCELLED",
1733 WorkflowStatusType::MaxRecoveryAttemptsExceeded => "MAX_RECOVERY_ATTEMPTS_EXCEEDED",
1734 }
1735 .to_string()
1736}
1737
1738fn parse_status(s: &str) -> WorkflowStatusType {
1739 match s {
1740 "PENDING" => WorkflowStatusType::Pending,
1741 "ENQUEUED" => WorkflowStatusType::Enqueued,
1742 "DELAYED" => WorkflowStatusType::Delayed,
1743 "SUCCESS" => WorkflowStatusType::Success,
1744 "ERROR" => WorkflowStatusType::Error,
1745 "CANCELLED" => WorkflowStatusType::Cancelled,
1746 "MAX_RECOVERY_ATTEMPTS_EXCEEDED" => WorkflowStatusType::MaxRecoveryAttemptsExceeded,
1747 other => {
1748 tracing::warn!(
1749 status = other,
1750 "unknown workflow status, defaulting to PENDING"
1751 );
1752 WorkflowStatusType::Pending
1753 }
1754 }
1755}
1756
1757fn schedule_status_to_str(status: ScheduleStatus) -> &'static str {
1758 match status {
1759 ScheduleStatus::Active => "ACTIVE",
1760 ScheduleStatus::Paused => "PAUSED",
1761 }
1762}
1763
1764fn parse_schedule_status(s: &str) -> ScheduleStatus {
1765 match s {
1766 "PAUSED" => ScheduleStatus::Paused,
1767 _ => ScheduleStatus::Active,
1768 }
1769}
1770
1771fn timestamp_ms(ms: i64) -> DateTime<Utc> {
1772 DateTime::<Utc>::from_timestamp_millis(ms)
1773 .unwrap_or_else(|| DateTime::<Utc>::from_timestamp(0, 0).expect("unix epoch"))
1774}
1775
1776fn row_to_workflow_status(r: &tokio_postgres::Row) -> WorkflowStatus {
1777 let roles_json: Option<String> = r.get(5);
1778 let authenticated_roles = roles_json
1779 .and_then(|s| serde_json::from_str::<Vec<String>>(&s).ok())
1780 .filter(|v| !v.is_empty());
1781
1782 let output_str: Option<String> = r.get(6);
1783 let output = output_str.and_then(|s| serde_json::from_str::<Interchange>(&s).ok());
1784
1785 let input_str: Option<String> = r.get(19);
1786 let input = input_str.and_then(|s| serde_json::from_str::<Interchange>(&s).ok());
1787
1788 let created_ms: i64 = r.get(9);
1789 let updated_ms: i64 = r.get(10);
1790 let timeout_ms: Option<i64> = r.get(15);
1791 let deadline_ms: Option<i64> = r.get(16);
1792 let started_ms: Option<i64> = r.get(17);
1793 let completed_ms: Option<i64> = r.get(25);
1794 let delay_ms: Option<i64> = r.get(29);
1795
1796 WorkflowStatus {
1797 id: r.get(0),
1798 status: parse_status(r.get(1)),
1799 name: r.get(2),
1800 authenticated_user: r.get(3),
1801 assumed_role: r.get(4),
1802 authenticated_roles,
1803 output,
1804 error: r.get(7),
1805 executor_id: r.get(8),
1806 created_at: timestamp_ms(created_ms),
1807 updated_at: timestamp_ms(updated_ms),
1808 application_version: r.try_get::<_, String>(11).unwrap_or_default(),
1809 application_id: r.get(12),
1810 attempts: r.get::<_, i64>(13),
1811 queue_name: r.get(14),
1812 timeout: timeout_ms
1813 .filter(|&m| m > 0)
1814 .map(|m| Duration::from_millis(m as u64)),
1815 deadline: deadline_ms.map(timestamp_ms),
1816 started_at: started_ms.map(timestamp_ms),
1817 deduplication_id: r.get(18),
1818 input,
1819 priority: r.get(20),
1820 queue_partition_key: r.get(21),
1821 forked_from: r.get(22),
1822 was_forked_from: r.get(23),
1823 parent_workflow_id: r.get(24),
1824 completed_at: completed_ms.map(timestamp_ms),
1825 class_name: r.get(26),
1826 config_name: r.get(27),
1827 serialization: r.get(28),
1828 delay_until: delay_ms.map(timestamp_ms),
1829 }
1830}
1831
1832fn row_to_workflow_schedule(r: &tokio_postgres::Row) -> WorkflowSchedule {
1833 let context_str: String = r.get(6);
1834 let last_fired_at_str: Option<String> = r.get(7);
1835 WorkflowSchedule {
1836 schedule_id: r.get(0),
1837 schedule_name: r.get(1),
1838 workflow_name: r.get(2),
1839 workflow_class_name: r.get(3),
1840 schedule: r.get(4),
1841 status: parse_schedule_status(r.get(5)),
1842 context: serde_json::from_str(&context_str).unwrap_or(Interchange::Null),
1843 last_fired_at: last_fired_at_str
1844 .and_then(|value| DateTime::parse_from_rfc3339(&value).ok())
1845 .map(|value| value.with_timezone(&Utc)),
1846 automatic_backfill: r.get(8),
1847 cron_timezone: r.get(9),
1848 queue_name: r.get(10),
1849 }
1850}
1851
1852fn row_to_queue_config(r: &tokio_postgres::Row) -> QueueConfig {
1853 QueueConfig {
1854 queue_id: r.get(0),
1855 name: r.get(1),
1856 concurrency: r.get(2),
1857 worker_concurrency: r.get(3),
1858 rate_limit_max: r.get(4),
1859 rate_limit_period_sec: r.get(5),
1860 priority_enabled: r.get(6),
1861 partition_queue: r.get(7),
1862 polling_interval_sec: r.get(8),
1863 }
1864}
1865
1866async fn gather_descendants(pool: &Pool, roots: &[String]) -> JournioResult<Vec<String>> {
1871 let client = pool.get().await.map_err(pool_err)?;
1872 let mut all: Vec<String> = roots.to_vec();
1873 let mut queue: Vec<String> = roots.to_vec();
1874 while let Some(parent) = queue.pop() {
1875 let query = "SELECT workflow_uuid FROM workflow_status WHERE parent_workflow_id = $1";
1876 let rows = client.query(query, &[&parent]).await.map_err(db_err)?;
1877 for row in rows {
1878 let id: String = row.get(0);
1879 if !all.contains(&id) {
1880 all.push(id.clone());
1881 queue.push(id);
1882 }
1883 }
1884 }
1885 Ok(all)
1886}