1use std::time::Duration;
4
5use async_trait::async_trait;
6use chrono::{DateTime, Utc};
7use serde::{Deserialize, Serialize};
8use uuid::Uuid;
9
10use crate::UpdateField;
11use crate::workflow::{ActivityOptions, WorkflowEvent};
12
13pub const DEFAULT_SNAPSHOT_INTERVAL: i32 = 1000;
16
17pub fn snapshot_interval_from_env() -> i32 {
19 std::env::var("DURABLE_SNAPSHOT_INTERVAL")
20 .ok()
21 .and_then(|v| v.parse().ok())
22 .unwrap_or(DEFAULT_SNAPSHOT_INTERVAL)
23}
24
25pub const DEFAULT_MAX_PENDING_TASKS_PER_WORKFLOW: u32 = 100;
28
29pub const DEFAULT_MAX_PENDING_STANDALONE_TASKS: u32 = 10_000;
32
33pub const WORKER_HEARTBEAT_TIMEOUT_SECS: i64 = 60;
36
37pub fn max_pending_tasks_per_workflow_from_env() -> u32 {
39 std::env::var("DURABLE_MAX_PENDING_TASKS_PER_WORKFLOW")
40 .ok()
41 .and_then(|v| v.parse().ok())
42 .unwrap_or(DEFAULT_MAX_PENDING_TASKS_PER_WORKFLOW)
43}
44
45#[derive(Debug, thiserror::Error)]
47pub enum StoreError {
48 #[error("workflow not found: {0}")]
50 WorkflowNotFound(Uuid),
51
52 #[error("task not found: {0}")]
54 TaskNotFound(Uuid),
55
56 #[error("task {0} not owned by worker (was reclaimed or already completed)")]
58 TaskNotOwned(Uuid),
59
60 #[error("circuit breaker not found: {0}")]
62 CircuitBreakerNotFound(String),
63
64 #[error("schedule not found: {0}")]
66 ScheduleNotFound(Uuid),
67
68 #[error("schedule execution not found: {0}")]
70 ScheduleExecutionNotFound(Uuid),
71
72 #[error("schedule limit exceeded: {current}/{limit} schedules")]
74 ScheduleLimitExceeded { current: u32, limit: u32 },
75
76 #[error(
78 "task queue limit exceeded for workflow {workflow_id}: {current}/{limit} pending tasks"
79 )]
80 TaskQueueLimitExceeded {
81 workflow_id: Uuid,
82 current: u32,
83 limit: u32,
84 },
85
86 #[error("standalone task queue limit exceeded: {current}/{limit} pending tasks")]
88 StandaloneTaskQueueLimitExceeded { current: u32, limit: u32 },
89
90 #[error("invalid cron expression: {0}")]
92 InvalidCronExpression(String),
93
94 #[error("cron interval too short: {actual}s < {minimum}s minimum")]
96 CronIntervalTooShort { actual: u64, minimum: u64 },
97
98 #[error("concurrency conflict: expected sequence {expected}, got {actual}")]
100 ConcurrencyConflict { expected: i32, actual: i32 },
101
102 #[error("database error: {0}")]
104 Database(String),
105
106 #[error("serialization error: {0}")]
108 Serialization(String),
109}
110
111#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
113#[serde(rename_all = "snake_case")]
114pub enum WorkflowStatus {
115 Pending,
117
118 Running,
120
121 Completed,
123
124 Failed,
126
127 Cancelled,
129
130 ContinuedAsNew,
132}
133
134impl WorkflowStatus {
135 pub fn is_terminal(&self) -> bool {
137 matches!(
138 self,
139 Self::Completed | Self::Failed | Self::Cancelled | Self::ContinuedAsNew
140 )
141 }
142}
143
144impl std::fmt::Display for WorkflowStatus {
145 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
146 match self {
147 Self::Pending => write!(f, "pending"),
148 Self::Running => write!(f, "running"),
149 Self::Completed => write!(f, "completed"),
150 Self::Failed => write!(f, "failed"),
151 Self::Cancelled => write!(f, "cancelled"),
152 Self::ContinuedAsNew => write!(f, "continued_as_new"),
153 }
154 }
155}
156
157#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
159#[serde(rename_all = "snake_case")]
160pub enum TaskStatus {
161 Pending,
162 Claimed,
163 Completed,
164 Failed,
165 Dead,
166 Cancelled,
167}
168
169#[derive(Debug, Clone)]
173pub struct TaskDefinition {
174 pub workflow_id: Option<Uuid>,
175 pub activity_id: String,
176 pub activity_type: String,
177 pub input: serde_json::Value,
178 pub options: ActivityOptions,
179}
180
181#[derive(Debug, Clone, Default)]
183pub struct ClaimedTask {
184 pub id: Uuid,
185 pub workflow_id: Option<Uuid>,
186 pub activity_id: String,
187 pub activity_type: String,
188 pub input: serde_json::Value,
189 pub options: ActivityOptions,
190 pub attempt: u32,
191 pub max_attempts: u32,
192 pub workflow_status: Option<WorkflowStatus>,
196}
197
198#[derive(Debug, Clone)]
200pub struct HeartbeatResponse {
201 pub accepted: bool,
203
204 pub should_cancel: bool,
206}
207
208#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
210pub struct WorkerHeartbeat {
211 pub draining: bool,
215}
216
217#[derive(Debug, Clone)]
219pub enum TaskFailureOutcome {
220 WillRetry { next_attempt: u32, delay: Duration },
222
223 MovedToDlq,
225
226 ExhaustedRetries,
228}
229
230#[derive(Debug, Clone)]
232pub struct DeadTaskInfo {
233 pub task_id: Uuid,
234 pub workflow_id: Option<Uuid>,
235 pub activity_id: String,
236 pub activity_type: String,
237 pub input: serde_json::Value,
239 pub last_error: Option<String>,
240}
241
242#[derive(Debug, Clone)]
251pub struct SealedTaskInfo {
252 pub task_id: Uuid,
253 pub workflow_id: Option<Uuid>,
254 pub activity_id: String,
255 pub activity_type: String,
256 pub input: serde_json::Value,
259 pub reason: String,
262 pub no_progress_count: u32,
264}
265
266pub const DEFAULT_NO_PROGRESS_SEAL_THRESHOLD: u32 = 3;
270
271pub fn no_progress_seal_threshold_from_env() -> u32 {
275 std::env::var("DURABLE_NO_PROGRESS_SEAL_THRESHOLD")
276 .ok()
277 .and_then(|v| v.parse::<u32>().ok())
278 .unwrap_or(DEFAULT_NO_PROGRESS_SEAL_THRESHOLD)
279 .max(1)
280}
281
282#[derive(Debug, Clone, Default)]
284pub struct ReclaimResult {
285 pub reclaimed_ids: Vec<Uuid>,
287 pub dead_tasks: Vec<DeadTaskInfo>,
289 pub sealed_tasks: Vec<SealedTaskInfo>,
292}
293
294#[derive(Debug, Clone, Default)]
296pub struct WorkerFilter {
297 pub status: Option<String>,
298 pub worker_group: Option<String>,
299}
300
301impl WorkerFilter {
302 pub fn active() -> Self {
303 Self {
304 status: Some("active".to_string()),
305 worker_group: None,
306 }
307 }
308}
309
310#[derive(Debug, Clone, Default)]
312pub struct WorkflowFilter {
313 pub status: Option<WorkflowStatus>,
314 pub workflow_type: Option<String>,
315}
316
317#[derive(Debug, Clone, Default)]
319pub struct TaskFilter {
320 pub status: Option<TaskStatus>,
321 pub activity_type: Option<String>,
322 pub workflow_id: Option<Uuid>,
323 pub standalone_only: bool,
325}
326
327#[derive(Debug, Clone)]
329pub struct TaskInfo {
330 pub id: Uuid,
331 pub workflow_id: Option<Uuid>,
332 pub activity_id: String,
333 pub activity_type: String,
334 pub status: TaskStatus,
335 pub priority: i32,
336 pub attempt: u32,
337 pub max_attempts: u32,
338 pub claimed_by: Option<String>,
339 pub last_error: Option<String>,
340 pub created_at: DateTime<Utc>,
341 pub claimed_at: Option<DateTime<Utc>>,
342 pub queue: Option<String>,
344}
345
346#[derive(Debug, Clone)]
348pub struct WorkflowInfoExtended {
349 pub id: Uuid,
350 pub workflow_type: String,
351 pub status: WorkflowStatus,
352 pub input: serde_json::Value,
353 pub result: Option<serde_json::Value>,
354 pub error: Option<crate::workflow::WorkflowError>,
355 pub created_at: DateTime<Utc>,
356 pub started_at: Option<DateTime<Utc>>,
357 pub completed_at: Option<DateTime<Utc>>,
358 pub continued_as_new_id: Option<Uuid>,
360}
361
362#[derive(Debug, Clone)]
364pub struct SystemHealth {
365 pub total_workers: usize,
366 pub active_workers: usize,
367 pub workers_accepting: usize,
368 pub total_capacity: usize,
369 pub current_load: usize,
370 pub pending_tasks: usize,
371 pub claimed_tasks: usize,
372 pub completed_tasks: usize,
373 pub failed_tasks: usize,
374 pub started_tasks: usize,
376 pub running_workflows: usize,
377 pub pending_workflows: usize,
378 pub completed_workflows: usize,
379 pub failed_workflows: usize,
380 pub started_workflows: usize,
382 pub dlq_size: usize,
383}
384
385#[derive(Debug, Clone)]
387pub struct WorkflowEventInfo {
388 pub id: i64,
389 pub workflow_id: Uuid,
390 pub sequence_num: i32,
391 pub event_type: String,
392 pub event_data: serde_json::Value,
393 pub created_at: DateTime<Utc>,
394}
395
396pub fn event_type_name(event: &WorkflowEvent) -> &'static str {
398 match event {
399 WorkflowEvent::WorkflowStarted { .. } => "workflow_started",
400 WorkflowEvent::WorkflowCompleted { .. } => "workflow_completed",
401 WorkflowEvent::WorkflowFailed { .. } => "workflow_failed",
402 WorkflowEvent::WorkflowCancelled { .. } => "workflow_cancelled",
403 WorkflowEvent::ActivityScheduled { .. } => "activity_scheduled",
404 WorkflowEvent::ActivityStarted { .. } => "activity_started",
405 WorkflowEvent::ActivityCompleted { .. } => "activity_completed",
406 WorkflowEvent::ActivityFailed { .. } => "activity_failed",
407 WorkflowEvent::ActivityTimedOut { .. } => "activity_timed_out",
408 WorkflowEvent::ActivityCancelled { .. } => "activity_cancelled",
409 WorkflowEvent::TimerStarted { .. } => "timer_started",
410 WorkflowEvent::TimerFired { .. } => "timer_fired",
411 WorkflowEvent::TimerCancelled { .. } => "timer_cancelled",
412 WorkflowEvent::ChildWorkflowStarted { .. } => "child_workflow_started",
413 WorkflowEvent::ChildWorkflowCompleted { .. } => "child_workflow_completed",
414 WorkflowEvent::ChildWorkflowFailed { .. } => "child_workflow_failed",
415 }
416}
417
418#[derive(Debug, Clone)]
420pub struct WorkerInfo {
421 pub id: String,
422 pub worker_group: Option<String>,
423 pub activity_types: Vec<String>,
424 pub max_concurrency: u32,
425 pub current_load: u32,
426 pub status: String,
427 pub accepting_tasks: bool,
428 pub backpressure_reason: Option<String>,
429 pub started_at: DateTime<Utc>,
430 pub last_heartbeat_at: DateTime<Utc>,
431 pub hostname: Option<String>,
432 pub version: Option<String>,
433 pub metadata: Option<serde_json::Value>,
434 pub tasks_completed: u64,
436 pub tasks_failed: u64,
438 pub avg_task_duration_ms: Option<u64>,
440}
441
442impl WorkerInfo {
443 pub fn new<I, S>(id: impl Into<String>, activity_types: I) -> Self
461 where
462 I: IntoIterator<Item = S>,
463 S: Into<String>,
464 {
465 let now = Utc::now();
466 Self {
467 id: id.into(),
468 worker_group: None,
469 activity_types: activity_types.into_iter().map(Into::into).collect(),
470 max_concurrency: 10,
471 current_load: 0,
472 status: "active".to_string(),
473 accepting_tasks: true,
474 backpressure_reason: None,
475 started_at: now,
476 last_heartbeat_at: now,
477 hostname: None,
478 version: None,
479 metadata: None,
480 tasks_completed: 0,
481 tasks_failed: 0,
482 avg_task_duration_ms: None,
483 }
484 }
485
486 pub fn protocol_version(&self) -> Option<u32> {
489 self.metadata_field(WORKER_META_PROTOCOL_VERSION)?
490 .as_u64()
491 .and_then(|version| u32::try_from(version).ok())
492 }
493
494 pub fn draining_since(&self) -> Option<DateTime<Utc>> {
496 self.metadata_time(WORKER_META_DRAINING_SINCE)
497 }
498
499 pub fn self_checks(&self) -> Option<(&serde_json::Value, DateTime<Utc>)> {
502 let checks = self.metadata_field(WORKER_META_SELF_CHECKS)?;
503 Some((checks, self.metadata_time(WORKER_META_SELF_CHECKED_AT)?))
504 }
505
506 fn metadata_field(&self, key: &str) -> Option<&serde_json::Value> {
507 self.metadata.as_ref()?.get(key)
508 }
509
510 fn metadata_time(&self, key: &str) -> Option<DateTime<Utc>> {
511 let value = self.metadata_field(key)?.as_str()?;
512 DateTime::parse_from_rfc3339(value)
513 .ok()
514 .map(|at| at.with_timezone(&Utc))
515 }
516}
517
518pub(crate) const WORKER_META_PROTOCOL_VERSION: &str = "protocol_version";
521pub(crate) const WORKER_META_DRAINING_SINCE: &str = "draining_since";
522pub(crate) const WORKER_META_SELF_CHECKS: &str = "self_checks";
523pub(crate) const WORKER_META_SELF_CHECKED_AT: &str = "self_checked_at";
524
525#[derive(Debug, Clone, Default)]
527pub struct CapacitySnapshot {
528 pub total_available: u32,
530 pub active_workers: u32,
532}
533
534#[derive(Debug, Clone, Default)]
536pub struct DlqFilter {
537 pub workflow_id: Option<Uuid>,
538 pub activity_type: Option<String>,
539}
540
541#[derive(Debug, Clone)]
543pub struct Pagination {
544 pub offset: u32,
545 pub limit: u32,
546}
547
548impl Default for Pagination {
549 fn default() -> Self {
550 Self {
551 offset: 0,
552 limit: 100,
553 }
554 }
555}
556
557#[derive(Debug, Clone)]
559pub struct DlqEntry {
560 pub id: Uuid,
561 pub original_task_id: Uuid,
562 pub workflow_id: Option<Uuid>,
563 pub activity_id: String,
564 pub activity_type: String,
565 pub input: serde_json::Value,
566 pub attempts: u32,
567 pub last_error: String,
568 pub error_history: Vec<String>,
569 pub dead_at: DateTime<Utc>,
570}
571
572#[derive(Debug, Clone, Serialize, Deserialize)]
574pub struct TraceContext {
575 pub trace_id: String,
576 pub span_id: String,
577 pub trace_flags: u8,
578}
579
580#[derive(Debug, Clone)]
582pub struct WorkflowInfo {
583 pub id: Uuid,
584 pub workflow_type: String,
585 pub status: WorkflowStatus,
586 pub input: serde_json::Value,
587 pub result: Option<serde_json::Value>,
588 pub error: Option<crate::workflow::WorkflowError>,
589 pub continued_as_new_id: Option<Uuid>,
591}
592
593#[derive(Debug, Clone)]
598pub struct WorkflowSnapshot {
599 pub workflow_id: Uuid,
601
602 pub sequence_num: i32,
605
606 pub snapshot_data: Vec<u8>,
608
609 pub created_at: DateTime<Utc>,
611}
612
613#[async_trait]
624pub trait EventLog: Send + Sync + 'static {
625 async fn create_workflow(
627 &self,
628 workflow_id: Uuid,
629 workflow_type: &str,
630 input: serde_json::Value,
631 trace_context: Option<&TraceContext>,
632 ) -> Result<(), StoreError>;
633
634 async fn get_workflow_status(&self, workflow_id: Uuid) -> Result<WorkflowStatus, StoreError>;
636
637 async fn get_workflow_info(&self, workflow_id: Uuid) -> Result<WorkflowInfo, StoreError>;
639
640 async fn append_events(
644 &self,
645 workflow_id: Uuid,
646 expected_sequence: i32,
647 events: Vec<WorkflowEvent>,
648 ) -> Result<i32, StoreError>;
649
650 async fn load_events(&self, workflow_id: Uuid)
652 -> Result<Vec<(i32, WorkflowEvent)>, StoreError>;
653
654 async fn count_events(&self, workflow_id: Uuid) -> Result<usize, StoreError> {
658 Ok(self.load_events(workflow_id).await?.len())
659 }
660
661 async fn count_events_after(
665 &self,
666 workflow_id: Uuid,
667 after_sequence: i32,
668 ) -> Result<usize, StoreError> {
669 Ok(self
670 .load_events_after(workflow_id, after_sequence)
671 .await?
672 .len())
673 }
674
675 async fn load_events_after(
679 &self,
680 workflow_id: Uuid,
681 after_sequence: i32,
682 ) -> Result<Vec<(i32, WorkflowEvent)>, StoreError> {
683 let all = self.load_events(workflow_id).await?;
685 Ok(all
686 .into_iter()
687 .filter(|(seq, _)| *seq > after_sequence)
688 .collect())
689 }
690
691 async fn save_snapshot(
696 &self,
697 workflow_id: Uuid,
698 sequence_num: i32,
699 snapshot_data: Vec<u8>,
700 ) -> Result<(), StoreError>;
701
702 async fn load_latest_snapshot(
706 &self,
707 workflow_id: Uuid,
708 ) -> Result<Option<WorkflowSnapshot>, StoreError>;
709
710 async fn delete_snapshots(&self, workflow_id: Uuid) -> Result<(), StoreError>;
712
713 async fn update_workflow_status(
715 &self,
716 workflow_id: Uuid,
717 status: WorkflowStatus,
718 result: Option<serde_json::Value>,
719 error: Option<crate::workflow::WorkflowError>,
720 ) -> Result<(), StoreError>;
721
722 async fn try_fail_workflow(
728 &self,
729 workflow_id: Uuid,
730 error: crate::workflow::WorkflowError,
731 ) -> Result<bool, StoreError>;
732
733 async fn try_start_new_run(&self, workflow_id: Uuid) -> Result<bool, StoreError>;
746
747 async fn cancel_workflow(&self, workflow_id: Uuid) -> Result<(), StoreError>;
749
750 async fn continue_as_new(
758 &self,
759 old_workflow_id: Uuid,
760 workflow_type: &str,
761 input: serde_json::Value,
762 snapshot_data: Vec<u8>,
763 ) -> Result<Uuid, StoreError>;
764}
765
766#[async_trait]
773pub trait TaskQueue: Send + Sync + 'static {
774 async fn enqueue_task(&self, task: TaskDefinition) -> Result<Uuid, StoreError>;
776
777 async fn claim_task(
780 &self,
781 worker_id: &str,
782 activity_types: &[String],
783 max_tasks: usize,
784 ) -> Result<Vec<ClaimedTask>, StoreError> {
785 self.claim_queue_tasks(worker_id, None, activity_types, max_tasks)
786 .await
787 }
788
789 async fn claim_queue_tasks(
796 &self,
797 worker_id: &str,
798 queue: Option<&str>,
799 activity_types: &[String],
800 max_tasks: usize,
801 ) -> Result<Vec<ClaimedTask>, StoreError>;
802
803 async fn heartbeat_task(
805 &self,
806 task_id: Uuid,
807 worker_id: &str,
808 details: Option<serde_json::Value>,
809 ) -> Result<HeartbeatResponse, StoreError>;
810
811 async fn complete_task(
818 &self,
819 task_id: Uuid,
820 worker_id: &str,
821 result: serde_json::Value,
822 ) -> Result<(), StoreError>;
823
824 async fn fail_task(
826 &self,
827 task_id: Uuid,
828 error: &str,
829 ) -> Result<TaskFailureOutcome, StoreError> {
830 self.fail_task_with_retry(task_id, error, true).await
831 }
832
833 async fn fail_task_with_retry(
835 &self,
836 task_id: Uuid,
837 error: &str,
838 retryable: bool,
839 ) -> Result<TaskFailureOutcome, StoreError>;
840
841 async fn cancel_pending_tasks_for_workflow(&self, workflow_id: Uuid)
846 -> Result<u64, StoreError>;
847
848 async fn get_task(&self, task_id: Uuid) -> Result<TaskInfo, StoreError>;
850
851 async fn reclaim_stale_tasks(
853 &self,
854 stale_threshold: Duration,
855 ) -> Result<ReclaimResult, StoreError>;
856
857 async fn list_tasks(
859 &self,
860 filter: TaskFilter,
861 pagination: Pagination,
862 ) -> Result<Vec<TaskInfo>, StoreError>;
863}
864
865#[async_trait]
871pub trait WorkerRegistry: Send + Sync + 'static {
872 async fn register_worker(&self, worker: WorkerInfo) -> Result<(), StoreError>;
874
875 async fn worker_heartbeat(
877 &self,
878 worker_id: &str,
879 current_load: usize,
880 accepting_tasks: bool,
881 ) -> Result<WorkerHeartbeat, StoreError>;
882
883 async fn list_workers(&self, filter: WorkerFilter) -> Result<Vec<WorkerInfo>, StoreError>;
885
886 async fn deregister_worker(&self, worker_id: &str) -> Result<usize, StoreError>;
889
890 async fn get_capacity_snapshot(&self) -> Result<CapacitySnapshot, StoreError>;
893
894 async fn drain_worker(&self, worker_id: &str) -> Result<(), StoreError>;
896
897 async fn resume_worker(&self, worker_id: &str) -> Result<(), StoreError>;
899
900 async fn record_worker_checks(
905 &self,
906 worker_id: &str,
907 checks: serde_json::Value,
908 ) -> Result<(), StoreError> {
909 let _ = (worker_id, checks);
910 Ok(())
911 }
912}
913
914#[async_trait]
919pub trait DeadLetters: Send + Sync + 'static {
920 async fn move_to_dlq(
922 &self,
923 task_id: Uuid,
924 error_history: Vec<String>,
925 ) -> Result<(), StoreError>;
926
927 async fn requeue_from_dlq(&self, dlq_id: Uuid) -> Result<Uuid, StoreError>;
929
930 async fn list_dlq(
932 &self,
933 filter: DlqFilter,
934 pagination: Pagination,
935 ) -> Result<Vec<DlqEntry>, StoreError>;
936}
937
938#[async_trait]
943pub trait CircuitBreakers: Send + Sync + 'static {
944 async fn create_circuit_breaker(
946 &self,
947 key: &str,
948 config: &crate::reliability::CircuitBreakerConfig,
949 ) -> Result<(), StoreError>;
950
951 async fn get_circuit_breaker(
953 &self,
954 key: &str,
955 ) -> Result<Option<CircuitBreakerState>, StoreError>;
956
957 async fn update_circuit_breaker(
959 &self,
960 key: &str,
961 state: crate::reliability::CircuitState,
962 failure_count: u32,
963 success_count: u32,
964 ) -> Result<(), StoreError>;
965
966 async fn list_circuit_breakers(&self) -> Result<Vec<CircuitBreakerState>, StoreError>;
968
969 async fn force_open_circuit_breaker(&self, key: &str) -> Result<(), StoreError>;
971
972 async fn force_close_circuit_breaker(&self, key: &str) -> Result<(), StoreError>;
974
975 async fn delete_circuit_breaker(&self, key: &str) -> Result<(), StoreError>;
977}
978
979#[async_trait]
984pub trait Schedules: Send + Sync + 'static {
985 fn schedule_transaction_pool(&self) -> Option<sqlx::PgPool> {
987 None
988 }
989
990 async fn create_schedule(&self, schedule: CreateScheduleRow) -> Result<Uuid, StoreError>;
992
993 async fn get_schedule(&self, id: Uuid) -> Result<ScheduleRow, StoreError>;
995
996 async fn list_schedules(
998 &self,
999 filter: ScheduleFilter,
1000 pagination: Pagination,
1001 ) -> Result<Vec<ScheduleRow>, StoreError>;
1002
1003 async fn count_schedules(&self, filter: ScheduleFilter) -> Result<u64, StoreError>;
1005
1006 async fn update_schedule(&self, id: Uuid, _update: UpdateSchedule) -> Result<(), StoreError>;
1008
1009 async fn delete_schedule(&self, id: Uuid) -> Result<(), StoreError>;
1011
1012 async fn claim_due_schedules(
1015 &self,
1016 scheduler_id: &str,
1017 limit: u32,
1018 ) -> Result<Vec<ScheduleRow>, StoreError>;
1019
1020 async fn update_next_trigger(&self, id: Uuid, _next: DateTime<Utc>) -> Result<(), StoreError>;
1022
1023 async fn skip_schedule_trigger(&self, id: Uuid) -> Result<(), StoreError>;
1025
1026 async fn release_schedule(&self, id: Uuid) -> Result<(), StoreError>;
1028
1029 async fn create_schedule_execution(
1031 &self,
1032 schedule_id: Uuid,
1033 scheduled_at: DateTime<Utc>,
1034 ) -> Result<Uuid, StoreError>;
1035
1036 async fn get_schedule_execution(&self, id: Uuid) -> Result<ScheduleExecutionRow, StoreError>;
1038
1039 async fn complete_schedule_execution(
1041 &self,
1042 execution_id: Uuid,
1043 target_id: Uuid,
1044 is_workflow: bool,
1045 ) -> Result<(), StoreError>;
1046
1047 async fn fail_schedule_execution(
1049 &self,
1050 execution_id: Uuid,
1051 error: &str,
1052 ) -> Result<(), StoreError>;
1053
1054 async fn skip_schedule_execution(
1056 &self,
1057 execution_id: Uuid,
1058 reason: &str,
1059 ) -> Result<(), StoreError>;
1060
1061 async fn list_schedule_executions(
1063 &self,
1064 filter: ScheduleExecutionFilter,
1065 pagination: Pagination,
1066 ) -> Result<Vec<ScheduleExecutionRow>, StoreError>;
1067
1068 async fn count_running_executions(&self, schedule_id: Uuid) -> Result<u32, StoreError>;
1070
1071 async fn last_scheduled_task_pending(&self, schedule_id: Uuid) -> Result<bool, StoreError>;
1075
1076 async fn get_schedule_stats(&self, schedule_id: Uuid) -> Result<ScheduleStats, StoreError>;
1078
1079 async fn register_scheduler_instance(
1081 &self,
1082 instance: SchedulerInstanceInfo,
1083 ) -> Result<(), StoreError>;
1084
1085 async fn heartbeat_scheduler_instance(
1087 &self,
1088 instance_id: &str,
1089 schedules_processed: u64,
1090 ) -> Result<(), StoreError>;
1091
1092 async fn list_scheduler_instances(&self) -> Result<Vec<SchedulerInstanceInfo>, StoreError>;
1094
1095 async fn deregister_scheduler_instance(&self, instance_id: &str) -> Result<(), StoreError>;
1097}
1098
1099#[async_trait]
1105pub trait DurableAdmin: EventLog + Send + Sync + 'static {
1106 async fn count_active_workflows(&self) -> Result<i64, StoreError>;
1108
1109 async fn list_workflows(
1111 &self,
1112 filter: WorkflowFilter,
1113 pagination: Pagination,
1114 ) -> Result<Vec<WorkflowInfoExtended>, StoreError>;
1115
1116 async fn get_workflow_extended(
1125 &self,
1126 workflow_id: Uuid,
1127 ) -> Result<Option<WorkflowInfoExtended>, StoreError>;
1128
1129 async fn get_workflow_events(
1131 &self,
1132 workflow_id: Uuid,
1133 ) -> Result<Vec<WorkflowEventInfo>, StoreError> {
1134 let events = self.load_events(workflow_id).await?;
1136 Ok(events
1137 .into_iter()
1138 .map(|(seq, event)| WorkflowEventInfo {
1139 id: seq as i64, workflow_id,
1141 sequence_num: seq,
1142 event_type: event_type_name(&event).to_string(),
1143 event_data: serde_json::to_value(&event).unwrap_or_default(),
1144 created_at: Utc::now(), })
1146 .collect())
1147 }
1148
1149 async fn count_workflow_events(&self, workflow_id: Uuid) -> Result<i64, StoreError> {
1152 let events = self.load_events(workflow_id).await?;
1153 Ok(events.len() as i64)
1154 }
1155
1156 async fn get_system_health(&self) -> Result<SystemHealth, StoreError>;
1158}
1159
1160pub trait WorkflowEventStore:
1168 EventLog
1169 + TaskQueue
1170 + WorkerRegistry
1171 + DeadLetters
1172 + CircuitBreakers
1173 + Schedules
1174 + DurableAdmin
1175 + Send
1176 + Sync
1177 + 'static
1178{
1179}
1180
1181impl<T> WorkflowEventStore for T where
1182 T: EventLog
1183 + TaskQueue
1184 + WorkerRegistry
1185 + DeadLetters
1186 + CircuitBreakers
1187 + Schedules
1188 + DurableAdmin
1189 + Send
1190 + Sync
1191 + 'static
1192{
1193}
1194
1195#[derive(Debug, Clone)]
1197pub struct CircuitBreakerState {
1198 pub key: String,
1199 pub state: crate::reliability::CircuitState,
1200 pub failure_count: u32,
1201 pub success_count: u32,
1202 pub last_failure_at: Option<chrono::DateTime<chrono::Utc>>,
1203 pub opened_at: Option<chrono::DateTime<chrono::Utc>>,
1204 pub half_open_at: Option<chrono::DateTime<chrono::Utc>>,
1205 pub updated_at: chrono::DateTime<chrono::Utc>,
1206}
1207
1208#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
1214#[serde(rename_all = "snake_case")]
1215pub enum ScheduleTargetType {
1216 Workflow,
1217 Activity,
1218}
1219
1220impl std::fmt::Display for ScheduleTargetType {
1221 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1222 match self {
1223 Self::Workflow => write!(f, "workflow"),
1224 Self::Activity => write!(f, "activity"),
1225 }
1226 }
1227}
1228
1229#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
1231#[serde(rename_all = "snake_case")]
1232pub enum ScheduleExecutionStatus {
1233 Pending,
1234 Running,
1235 Completed,
1236 Failed,
1237 Skipped,
1238}
1239
1240impl std::fmt::Display for ScheduleExecutionStatus {
1241 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1242 match self {
1243 Self::Pending => write!(f, "pending"),
1244 Self::Running => write!(f, "running"),
1245 Self::Completed => write!(f, "completed"),
1246 Self::Failed => write!(f, "failed"),
1247 Self::Skipped => write!(f, "skipped"),
1248 }
1249 }
1250}
1251
1252#[derive(Debug, Clone)]
1254pub struct ScheduleRow {
1255 pub id: Uuid,
1256 pub name: String,
1257 pub description: Option<String>,
1258 pub cron_expression: String,
1259 pub timezone: String,
1260 pub target_type: ScheduleTargetType,
1261 pub target_name: String,
1262 pub target_input: serde_json::Value,
1263 pub enabled: bool,
1264 pub max_concurrent: Option<u32>,
1265 pub catch_up_missed: bool,
1266 pub max_catch_up: Option<u32>,
1267 pub retry_policy: Option<serde_json::Value>,
1268 pub last_triggered_at: Option<DateTime<Utc>>,
1269 pub next_trigger_at: Option<DateTime<Utc>>,
1270 pub claimed_by: Option<String>,
1271 pub claimed_at: Option<DateTime<Utc>>,
1272 pub created_at: DateTime<Utc>,
1273 pub updated_at: DateTime<Utc>,
1274}
1275
1276#[derive(Debug, Clone)]
1278pub struct CreateScheduleRow {
1279 pub name: String,
1280 pub description: Option<String>,
1281 pub cron_expression: String,
1282 pub timezone: String,
1283 pub target_type: ScheduleTargetType,
1284 pub target_name: String,
1285 pub target_input: serde_json::Value,
1286 pub enabled: bool,
1287 pub max_concurrent: Option<u32>,
1288 pub catch_up_missed: bool,
1289 pub max_catch_up: Option<u32>,
1290 pub retry_policy: Option<serde_json::Value>,
1291 pub next_trigger_at: Option<DateTime<Utc>>,
1292}
1293
1294#[derive(Debug, Clone, Default)]
1296pub struct UpdateSchedule {
1297 pub name: Option<String>,
1298 pub description: UpdateField<String>,
1299 pub cron_expression: Option<String>,
1300 pub timezone: Option<String>,
1301 pub target_type: Option<ScheduleTargetType>,
1302 pub target_name: Option<String>,
1303 pub target_input: Option<serde_json::Value>,
1304 pub enabled: Option<bool>,
1305 pub max_concurrent: UpdateField<u32>,
1306 pub catch_up_missed: Option<bool>,
1307 pub max_catch_up: UpdateField<u32>,
1308 pub retry_policy: UpdateField<serde_json::Value>,
1309 pub next_trigger_at: UpdateField<DateTime<Utc>>,
1310}
1311
1312#[derive(Debug, Clone, Default)]
1314pub struct ScheduleFilter {
1315 pub enabled: Option<bool>,
1316 pub target_type: Option<ScheduleTargetType>,
1317}
1318
1319#[derive(Debug, Clone)]
1321pub struct ScheduleExecutionRow {
1322 pub id: Uuid,
1323 pub schedule_id: Uuid,
1324 pub scheduled_at: DateTime<Utc>,
1325 pub started_at: DateTime<Utc>,
1326 pub completed_at: Option<DateTime<Utc>>,
1327 pub status: ScheduleExecutionStatus,
1328 pub workflow_id: Option<Uuid>,
1329 pub task_id: Option<Uuid>,
1330 pub error: Option<String>,
1331 pub duration_ms: Option<i32>,
1332 pub created_at: DateTime<Utc>,
1333}
1334
1335#[derive(Debug, Clone, Default)]
1337pub struct ScheduleStats {
1338 pub total_executions: u64,
1339 pub successful_executions: u64,
1340 pub failed_executions: u64,
1341 pub skipped_executions: u64,
1342 pub avg_duration_ms: Option<u64>,
1343 pub last_execution_status: Option<ScheduleExecutionStatus>,
1344}
1345
1346#[derive(Debug, Clone, Default)]
1348pub struct ScheduleExecutionFilter {
1349 pub schedule_id: Option<Uuid>,
1350 pub status: Option<ScheduleExecutionStatus>,
1351}
1352
1353#[derive(Debug, Clone)]
1355pub struct SchedulerInstanceInfo {
1356 pub instance_id: String,
1357 pub started_at: DateTime<Utc>,
1358 pub last_heartbeat_at: DateTime<Utc>,
1359 pub schedules_processed: u64,
1360 pub hostname: Option<String>,
1361 pub version: Option<String>,
1362}