1use async_trait::async_trait;
7use std::cmp::Ordering;
8
9use chrono::{DateTime, SecondsFormat, Utc};
10use serde_json::Value;
11
12use crate::channel::ChannelKind;
13use crate::context::{
14 WorkflowBranchCursorSelection, WorkflowContext, WorkflowCursor, WorkflowCursorFrame,
15 WorkflowPendingWait,
16};
17use crate::payment::{ChargeRequest, ChargeResponse, Currency, OrderOrigin, PaymentKind};
18use crate::registry::{StateTransition, WorkflowDefinition};
19use crate::runtime::{
20 WorkflowLease, WorkflowOperation, WorkflowOperationKind, WorkflowOperationStatus,
21 WorkflowRunOptions, WorkflowSideEffect, WorkflowSideEffectKind, WorkflowSideEffectStatus,
22 WorkflowTimeoutOutcome,
23};
24use crate::step::{WorkflowBranchCondition, WorkflowStep};
25
26const WORKFLOW_QUERY_WIRE_MAGIC: &str = "QAIL-CMD/1\n";
27
28#[derive(Debug, Clone, PartialEq, Eq)]
30pub struct LegacyQueryPayloadIssue {
31 pub transition_from: String,
33 pub transition_to: String,
35 pub step_path: String,
37 pub payload_preview: String,
39}
40
41pub fn collect_legacy_query_payload_issues(
49 definition: &WorkflowDefinition,
50) -> Vec<LegacyQueryPayloadIssue> {
51 let mut out = Vec::new();
52 for transition in &definition.transitions {
53 collect_legacy_query_payload_issues_in_steps(
54 &transition.from,
55 &transition.to,
56 &transition.steps,
57 "",
58 &mut out,
59 );
60 }
61 out
62}
63
64fn collect_legacy_query_payload_issues_in_steps(
65 transition_from: &str,
66 transition_to: &str,
67 steps: &[WorkflowStep],
68 path_prefix: &str,
69 out: &mut Vec<LegacyQueryPayloadIssue>,
70) {
71 for (idx, step) in steps.iter().enumerate() {
72 let step_path = if path_prefix.is_empty() {
73 format!("steps[{idx}]")
74 } else {
75 format!("{path_prefix}.steps[{idx}]")
76 };
77
78 match step {
79 WorkflowStep::Query { cmd_json, .. } => {
80 if !is_current_workflow_query_wire(cmd_json) {
81 out.push(LegacyQueryPayloadIssue {
82 transition_from: transition_from.to_string(),
83 transition_to: transition_to.to_string(),
84 step_path,
85 payload_preview: summarize_payload_preview(cmd_json),
86 });
87 }
88 }
89 WorkflowStep::Wait { on_timeout, .. } => {
90 collect_legacy_query_payload_issues_in_steps(
91 transition_from,
92 transition_to,
93 on_timeout,
94 &format!("{step_path}.on_timeout"),
95 out,
96 );
97 }
98 WorkflowStep::Branch {
99 branches, default, ..
100 } => {
101 for (branch_idx, (branch_value, branch_steps)) in branches.iter().enumerate() {
102 collect_legacy_query_payload_issues_in_steps(
103 transition_from,
104 transition_to,
105 branch_steps,
106 &format!("{step_path}.branches[{branch_idx}:{branch_value}]"),
107 out,
108 );
109 }
110 collect_legacy_query_payload_issues_in_steps(
111 transition_from,
112 transition_to,
113 default,
114 &format!("{step_path}.default"),
115 out,
116 );
117 }
118 WorkflowStep::BranchWhen {
119 branches, default, ..
120 } => {
121 for (branch_idx, (condition, branch_steps)) in branches.iter().enumerate() {
122 collect_legacy_query_payload_issues_in_steps(
123 transition_from,
124 transition_to,
125 branch_steps,
126 &format!("{step_path}.branch_when[{branch_idx}:{condition:?}]"),
127 out,
128 );
129 }
130 collect_legacy_query_payload_issues_in_steps(
131 transition_from,
132 transition_to,
133 default,
134 &format!("{step_path}.default"),
135 out,
136 );
137 }
138 WorkflowStep::ForEach { steps: nested, .. } => {
139 collect_legacy_query_payload_issues_in_steps(
140 transition_from,
141 transition_to,
142 nested,
143 &format!("{step_path}.for_each"),
144 out,
145 );
146 }
147 WorkflowStep::Notify { .. }
148 | WorkflowStep::Transition { .. }
149 | WorkflowStep::Log { .. }
150 | WorkflowStep::Charge { .. } => {}
151 }
152 }
153}
154
155fn summarize_payload_preview(cmd_json: &str) -> String {
156 const MAX_CHARS: usize = 64;
157 let escaped = cmd_json.replace('\n', "\\n");
158 let mut preview = escaped.chars().take(MAX_CHARS).collect::<String>();
159 if escaped.chars().count() > MAX_CHARS {
160 preview.push_str("...");
161 }
162 preview
163}
164
165fn is_current_workflow_query_wire(cmd_json: &str) -> bool {
166 cmd_json.starts_with(WORKFLOW_QUERY_WIRE_MAGIC)
167}
168
169#[derive(Debug)]
171pub enum WorkflowError {
172 QueryFailed(String),
174 NotifyFailed(String),
176 NoTransition {
178 state: String,
180 workflow: String,
182 },
183 MissingContextKey(String),
185 AlreadyTerminal(String),
187 PersistenceFailed(String),
189 Timeout {
191 event: String,
193 },
194 ChargeFailed(String),
196 Other(String),
198}
199
200impl std::fmt::Display for WorkflowError {
201 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
202 match self {
203 WorkflowError::QueryFailed(e) => write!(f, "Query failed: {}", e),
204 WorkflowError::NotifyFailed(e) => write!(f, "Notification failed: {}", e),
205 WorkflowError::NoTransition { state, workflow } => {
206 write!(
207 f,
208 "No transition from '{}' in workflow '{}'",
209 state, workflow
210 )
211 }
212 WorkflowError::MissingContextKey(k) => write!(f, "Context key not found: {}", k),
213 WorkflowError::AlreadyTerminal(s) => write!(f, "Workflow already terminal: {}", s),
214 WorkflowError::PersistenceFailed(e) => write!(f, "Persistence failed: {}", e),
215 WorkflowError::Timeout { event } => write!(f, "Timeout waiting for: {}", event),
216 WorkflowError::ChargeFailed(e) => write!(f, "Charge failed: {}", e),
217 WorkflowError::Other(e) => write!(f, "{}", e),
218 }
219 }
220}
221
222impl std::error::Error for WorkflowError {}
223
224#[async_trait]
261pub trait WorkflowExecutor: Send + Sync {
262 async fn execute_query(&self, cmd_json: &str) -> Result<Value, WorkflowError>;
264
265 async fn send_notification(
267 &self,
268 channel: &ChannelKind,
269 recipient: &str,
270 template: &str,
271 params: &Value,
272 ) -> Result<(), WorkflowError>;
273
274 async fn save_state(&self, ctx: &WorkflowContext) -> Result<(), WorkflowError>;
276
277 async fn load_state(&self, workflow_id: &str)
279 -> Result<Option<WorkflowContext>, WorkflowError>;
280
281 async fn create_charge(
288 &self,
289 provider: &PaymentKind,
290 request: ChargeRequest,
291 ) -> Result<ChargeResponse, WorkflowError>;
292
293 async fn acquire_workflow_lease(&self, _lease: &WorkflowLease) -> Result<bool, WorkflowError> {
298 Ok(true)
299 }
300
301 async fn release_workflow_lease(&self, _lease: &WorkflowLease) -> Result<(), WorkflowError> {
303 Ok(())
304 }
305
306 async fn begin_workflow_operation(
310 &self,
311 _operation: &WorkflowOperation,
312 ) -> Result<WorkflowOperationStatus, WorkflowError> {
313 Ok(WorkflowOperationStatus::Started)
314 }
315
316 async fn complete_workflow_operation(
318 &self,
319 _operation: &WorkflowOperation,
320 _state: &str,
321 ) -> Result<(), WorkflowError> {
322 Ok(())
323 }
324
325 async fn fail_workflow_operation(
327 &self,
328 _operation: &WorkflowOperation,
329 _error: &str,
330 ) -> Result<(), WorkflowError> {
331 Ok(())
332 }
333
334 async fn load_due_workflow_timeouts(
340 &self,
341 _workflow_name: &str,
342 _now: DateTime<Utc>,
343 _limit: usize,
344 ) -> Result<Vec<String>, WorkflowError> {
345 Ok(Vec::new())
346 }
347
348 async fn begin_workflow_side_effect(
350 &self,
351 _operation: &WorkflowSideEffect,
352 ) -> Result<WorkflowSideEffectStatus, WorkflowError> {
353 Ok(WorkflowSideEffectStatus::Execute)
354 }
355
356 async fn complete_workflow_side_effect(
358 &self,
359 _operation: &WorkflowSideEffect,
360 _result: Option<&Value>,
361 ) -> Result<(), WorkflowError> {
362 Ok(())
363 }
364
365 async fn fail_workflow_side_effect(
372 &self,
373 _operation: &WorkflowSideEffect,
374 _error: &str,
375 ) -> Result<(), WorkflowError> {
376 Ok(())
377 }
378
379 async fn execute_query_once(
381 &self,
382 operation: &WorkflowSideEffect,
383 cmd_json: &str,
384 ) -> Result<Value, WorkflowError> {
385 match self.begin_workflow_side_effect(operation).await? {
386 WorkflowSideEffectStatus::Execute => {
387 let result = match self.execute_query(cmd_json).await {
388 Ok(result) => result,
389 Err(err) => {
390 let _ = self
391 .fail_workflow_side_effect(operation, &err.to_string())
392 .await;
393 return Err(err);
394 }
395 };
396 self.complete_workflow_side_effect(operation, Some(&result))
397 .await?;
398 Ok(result)
399 }
400 WorkflowSideEffectStatus::AlreadyCompleted { result } => result.ok_or_else(|| {
401 WorkflowError::Other(format!(
402 "Workflow side effect '{}' completed without a stored query result",
403 operation.operation_id
404 ))
405 }),
406 }
407 }
408
409 async fn send_notification_once(
411 &self,
412 operation: &WorkflowSideEffect,
413 channel: &ChannelKind,
414 recipient: &str,
415 template: &str,
416 params: &Value,
417 ) -> Result<(), WorkflowError> {
418 match self.begin_workflow_side_effect(operation).await? {
419 WorkflowSideEffectStatus::Execute => {
420 if let Err(err) = self
421 .send_notification(channel, recipient, template, params)
422 .await
423 {
424 let _ = self
425 .fail_workflow_side_effect(operation, &err.to_string())
426 .await;
427 return Err(err);
428 }
429 self.complete_workflow_side_effect(operation, None).await
430 }
431 WorkflowSideEffectStatus::AlreadyCompleted { .. } => Ok(()),
432 }
433 }
434
435 async fn create_charge_once(
437 &self,
438 operation: &WorkflowSideEffect,
439 provider: &PaymentKind,
440 mut request: ChargeRequest,
441 ) -> Result<ChargeResponse, WorkflowError> {
442 match self.begin_workflow_side_effect(operation).await? {
443 WorkflowSideEffectStatus::Execute => {
444 if request.idempotency_key.is_none() {
445 request.idempotency_key = Some(operation.operation_id.clone());
446 }
447 let response = match self.create_charge(provider, request).await {
448 Ok(response) => response,
449 Err(err) => {
450 let _ = self
451 .fail_workflow_side_effect(operation, &err.to_string())
452 .await;
453 return Err(err);
454 }
455 };
456 let response_json = serde_json::to_value(&response)
457 .map_err(|e| WorkflowError::Other(e.to_string()))?;
458 self.complete_workflow_side_effect(operation, Some(&response_json))
459 .await?;
460 Ok(response)
461 }
462 WorkflowSideEffectStatus::AlreadyCompleted {
463 result: Some(result),
464 } => serde_json::from_value(result).map_err(|e| {
465 WorkflowError::Other(format!(
466 "Workflow side effect '{}' stored an invalid charge result: {e}",
467 operation.operation_id
468 ))
469 }),
470 WorkflowSideEffectStatus::AlreadyCompleted { result: None } => {
471 Err(WorkflowError::Other(format!(
472 "Workflow side effect '{}' completed without a stored charge result",
473 operation.operation_id
474 )))
475 }
476 }
477 }
478}
479
480#[derive(Debug, Clone, PartialEq, Eq)]
481enum StepOutcome {
482 Continue,
483 Paused(StepPause),
484}
485
486#[derive(Debug, Clone, PartialEq, Eq)]
487struct StepPause {
488 frames: Vec<WorkflowCursorFrame>,
489 wait: WorkflowPendingWait,
490}
491
492struct RuntimeGuard {
493 lease: Option<WorkflowLease>,
494 operation: Option<WorkflowOperation>,
495}
496
497enum RuntimeEntry {
498 Active(RuntimeGuard),
499 Completed(String),
500}
501
502#[derive(Debug, Clone)]
503enum StepListCursorKind {
504 Steps,
505 Branch {
506 selection: WorkflowBranchCursorSelection,
507 },
508 ForEach {
509 item_index: usize,
510 item: Value,
511 },
512}
513
514#[derive(Debug, Clone, Copy, PartialEq, Eq)]
515enum RunMode<'a> {
516 Normal,
517 EventResume { event: &'a str },
518}
519
520const FOR_EACH_ITEM_KEY: &str = "item";
521const RESUME_EVENT_KEY: &str = "event";
522const TIMEOUT_FALLBACK_KEY: &str = "__qail_timeout_fallback";
523const RESERVED_CONTEXT_KEYS: &[&str] = &[FOR_EACH_ITEM_KEY, RESUME_EVENT_KEY, TIMEOUT_FALLBACK_KEY];
524
525#[derive(Debug, Clone)]
526struct StepExecutionScope<'a> {
527 state: &'a str,
528 path_prefix: Vec<WorkflowCursorFrame>,
529 list_kind: StepListCursorKind,
530 checkpoint_steps: bool,
531 checkpoint_timeout: bool,
532}
533
534impl<'a> StepExecutionScope<'a> {
535 fn new(state: &'a str, list_kind: StepListCursorKind, checkpoint_steps: bool) -> Self {
536 Self {
537 state,
538 path_prefix: Vec::new(),
539 list_kind,
540 checkpoint_steps,
541 checkpoint_timeout: false,
542 }
543 }
544
545 fn with_timeout_checkpoint(mut self) -> Self {
546 self.checkpoint_timeout = true;
547 self
548 }
549
550 fn child(&self, step_index: usize, list_kind: StepListCursorKind) -> Self {
551 Self {
552 state: self.state,
553 path_prefix: cursor_frames_for_index(&self.path_prefix, &self.list_kind, step_index),
554 list_kind,
555 checkpoint_steps: self.checkpoint_steps,
556 checkpoint_timeout: self.checkpoint_timeout,
557 }
558 }
559}
560
561impl RuntimeGuard {
562 async fn enter<E: WorkflowExecutor>(
563 executor: &E,
564 workflow_name: &str,
565 workflow_id: &str,
566 kind: WorkflowOperationKind,
567 options: WorkflowRunOptions,
568 ) -> Result<RuntimeEntry, WorkflowError> {
569 let lease_options = match options.lease {
570 Some(lease_options) => {
571 ensure_definition_text(&lease_options.owner, "Workflow lease owner")?;
572 if lease_options.ttl.is_zero() {
573 return Err(WorkflowError::Other(
574 "Workflow lease TTL must be greater than zero".to_string(),
575 ));
576 }
577 Some(lease_options)
578 }
579 None => None,
580 };
581
582 let operation = match options.idempotency_key {
583 Some(idempotency_key) => {
584 ensure_definition_text(&idempotency_key, "Workflow idempotency key")?;
585 let operation = WorkflowOperation {
586 workflow_name: workflow_name.to_string(),
587 workflow_id: workflow_id.to_string(),
588 idempotency_key,
589 kind: kind.clone(),
590 };
591 let operation_status = match executor.begin_workflow_operation(&operation).await {
592 Ok(status) => status,
593 Err(err) => return Err(err),
594 };
595 match operation_status {
596 WorkflowOperationStatus::Started => Some(operation),
597 WorkflowOperationStatus::InProgress => {
598 return Err(WorkflowError::Other(format!(
599 "Workflow operation '{}' is already in progress",
600 operation.idempotency_key
601 )));
602 }
603 WorkflowOperationStatus::Completed { state } => {
604 ensure_definition_text(&state, "Workflow completed operation state")?;
605 return Ok(RuntimeEntry::Completed(state));
606 }
607 }
608 }
609 None => None,
610 };
611
612 let lease = lease_options.map(|lease_options| WorkflowLease {
613 workflow_id: workflow_id.to_string(),
614 owner: runtime_lease_owner(
615 &lease_options.owner,
616 workflow_name,
617 workflow_id,
618 &kind,
619 operation
620 .as_ref()
621 .map(|operation| operation.idempotency_key.as_str()),
622 Utc::now(),
623 ),
624 ttl: lease_options.ttl,
625 });
626
627 let lease = match lease {
628 Some(lease) => match executor.acquire_workflow_lease(&lease).await {
629 Ok(true) => Some(lease),
630 Ok(false) => {
631 let error = format!("Workflow '{}' is already locked", workflow_id);
632 if let Some(operation) = &operation {
633 let _ = executor.fail_workflow_operation(operation, &error).await;
634 }
635 return Err(WorkflowError::Other(error));
636 }
637 Err(err) => {
638 if let Some(operation) = &operation {
639 let _ = executor
640 .fail_workflow_operation(operation, &err.to_string())
641 .await;
642 }
643 return Err(err);
644 }
645 },
646 None => None,
647 };
648
649 Ok(RuntimeEntry::Active(Self { lease, operation }))
650 }
651
652 async fn finish<E: WorkflowExecutor>(
653 self,
654 executor: &E,
655 result: &Result<String, WorkflowError>,
656 ) -> Result<(), WorkflowError> {
657 let operation_result = match &self.operation {
658 Some(operation) => match result {
659 Ok(state) => executor.complete_workflow_operation(operation, state).await,
660 Err(err) => {
661 executor
662 .fail_workflow_operation(operation, &err.to_string())
663 .await
664 }
665 },
666 None => Ok(()),
667 };
668
669 let release_result = match &self.lease {
670 Some(lease) => executor.release_workflow_lease(lease).await,
671 None => Ok(()),
672 };
673
674 match (operation_result, release_result) {
675 (Err(err), _) => Err(err),
676 (Ok(()), Err(err)) => Err(err),
677 (Ok(()), Ok(())) => Ok(()),
678 }
679 }
680}
681
682fn runtime_lease_owner(
683 owner: &str,
684 workflow_name: &str,
685 workflow_id: &str,
686 kind: &WorkflowOperationKind,
687 idempotency_key: Option<&str>,
688 acquired_at: DateTime<Utc>,
689) -> String {
690 serde_json::json!([
691 "qail-workflow-lease",
692 1,
693 owner,
694 workflow_name,
695 workflow_id,
696 operation_kind_lease_key(kind),
697 idempotency_key,
698 acquired_at.to_rfc3339_opts(SecondsFormat::Micros, true)
699 ])
700 .to_string()
701}
702
703fn operation_kind_lease_key(kind: &WorkflowOperationKind) -> serde_json::Value {
704 match kind {
705 WorkflowOperationKind::Run => serde_json::json!(["run"]),
706 WorkflowOperationKind::Resume { event } => serde_json::json!(["resume", event]),
707 WorkflowOperationKind::Timeout => serde_json::json!(["timeout"]),
708 }
709}
710
711async fn finish_runtime_operation<E: WorkflowExecutor>(
712 executor: &E,
713 guard: RuntimeGuard,
714 result: Result<String, WorkflowError>,
715) -> Result<String, WorkflowError> {
716 let finish_result = guard.finish(executor, &result).await;
717 match (result, finish_result) {
718 (Ok(state), Ok(())) => Ok(state),
719 (Ok(_), Err(err)) => Err(err),
720 (Err(err), Ok(())) | (Err(err), Err(_)) => Err(err),
721 }
722}
723
724fn invalid_cursor(message: impl Into<String>) -> WorkflowError {
725 WorkflowError::Other(format!(
726 "Invalid workflow resume cursor: {}",
727 message.into()
728 ))
729}
730
731fn set_timeout_fallback(
732 ctx: &mut WorkflowContext,
733 wait: &WorkflowPendingWait,
734) -> Result<(), WorkflowError> {
735 let value = serde_json::to_value(wait)
736 .map_err(|e| WorkflowError::Other(format!("Failed to serialize timeout cursor: {e}")))?;
737 ctx.set(TIMEOUT_FALLBACK_KEY, value);
738 Ok(())
739}
740
741fn timeout_fallback_from_context(
742 ctx: &WorkflowContext,
743) -> Result<Option<WorkflowPendingWait>, WorkflowError> {
744 let Some(value) = ctx.get(TIMEOUT_FALLBACK_KEY) else {
745 return Ok(None);
746 };
747 serde_json::from_value(value.clone())
748 .map(Some)
749 .map_err(|e| invalid_cursor(format!("timeout fallback metadata is invalid: {e}")))
750}
751
752fn clear_timeout_fallback(ctx: &mut WorkflowContext) {
753 if ctx.data.remove(TIMEOUT_FALLBACK_KEY).is_some() {
754 ctx.updated_at = Utc::now();
755 }
756}
757
758fn json_type_name(value: &Value) -> &'static str {
759 match value {
760 Value::Null => "null",
761 Value::Bool(_) => "boolean",
762 Value::Number(_) => "number",
763 Value::String(_) => "string",
764 Value::Array(_) => "array",
765 Value::Object(_) => "object",
766 }
767}
768
769fn ensure_no_child_cursor(
770 step_name: &str,
771 cursor_frames: &[WorkflowCursorFrame],
772) -> Result<(), WorkflowError> {
773 if cursor_frames.is_empty() {
774 Ok(())
775 } else {
776 Err(invalid_cursor(format!(
777 "{step_name} cannot resume nested cursor frames"
778 )))
779 }
780}
781
782fn ensure_step_position(
783 step: &WorkflowStep,
784 idx: usize,
785 step_count: usize,
786) -> Result<(), WorkflowError> {
787 if matches!(step, WorkflowStep::Transition { .. }) && idx + 1 < step_count {
788 return Err(WorkflowError::Other(
789 "Transition steps must be the final step in their block; move follow-up work before the Transition or into the target state's transition".to_string(),
790 ));
791 }
792 Ok(())
793}
794
795fn ensure_wait_event_name(event: &str) -> Result<(), WorkflowError> {
796 if event.trim().is_empty() {
797 return Err(WorkflowError::Other(
798 "Wait event name must not be empty".to_string(),
799 ));
800 }
801 if event.trim() != event {
802 return Err(WorkflowError::Other(
803 "Wait event name must not have leading or trailing whitespace".to_string(),
804 ));
805 }
806 Ok(())
807}
808
809fn ensure_definition_text(value: &str, usage: &str) -> Result<(), WorkflowError> {
810 if value.trim().is_empty() {
811 return Err(WorkflowError::Other(format!("{usage} must not be empty")));
812 }
813 if value.trim() != value {
814 return Err(WorkflowError::Other(format!(
815 "{usage} must not have leading or trailing whitespace"
816 )));
817 }
818 Ok(())
819}
820
821fn validate_workflow_context_identity(ctx: &WorkflowContext) -> Result<(), WorkflowError> {
822 ensure_definition_text(&ctx.workflow_id, "Workflow id")?;
823 ensure_definition_text(&ctx.current_state, "Workflow current state")?;
824 Ok(())
825}
826
827fn ensure_user_context_key(key: &str, usage: &str) -> Result<(), WorkflowError> {
828 ensure_definition_text(key, usage)?;
829 if key.contains('.') {
830 return Err(WorkflowError::Other(format!(
831 "Workflow {usage} must be a top-level context key; dot notation is only supported for lookups"
832 )));
833 }
834 if RESERVED_CONTEXT_KEYS.contains(&key) {
835 return Err(WorkflowError::Other(format!(
836 "Workflow {usage} uses reserved context key '{key}'"
837 )));
838 }
839 Ok(())
840}
841
842fn ensure_context_lookup_key(key: &str, usage: &str) -> Result<(), WorkflowError> {
843 ensure_definition_text(key, usage)?;
844 if key.split('.').any(str::is_empty) {
845 return Err(WorkflowError::Other(format!(
846 "{usage} must not contain empty dot-notation path segments"
847 )));
848 }
849 Ok(())
850}
851
852fn ensure_optional_user_context_key(
853 key: Option<&String>,
854 usage: &str,
855) -> Result<(), WorkflowError> {
856 if let Some(key) = key {
857 ensure_user_context_key(key, usage)?;
858 }
859 Ok(())
860}
861
862fn ensure_cursor_has_frames(cursor: &WorkflowCursor) -> Result<(), WorkflowError> {
863 if cursor.frames.is_empty() {
864 return Err(invalid_cursor("resume cursor has no frames"));
865 }
866 Ok(())
867}
868
869fn ensure_unique_branch_values(
870 condition_key: &str,
871 branches: &[(String, Vec<WorkflowStep>)],
872) -> Result<(), WorkflowError> {
873 let mut seen = std::collections::HashSet::new();
874 for (branch_value, _) in branches {
875 if !seen.insert(branch_value.as_str()) {
876 return Err(WorkflowError::Other(format!(
877 "Ambiguous workflow branch for '{condition_key}': duplicate value '{branch_value}'"
878 )));
879 }
880 }
881 Ok(())
882}
883
884fn ensure_unique_branch_conditions(
885 condition_key: &str,
886 branches: &[(WorkflowBranchCondition, Vec<WorkflowStep>)],
887) -> Result<(), WorkflowError> {
888 let mut seen = std::collections::HashSet::new();
889 for (condition, _) in branches {
890 validate_branch_condition(condition_key, condition)?;
891 if !seen.insert(condition) {
892 return Err(WorkflowError::Other(format!(
893 "Ambiguous workflow branch for '{condition_key}': duplicate condition '{condition:?}'"
894 )));
895 }
896 }
897 Ok(())
898}
899
900fn validate_branch_condition(
901 condition_key: &str,
902 condition: &WorkflowBranchCondition,
903) -> Result<(), WorkflowError> {
904 match condition {
905 WorkflowBranchCondition::OneOf(expected_values) if expected_values.is_empty() => {
906 Err(WorkflowError::Other(format!(
907 "Ambiguous workflow branch for '{condition_key}': OneOf condition must include at least one value"
908 )))
909 }
910 WorkflowBranchCondition::StringContains(needle) if needle.is_empty() => {
911 Err(WorkflowError::Other(format!(
912 "Ambiguous workflow branch for '{condition_key}': StringContains condition must not be empty"
913 )))
914 }
915 _ => Ok(()),
916 }
917}
918
919fn branch_condition_matches(condition: &WorkflowBranchCondition, value: Option<&Value>) -> bool {
920 match condition {
921 WorkflowBranchCondition::Exists => value.is_some(),
922 WorkflowBranchCondition::Missing => value.is_none(),
923 WorkflowBranchCondition::Null => matches!(value, Some(Value::Null)),
924 WorkflowBranchCondition::Bool(expected) => {
925 value.and_then(Value::as_bool) == Some(*expected)
926 }
927 WorkflowBranchCondition::Equals(expected) => {
928 value.map(branch_condition_text).as_deref() == Some(expected.as_str())
929 }
930 WorkflowBranchCondition::NotEquals(expected) => value
931 .map(branch_condition_text)
932 .is_some_and(|actual| actual != *expected),
933 WorkflowBranchCondition::OneOf(expected_values) => value
934 .map(branch_condition_text)
935 .is_some_and(|actual| expected_values.iter().any(|expected| expected == &actual)),
936 WorkflowBranchCondition::NumberGt(expected) => value
937 .and_then(|value| branch_condition_number_cmp(value, *expected))
938 .is_some_and(|ordering| ordering == Ordering::Greater),
939 WorkflowBranchCondition::NumberGte(expected) => value
940 .and_then(|value| branch_condition_number_cmp(value, *expected))
941 .is_some_and(|ordering| ordering != Ordering::Less),
942 WorkflowBranchCondition::NumberLt(expected) => value
943 .and_then(|value| branch_condition_number_cmp(value, *expected))
944 .is_some_and(|ordering| ordering == Ordering::Less),
945 WorkflowBranchCondition::NumberLte(expected) => value
946 .and_then(|value| branch_condition_number_cmp(value, *expected))
947 .is_some_and(|ordering| ordering != Ordering::Greater),
948 WorkflowBranchCondition::StringContains(needle) => value
949 .map(branch_condition_text)
950 .is_some_and(|actual| actual.contains(needle)),
951 }
952}
953
954fn branch_condition_text(value: &Value) -> String {
955 match value {
956 Value::String(value) => value.clone(),
957 _ => value.to_string(),
958 }
959}
960
961fn branch_condition_number_cmp(value: &Value, expected: i64) -> Option<Ordering> {
962 let Value::Number(number) = value else {
963 return None;
964 };
965
966 if let Some(actual) = number.as_i64() {
967 return Some(actual.cmp(&expected));
968 }
969
970 if let Some(actual) = number.as_u64() {
971 return Some(if expected < 0 {
972 Ordering::Greater
973 } else {
974 actual.cmp(&(expected as u64))
975 });
976 }
977
978 let actual = number.as_f64()?;
979 actual.partial_cmp(&(expected as f64))
980}
981
982fn find_single_transition<'a>(
983 definition: &'a WorkflowDefinition,
984 state: &str,
985) -> Result<Option<&'a StateTransition>, WorkflowError> {
986 let mut matches = definition.transitions.iter().filter(|t| t.from == state);
987 let Some(first) = matches.next() else {
988 return Ok(None);
989 };
990 if matches.next().is_some() {
991 let count = definition.transitions_from(state).len();
992 return Err(WorkflowError::Other(format!(
993 "Ambiguous workflow definition '{}': state '{state}' has {count} outgoing transitions; use a Branch step or split states explicitly",
994 definition.name
995 )));
996 }
997 Ok(Some(first))
998}
999
1000fn validate_workflow_definition(definition: &WorkflowDefinition) -> Result<(), WorkflowError> {
1001 ensure_definition_text(&definition.name, "Workflow definition name")?;
1002 ensure_definition_text(&definition.initial_state, "Workflow initial state")?;
1003 if let Some(version) = &definition.version {
1004 ensure_definition_text(version, "Workflow definition version")?;
1005 }
1006
1007 let mut seen_states = std::collections::HashMap::<&str, usize>::new();
1008 for transition in &definition.transitions {
1009 ensure_definition_text(&transition.from, "Workflow transition source state")?;
1010 ensure_definition_text(&transition.to, "Workflow transition target state")?;
1011 *seen_states.entry(&transition.from).or_default() += 1;
1012 validate_workflow_steps(&transition.steps)?;
1013 }
1014
1015 if let Some((state, count)) = seen_states.into_iter().find(|(_, count)| *count > 1) {
1016 return Err(WorkflowError::Other(format!(
1017 "Ambiguous workflow definition '{}': state '{state}' has {count} outgoing transitions; use a Branch step or split states explicitly",
1018 definition.name
1019 )));
1020 }
1021
1022 Ok(())
1023}
1024
1025fn ensure_context_definition(
1026 ctx: &mut WorkflowContext,
1027 definition: &WorkflowDefinition,
1028) -> Result<(), WorkflowError> {
1029 if let Some(context_name) = &ctx.definition_name
1030 && context_name != &definition.name
1031 {
1032 return Err(WorkflowError::Other(format!(
1033 "Workflow context '{}' belongs to definition '{context_name}', not '{}'",
1034 ctx.workflow_id, definition.name
1035 )));
1036 }
1037
1038 if let Some(context_version) = &ctx.definition_version
1039 && Some(context_version) != definition.version.as_ref()
1040 {
1041 return Err(WorkflowError::Other(format!(
1042 "Workflow context '{}' uses definition version '{}', not '{}'",
1043 ctx.workflow_id,
1044 context_version,
1045 definition.version.as_deref().unwrap_or("<unversioned>")
1046 )));
1047 }
1048
1049 if ctx.definition_name.is_none() {
1050 ctx.definition_name = Some(definition.name.clone());
1051 ctx.updated_at = Utc::now();
1052 }
1053 if ctx.definition_version.is_none()
1054 && let Some(version) = &definition.version
1055 {
1056 ctx.definition_version = Some(version.clone());
1057 ctx.updated_at = Utc::now();
1058 }
1059
1060 Ok(())
1061}
1062
1063fn validate_workflow_steps(steps: &[WorkflowStep]) -> Result<(), WorkflowError> {
1064 for (idx, step) in steps.iter().enumerate() {
1065 ensure_step_position(step, idx, steps.len())?;
1066 match step {
1067 WorkflowStep::Query { cmd_json, store_as } => {
1068 normalize_query_wire_for_execution(cmd_json)?;
1069 ensure_optional_user_context_key(store_as.as_ref(), "Query store_as")?;
1070 }
1071 WorkflowStep::Wait {
1072 event, on_timeout, ..
1073 } => {
1074 ensure_wait_event_name(event)?;
1075 if steps_contain_wait(on_timeout) {
1076 return Err(WorkflowError::Other(
1077 "Wait steps inside on_timeout fallback are not supported".to_string(),
1078 ));
1079 }
1080 validate_workflow_steps(on_timeout)?;
1081 }
1082 WorkflowStep::Branch {
1083 condition_key,
1084 branches,
1085 default,
1086 } => {
1087 ensure_context_lookup_key(condition_key, "Branch condition_key")?;
1088 ensure_unique_branch_values(condition_key, branches)?;
1089 for (_, branch_steps) in branches {
1090 validate_workflow_steps(branch_steps)?;
1091 }
1092 validate_workflow_steps(default)?;
1093 }
1094 WorkflowStep::BranchWhen {
1095 condition_key,
1096 branches,
1097 default,
1098 } => {
1099 ensure_context_lookup_key(condition_key, "BranchWhen condition_key")?;
1100 ensure_unique_branch_conditions(condition_key, branches)?;
1101 for (_, branch_steps) in branches {
1102 validate_workflow_steps(branch_steps)?;
1103 }
1104 validate_workflow_steps(default)?;
1105 }
1106 WorkflowStep::ForEach { list_key, steps } => {
1107 ensure_context_lookup_key(list_key, "ForEach list_key")?;
1108 validate_workflow_steps(steps)?;
1109 }
1110 WorkflowStep::Charge {
1111 amount_key,
1112 reference_key,
1113 description_key,
1114 payment_method_key,
1115 order_origin_key,
1116 store_as,
1117 ..
1118 } => {
1119 ensure_context_lookup_key(amount_key, "Charge amount_key")?;
1120 ensure_context_lookup_key(reference_key, "Charge reference_key")?;
1121 if let Some(description_key) = description_key {
1122 ensure_context_lookup_key(description_key, "Charge description_key")?;
1123 }
1124 if let Some(payment_method_key) = payment_method_key {
1125 ensure_context_lookup_key(payment_method_key, "Charge payment_method_key")?;
1126 }
1127 if let Some(order_origin_key) = order_origin_key {
1128 ensure_context_lookup_key(order_origin_key, "Charge order_origin_key")?;
1129 }
1130 ensure_optional_user_context_key(store_as.as_ref(), "Charge store_as")?;
1131 }
1132 WorkflowStep::Notify {
1133 template,
1134 recipient_key,
1135 payload_key,
1136 ..
1137 } => {
1138 ensure_definition_text(template, "Notify template")?;
1139 ensure_context_lookup_key(recipient_key, "Notify recipient_key")?;
1140 if let Some(payload_key) = payload_key {
1141 ensure_context_lookup_key(payload_key, "Notify payload_key")?;
1142 }
1143 }
1144 WorkflowStep::Transition { to } => {
1145 ensure_definition_text(to, "Transition target state")?;
1146 }
1147 WorkflowStep::Log { .. } => {}
1148 }
1149 }
1150
1151 Ok(())
1152}
1153
1154fn deadline_from_timeout(timeout: &std::time::Duration) -> Result<DateTime<Utc>, WorkflowError> {
1155 let timeout = chrono::Duration::from_std(*timeout).map_err(|_| {
1156 WorkflowError::Other("Wait timeout duration is too large to represent".to_string())
1157 })?;
1158 Utc::now()
1159 .checked_add_signed(timeout)
1160 .ok_or_else(|| WorkflowError::Other("Wait timeout deadline overflowed".to_string()))
1161}
1162
1163fn cursor_frame_for_index(kind: &StepListCursorKind, index: usize) -> WorkflowCursorFrame {
1164 match kind {
1165 StepListCursorKind::Steps => WorkflowCursorFrame::Steps { index },
1166 StepListCursorKind::Branch { selection } => WorkflowCursorFrame::Branch {
1167 selection: selection.clone(),
1168 index,
1169 },
1170 StepListCursorKind::ForEach { item_index, item } => WorkflowCursorFrame::ForEachItem {
1171 item_index: *item_index,
1172 index,
1173 item: item.clone(),
1174 },
1175 }
1176}
1177
1178fn cursor_frames_for_index(
1179 path_prefix: &[WorkflowCursorFrame],
1180 kind: &StepListCursorKind,
1181 index: usize,
1182) -> Vec<WorkflowCursorFrame> {
1183 let mut frames = Vec::with_capacity(path_prefix.len() + 1);
1184 frames.extend_from_slice(path_prefix);
1185 frames.push(cursor_frame_for_index(kind, index));
1186 frames
1187}
1188
1189async fn checkpoint_cursor<E: WorkflowExecutor>(
1190 executor: &E,
1191 ctx: &mut WorkflowContext,
1192 state: &str,
1193 frames: Vec<WorkflowCursorFrame>,
1194 wait: Option<WorkflowPendingWait>,
1195 timeout_fallback: bool,
1196) -> Result<(), WorkflowError> {
1197 if !timeout_fallback {
1198 clear_timeout_fallback(ctx);
1199 }
1200 let wait = if timeout_fallback && wait.is_none() {
1201 timeout_fallback_from_context(ctx)?
1202 } else {
1203 wait
1204 };
1205 if ctx.current_state == state {
1206 ctx.set_cursor(WorkflowCursor {
1207 state: state.to_string(),
1208 frames,
1209 wait,
1210 });
1211 } else {
1212 ctx.clear_cursor();
1213 if timeout_fallback {
1214 clear_timeout_fallback(ctx);
1215 }
1216 }
1217 executor.save_state(ctx).await
1218}
1219
1220async fn checkpoint_completed_step<E: WorkflowExecutor>(
1221 executor: &E,
1222 ctx: &mut WorkflowContext,
1223 state: &str,
1224 path_prefix: &[WorkflowCursorFrame],
1225 kind: &StepListCursorKind,
1226 next_index: usize,
1227 timeout_fallback: bool,
1228) -> Result<(), WorkflowError> {
1229 let frames = cursor_frames_for_index(path_prefix, kind, next_index);
1230 checkpoint_cursor(executor, ctx, state, frames, None, timeout_fallback).await
1231}
1232
1233fn restore_for_each_item(ctx: &mut WorkflowContext, previous_item: Option<Value>) {
1234 match previous_item {
1235 Some(item) => {
1236 ctx.data.insert(FOR_EACH_ITEM_KEY.to_string(), item);
1237 }
1238 None => {
1239 ctx.data.remove(FOR_EACH_ITEM_KEY);
1240 }
1241 }
1242 ctx.updated_at = Utc::now();
1243}
1244
1245fn side_effect_operation(
1246 ctx: &WorkflowContext,
1247 scope: &StepExecutionScope<'_>,
1248 step_index: usize,
1249 kind: WorkflowSideEffectKind,
1250) -> WorkflowSideEffect {
1251 let frames = cursor_frames_for_index(&scope.path_prefix, &scope.list_kind, step_index);
1252 let step_path = render_cursor_path(&frames);
1253 WorkflowSideEffect {
1254 workflow_id: ctx.workflow_id.clone(),
1255 state: scope.state.to_string(),
1256 step_path: step_path.clone(),
1257 kind,
1258 operation_id: side_effect_operation_id(
1259 &ctx.workflow_id,
1260 scope.state,
1261 ctx.transition_count(),
1262 kind,
1263 &step_path,
1264 ),
1265 }
1266}
1267
1268fn side_effect_operation_id(
1269 workflow_id: &str,
1270 state: &str,
1271 state_generation: usize,
1272 kind: WorkflowSideEffectKind,
1273 step_path: &str,
1274) -> String {
1275 serde_json::json!([
1276 "qail-workflow-side-effect",
1277 2,
1278 workflow_id,
1279 state,
1280 state_generation,
1281 kind.as_str(),
1282 step_path
1283 ])
1284 .to_string()
1285}
1286
1287fn render_cursor_path(frames: &[WorkflowCursorFrame]) -> String {
1288 frames
1289 .iter()
1290 .map(render_cursor_frame)
1291 .collect::<Vec<_>>()
1292 .join("/")
1293}
1294
1295fn render_cursor_frame(frame: &WorkflowCursorFrame) -> String {
1296 match frame {
1297 WorkflowCursorFrame::Steps { index } => format!("steps[{index}]"),
1298 WorkflowCursorFrame::Branch { selection, index } => {
1299 format!(
1300 "branch[{}].steps[{index}]",
1301 render_branch_selection(selection)
1302 )
1303 }
1304 WorkflowCursorFrame::ForEach { item_index, index } => {
1305 format!("for_each[{item_index}].steps[{index}]")
1306 }
1307 WorkflowCursorFrame::ForEachItem {
1308 item_index, index, ..
1309 } => format!("for_each[{item_index}].steps[{index}]"),
1310 }
1311}
1312
1313fn render_branch_selection(selection: &WorkflowBranchCursorSelection) -> String {
1314 match selection {
1315 WorkflowBranchCursorSelection::Branch(index) => index.to_string(),
1316 WorkflowBranchCursorSelection::Default => "default".to_string(),
1317 }
1318}
1319
1320fn log_value_to_string(value: &Value) -> String {
1321 match value {
1322 Value::String(value) => value.clone(),
1323 _ => value.to_string(),
1324 }
1325}
1326
1327fn resolve_log_message(ctx: &WorkflowContext, message: &str) -> String {
1328 let mut resolved = String::with_capacity(message.len());
1329 let mut rest = message;
1330
1331 while let Some(start) = rest.find('{') {
1332 resolved.push_str(&rest[..start]);
1333 let after_open = &rest[start + 1..];
1334 let Some(end) = after_open.find('}') else {
1335 resolved.push_str(&rest[start..]);
1336 return resolved;
1337 };
1338
1339 let key = &after_open[..end];
1340 if !key.is_empty()
1341 && let Some(value) = ctx.get(key)
1342 {
1343 resolved.push_str(&log_value_to_string(value));
1344 } else {
1345 resolved.push('{');
1346 resolved.push_str(key);
1347 resolved.push('}');
1348 }
1349 rest = &after_open[end + 1..];
1350 }
1351
1352 resolved.push_str(rest);
1353 resolved
1354}
1355
1356fn execute_steps<'a, E: WorkflowExecutor>(
1357 executor: &'a E,
1358 steps: &'a [WorkflowStep],
1359 ctx: &'a mut WorkflowContext,
1360 start_index: usize,
1361 cursor_frames: &'a [WorkflowCursorFrame],
1362 scope: StepExecutionScope<'a>,
1363) -> std::pin::Pin<
1364 Box<dyn std::future::Future<Output = Result<Option<StepPause>, WorkflowError>> + Send + 'a>,
1365> {
1366 Box::pin(async move {
1367 if start_index > steps.len() {
1368 return Err(invalid_cursor(format!(
1369 "step index {start_index} is past step count {}",
1370 steps.len()
1371 )));
1372 }
1373 if start_index == steps.len() && !cursor_frames.is_empty() {
1374 return Err(invalid_cursor(
1375 "cursor contains child frames after the step list ended",
1376 ));
1377 }
1378
1379 for (idx, step) in steps.iter().enumerate().skip(start_index) {
1380 ensure_step_position(step, idx, steps.len())?;
1381 let step_cursor = if idx == start_index {
1382 cursor_frames
1383 } else {
1384 &[]
1385 };
1386 match execute_step(executor, step, ctx, idx, step_cursor, &scope).await? {
1387 StepOutcome::Continue => {
1388 if scope.checkpoint_steps {
1389 checkpoint_completed_step(
1390 executor,
1391 ctx,
1392 scope.state,
1393 &scope.path_prefix,
1394 &scope.list_kind,
1395 idx + 1,
1396 scope.checkpoint_timeout,
1397 )
1398 .await?;
1399 }
1400 if ctx.current_state != scope.state {
1401 return Ok(None);
1402 }
1403 }
1404 StepOutcome::Paused(pause) => return Ok(Some(pause)),
1405 }
1406 }
1407
1408 Ok(None)
1409 })
1410}
1411
1412fn selected_branch_steps<'a>(
1413 branches: &'a [(String, Vec<WorkflowStep>)],
1414 default: &'a [WorkflowStep],
1415 selection: &WorkflowBranchCursorSelection,
1416) -> Result<&'a [WorkflowStep], WorkflowError> {
1417 match selection {
1418 WorkflowBranchCursorSelection::Branch(idx) => branches
1419 .get(*idx)
1420 .map(|(_, steps)| steps.as_slice())
1421 .ok_or_else(|| invalid_cursor(format!("branch index {idx} no longer exists"))),
1422 WorkflowBranchCursorSelection::Default => Ok(default),
1423 }
1424}
1425
1426fn selected_branch_when_steps<'a>(
1427 branches: &'a [(WorkflowBranchCondition, Vec<WorkflowStep>)],
1428 default: &'a [WorkflowStep],
1429 selection: &WorkflowBranchCursorSelection,
1430) -> Result<&'a [WorkflowStep], WorkflowError> {
1431 match selection {
1432 WorkflowBranchCursorSelection::Branch(idx) => branches
1433 .get(*idx)
1434 .map(|(_, steps)| steps.as_slice())
1435 .ok_or_else(|| invalid_cursor(format!("branch index {idx} no longer exists"))),
1436 WorkflowBranchCursorSelection::Default => Ok(default),
1437 }
1438}
1439
1440fn validate_branch_resume_selection(
1441 branches: &[(String, Vec<WorkflowStep>)],
1442 selection: &WorkflowBranchCursorSelection,
1443 condition_key: &str,
1444 condition_value: &str,
1445) -> Result<(), WorkflowError> {
1446 match selection {
1447 WorkflowBranchCursorSelection::Branch(idx) => {
1448 let Some((branch_value, _)) = branches.get(*idx) else {
1449 return Err(invalid_cursor(format!(
1450 "branch index {idx} no longer exists"
1451 )));
1452 };
1453 if branch_value != condition_value {
1454 return Err(invalid_cursor(format!(
1455 "branch cursor selected index {idx} for value '{branch_value}', \
1456 but current condition '{condition_key}' is '{condition_value}'"
1457 )));
1458 }
1459 }
1460 WorkflowBranchCursorSelection::Default => {
1461 if branches
1462 .iter()
1463 .any(|(branch_value, _)| branch_value == condition_value)
1464 {
1465 return Err(invalid_cursor(format!(
1466 "default branch cursor no longer matches condition '{condition_key}' \
1467 value '{condition_value}'"
1468 )));
1469 }
1470 }
1471 }
1472 Ok(())
1473}
1474
1475fn validate_branch_when_resume_selection(
1476 branches: &[(WorkflowBranchCondition, Vec<WorkflowStep>)],
1477 selection: &WorkflowBranchCursorSelection,
1478 condition_key: &str,
1479 condition_value: Option<&Value>,
1480) -> Result<(), WorkflowError> {
1481 match selection {
1482 WorkflowBranchCursorSelection::Branch(idx) => {
1483 let Some((condition, _)) = branches.get(*idx) else {
1484 return Err(invalid_cursor(format!(
1485 "branch index {idx} no longer exists"
1486 )));
1487 };
1488 if !branch_condition_matches(condition, condition_value) {
1489 return Err(invalid_cursor(format!(
1490 "branch cursor selected index {idx} for condition '{condition:?}', \
1491 but current condition '{condition_key}' no longer matches"
1492 )));
1493 }
1494 }
1495 WorkflowBranchCursorSelection::Default => {
1496 if branches
1497 .iter()
1498 .any(|(condition, _)| branch_condition_matches(condition, condition_value))
1499 {
1500 return Err(invalid_cursor(format!(
1501 "default branch cursor no longer matches condition '{condition_key}'"
1502 )));
1503 }
1504 }
1505 }
1506 Ok(())
1507}
1508
1509fn execute_step<'a, E: WorkflowExecutor>(
1511 executor: &'a E,
1512 step: &'a WorkflowStep,
1513 ctx: &'a mut WorkflowContext,
1514 step_index: usize,
1515 cursor_frames: &'a [WorkflowCursorFrame],
1516 scope: &'a StepExecutionScope<'a>,
1517) -> std::pin::Pin<
1518 Box<dyn std::future::Future<Output = Result<StepOutcome, WorkflowError>> + Send + 'a>,
1519> {
1520 Box::pin(async move {
1521 match step {
1522 WorkflowStep::Query { cmd_json, store_as } => {
1523 ensure_no_child_cursor("Query", cursor_frames)?;
1524 ensure_optional_user_context_key(store_as.as_ref(), "Query store_as")?;
1525 let cmd_wire = normalize_query_wire_for_execution(cmd_json)?;
1526 let operation =
1527 side_effect_operation(ctx, scope, step_index, WorkflowSideEffectKind::Query);
1528 let result = executor.execute_query_once(&operation, &cmd_wire).await?;
1529 if let Some(key) = store_as {
1530 ctx.set(key, result);
1531 }
1532 }
1533
1534 WorkflowStep::Notify {
1535 channel,
1536 template,
1537 recipient_key,
1538 payload_key,
1539 } => {
1540 ensure_no_child_cursor("Notify", cursor_frames)?;
1541 let recipient = ctx
1542 .get_str(recipient_key)
1543 .ok_or_else(|| WorkflowError::MissingContextKey(recipient_key.clone()))?
1544 .to_string();
1545
1546 let params = payload_key
1547 .as_ref()
1548 .map(|key| {
1549 ctx.get(key)
1550 .cloned()
1551 .ok_or_else(|| WorkflowError::MissingContextKey(key.clone()))
1552 })
1553 .transpose()?
1554 .unwrap_or(Value::Object(serde_json::Map::new()));
1555
1556 let operation =
1557 side_effect_operation(ctx, scope, step_index, WorkflowSideEffectKind::Notify);
1558 executor
1559 .send_notification_once(&operation, channel, &recipient, template, ¶ms)
1560 .await?;
1561 }
1562
1563 WorkflowStep::Wait {
1564 event,
1565 timeout,
1566 on_timeout,
1567 } => {
1568 ensure_no_child_cursor("Wait", cursor_frames)?;
1569 ensure_wait_event_name(event)?;
1570 return Ok(StepOutcome::Paused(StepPause {
1573 frames: cursor_frames_for_index(
1574 &scope.path_prefix,
1575 &scope.list_kind,
1576 step_index + 1,
1577 ),
1578 wait: WorkflowPendingWait {
1579 event: event.clone(),
1580 deadline_at: deadline_from_timeout(timeout)?,
1581 on_timeout: on_timeout.clone(),
1582 },
1583 }));
1584 }
1585
1586 WorkflowStep::Branch {
1587 condition_key,
1588 branches,
1589 default,
1590 } => {
1591 ensure_unique_branch_values(condition_key, branches)?;
1592 let (selection, selected_steps, start_index, nested_cursor) = match cursor_frames
1593 .first()
1594 {
1595 Some(WorkflowCursorFrame::Branch { selection, index }) => {
1596 let condition_value = ctx
1597 .get_str(condition_key)
1598 .ok_or_else(|| WorkflowError::MissingContextKey(condition_key.clone()))?
1599 .to_string();
1600 validate_branch_resume_selection(
1601 branches,
1602 selection,
1603 condition_key,
1604 &condition_value,
1605 )?;
1606 (
1607 selection.clone(),
1608 selected_branch_steps(branches, default, selection)?,
1609 *index,
1610 &cursor_frames[1..],
1611 )
1612 }
1613 Some(_) => {
1614 return Err(invalid_cursor(
1615 "expected Branch frame for nested branch resume",
1616 ));
1617 }
1618 None => {
1619 let value = ctx
1620 .get_str(condition_key)
1621 .ok_or_else(|| WorkflowError::MissingContextKey(condition_key.clone()))?
1622 .to_string();
1623 match branches.iter().enumerate().find(|(_, (k, _))| k == &value) {
1624 Some((idx, (_, steps))) => (
1625 WorkflowBranchCursorSelection::Branch(idx),
1626 steps.as_slice(),
1627 0,
1628 &[][..],
1629 ),
1630 None => (
1631 WorkflowBranchCursorSelection::Default,
1632 default.as_slice(),
1633 0,
1634 &[][..],
1635 ),
1636 }
1637 }
1638 };
1639
1640 if let Some(pause) = execute_steps(
1641 executor,
1642 selected_steps,
1643 ctx,
1644 start_index,
1645 nested_cursor,
1646 scope.child(step_index, StepListCursorKind::Branch { selection }),
1647 )
1648 .await?
1649 {
1650 return Ok(StepOutcome::Paused(pause));
1651 }
1652 }
1653
1654 WorkflowStep::BranchWhen {
1655 condition_key,
1656 branches,
1657 default,
1658 } => {
1659 ensure_unique_branch_conditions(condition_key, branches)?;
1660 let condition_value = ctx.get(condition_key).cloned();
1661 let (selection, selected_steps, start_index, nested_cursor) =
1662 match cursor_frames.first() {
1663 Some(WorkflowCursorFrame::Branch { selection, index }) => {
1664 validate_branch_when_resume_selection(
1665 branches,
1666 selection,
1667 condition_key,
1668 condition_value.as_ref(),
1669 )?;
1670 (
1671 selection.clone(),
1672 selected_branch_when_steps(branches, default, selection)?,
1673 *index,
1674 &cursor_frames[1..],
1675 )
1676 }
1677 Some(_) => {
1678 return Err(invalid_cursor(
1679 "expected Branch frame for nested branch resume",
1680 ));
1681 }
1682 None => match branches.iter().enumerate().find(|(_, (condition, _))| {
1683 branch_condition_matches(condition, condition_value.as_ref())
1684 }) {
1685 Some((idx, (_, steps))) => (
1686 WorkflowBranchCursorSelection::Branch(idx),
1687 steps.as_slice(),
1688 0,
1689 &[][..],
1690 ),
1691 None => (
1692 WorkflowBranchCursorSelection::Default,
1693 default.as_slice(),
1694 0,
1695 &[][..],
1696 ),
1697 },
1698 };
1699
1700 if let Some(pause) = execute_steps(
1701 executor,
1702 selected_steps,
1703 ctx,
1704 start_index,
1705 nested_cursor,
1706 scope.child(step_index, StepListCursorKind::Branch { selection }),
1707 )
1708 .await?
1709 {
1710 return Ok(StepOutcome::Paused(pause));
1711 }
1712 }
1713
1714 WorkflowStep::ForEach { list_key, steps } => {
1715 let list = ctx
1716 .get(list_key)
1717 .cloned()
1718 .ok_or_else(|| WorkflowError::MissingContextKey(list_key.clone()))?;
1719
1720 let Value::Array(items) = list else {
1721 return Err(WorkflowError::Other(format!(
1722 "Expected JSON array for ForEach list '{}', got {}",
1723 list_key,
1724 json_type_name(&list)
1725 )));
1726 };
1727
1728 let (start_item_index, start_step_index, nested_cursor) = match cursor_frames
1729 .first()
1730 {
1731 Some(WorkflowCursorFrame::ForEachItem {
1732 item_index,
1733 index,
1734 item,
1735 }) => {
1736 if *item_index >= items.len() {
1737 return Err(invalid_cursor(format!(
1738 "for_each item index {item_index} is past item count {}",
1739 items.len()
1740 )));
1741 }
1742 if items.get(*item_index) != Some(item) {
1743 return Err(invalid_cursor(format!(
1744 "for_each cursor item at index {item_index} changed while the workflow was paused"
1745 )));
1746 }
1747 (*item_index, *index, &cursor_frames[1..])
1748 }
1749 Some(WorkflowCursorFrame::ForEach { .. }) => {
1750 return Err(invalid_cursor(
1751 "legacy for_each cursor without item snapshot cannot be resumed safely",
1752 ));
1753 }
1754 Some(_) => {
1755 return Err(invalid_cursor(
1756 "expected ForEach frame for nested loop resume",
1757 ));
1758 }
1759 None => (0, 0, &[][..]),
1760 };
1761
1762 for (item_index, item) in items.into_iter().enumerate().skip(start_item_index) {
1763 let item_snapshot = item.clone();
1764 let previous_item = ctx.data.insert(FOR_EACH_ITEM_KEY.to_string(), item);
1765 ctx.updated_at = Utc::now();
1766 let item_step_start = if item_index == start_item_index {
1767 start_step_index
1768 } else {
1769 0
1770 };
1771 let item_cursor = if item_index == start_item_index {
1772 nested_cursor
1773 } else {
1774 &[]
1775 };
1776
1777 match execute_steps(
1778 executor,
1779 steps,
1780 ctx,
1781 item_step_start,
1782 item_cursor,
1783 scope.child(
1784 step_index,
1785 StepListCursorKind::ForEach {
1786 item_index,
1787 item: item_snapshot,
1788 },
1789 ),
1790 )
1791 .await
1792 {
1793 Ok(Some(pause)) => {
1794 restore_for_each_item(ctx, previous_item);
1795 return Ok(StepOutcome::Paused(pause));
1796 }
1797 Ok(None) => {
1798 restore_for_each_item(ctx, previous_item);
1799 }
1800 Err(err) => {
1801 restore_for_each_item(ctx, previous_item);
1802 return Err(err);
1803 }
1804 }
1805 }
1806 }
1807
1808 WorkflowStep::Transition { to } => {
1809 ensure_no_child_cursor("Transition", cursor_frames)?;
1810 ctx.transition_to(to, None);
1811 }
1812
1813 WorkflowStep::Log { message } => {
1814 ensure_no_child_cursor("Log", cursor_frames)?;
1815 let resolved = resolve_log_message(ctx, message);
1816 eprintln!("[workflow:{}] {}", ctx.workflow_id, resolved);
1818 }
1819
1820 WorkflowStep::Charge {
1821 provider,
1822 amount_key,
1823 reference_key,
1824 description_key,
1825 payment_method_key,
1826 order_origin_key,
1827 store_as,
1828 } => {
1829 ensure_no_child_cursor("Charge", cursor_frames)?;
1830 ensure_optional_user_context_key(store_as.as_ref(), "Charge store_as")?;
1831 let amount = resolve_charge_amount(ctx, amount_key)?;
1832
1833 let reference_id = ctx
1834 .get_str(reference_key)
1835 .ok_or_else(|| WorkflowError::MissingContextKey(reference_key.clone()))?
1836 .to_string();
1837
1838 let description = resolve_optional_charge_string(ctx, description_key.as_ref())?;
1839 let payment_method =
1840 resolve_optional_charge_string(ctx, payment_method_key.as_ref())?;
1841 let order_origin = resolve_optional_order_origin(ctx, order_origin_key.as_ref())?;
1842
1843 let request = ChargeRequest {
1844 amount,
1845 currency: Currency::default(),
1846 reference_id,
1847 description,
1848 payment_method,
1849 order_origin,
1850 idempotency_key: None,
1851 return_url: None,
1852 metadata: None,
1853 };
1854
1855 let operation =
1856 side_effect_operation(ctx, scope, step_index, WorkflowSideEffectKind::Charge);
1857 let response = executor
1858 .create_charge_once(&operation, provider, request.clone())
1859 .await?;
1860
1861 if let Some(key) = store_as {
1862 let response_json = serde_json::to_value(response.display_for(&request))
1863 .map_err(|e| WorkflowError::Other(e.to_string()))?;
1864 ctx.set(key, response_json);
1865 }
1866 }
1867 }
1868
1869 Ok(StepOutcome::Continue)
1870 })
1871}
1872
1873fn resolve_charge_amount(ctx: &WorkflowContext, amount_key: &str) -> Result<i64, WorkflowError> {
1874 let value = ctx.get(amount_key).ok_or_else(|| {
1875 WorkflowError::MissingContextKey(format!("{amount_key} (expected numeric amount)"))
1876 })?;
1877
1878 let amount = match value {
1879 Value::Number(number) => charge_amount_from_number(number, amount_key)?,
1880 _ => {
1881 return Err(WorkflowError::MissingContextKey(format!(
1882 "{amount_key} (expected numeric amount)"
1883 )));
1884 }
1885 };
1886
1887 if amount <= 0 {
1888 return Err(invalid_charge_amount(
1889 amount_key,
1890 "amount must be greater than zero",
1891 ));
1892 }
1893
1894 Ok(amount)
1895}
1896
1897fn resolve_optional_charge_string(
1898 ctx: &WorkflowContext,
1899 key: Option<&String>,
1900) -> Result<Option<String>, WorkflowError> {
1901 let Some(key) = key else {
1902 return Ok(None);
1903 };
1904 ctx.get_str(key)
1905 .map(|value| Some(value.to_string()))
1906 .ok_or_else(|| WorkflowError::MissingContextKey(format!("{key} (expected string)")))
1907}
1908
1909fn resolve_optional_order_origin(
1910 ctx: &WorkflowContext,
1911 key: Option<&String>,
1912) -> Result<Option<OrderOrigin>, WorkflowError> {
1913 let Some(key) = key else {
1914 return Ok(None);
1915 };
1916 let value = ctx.get_str(key).ok_or_else(|| {
1917 WorkflowError::MissingContextKey(format!("{key} (expected order origin)"))
1918 })?;
1919
1920 OrderOrigin::parse(value).map(Some).ok_or_else(|| {
1921 WorkflowError::Other(format!(
1922 "Charge order_origin_key '{key}' must resolve to one of: whatsapp, mcp, web, ios_app, android_app, api"
1923 ))
1924 })
1925}
1926
1927fn charge_amount_from_number(
1928 number: &serde_json::Number,
1929 amount_key: &str,
1930) -> Result<i64, WorkflowError> {
1931 if let Some(value) = number.as_i64() {
1932 return Ok(value);
1933 }
1934
1935 if let Some(value) = number.as_u64() {
1936 return i64::try_from(value).map_err(|_| {
1937 invalid_charge_amount(amount_key, "integer amount must fit in signed 64-bit range")
1938 });
1939 }
1940
1941 if let Some(value) = number.as_f64() {
1942 const MAX_SAFE_JSON_INTEGER: f64 = 9_007_199_254_740_991.0;
1943
1944 if !value.is_finite() {
1945 return Err(invalid_charge_amount(
1946 amount_key,
1947 "float amount must be finite",
1948 ));
1949 }
1950
1951 if value.fract() != 0.0 {
1952 return Err(invalid_charge_amount(
1953 amount_key,
1954 "float amount must be an integer in the smallest currency unit",
1955 ));
1956 }
1957
1958 if !(-MAX_SAFE_JSON_INTEGER..=MAX_SAFE_JSON_INTEGER).contains(&value) {
1959 return Err(invalid_charge_amount(
1960 amount_key,
1961 "float amount exceeds JSON safe integer range",
1962 ));
1963 }
1964
1965 return Ok(value as i64);
1966 }
1967
1968 Err(invalid_charge_amount(
1969 amount_key,
1970 "amount number is not representable",
1971 ))
1972}
1973
1974fn invalid_charge_amount(amount_key: &str, message: &str) -> WorkflowError {
1975 WorkflowError::Other(format!(
1976 "Invalid charge amount at '{amount_key}': {message}"
1977 ))
1978}
1979
1980fn normalize_query_wire_for_execution(cmd_json: &str) -> Result<String, WorkflowError> {
1981 if !is_current_workflow_query_wire(cmd_json) {
1982 return Err(WorkflowError::QueryFailed(
1983 "Legacy workflow query payload detected: cmd_json must use QAIL wire text \
1984 (QAIL-CMD/1). Migrate persisted workflow rows to wire text or purge/restart pending workflows."
1985 .to_string(),
1986 ));
1987 }
1988
1989 let cmd = qail_core::wire::decode_cmd_text(cmd_json).map_err(|e| {
1990 WorkflowError::QueryFailed(format!(
1991 "Invalid workflow query wire payload (expected QAIL-CMD/1): {}",
1992 e
1993 ))
1994 })?;
1995
1996 Ok(qail_core::wire::encode_cmd_text(&cmd))
1998}
1999
2000pub async fn run_workflow<E: WorkflowExecutor>(
2021 executor: &E,
2022 definition: &WorkflowDefinition,
2023 ctx: &mut WorkflowContext,
2024) -> Result<String, WorkflowError> {
2025 run_workflow_with_options(executor, definition, ctx, WorkflowRunOptions::default()).await
2026}
2027
2028pub async fn run_workflow_with_options<E: WorkflowExecutor>(
2030 executor: &E,
2031 definition: &WorkflowDefinition,
2032 ctx: &mut WorkflowContext,
2033 options: WorkflowRunOptions,
2034) -> Result<String, WorkflowError> {
2035 validate_workflow_definition(definition)?;
2036 validate_workflow_context_identity(ctx)?;
2037 let guard = match RuntimeGuard::enter(
2038 executor,
2039 &definition.name,
2040 &ctx.workflow_id,
2041 WorkflowOperationKind::Run,
2042 options,
2043 )
2044 .await?
2045 {
2046 RuntimeEntry::Active(guard) => guard,
2047 RuntimeEntry::Completed(state) => return Ok(state),
2048 };
2049
2050 let result = run_workflow_inner(executor, definition, ctx, RunMode::Normal).await;
2051 finish_runtime_operation(executor, guard, result).await
2052}
2053
2054async fn run_workflow_inner<'a, E: WorkflowExecutor>(
2055 executor: &E,
2056 definition: &WorkflowDefinition,
2057 ctx: &mut WorkflowContext,
2058 mode: RunMode<'a>,
2059) -> Result<String, WorkflowError> {
2060 validate_workflow_definition(definition)?;
2061 validate_workflow_context_identity(ctx)?;
2062 ensure_context_definition(ctx, definition)?;
2063 let run_start_transition_count = ctx.transition_count();
2064 let mut pending_cursor_frames = match ctx.cursor.clone() {
2065 Some(cursor) if cursor.state == ctx.current_state => {
2066 ensure_cursor_has_frames(&cursor)?;
2067 if timeout_fallback_from_context(ctx)?.is_some() {
2068 return Err(WorkflowError::Other(
2069 "Workflow is executing a timeout fallback; resume with timeout_workflow"
2070 .to_string(),
2071 ));
2072 }
2073 match (&cursor.wait, mode) {
2074 (Some(wait), RunMode::Normal) => {
2075 return Err(WorkflowError::Other(format!(
2076 "Workflow is paused awaiting event '{}'; resume with a matching event",
2077 wait.event
2078 )));
2079 }
2080 (Some(wait), RunMode::EventResume { event }) => {
2081 if wait.event != event {
2082 return Err(WorkflowError::Other(format!(
2083 "Workflow is waiting for event '{}', received '{}'",
2084 wait.event, event
2085 )));
2086 }
2087 if Utc::now() > wait.deadline_at {
2088 return Err(WorkflowError::Timeout {
2089 event: wait.event.clone(),
2090 });
2091 }
2092 ctx.clear_cursor();
2093 Some(cursor.frames)
2094 }
2095 (None, RunMode::EventResume { .. }) => {
2096 return Err(WorkflowError::Other(
2097 "Workflow is not waiting for an external event".to_string(),
2098 ));
2099 }
2100 (None, RunMode::Normal) => {
2101 ctx.clear_cursor();
2102 Some(cursor.frames)
2103 }
2104 }
2105 }
2106 Some(_) => match mode {
2107 RunMode::Normal => {
2108 ctx.clear_cursor();
2109 None
2110 }
2111 RunMode::EventResume { .. } => {
2112 return Err(invalid_cursor(
2113 "resume cursor state does not match current workflow state",
2114 ));
2115 }
2116 },
2117 None => match mode {
2118 RunMode::Normal => None,
2119 RunMode::EventResume { .. } => {
2120 return Err(WorkflowError::Other(
2121 "Workflow is not waiting for an external event".to_string(),
2122 ));
2123 }
2124 },
2125 };
2126
2127 loop {
2128 let transition = find_single_transition(definition, &ctx.current_state)?;
2130
2131 let transition = match transition {
2132 Some(t) => t,
2133 None => {
2134 executor.save_state(ctx).await?;
2136 return Ok(ctx.current_state.clone());
2137 }
2138 };
2139
2140 if ctx
2142 .transition_count()
2143 .saturating_sub(run_start_transition_count)
2144 >= 50
2145 {
2146 return Err(WorkflowError::Other(
2147 "Maximum transition count exceeded (50)".into(),
2148 ));
2149 }
2150
2151 let cursor_frames = pending_cursor_frames.take().unwrap_or_default();
2152 let (start_index, nested_cursor) = match cursor_frames.first() {
2153 Some(WorkflowCursorFrame::Steps { index }) => (*index, &cursor_frames[1..]),
2154 Some(_) => {
2155 return Err(invalid_cursor(
2156 "top-level transition resume must start with a Steps frame",
2157 ));
2158 }
2159 None => (0, &[][..]),
2160 };
2161
2162 if let Some(pause) = execute_steps(
2163 executor,
2164 &transition.steps,
2165 ctx,
2166 start_index,
2167 nested_cursor,
2168 StepExecutionScope::new(&transition.from, StepListCursorKind::Steps, true),
2169 )
2170 .await?
2171 {
2172 clear_timeout_fallback(ctx);
2173 ctx.set_cursor(WorkflowCursor {
2174 state: transition.from.clone(),
2175 frames: pause.frames,
2176 wait: Some(pause.wait),
2177 });
2178 executor.save_state(ctx).await?;
2179 return Ok(ctx.current_state.clone());
2180 }
2181
2182 if ctx.current_state == transition.from {
2186 ctx.transition_to(&transition.to, None);
2188 }
2189
2190 executor.save_state(ctx).await?;
2192 }
2193}
2194
2195pub async fn resume_workflow<E: WorkflowExecutor>(
2200 executor: &E,
2201 definition: &WorkflowDefinition,
2202 workflow_id: &str,
2203 event_data: Value,
2204) -> Result<String, WorkflowError> {
2205 resume_workflow_with_options(
2206 executor,
2207 definition,
2208 workflow_id,
2209 event_data,
2210 WorkflowRunOptions::default(),
2211 )
2212 .await
2213}
2214
2215pub async fn resume_workflow_with_options<E: WorkflowExecutor>(
2217 executor: &E,
2218 definition: &WorkflowDefinition,
2219 workflow_id: &str,
2220 event_data: Value,
2221 options: WorkflowRunOptions,
2222) -> Result<String, WorkflowError> {
2223 let event_name = extract_resume_event_name(&event_data)?;
2224 resume_workflow_with_event_and_options(
2225 executor,
2226 definition,
2227 workflow_id,
2228 &event_name,
2229 event_data,
2230 options,
2231 )
2232 .await
2233}
2234
2235pub async fn resume_workflow_with_event<E: WorkflowExecutor>(
2237 executor: &E,
2238 definition: &WorkflowDefinition,
2239 workflow_id: &str,
2240 event_name: &str,
2241 event_data: Value,
2242) -> Result<String, WorkflowError> {
2243 resume_workflow_with_event_and_options(
2244 executor,
2245 definition,
2246 workflow_id,
2247 event_name,
2248 event_data,
2249 WorkflowRunOptions::default(),
2250 )
2251 .await
2252}
2253
2254pub async fn resume_workflow_with_event_and_options<E: WorkflowExecutor>(
2256 executor: &E,
2257 definition: &WorkflowDefinition,
2258 workflow_id: &str,
2259 event_name: &str,
2260 event_data: Value,
2261 options: WorkflowRunOptions,
2262) -> Result<String, WorkflowError> {
2263 validate_workflow_definition(definition)?;
2264 ensure_definition_text(workflow_id, "Workflow id")?;
2265 ensure_wait_event_name(event_name)?;
2266
2267 let guard = match RuntimeGuard::enter(
2268 executor,
2269 &definition.name,
2270 workflow_id,
2271 WorkflowOperationKind::Resume {
2272 event: event_name.to_string(),
2273 },
2274 options,
2275 )
2276 .await?
2277 {
2278 RuntimeEntry::Active(guard) => guard,
2279 RuntimeEntry::Completed(state) => return Ok(state),
2280 };
2281
2282 let mut ctx = executor
2284 .load_state(workflow_id)
2285 .await?
2286 .ok_or_else(|| WorkflowError::Other(format!("Workflow not found: {}", workflow_id)))?;
2287
2288 ctx.set(RESUME_EVENT_KEY, event_data);
2290
2291 let result = run_workflow_inner(
2293 executor,
2294 definition,
2295 &mut ctx,
2296 RunMode::EventResume { event: event_name },
2297 )
2298 .await;
2299 finish_runtime_operation(executor, guard, result).await
2300}
2301
2302pub async fn timeout_workflow<E: WorkflowExecutor>(
2304 executor: &E,
2305 definition: &WorkflowDefinition,
2306 workflow_id: &str,
2307) -> Result<String, WorkflowError> {
2308 timeout_workflow_with_options(
2309 executor,
2310 definition,
2311 workflow_id,
2312 WorkflowRunOptions::default(),
2313 )
2314 .await
2315}
2316
2317pub async fn timeout_workflow_with_options<E: WorkflowExecutor>(
2319 executor: &E,
2320 definition: &WorkflowDefinition,
2321 workflow_id: &str,
2322 options: WorkflowRunOptions,
2323) -> Result<String, WorkflowError> {
2324 timeout_workflow_with_options_at(executor, definition, workflow_id, options, Utc::now()).await
2325}
2326
2327async fn timeout_workflow_with_options_at<E: WorkflowExecutor>(
2328 executor: &E,
2329 definition: &WorkflowDefinition,
2330 workflow_id: &str,
2331 options: WorkflowRunOptions,
2332 now: DateTime<Utc>,
2333) -> Result<String, WorkflowError> {
2334 validate_workflow_definition(definition)?;
2335 ensure_definition_text(workflow_id, "Workflow id")?;
2336 let guard = match RuntimeGuard::enter(
2337 executor,
2338 &definition.name,
2339 workflow_id,
2340 WorkflowOperationKind::Timeout,
2341 options,
2342 )
2343 .await?
2344 {
2345 RuntimeEntry::Active(guard) => guard,
2346 RuntimeEntry::Completed(state) => return Ok(state),
2347 };
2348
2349 let result = timeout_workflow_inner(executor, definition, workflow_id, now).await;
2350 finish_runtime_operation(executor, guard, result).await
2351}
2352
2353pub async fn timeout_due_workflows<E: WorkflowExecutor>(
2359 executor: &E,
2360 definition: &WorkflowDefinition,
2361 now: DateTime<Utc>,
2362 limit: usize,
2363 options: WorkflowRunOptions,
2364) -> Result<Vec<WorkflowTimeoutOutcome>, WorkflowError> {
2365 if limit == 0 {
2366 return Ok(Vec::new());
2367 }
2368 validate_workflow_definition(definition)?;
2369
2370 let workflow_ids = executor
2371 .load_due_workflow_timeouts(&definition.name, now, limit)
2372 .await?;
2373 let mut outcomes = Vec::with_capacity(workflow_ids.len().min(limit));
2374
2375 for workflow_id in workflow_ids.into_iter().take(limit) {
2376 let run_options = timeout_options_for_workflow(&options, definition, &workflow_id, now);
2377 let outcome = match timeout_workflow_with_options_at(
2378 executor,
2379 definition,
2380 &workflow_id,
2381 run_options,
2382 now,
2383 )
2384 .await
2385 {
2386 Ok(state) => WorkflowTimeoutOutcome {
2387 workflow_id,
2388 state: Some(state),
2389 error: None,
2390 },
2391 Err(err) => WorkflowTimeoutOutcome {
2392 workflow_id,
2393 state: None,
2394 error: Some(err.to_string()),
2395 },
2396 };
2397 outcomes.push(outcome);
2398 }
2399
2400 Ok(outcomes)
2401}
2402
2403fn timeout_options_for_workflow(
2404 options: &WorkflowRunOptions,
2405 definition: &WorkflowDefinition,
2406 workflow_id: &str,
2407 now: DateTime<Utc>,
2408) -> WorkflowRunOptions {
2409 let mut run_options = options.clone();
2410 run_options.idempotency_key = match &options.idempotency_key {
2411 Some(prefix) if prefix.trim().is_empty() => Some(prefix.clone()),
2412 Some(prefix) => Some(timeout_operation_idempotency_key(prefix, workflow_id, now)),
2413 None => Some(timeout_operation_idempotency_key(
2414 &definition.name,
2415 workflow_id,
2416 now,
2417 )),
2418 };
2419 run_options
2420}
2421
2422fn timeout_operation_idempotency_key(scope: &str, workflow_id: &str, now: DateTime<Utc>) -> String {
2423 serde_json::json!([
2424 "qail-workflow-timeout",
2425 2,
2426 scope,
2427 workflow_id,
2428 now.to_rfc3339_opts(SecondsFormat::Micros, true)
2429 ])
2430 .to_string()
2431}
2432
2433async fn timeout_workflow_inner<E: WorkflowExecutor>(
2434 executor: &E,
2435 definition: &WorkflowDefinition,
2436 workflow_id: &str,
2437 now: DateTime<Utc>,
2438) -> Result<String, WorkflowError> {
2439 let mut ctx = executor
2440 .load_state(workflow_id)
2441 .await?
2442 .ok_or_else(|| WorkflowError::Other(format!("Workflow not found: {}", workflow_id)))?;
2443 validate_workflow_context_identity(&ctx)?;
2444 ensure_context_definition(&mut ctx, definition)?;
2445
2446 let cursor = ctx
2447 .cursor
2448 .clone()
2449 .ok_or_else(|| WorkflowError::Other("Workflow is not paused at a Wait step".to_string()))?;
2450 ensure_cursor_has_frames(&cursor)?;
2451 if cursor.state != ctx.current_state {
2452 return Err(invalid_cursor(
2453 "timeout cursor state does not match current workflow state",
2454 ));
2455 }
2456 let timeout_fallback = timeout_fallback_from_context(&ctx)?;
2457 if let (Some(cursor_wait), Some(timeout)) = (&cursor.wait, &timeout_fallback)
2458 && cursor_wait != timeout
2459 {
2460 return Err(invalid_cursor(
2461 "timeout cursor wait metadata does not match timeout fallback metadata",
2462 ));
2463 }
2464
2465 let (wait, cursor_frames) = if let Some(timeout) = timeout_fallback {
2466 (timeout, cursor.frames.clone())
2467 } else {
2468 let wait = cursor.wait.clone().ok_or_else(|| {
2469 WorkflowError::Other("Workflow is not waiting for a timeout".to_string())
2470 })?;
2471 (wait, Vec::new())
2472 };
2473
2474 ensure_wait_event_name(&wait.event)?;
2475 if now < wait.deadline_at {
2476 return Err(WorkflowError::Other(format!(
2477 "Workflow wait for event '{}' has not timed out",
2478 wait.event
2479 )));
2480 }
2481 if wait.on_timeout.is_empty() {
2482 return Err(WorkflowError::Timeout { event: wait.event });
2483 }
2484 if steps_contain_wait(&wait.on_timeout) {
2485 return Err(WorkflowError::Other(
2486 "Wait steps inside on_timeout fallback are not supported".to_string(),
2487 ));
2488 }
2489 validate_workflow_steps(&wait.on_timeout)?;
2490
2491 set_timeout_fallback(&mut ctx, &wait)?;
2492 ctx.clear_cursor();
2493 ctx.set(
2494 RESUME_EVENT_KEY,
2495 serde_json::json!({
2496 "event": wait.event.clone(),
2497 "timeout": true,
2498 }),
2499 );
2500
2501 let (start_index, nested_cursor) = match cursor_frames.first() {
2502 Some(WorkflowCursorFrame::Steps { index }) => (*index, &cursor_frames[1..]),
2503 Some(_) => {
2504 return Err(invalid_cursor(
2505 "timeout fallback resume must start with a Steps frame",
2506 ));
2507 }
2508 None => (0, &[][..]),
2509 };
2510
2511 if execute_steps(
2512 executor,
2513 &wait.on_timeout,
2514 &mut ctx,
2515 start_index,
2516 nested_cursor,
2517 StepExecutionScope::new(&cursor.state, StepListCursorKind::Steps, true)
2518 .with_timeout_checkpoint(),
2519 )
2520 .await?
2521 .is_some()
2522 {
2523 return Err(WorkflowError::Other(
2524 "on_timeout fallback paused unexpectedly".to_string(),
2525 ));
2526 }
2527
2528 clear_timeout_fallback(&mut ctx);
2529 ctx.clear_cursor();
2530 if ctx.current_state == cursor.state {
2531 executor.save_state(&ctx).await?;
2532 return Ok(ctx.current_state.clone());
2533 }
2534
2535 run_workflow_inner(executor, definition, &mut ctx, RunMode::Normal).await
2536}
2537
2538fn extract_resume_event_name(event_data: &Value) -> Result<String, WorkflowError> {
2539 let event = ["event", "event_name", "type"]
2540 .iter()
2541 .find_map(|key| event_data.get(*key).and_then(Value::as_str))
2542 .ok_or_else(|| {
2543 WorkflowError::Other(
2544 "Resume event data must include a string 'event' field".to_string(),
2545 )
2546 })?;
2547 ensure_wait_event_name(event)?;
2548 Ok(event.to_string())
2549}
2550
2551fn steps_contain_wait(steps: &[WorkflowStep]) -> bool {
2552 steps.iter().any(|step| match step {
2553 WorkflowStep::Wait { .. } => true,
2554 WorkflowStep::Branch {
2555 branches, default, ..
2556 } => {
2557 branches
2558 .iter()
2559 .any(|(_, branch_steps)| steps_contain_wait(branch_steps))
2560 || steps_contain_wait(default)
2561 }
2562 WorkflowStep::BranchWhen {
2563 branches, default, ..
2564 } => {
2565 branches
2566 .iter()
2567 .any(|(_, branch_steps)| steps_contain_wait(branch_steps))
2568 || steps_contain_wait(default)
2569 }
2570 WorkflowStep::ForEach { steps, .. } => steps_contain_wait(steps),
2571 WorkflowStep::Query { .. }
2572 | WorkflowStep::Notify { .. }
2573 | WorkflowStep::Transition { .. }
2574 | WorkflowStep::Log { .. }
2575 | WorkflowStep::Charge { .. } => false,
2576 })
2577}
2578
2579#[cfg(test)]
2580mod tests {
2581 use super::*;
2582 use crate::context::StateChange;
2583 use crate::payment::{ChargeResponse, ChargeStatus, PaymentKind};
2584
2585 struct MockExecutor {
2586 queries: std::sync::Mutex<Vec<String>>,
2587 query_failures: std::sync::Mutex<std::collections::HashSet<String>>,
2588 notifications: std::sync::Mutex<Vec<(String, String)>>,
2589 charges: std::sync::Mutex<Vec<ChargeRequest>>,
2590 saved_state: std::sync::Mutex<Option<WorkflowContext>>,
2591 reject_leases: std::sync::Mutex<bool>,
2592 acquired_leases: std::sync::Mutex<Vec<WorkflowLease>>,
2593 released_leases: std::sync::Mutex<Vec<WorkflowLease>>,
2594 workflow_operations:
2595 std::sync::Mutex<std::collections::HashMap<String, WorkflowOperationStatus>>,
2596 workflow_operation_begin_failures: std::sync::Mutex<std::collections::HashSet<String>>,
2597 workflow_operation_complete_failures: std::sync::Mutex<std::collections::HashSet<String>>,
2598 workflow_operation_fail_failures: std::sync::Mutex<std::collections::HashSet<String>>,
2599 completed_workflow_operations: std::sync::Mutex<Vec<(String, String)>>,
2600 failed_workflow_operations: std::sync::Mutex<Vec<(String, String)>>,
2601 due_timeout_workflow_ids: std::sync::Mutex<Vec<String>>,
2602 skip_notify_side_effects: std::sync::Mutex<bool>,
2603 completed_side_effects: std::sync::Mutex<Vec<String>>,
2604 failed_side_effects: std::sync::Mutex<Vec<(String, String)>>,
2605 }
2606
2607 impl MockExecutor {
2608 fn new() -> Self {
2609 Self {
2610 queries: std::sync::Mutex::new(Vec::new()),
2611 query_failures: std::sync::Mutex::new(std::collections::HashSet::new()),
2612 notifications: std::sync::Mutex::new(Vec::new()),
2613 charges: std::sync::Mutex::new(Vec::new()),
2614 saved_state: std::sync::Mutex::new(None),
2615 reject_leases: std::sync::Mutex::new(false),
2616 acquired_leases: std::sync::Mutex::new(Vec::new()),
2617 released_leases: std::sync::Mutex::new(Vec::new()),
2618 workflow_operations: std::sync::Mutex::new(std::collections::HashMap::new()),
2619 workflow_operation_begin_failures: std::sync::Mutex::new(
2620 std::collections::HashSet::new(),
2621 ),
2622 workflow_operation_complete_failures: std::sync::Mutex::new(
2623 std::collections::HashSet::new(),
2624 ),
2625 workflow_operation_fail_failures: std::sync::Mutex::new(
2626 std::collections::HashSet::new(),
2627 ),
2628 completed_workflow_operations: std::sync::Mutex::new(Vec::new()),
2629 failed_workflow_operations: std::sync::Mutex::new(Vec::new()),
2630 due_timeout_workflow_ids: std::sync::Mutex::new(Vec::new()),
2631 skip_notify_side_effects: std::sync::Mutex::new(false),
2632 completed_side_effects: std::sync::Mutex::new(Vec::new()),
2633 failed_side_effects: std::sync::Mutex::new(Vec::new()),
2634 }
2635 }
2636
2637 fn fail_query(&self, cmd_wire: impl Into<String>) {
2638 self.query_failures.lock().unwrap().insert(cmd_wire.into());
2639 }
2640
2641 fn reject_workflow_leases(&self) {
2642 *self.reject_leases.lock().unwrap() = true;
2643 }
2644
2645 fn set_workflow_operation_status(
2646 &self,
2647 idempotency_key: impl Into<String>,
2648 status: WorkflowOperationStatus,
2649 ) {
2650 self.workflow_operations
2651 .lock()
2652 .unwrap()
2653 .insert(idempotency_key.into(), status);
2654 }
2655
2656 fn fail_workflow_operation_begin(&self, idempotency_key: impl Into<String>) {
2657 self.workflow_operation_begin_failures
2658 .lock()
2659 .unwrap()
2660 .insert(idempotency_key.into());
2661 }
2662
2663 fn fail_workflow_operation_complete(&self, idempotency_key: impl Into<String>) {
2664 self.workflow_operation_complete_failures
2665 .lock()
2666 .unwrap()
2667 .insert(idempotency_key.into());
2668 }
2669
2670 fn fail_workflow_operation_fail(&self, idempotency_key: impl Into<String>) {
2671 self.workflow_operation_fail_failures
2672 .lock()
2673 .unwrap()
2674 .insert(idempotency_key.into());
2675 }
2676
2677 fn skip_notify_side_effects(&self) {
2678 *self.skip_notify_side_effects.lock().unwrap() = true;
2679 }
2680
2681 fn set_due_timeout_workflow_ids(&self, workflow_ids: Vec<&str>) {
2682 *self.due_timeout_workflow_ids.lock().unwrap() =
2683 workflow_ids.into_iter().map(ToString::to_string).collect();
2684 }
2685 }
2686
2687 fn charge_only_workflow(name: &str) -> WorkflowDefinition {
2688 WorkflowDefinition::new(name)
2689 .initial_state("created")
2690 .transition(
2691 "created",
2692 "awaiting_payment",
2693 vec![WorkflowStep::charge(
2694 PaymentKind::Xendit,
2695 "order.total",
2696 "order.id",
2697 Some("charge"),
2698 )],
2699 )
2700 }
2701
2702 #[async_trait]
2703 impl WorkflowExecutor for MockExecutor {
2704 async fn execute_query(&self, cmd_json: &str) -> Result<Value, WorkflowError> {
2705 self.queries.lock().unwrap().push(cmd_json.to_string());
2706 if self.query_failures.lock().unwrap().contains(cmd_json) {
2707 return Err(WorkflowError::QueryFailed(
2708 "forced query failure".to_string(),
2709 ));
2710 }
2711 Ok(serde_json::json!([
2713 {"id": "op-1", "phone": "+628111"},
2714 {"id": "op-2", "phone": "+628222"},
2715 ]))
2716 }
2717
2718 async fn send_notification(
2719 &self,
2720 _channel: &ChannelKind,
2721 recipient: &str,
2722 template: &str,
2723 _params: &Value,
2724 ) -> Result<(), WorkflowError> {
2725 self.notifications
2726 .lock()
2727 .unwrap()
2728 .push((recipient.to_string(), template.to_string()));
2729 Ok(())
2730 }
2731
2732 async fn save_state(&self, ctx: &WorkflowContext) -> Result<(), WorkflowError> {
2733 *self.saved_state.lock().unwrap() = Some(ctx.clone());
2734 Ok(())
2735 }
2736
2737 async fn load_state(
2738 &self,
2739 workflow_id: &str,
2740 ) -> Result<Option<WorkflowContext>, WorkflowError> {
2741 Ok(self
2742 .saved_state
2743 .lock()
2744 .unwrap()
2745 .as_ref()
2746 .filter(|ctx| ctx.workflow_id == workflow_id)
2747 .cloned())
2748 }
2749
2750 async fn create_charge(
2751 &self,
2752 _provider: &PaymentKind,
2753 request: ChargeRequest,
2754 ) -> Result<ChargeResponse, WorkflowError> {
2755 self.charges.lock().unwrap().push(request.clone());
2756 Ok(ChargeResponse {
2757 charge_id: format!("mock-charge-{}", request.reference_id),
2758 status: ChargeStatus::Pending,
2759 redirect_url: None,
2760 qr_code: Some("00020101021226610014ID.CO.MOCK".into()),
2761 payment_code: None,
2762 expires_at: Some("2026-02-13T16:00:00Z".into()),
2763 raw: Some(serde_json::json!({"provider_secret": "hidden"})),
2764 })
2765 }
2766
2767 async fn acquire_workflow_lease(
2768 &self,
2769 lease: &WorkflowLease,
2770 ) -> Result<bool, WorkflowError> {
2771 if *self.reject_leases.lock().unwrap() {
2772 return Ok(false);
2773 }
2774 self.acquired_leases.lock().unwrap().push(lease.clone());
2775 Ok(true)
2776 }
2777
2778 async fn release_workflow_lease(&self, lease: &WorkflowLease) -> Result<(), WorkflowError> {
2779 self.released_leases.lock().unwrap().push(lease.clone());
2780 Ok(())
2781 }
2782
2783 async fn begin_workflow_operation(
2784 &self,
2785 operation: &WorkflowOperation,
2786 ) -> Result<WorkflowOperationStatus, WorkflowError> {
2787 if self
2788 .workflow_operation_begin_failures
2789 .lock()
2790 .unwrap()
2791 .contains(&operation.idempotency_key)
2792 {
2793 return Err(WorkflowError::Other(
2794 "forced operation begin failure".to_string(),
2795 ));
2796 }
2797 Ok(self
2798 .workflow_operations
2799 .lock()
2800 .unwrap()
2801 .get(&operation.idempotency_key)
2802 .cloned()
2803 .unwrap_or(WorkflowOperationStatus::Started))
2804 }
2805
2806 async fn complete_workflow_operation(
2807 &self,
2808 operation: &WorkflowOperation,
2809 state: &str,
2810 ) -> Result<(), WorkflowError> {
2811 if self
2812 .workflow_operation_complete_failures
2813 .lock()
2814 .unwrap()
2815 .contains(&operation.idempotency_key)
2816 {
2817 return Err(WorkflowError::Other(
2818 "forced operation complete failure".to_string(),
2819 ));
2820 }
2821 self.completed_workflow_operations
2822 .lock()
2823 .unwrap()
2824 .push((operation.idempotency_key.clone(), state.to_string()));
2825 Ok(())
2826 }
2827
2828 async fn fail_workflow_operation(
2829 &self,
2830 operation: &WorkflowOperation,
2831 error: &str,
2832 ) -> Result<(), WorkflowError> {
2833 if self
2834 .workflow_operation_fail_failures
2835 .lock()
2836 .unwrap()
2837 .contains(&operation.idempotency_key)
2838 {
2839 return Err(WorkflowError::Other(
2840 "forced operation fail failure".to_string(),
2841 ));
2842 }
2843 self.failed_workflow_operations
2844 .lock()
2845 .unwrap()
2846 .push((operation.idempotency_key.clone(), error.to_string()));
2847 Ok(())
2848 }
2849
2850 async fn load_due_workflow_timeouts(
2851 &self,
2852 _workflow_name: &str,
2853 _now: DateTime<Utc>,
2854 limit: usize,
2855 ) -> Result<Vec<String>, WorkflowError> {
2856 Ok(self
2857 .due_timeout_workflow_ids
2858 .lock()
2859 .unwrap()
2860 .iter()
2861 .take(limit)
2862 .cloned()
2863 .collect())
2864 }
2865
2866 async fn begin_workflow_side_effect(
2867 &self,
2868 operation: &WorkflowSideEffect,
2869 ) -> Result<WorkflowSideEffectStatus, WorkflowError> {
2870 if operation.kind == WorkflowSideEffectKind::Notify
2871 && *self.skip_notify_side_effects.lock().unwrap()
2872 {
2873 return Ok(WorkflowSideEffectStatus::AlreadyCompleted { result: None });
2874 }
2875 Ok(WorkflowSideEffectStatus::Execute)
2876 }
2877
2878 async fn complete_workflow_side_effect(
2879 &self,
2880 operation: &WorkflowSideEffect,
2881 _result: Option<&Value>,
2882 ) -> Result<(), WorkflowError> {
2883 self.completed_side_effects
2884 .lock()
2885 .unwrap()
2886 .push(operation.operation_id.clone());
2887 Ok(())
2888 }
2889
2890 async fn fail_workflow_side_effect(
2891 &self,
2892 operation: &WorkflowSideEffect,
2893 error: &str,
2894 ) -> Result<(), WorkflowError> {
2895 self.failed_side_effects
2896 .lock()
2897 .unwrap()
2898 .push((operation.operation_id.clone(), error.to_string()));
2899 Ok(())
2900 }
2901 }
2902
2903 #[tokio::test]
2904 async fn test_simple_workflow() {
2905 let executor = MockExecutor::new();
2906
2907 let wf = WorkflowDefinition::new("test")
2908 .initial_state("start")
2909 .transition(
2910 "start",
2911 "done",
2912 vec![
2913 WorkflowStep::log("Starting workflow"),
2914 WorkflowStep::transition("done"),
2915 ],
2916 );
2917
2918 let mut ctx = WorkflowContext::new("wf-001", "start");
2919 let result = run_workflow(&executor, &wf, &mut ctx).await.unwrap();
2920 assert_eq!(result, "done");
2921 assert_eq!(ctx.transition_count(), 1);
2922 }
2923
2924 #[tokio::test]
2925 async fn versioned_workflow_stamps_context_metadata() {
2926 let executor = MockExecutor::new();
2927
2928 let wf = WorkflowDefinition::new("versioned_booking")
2929 .version("v1")
2930 .initial_state("start")
2931 .transition("start", "done", vec![WorkflowStep::transition("done")]);
2932
2933 let mut ctx = WorkflowContext::new("wf-versioned", "start");
2934 let result = run_workflow(&executor, &wf, &mut ctx).await.unwrap();
2935
2936 assert_eq!(result, "done");
2937 assert_eq!(ctx.definition_name.as_deref(), Some("versioned_booking"));
2938 assert_eq!(ctx.definition_version.as_deref(), Some("v1"));
2939 }
2940
2941 #[tokio::test]
2942 async fn versioned_resume_rejects_definition_version_drift() {
2943 let executor = MockExecutor::new();
2944
2945 let wf_v1 = WorkflowDefinition::new("versioned_resume")
2946 .version("v1")
2947 .initial_state("active")
2948 .transition(
2949 "active",
2950 "done",
2951 vec![
2952 WorkflowStep::wait("operator.accepted", std::time::Duration::from_secs(3600)),
2953 WorkflowStep::notify(ChannelKind::Email, "accepted", "customer.email"),
2954 WorkflowStep::transition("done"),
2955 ],
2956 );
2957 let wf_v2 = WorkflowDefinition::new("versioned_resume")
2958 .version("v2")
2959 .initial_state("active")
2960 .transition(
2961 "active",
2962 "done",
2963 vec![
2964 WorkflowStep::wait("operator.accepted", std::time::Duration::from_secs(3600)),
2965 WorkflowStep::notify(ChannelKind::Email, "accepted", "customer.email"),
2966 WorkflowStep::transition("done"),
2967 ],
2968 );
2969
2970 let mut ctx = WorkflowContext::new("wf-version-drift", "active");
2971 ctx.set(
2972 "customer",
2973 serde_json::json!({"email": "guest@example.com"}),
2974 );
2975 let result = run_workflow(&executor, &wf_v1, &mut ctx).await.unwrap();
2976 assert_eq!(result, "active");
2977
2978 let err = resume_workflow(
2979 &executor,
2980 &wf_v2,
2981 "wf-version-drift",
2982 serde_json::json!({"event": "operator.accepted"}),
2983 )
2984 .await
2985 .expect_err("version drift must fail before replay");
2986
2987 match err {
2988 WorkflowError::Other(msg) => {
2989 assert!(msg.contains("definition version 'v1'"), "got: {msg}");
2990 assert!(msg.contains("not 'v2'"), "got: {msg}");
2991 }
2992 other => panic!("expected version drift error, got {other:?}"),
2993 }
2994 assert!(executor.notifications.lock().unwrap().is_empty());
2995 }
2996
2997 #[tokio::test]
2998 async fn run_with_options_acquires_lease_and_completes_operation() {
2999 let executor = MockExecutor::new();
3000
3001 let wf = WorkflowDefinition::new("runtime_guarded_run")
3002 .initial_state("start")
3003 .transition("start", "done", vec![WorkflowStep::transition("done")]);
3004
3005 let mut ctx = WorkflowContext::new("wf-runtime-run", "start");
3006 let options = WorkflowRunOptions::default()
3007 .with_lease("worker-a", std::time::Duration::from_secs(30))
3008 .with_idempotency_key("run-1");
3009
3010 let result = run_workflow_with_options(&executor, &wf, &mut ctx, options)
3011 .await
3012 .unwrap();
3013
3014 assert_eq!(result, "done");
3015 let acquired_leases = executor.acquired_leases.lock().unwrap();
3016 let released_leases = executor.released_leases.lock().unwrap();
3017 assert_eq!(acquired_leases.len(), 1);
3018 assert_eq!(released_leases.len(), 1);
3019 assert_eq!(released_leases[0].owner, acquired_leases[0].owner);
3020 assert_ne!(
3021 acquired_leases[0].owner, "worker-a",
3022 "runtime lease owner must include a fencing token"
3023 );
3024 let lease_owner: serde_json::Value =
3025 serde_json::from_str(&acquired_leases[0].owner).unwrap();
3026 let lease_owner = lease_owner.as_array().unwrap();
3027 assert_eq!(lease_owner[0], "qail-workflow-lease");
3028 assert_eq!(lease_owner[2], "worker-a");
3029 assert_eq!(lease_owner[3], "runtime_guarded_run");
3030 assert_eq!(lease_owner[4], "wf-runtime-run");
3031 assert_eq!(lease_owner[6], "run-1");
3032 drop(acquired_leases);
3033 drop(released_leases);
3034 assert_eq!(
3035 executor
3036 .completed_workflow_operations
3037 .lock()
3038 .unwrap()
3039 .as_slice(),
3040 &[("run-1".to_string(), "done".to_string())]
3041 );
3042 }
3043
3044 #[test]
3045 fn runtime_lease_owner_changes_per_acquire_time() {
3046 let first_at = DateTime::parse_from_rfc3339("2026-06-17T12:00:00Z")
3047 .unwrap()
3048 .with_timezone(&Utc);
3049 let second_at = DateTime::parse_from_rfc3339("2026-06-17T12:00:01Z")
3050 .unwrap()
3051 .with_timezone(&Utc);
3052
3053 let first = runtime_lease_owner(
3054 "worker-a",
3055 "booking",
3056 "wf-1",
3057 &WorkflowOperationKind::Resume {
3058 event: "vendor.accepted".to_string(),
3059 },
3060 Some("event-1"),
3061 first_at,
3062 );
3063 let second = runtime_lease_owner(
3064 "worker-a",
3065 "booking",
3066 "wf-1",
3067 &WorkflowOperationKind::Resume {
3068 event: "vendor.accepted".to_string(),
3069 },
3070 Some("event-1"),
3071 second_at,
3072 );
3073
3074 assert_ne!(
3075 first, second,
3076 "stale release calls must not share the same lease owner as later acquisitions"
3077 );
3078 }
3079
3080 #[tokio::test]
3081 async fn lease_rejection_stops_workflow_before_side_effects() {
3082 let executor = MockExecutor::new();
3083 executor.reject_workflow_leases();
3084
3085 let wf = WorkflowDefinition::new("lease_guard")
3086 .initial_state("active")
3087 .transition(
3088 "active",
3089 "done",
3090 vec![
3091 WorkflowStep::notify(ChannelKind::Email, "should_not_send", "customer.email"),
3092 WorkflowStep::transition("done"),
3093 ],
3094 );
3095
3096 let mut ctx = WorkflowContext::new("wf-lease-guard", "active");
3097 ctx.set(
3098 "customer",
3099 serde_json::json!({"email": "guest@example.com"}),
3100 );
3101 let options = WorkflowRunOptions::default()
3102 .with_lease("worker-a", std::time::Duration::from_secs(30));
3103
3104 let err = run_workflow_with_options(&executor, &wf, &mut ctx, options)
3105 .await
3106 .expect_err("lease rejection must fail before executing steps");
3107
3108 match err {
3109 WorkflowError::Other(msg) => {
3110 assert!(msg.contains("already locked"), "got: {msg}");
3111 }
3112 other => panic!("expected lock error, got {other:?}"),
3113 }
3114 assert_eq!(ctx.current_state, "active");
3115 assert!(executor.notifications.lock().unwrap().is_empty());
3116 }
3117
3118 #[tokio::test]
3119 async fn resume_idempotency_completion_short_circuits_before_load_and_side_effects() {
3120 let executor = MockExecutor::new();
3121 executor.set_workflow_operation_status(
3122 "event-1",
3123 WorkflowOperationStatus::Completed {
3124 state: "done".to_string(),
3125 },
3126 );
3127
3128 let wf = WorkflowDefinition::new("resume_idempotency_guard")
3129 .initial_state("active")
3130 .transition(
3131 "active",
3132 "done",
3133 vec![
3134 WorkflowStep::wait("payment.success", std::time::Duration::from_secs(3600)),
3135 WorkflowStep::notify(ChannelKind::Email, "paid", "customer.email"),
3136 WorkflowStep::transition("done"),
3137 ],
3138 );
3139
3140 let result = resume_workflow_with_options(
3141 &executor,
3142 &wf,
3143 "missing-workflow",
3144 serde_json::json!({"event": "payment.success"}),
3145 WorkflowRunOptions::default().with_idempotency_key("event-1"),
3146 )
3147 .await
3148 .unwrap();
3149
3150 assert_eq!(result, "done");
3151 assert!(executor.notifications.lock().unwrap().is_empty());
3152 assert!(
3153 executor
3154 .completed_workflow_operations
3155 .lock()
3156 .unwrap()
3157 .is_empty()
3158 );
3159 }
3160
3161 #[tokio::test]
3162 async fn completed_operation_short_circuits_before_rejected_lease() {
3163 let executor = MockExecutor::new();
3164 executor.reject_workflow_leases();
3165 executor.set_workflow_operation_status(
3166 "run-replay-1",
3167 WorkflowOperationStatus::Completed {
3168 state: "done".to_string(),
3169 },
3170 );
3171
3172 let wf = WorkflowDefinition::new("completed_before_lease")
3173 .initial_state("active")
3174 .transition(
3175 "active",
3176 "done",
3177 vec![
3178 WorkflowStep::notify(ChannelKind::Email, "should_not_send", "customer.email"),
3179 WorkflowStep::transition("done"),
3180 ],
3181 );
3182
3183 let mut ctx = WorkflowContext::new("wf-completed-before-lease", "active");
3184 ctx.set(
3185 "customer",
3186 serde_json::json!({"email": "guest@example.com"}),
3187 );
3188
3189 let result = run_workflow_with_options(
3190 &executor,
3191 &wf,
3192 &mut ctx,
3193 WorkflowRunOptions::default()
3194 .with_idempotency_key("run-replay-1")
3195 .with_lease("worker-a", std::time::Duration::from_secs(30)),
3196 )
3197 .await
3198 .unwrap();
3199
3200 assert_eq!(result, "done");
3201 assert!(executor.acquired_leases.lock().unwrap().is_empty());
3202 assert!(executor.released_leases.lock().unwrap().is_empty());
3203 assert!(executor.notifications.lock().unwrap().is_empty());
3204 assert!(
3205 executor
3206 .completed_workflow_operations
3207 .lock()
3208 .unwrap()
3209 .is_empty()
3210 );
3211 }
3212
3213 #[tokio::test]
3214 async fn started_operation_is_failed_when_lease_is_rejected() {
3215 let executor = MockExecutor::new();
3216 executor.reject_workflow_leases();
3217
3218 let wf = WorkflowDefinition::new("started_then_lease_rejected")
3219 .initial_state("active")
3220 .transition(
3221 "active",
3222 "done",
3223 vec![
3224 WorkflowStep::notify(ChannelKind::Email, "should_not_send", "customer.email"),
3225 WorkflowStep::transition("done"),
3226 ],
3227 );
3228
3229 let mut ctx = WorkflowContext::new("wf-started-lease-rejected", "active");
3230 ctx.set(
3231 "customer",
3232 serde_json::json!({"email": "guest@example.com"}),
3233 );
3234
3235 let err = run_workflow_with_options(
3236 &executor,
3237 &wf,
3238 &mut ctx,
3239 WorkflowRunOptions::default()
3240 .with_idempotency_key("run-lock-1")
3241 .with_lease("worker-a", std::time::Duration::from_secs(30)),
3242 )
3243 .await
3244 .expect_err("new idempotent operation must not execute without its lease");
3245
3246 match err {
3247 WorkflowError::Other(msg) => {
3248 assert!(msg.contains("already locked"), "got: {msg}");
3249 }
3250 other => panic!("expected lock error, got {other:?}"),
3251 }
3252 assert_eq!(ctx.current_state, "active");
3253 assert!(executor.acquired_leases.lock().unwrap().is_empty());
3254 assert!(executor.notifications.lock().unwrap().is_empty());
3255 assert_eq!(
3256 executor
3257 .failed_workflow_operations
3258 .lock()
3259 .unwrap()
3260 .as_slice(),
3261 &[(
3262 "run-lock-1".to_string(),
3263 "Workflow 'wf-started-lease-rejected' is already locked".to_string()
3264 )]
3265 );
3266 }
3267
3268 #[tokio::test]
3269 async fn empty_idempotency_key_does_not_release_unacquired_lease() {
3270 let executor = MockExecutor::new();
3271
3272 let wf = WorkflowDefinition::new("empty_idempotency_key")
3273 .initial_state("active")
3274 .transition("active", "done", vec![WorkflowStep::transition("done")]);
3275
3276 let mut ctx = WorkflowContext::new("wf-empty-idempotency", "active");
3277 let err = run_workflow_with_options(
3278 &executor,
3279 &wf,
3280 &mut ctx,
3281 WorkflowRunOptions::default()
3282 .with_idempotency_key(" ")
3283 .with_lease("worker-a", std::time::Duration::from_secs(30)),
3284 )
3285 .await
3286 .expect_err("empty idempotency key must fail before lease acquisition");
3287
3288 match err {
3289 WorkflowError::Other(msg) => {
3290 assert!(
3291 msg.contains("idempotency key must not be empty"),
3292 "got: {msg}"
3293 );
3294 }
3295 other => panic!("expected idempotency validation error, got {other:?}"),
3296 }
3297 assert!(executor.acquired_leases.lock().unwrap().is_empty());
3298 assert!(executor.released_leases.lock().unwrap().is_empty());
3299 assert_eq!(ctx.current_state, "active");
3300 }
3301
3302 #[tokio::test]
3303 async fn padded_runtime_guard_identity_fails_before_side_effects() {
3304 let executor = MockExecutor::new();
3305
3306 let wf = WorkflowDefinition::new("padded_runtime_guard")
3307 .initial_state("active")
3308 .transition(
3309 "active",
3310 "done",
3311 vec![
3312 WorkflowStep::notify(ChannelKind::Email, "should_not_send", "customer.email"),
3313 WorkflowStep::transition("done"),
3314 ],
3315 );
3316
3317 let mut padded_key_ctx = WorkflowContext::new("wf-padded-idempotency", "active");
3318 padded_key_ctx.set(
3319 "customer",
3320 serde_json::json!({"email": "guest@example.com"}),
3321 );
3322 let err = run_workflow_with_options(
3323 &executor,
3324 &wf,
3325 &mut padded_key_ctx,
3326 WorkflowRunOptions::default()
3327 .with_idempotency_key(" run-1 ")
3328 .with_lease("worker-a", std::time::Duration::from_secs(30)),
3329 )
3330 .await
3331 .expect_err("padded idempotency key must fail before side effects");
3332 match err {
3333 WorkflowError::Other(msg) => {
3334 assert!(
3335 msg.contains("Workflow idempotency key must not have leading"),
3336 "got: {msg}"
3337 );
3338 }
3339 other => panic!("expected idempotency key validation error, got {other:?}"),
3340 }
3341
3342 let mut padded_owner_ctx = WorkflowContext::new("wf-padded-owner", "active");
3343 padded_owner_ctx.set(
3344 "customer",
3345 serde_json::json!({"email": "guest@example.com"}),
3346 );
3347 let err = run_workflow_with_options(
3348 &executor,
3349 &wf,
3350 &mut padded_owner_ctx,
3351 WorkflowRunOptions::default()
3352 .with_idempotency_key("run-2")
3353 .with_lease(" worker-a ", std::time::Duration::from_secs(30)),
3354 )
3355 .await
3356 .expect_err("padded lease owner must fail before side effects");
3357 match err {
3358 WorkflowError::Other(msg) => {
3359 assert!(
3360 msg.contains("Workflow lease owner must not have leading"),
3361 "got: {msg}"
3362 );
3363 }
3364 other => panic!("expected lease owner validation error, got {other:?}"),
3365 }
3366
3367 assert!(executor.acquired_leases.lock().unwrap().is_empty());
3368 assert!(executor.released_leases.lock().unwrap().is_empty());
3369 assert!(executor.notifications.lock().unwrap().is_empty());
3370 }
3371
3372 #[tokio::test]
3373 async fn operation_begin_failure_does_not_release_unacquired_lease() {
3374 let executor = MockExecutor::new();
3375 executor.fail_workflow_operation_begin("begin-fails-1");
3376
3377 let wf = WorkflowDefinition::new("operation_begin_failure")
3378 .initial_state("active")
3379 .transition("active", "done", vec![WorkflowStep::transition("done")]);
3380
3381 let mut ctx = WorkflowContext::new("wf-begin-failure", "active");
3382 let err = run_workflow_with_options(
3383 &executor,
3384 &wf,
3385 &mut ctx,
3386 WorkflowRunOptions::default()
3387 .with_idempotency_key("begin-fails-1")
3388 .with_lease("worker-a", std::time::Duration::from_secs(30)),
3389 )
3390 .await
3391 .expect_err("operation begin failure must happen before lease acquisition");
3392
3393 match err {
3394 WorkflowError::Other(msg) => {
3395 assert!(msg.contains("forced operation begin failure"), "got: {msg}");
3396 }
3397 other => panic!("expected operation begin error, got {other:?}"),
3398 }
3399 assert!(executor.acquired_leases.lock().unwrap().is_empty());
3400 assert!(executor.released_leases.lock().unwrap().is_empty());
3401 assert_eq!(ctx.current_state, "active");
3402 }
3403
3404 #[tokio::test]
3405 async fn completed_operation_with_invalid_state_is_rejected_before_lease() {
3406 let executor = MockExecutor::new();
3407 executor.set_workflow_operation_status(
3408 "completed-invalid-state-1",
3409 WorkflowOperationStatus::Completed {
3410 state: " ".to_string(),
3411 },
3412 );
3413
3414 let wf = WorkflowDefinition::new("completed_invalid_state")
3415 .initial_state("active")
3416 .transition("active", "done", vec![WorkflowStep::transition("done")]);
3417
3418 let mut ctx = WorkflowContext::new("wf-completed-invalid-state", "active");
3419 let err = run_workflow_with_options(
3420 &executor,
3421 &wf,
3422 &mut ctx,
3423 WorkflowRunOptions::default()
3424 .with_idempotency_key("completed-invalid-state-1")
3425 .with_lease("worker-a", std::time::Duration::from_secs(30)),
3426 )
3427 .await
3428 .expect_err("invalid completed replay state must be rejected");
3429
3430 match err {
3431 WorkflowError::Other(msg) => {
3432 assert!(
3433 msg.contains("Workflow completed operation state must not be empty"),
3434 "got: {msg}"
3435 );
3436 }
3437 other => panic!("expected completed state validation error, got {other:?}"),
3438 }
3439 assert!(executor.acquired_leases.lock().unwrap().is_empty());
3440 assert!(executor.released_leases.lock().unwrap().is_empty());
3441 assert_eq!(ctx.current_state, "active");
3442 }
3443
3444 #[tokio::test]
3445 async fn invalid_definition_fails_before_runtime_guards() {
3446 let executor = MockExecutor::new();
3447
3448 let wf = WorkflowDefinition::new("invalid_definition")
3449 .version(" v2")
3450 .initial_state("active")
3451 .transition(
3452 "active",
3453 "done",
3454 vec![
3455 WorkflowStep::notify(ChannelKind::Email, "should_not_send", "customer.email"),
3456 WorkflowStep::transition("done"),
3457 ],
3458 );
3459
3460 let mut ctx = WorkflowContext::new("wf-invalid-definition", "active");
3461 ctx.set(
3462 "customer",
3463 serde_json::json!({"email": "guest@example.com"}),
3464 );
3465
3466 let err = run_workflow_with_options(
3467 &executor,
3468 &wf,
3469 &mut ctx,
3470 WorkflowRunOptions::default()
3471 .with_idempotency_key("invalid-run-1")
3472 .with_lease("worker-a", std::time::Duration::from_secs(30)),
3473 )
3474 .await
3475 .expect_err("invalid definition must fail before runtime guards");
3476
3477 match err {
3478 WorkflowError::Other(msg) => {
3479 assert!(
3480 msg.contains("Workflow definition version must not have leading"),
3481 "got: {msg}"
3482 );
3483 }
3484 other => panic!("expected definition validation error, got {other:?}"),
3485 }
3486 assert!(executor.acquired_leases.lock().unwrap().is_empty());
3487 assert!(executor.released_leases.lock().unwrap().is_empty());
3488 assert!(
3489 executor
3490 .completed_workflow_operations
3491 .lock()
3492 .unwrap()
3493 .is_empty()
3494 );
3495 assert!(
3496 executor
3497 .failed_workflow_operations
3498 .lock()
3499 .unwrap()
3500 .is_empty()
3501 );
3502 assert!(executor.notifications.lock().unwrap().is_empty());
3503 }
3504
3505 #[tokio::test]
3506 async fn empty_workflow_id_fails_before_runtime_guards() {
3507 let executor = MockExecutor::new();
3508
3509 let wf = WorkflowDefinition::new("empty_workflow_id")
3510 .initial_state("active")
3511 .transition("active", "done", vec![WorkflowStep::transition("done")]);
3512
3513 let mut ctx = WorkflowContext::new(" ", "active");
3514 let err = run_workflow_with_options(
3515 &executor,
3516 &wf,
3517 &mut ctx,
3518 WorkflowRunOptions::default()
3519 .with_idempotency_key("empty-id-1")
3520 .with_lease("worker-a", std::time::Duration::from_secs(30)),
3521 )
3522 .await
3523 .expect_err("empty workflow id must fail before runtime guards");
3524
3525 match err {
3526 WorkflowError::Other(msg) => {
3527 assert!(msg.contains("Workflow id must not be empty"), "got: {msg}");
3528 }
3529 other => panic!("expected workflow id validation error, got {other:?}"),
3530 }
3531 assert!(executor.acquired_leases.lock().unwrap().is_empty());
3532 assert!(executor.released_leases.lock().unwrap().is_empty());
3533 assert!(
3534 executor
3535 .completed_workflow_operations
3536 .lock()
3537 .unwrap()
3538 .is_empty()
3539 );
3540 assert!(
3541 executor
3542 .failed_workflow_operations
3543 .lock()
3544 .unwrap()
3545 .is_empty()
3546 );
3547 }
3548
3549 #[tokio::test]
3550 async fn empty_resume_workflow_id_fails_before_runtime_guards() {
3551 let executor = MockExecutor::new();
3552
3553 let wf = WorkflowDefinition::new("empty_resume_id")
3554 .initial_state("active")
3555 .transition(
3556 "active",
3557 "done",
3558 vec![
3559 WorkflowStep::wait("vendor.ready", std::time::Duration::from_secs(3600)),
3560 WorkflowStep::transition("done"),
3561 ],
3562 );
3563
3564 let err = resume_workflow_with_options(
3565 &executor,
3566 &wf,
3567 " ",
3568 serde_json::json!({"event": "vendor.ready"}),
3569 WorkflowRunOptions::default()
3570 .with_idempotency_key("empty-resume-id-1")
3571 .with_lease("worker-a", std::time::Duration::from_secs(30)),
3572 )
3573 .await
3574 .expect_err("empty workflow id must fail before runtime guards");
3575
3576 match err {
3577 WorkflowError::Other(msg) => {
3578 assert!(msg.contains("Workflow id must not be empty"), "got: {msg}");
3579 }
3580 other => panic!("expected workflow id validation error, got {other:?}"),
3581 }
3582 assert!(executor.acquired_leases.lock().unwrap().is_empty());
3583 assert!(executor.released_leases.lock().unwrap().is_empty());
3584 assert!(
3585 executor
3586 .completed_workflow_operations
3587 .lock()
3588 .unwrap()
3589 .is_empty()
3590 );
3591 assert!(
3592 executor
3593 .failed_workflow_operations
3594 .lock()
3595 .unwrap()
3596 .is_empty()
3597 );
3598 }
3599
3600 #[tokio::test]
3601 async fn operation_complete_failure_still_releases_lease() {
3602 let executor = MockExecutor::new();
3603 executor.fail_workflow_operation_complete("complete-fails-1");
3604
3605 let wf = WorkflowDefinition::new("complete_failure_release")
3606 .initial_state("active")
3607 .transition("active", "done", vec![WorkflowStep::transition("done")]);
3608
3609 let mut ctx = WorkflowContext::new("wf-complete-failure-release", "active");
3610 let err = run_workflow_with_options(
3611 &executor,
3612 &wf,
3613 &mut ctx,
3614 WorkflowRunOptions::default()
3615 .with_idempotency_key("complete-fails-1")
3616 .with_lease("worker-a", std::time::Duration::from_secs(30)),
3617 )
3618 .await
3619 .expect_err("operation completion failure should be returned");
3620
3621 match err {
3622 WorkflowError::Other(msg) => {
3623 assert!(
3624 msg.contains("forced operation complete failure"),
3625 "got: {msg}"
3626 );
3627 }
3628 other => panic!("expected completion failure, got {other:?}"),
3629 }
3630 assert_eq!(ctx.current_state, "done");
3631 assert_eq!(executor.acquired_leases.lock().unwrap().len(), 1);
3632 assert_eq!(executor.released_leases.lock().unwrap().len(), 1);
3633 }
3634
3635 #[tokio::test]
3636 async fn operation_fail_failure_still_releases_lease() {
3637 let executor = MockExecutor::new();
3638 executor.fail_workflow_operation_fail("fail-fails-1");
3639
3640 let wf = WorkflowDefinition::new("fail_failure_release")
3641 .initial_state("active")
3642 .transition(
3643 "active",
3644 "done",
3645 vec![
3646 WorkflowStep::notify(ChannelKind::Email, "missing_recipient", "customer.email"),
3647 WorkflowStep::transition("done"),
3648 ],
3649 );
3650
3651 let mut ctx = WorkflowContext::new("wf-fail-failure-release", "active");
3652 let err = run_workflow_with_options(
3653 &executor,
3654 &wf,
3655 &mut ctx,
3656 WorkflowRunOptions::default()
3657 .with_idempotency_key("fail-fails-1")
3658 .with_lease("worker-a", std::time::Duration::from_secs(30)),
3659 )
3660 .await
3661 .expect_err("original workflow error should be returned");
3662
3663 match err {
3664 WorkflowError::MissingContextKey(key) => {
3665 assert_eq!(key, "customer.email");
3666 }
3667 other => panic!("expected original workflow error, got {other:?}"),
3668 }
3669 assert_eq!(ctx.current_state, "active");
3670 assert_eq!(executor.acquired_leases.lock().unwrap().len(), 1);
3671 assert_eq!(executor.released_leases.lock().unwrap().len(), 1);
3672 }
3673
3674 #[tokio::test]
3675 async fn completed_notify_side_effect_skips_provider_call() {
3676 let executor = MockExecutor::new();
3677 executor.skip_notify_side_effects();
3678
3679 let wf = WorkflowDefinition::new("notify_side_effect_replay")
3680 .initial_state("active")
3681 .transition(
3682 "active",
3683 "done",
3684 vec![
3685 WorkflowStep::notify(ChannelKind::Email, "booking_confirmed", "customer.email"),
3686 WorkflowStep::transition("done"),
3687 ],
3688 );
3689
3690 let mut ctx = WorkflowContext::new("wf-notify-side-effect", "active");
3691 ctx.set(
3692 "customer",
3693 serde_json::json!({"email": "guest@example.com"}),
3694 );
3695
3696 let result = run_workflow(&executor, &wf, &mut ctx).await.unwrap();
3697
3698 assert_eq!(result, "done");
3699 assert!(
3700 executor.notifications.lock().unwrap().is_empty(),
3701 "completed notify side effect must not call the provider again"
3702 );
3703 }
3704
3705 #[tokio::test]
3706 async fn transition_limit_counts_only_current_run() {
3707 let executor = MockExecutor::new();
3708
3709 let wf = WorkflowDefinition::new("aged_workflow")
3710 .initial_state("active")
3711 .transition("active", "done", vec![WorkflowStep::transition("done")]);
3712
3713 let mut ctx = WorkflowContext::new("wf-aged-history", "active");
3714 for idx in 0..51 {
3715 ctx.history.push(StateChange {
3716 from: format!("past_{idx}"),
3717 to: format!("past_{}", idx + 1),
3718 at: Utc::now(),
3719 reason: None,
3720 });
3721 }
3722
3723 let result = run_workflow(&executor, &wf, &mut ctx)
3724 .await
3725 .expect("old workflow history must not trip per-run loop guard");
3726
3727 assert_eq!(result, "done");
3728 assert_eq!(ctx.transition_count(), 52);
3729 }
3730
3731 #[tokio::test]
3732 async fn transition_limit_blocks_next_transition_before_side_effects() {
3733 let executor = MockExecutor::new();
3734
3735 let mut wf = WorkflowDefinition::new("runaway_workflow").initial_state("s0");
3736 for idx in 0..50 {
3737 wf = wf.transition(
3738 format!("s{idx}"),
3739 format!("s{}", idx + 1),
3740 vec![WorkflowStep::transition(&format!("s{}", idx + 1))],
3741 );
3742 }
3743 wf = wf.transition(
3744 "s50",
3745 "s51",
3746 vec![
3747 WorkflowStep::notify(ChannelKind::Email, "runaway_side_effect", "ops.email"),
3748 WorkflowStep::transition("s51"),
3749 ],
3750 );
3751
3752 let mut ctx = WorkflowContext::new("wf-runaway", "s0");
3753 ctx.set("ops", serde_json::json!({"email": "ops@example.com"}));
3754
3755 let err = run_workflow(&executor, &wf, &mut ctx)
3756 .await
3757 .expect_err("the 51st transition must be blocked before side effects");
3758
3759 match err {
3760 WorkflowError::Other(msg) => {
3761 assert!(
3762 msg.contains("Maximum transition count exceeded"),
3763 "got: {msg}"
3764 );
3765 }
3766 other => panic!("expected transition limit error, got {other:?}"),
3767 }
3768 assert_eq!(ctx.current_state, "s50");
3769 assert_eq!(ctx.transition_count(), 50);
3770 assert!(
3771 executor.notifications.lock().unwrap().is_empty(),
3772 "blocked transition side effects must not execute"
3773 );
3774 let saved = executor
3775 .saved_state
3776 .lock()
3777 .unwrap()
3778 .clone()
3779 .expect("state after the last allowed transition should be saved");
3780 assert_eq!(saved.current_state, "s50");
3781 }
3782
3783 #[tokio::test]
3784 async fn test_workflow_with_branch() {
3785 let executor = MockExecutor::new();
3786
3787 let wf = WorkflowDefinition::new("branching")
3788 .initial_state("pending")
3789 .transition(
3790 "pending",
3791 "resolved",
3792 vec![
3793 WorkflowStep::branch(
3794 "reason",
3795 vec![
3796 ("full", vec![WorkflowStep::log("Fully booked")]),
3797 ("maintenance", vec![WorkflowStep::log("Under maintenance")]),
3798 ],
3799 vec![WorkflowStep::log("Unknown reason")],
3800 ),
3801 WorkflowStep::transition("resolved"),
3802 ],
3803 );
3804
3805 let mut ctx = WorkflowContext::new("wf-002", "pending");
3806 ctx.set("reason", Value::String("maintenance".into()));
3807
3808 let result = run_workflow(&executor, &wf, &mut ctx).await.unwrap();
3809 assert_eq!(result, "resolved");
3810 }
3811
3812 #[tokio::test]
3813 async fn branch_when_routes_by_typed_numeric_condition() {
3814 let executor = MockExecutor::new();
3815
3816 let wf = WorkflowDefinition::new("typed_branching")
3817 .initial_state("pending")
3818 .transition(
3819 "pending",
3820 "resolved",
3821 vec![WorkflowStep::branch_when(
3822 "payment.attempts",
3823 vec![(
3824 WorkflowBranchCondition::NumberGte(3),
3825 vec![WorkflowStep::transition("manual_review")],
3826 )],
3827 vec![WorkflowStep::transition("retry")],
3828 )],
3829 );
3830
3831 let mut ctx = WorkflowContext::new("wf-typed-branch", "pending");
3832 ctx.set("payment", serde_json::json!({"attempts": 3}));
3833
3834 let result = run_workflow(&executor, &wf, &mut ctx).await.unwrap();
3835
3836 assert_eq!(result, "manual_review");
3837 assert_eq!(ctx.current_state, "manual_review");
3838 }
3839
3840 #[tokio::test]
3841 async fn branch_when_routes_by_decimal_numeric_condition() {
3842 let executor = MockExecutor::new();
3843
3844 let wf = WorkflowDefinition::new("decimal_branching")
3845 .initial_state("pending")
3846 .transition(
3847 "pending",
3848 "resolved",
3849 vec![WorkflowStep::branch_when(
3850 "payment.risk_score",
3851 vec![(
3852 WorkflowBranchCondition::NumberGt(10),
3853 vec![WorkflowStep::transition("manual_review")],
3854 )],
3855 vec![WorkflowStep::transition("retry")],
3856 )],
3857 );
3858
3859 let mut ctx = WorkflowContext::new("wf-decimal-branch", "pending");
3860 ctx.set("payment", serde_json::json!({"risk_score": 10.5}));
3861
3862 let result = run_workflow(&executor, &wf, &mut ctx).await.unwrap();
3863
3864 assert_eq!(result, "manual_review");
3865 assert_eq!(ctx.current_state, "manual_review");
3866 }
3867
3868 #[tokio::test]
3869 async fn branch_when_resume_rejects_condition_drift_before_side_effects() {
3870 let executor = MockExecutor::new();
3871
3872 let wf = WorkflowDefinition::new("branch_when_drift_source")
3873 .initial_state("active")
3874 .transition(
3875 "active",
3876 "done",
3877 vec![WorkflowStep::branch_when(
3878 "payment.attempts",
3879 vec![(
3880 WorkflowBranchCondition::NumberGte(3),
3881 vec![
3882 WorkflowStep::wait(
3883 "manual.approved",
3884 std::time::Duration::from_secs(3600),
3885 ),
3886 WorkflowStep::notify(
3887 ChannelKind::Email,
3888 "manual_review_approved",
3889 "customer.email",
3890 ),
3891 ],
3892 )],
3893 vec![],
3894 )],
3895 );
3896
3897 let mut ctx = WorkflowContext::new("wf-branch-when-drift", "active");
3898 ctx.set(
3899 "payment",
3900 serde_json::json!({
3901 "attempts": 3
3902 }),
3903 );
3904 ctx.set(
3905 "customer",
3906 serde_json::json!({"email": "guest@example.com"}),
3907 );
3908
3909 let result = run_workflow(&executor, &wf, &mut ctx).await.unwrap();
3910 assert_eq!(result, "active");
3911
3912 {
3913 let mut saved = executor.saved_state.lock().unwrap();
3914 let saved = saved.as_mut().expect("paused state should be saved");
3915 saved.set(
3916 "payment",
3917 serde_json::json!({
3918 "attempts": 1
3919 }),
3920 );
3921 }
3922
3923 let err = resume_workflow(
3924 &executor,
3925 &wf,
3926 "wf-branch-when-drift",
3927 serde_json::json!({"event": "manual.approved"}),
3928 )
3929 .await
3930 .expect_err("branch predicate drift must reject resume");
3931
3932 match err {
3933 WorkflowError::Other(msg) => {
3934 assert!(msg.contains("Invalid workflow resume cursor"), "got: {msg}");
3935 assert!(msg.contains("no longer matches"), "got: {msg}");
3936 }
3937 other => panic!("expected invalid cursor error, got {other:?}"),
3938 }
3939 assert!(executor.notifications.lock().unwrap().is_empty());
3940 }
3941
3942 #[tokio::test]
3943 async fn branch_missing_context_key_errors_instead_of_taking_default() {
3944 let executor = MockExecutor::new();
3945
3946 let wf = WorkflowDefinition::new("branching_missing_key")
3947 .initial_state("pending")
3948 .transition(
3949 "pending",
3950 "resolved",
3951 vec![WorkflowStep::branch(
3952 "payment.status",
3953 vec![("paid", vec![WorkflowStep::transition("resolved")])],
3954 vec![WorkflowStep::transition("fallback")],
3955 )],
3956 );
3957
3958 let mut ctx = WorkflowContext::new("wf-missing-branch-key", "pending");
3959 let err = run_workflow(&executor, &wf, &mut ctx)
3960 .await
3961 .expect_err("missing branch context must not route to default");
3962
3963 match err {
3964 WorkflowError::MissingContextKey(key) => assert_eq!(key, "payment.status"),
3965 other => panic!("expected MissingContextKey, got {other:?}"),
3966 }
3967 assert_eq!(ctx.current_state, "pending");
3968 }
3969
3970 #[tokio::test]
3971 async fn duplicate_transitions_from_same_state_fail_before_side_effects() {
3972 let executor = MockExecutor::new();
3973
3974 let wf = WorkflowDefinition::new("duplicate_transition_guard")
3975 .initial_state("active")
3976 .transition(
3977 "active",
3978 "vendor_a",
3979 vec![WorkflowStep::notify(
3980 ChannelKind::WhatsApp,
3981 "vendor_a",
3982 "vendor.phone",
3983 )],
3984 )
3985 .transition(
3986 "active",
3987 "vendor_b",
3988 vec![WorkflowStep::notify(
3989 ChannelKind::WhatsApp,
3990 "vendor_b",
3991 "vendor.phone",
3992 )],
3993 );
3994
3995 let mut ctx = WorkflowContext::new("wf-duplicate-transition", "active");
3996 ctx.set("vendor", serde_json::json!({"phone": "+628111"}));
3997
3998 let err = run_workflow(&executor, &wf, &mut ctx)
3999 .await
4000 .expect_err("duplicate transitions must fail closed");
4001
4002 match err {
4003 WorkflowError::Other(msg) => {
4004 assert!(msg.contains("Ambiguous workflow definition"), "got: {msg}");
4005 assert!(msg.contains("active"), "got: {msg}");
4006 assert!(msg.contains("2 outgoing transitions"), "got: {msg}");
4007 }
4008 other => panic!("expected ambiguous transition error, got {other:?}"),
4009 }
4010 assert!(executor.notifications.lock().unwrap().is_empty());
4011 }
4012
4013 #[tokio::test]
4014 async fn duplicate_future_transitions_fail_before_current_side_effects() {
4015 let executor = MockExecutor::new();
4016
4017 let wf = WorkflowDefinition::new("future_duplicate_transition_guard")
4018 .initial_state("active")
4019 .transition(
4020 "active",
4021 "routing",
4022 vec![
4023 WorkflowStep::notify(ChannelKind::WhatsApp, "before_bad_state", "vendor.phone"),
4024 WorkflowStep::transition("routing"),
4025 ],
4026 )
4027 .transition(
4028 "routing",
4029 "vendor_a",
4030 vec![WorkflowStep::notify(
4031 ChannelKind::WhatsApp,
4032 "vendor_a",
4033 "vendor.phone",
4034 )],
4035 )
4036 .transition(
4037 "routing",
4038 "vendor_b",
4039 vec![WorkflowStep::notify(
4040 ChannelKind::WhatsApp,
4041 "vendor_b",
4042 "vendor.phone",
4043 )],
4044 );
4045
4046 let mut ctx = WorkflowContext::new("wf-future-duplicate-transition", "active");
4047 ctx.set("vendor", serde_json::json!({"phone": "+628111"}));
4048
4049 let err = run_workflow(&executor, &wf, &mut ctx)
4050 .await
4051 .expect_err("future duplicate transitions must fail before current side effects");
4052
4053 match err {
4054 WorkflowError::Other(msg) => {
4055 assert!(msg.contains("Ambiguous workflow definition"), "got: {msg}");
4056 assert!(msg.contains("routing"), "got: {msg}");
4057 }
4058 other => panic!("expected ambiguous transition error, got {other:?}"),
4059 }
4060 assert_eq!(ctx.current_state, "active");
4061 assert!(
4062 executor.notifications.lock().unwrap().is_empty(),
4063 "definition validation must run before the initial notification"
4064 );
4065 }
4066
4067 #[tokio::test]
4068 async fn duplicate_branch_values_fail_before_side_effects() {
4069 let executor = MockExecutor::new();
4070
4071 let wf = WorkflowDefinition::new("duplicate_branch_guard")
4072 .initial_state("active")
4073 .transition(
4074 "active",
4075 "done",
4076 vec![WorkflowStep::branch(
4077 "decision",
4078 vec![
4079 (
4080 "accepted",
4081 vec![WorkflowStep::notify(
4082 ChannelKind::WhatsApp,
4083 "accepted_a",
4084 "vendor.phone",
4085 )],
4086 ),
4087 (
4088 "accepted",
4089 vec![WorkflowStep::notify(
4090 ChannelKind::WhatsApp,
4091 "accepted_b",
4092 "vendor.phone",
4093 )],
4094 ),
4095 ],
4096 vec![],
4097 )],
4098 );
4099
4100 let mut ctx = WorkflowContext::new("wf-duplicate-branch", "active");
4101 ctx.set("decision", Value::String("accepted".to_string()));
4102 ctx.set("vendor", serde_json::json!({"phone": "+628111"}));
4103
4104 let err = run_workflow(&executor, &wf, &mut ctx)
4105 .await
4106 .expect_err("duplicate branch values must fail closed");
4107
4108 match err {
4109 WorkflowError::Other(msg) => {
4110 assert!(msg.contains("Ambiguous workflow branch"), "got: {msg}");
4111 assert!(msg.contains("accepted"), "got: {msg}");
4112 }
4113 other => panic!("expected ambiguous branch error, got {other:?}"),
4114 }
4115 assert!(executor.notifications.lock().unwrap().is_empty());
4116 }
4117
4118 #[tokio::test]
4119 async fn branch_when_rejects_empty_string_contains_before_side_effects() {
4120 let executor = MockExecutor::new();
4121
4122 let wf = WorkflowDefinition::new("empty_string_contains_guard")
4123 .initial_state("active")
4124 .transition(
4125 "active",
4126 "done",
4127 vec![
4128 WorkflowStep::notify(ChannelKind::Email, "before_bad_branch", "ops.email"),
4129 WorkflowStep::branch_when(
4130 "decision",
4131 vec![(
4132 WorkflowBranchCondition::StringContains(String::new()),
4133 vec![WorkflowStep::transition("catch_all")],
4134 )],
4135 vec![WorkflowStep::transition("done")],
4136 ),
4137 ],
4138 );
4139
4140 let mut ctx = WorkflowContext::new("wf-empty-string-contains", "active");
4141 ctx.set("ops", serde_json::json!({"email": "ops@example.com"}));
4142 ctx.set("decision", Value::String("accepted".to_string()));
4143
4144 let err = run_workflow(&executor, &wf, &mut ctx)
4145 .await
4146 .expect_err("empty StringContains predicate must fail before notification");
4147
4148 match err {
4149 WorkflowError::Other(msg) => {
4150 assert!(
4151 msg.contains("StringContains condition must not be empty"),
4152 "got: {msg}"
4153 );
4154 }
4155 other => panic!("expected branch condition validation error, got {other:?}"),
4156 }
4157 assert!(executor.notifications.lock().unwrap().is_empty());
4158 }
4159
4160 #[tokio::test]
4161 async fn branch_when_rejects_empty_one_of_before_side_effects() {
4162 let executor = MockExecutor::new();
4163
4164 let wf = WorkflowDefinition::new("empty_one_of_guard")
4165 .initial_state("active")
4166 .transition(
4167 "active",
4168 "done",
4169 vec![
4170 WorkflowStep::notify(ChannelKind::Email, "before_bad_branch", "ops.email"),
4171 WorkflowStep::branch_when(
4172 "decision",
4173 vec![(
4174 WorkflowBranchCondition::OneOf(Vec::new()),
4175 vec![WorkflowStep::transition("impossible")],
4176 )],
4177 vec![WorkflowStep::transition("done")],
4178 ),
4179 ],
4180 );
4181
4182 let mut ctx = WorkflowContext::new("wf-empty-one-of", "active");
4183 ctx.set("ops", serde_json::json!({"email": "ops@example.com"}));
4184 ctx.set("decision", Value::String("accepted".to_string()));
4185
4186 let err = run_workflow(&executor, &wf, &mut ctx)
4187 .await
4188 .expect_err("empty OneOf predicate must fail before notification");
4189
4190 match err {
4191 WorkflowError::Other(msg) => {
4192 assert!(
4193 msg.contains("OneOf condition must include at least one value"),
4194 "got: {msg}"
4195 );
4196 }
4197 other => panic!("expected branch condition validation error, got {other:?}"),
4198 }
4199 assert!(executor.notifications.lock().unwrap().is_empty());
4200 }
4201
4202 #[tokio::test]
4203 async fn invalid_transition_target_fails_before_side_effects() {
4204 let executor = MockExecutor::new();
4205
4206 let wf = WorkflowDefinition::new("invalid_transition_target")
4207 .initial_state("active")
4208 .transition(
4209 "active",
4210 "done",
4211 vec![
4212 WorkflowStep::notify(ChannelKind::Email, "before_bad_transition", "ops.email"),
4213 WorkflowStep::transition(" "),
4214 ],
4215 );
4216
4217 let mut ctx = WorkflowContext::new("wf-invalid-transition-target", "active");
4218 ctx.set("ops", serde_json::json!({"email": "ops@example.com"}));
4219
4220 let err = run_workflow(&executor, &wf, &mut ctx)
4221 .await
4222 .expect_err("invalid transition target must fail during definition validation");
4223
4224 match err {
4225 WorkflowError::Other(msg) => {
4226 assert!(
4227 msg.contains("Transition target state must not be empty"),
4228 "got: {msg}"
4229 );
4230 }
4231 other => panic!("expected transition target validation error, got {other:?}"),
4232 }
4233 assert!(executor.notifications.lock().unwrap().is_empty());
4234 }
4235
4236 #[tokio::test]
4237 async fn invalid_notify_recipient_key_fails_before_side_effects() {
4238 let executor = MockExecutor::new();
4239
4240 let wf = WorkflowDefinition::new("invalid_notify_recipient")
4241 .initial_state("active")
4242 .transition(
4243 "active",
4244 "done",
4245 vec![
4246 WorkflowStep::notify(ChannelKind::Email, "bad_notify", " "),
4247 WorkflowStep::transition("done"),
4248 ],
4249 );
4250
4251 let mut ctx = WorkflowContext::new("wf-invalid-notify-recipient", "active");
4252 let err = run_workflow(&executor, &wf, &mut ctx)
4253 .await
4254 .expect_err("invalid notify recipient key must fail before send");
4255
4256 match err {
4257 WorkflowError::Other(msg) => {
4258 assert!(
4259 msg.contains("Notify recipient_key must not be empty"),
4260 "got: {msg}"
4261 );
4262 }
4263 other => panic!("expected notify recipient validation error, got {other:?}"),
4264 }
4265 assert!(executor.notifications.lock().unwrap().is_empty());
4266 }
4267
4268 #[tokio::test]
4269 async fn malformed_lookup_key_fails_before_side_effects() {
4270 let executor = MockExecutor::new();
4271
4272 let wf = WorkflowDefinition::new("malformed_lookup_key_guard")
4273 .initial_state("active")
4274 .transition(
4275 "active",
4276 "done",
4277 vec![
4278 WorkflowStep::notify(ChannelKind::Email, "before_bad_lookup", "ops.email"),
4279 WorkflowStep::notify(ChannelKind::Email, "bad_lookup", "customer."),
4280 WorkflowStep::transition("done"),
4281 ],
4282 );
4283
4284 let mut ctx = WorkflowContext::new("wf-malformed-lookup-key", "active");
4285 ctx.set("ops", serde_json::json!({"email": "ops@example.com"}));
4286 ctx.set(
4287 "customer",
4288 serde_json::json!({"email": "guest@example.com"}),
4289 );
4290
4291 let err = run_workflow(&executor, &wf, &mut ctx)
4292 .await
4293 .expect_err("malformed lookup key must fail before first notification");
4294
4295 match err {
4296 WorkflowError::Other(msg) => {
4297 assert!(
4298 msg.contains("empty dot-notation path segments"),
4299 "got: {msg}"
4300 );
4301 }
4302 other => panic!("expected malformed lookup key error, got {other:?}"),
4303 }
4304 assert!(
4305 executor.notifications.lock().unwrap().is_empty(),
4306 "definition validation must fail before earlier notification side effects"
4307 );
4308 }
4309
4310 #[tokio::test]
4311 async fn invalid_charge_amount_key_fails_before_provider_call() {
4312 let executor = MockExecutor::new();
4313
4314 let wf = WorkflowDefinition::new("invalid_charge_amount_key")
4315 .initial_state("created")
4316 .transition(
4317 "created",
4318 "awaiting_payment",
4319 vec![WorkflowStep::charge(
4320 PaymentKind::Xendit,
4321 " ",
4322 "order.id",
4323 Some("charge"),
4324 )],
4325 );
4326
4327 let mut ctx = WorkflowContext::new("wf-invalid-charge-amount", "created");
4328 ctx.set(
4329 "order",
4330 serde_json::json!({"id": "booking-1", "total": 125_000}),
4331 );
4332 let err = run_workflow(&executor, &wf, &mut ctx)
4333 .await
4334 .expect_err("invalid charge amount key must fail before provider call");
4335
4336 match err {
4337 WorkflowError::Other(msg) => {
4338 assert!(
4339 msg.contains("Charge amount_key must not be empty"),
4340 "got: {msg}"
4341 );
4342 }
4343 other => panic!("expected charge amount key validation error, got {other:?}"),
4344 }
4345 assert!(executor.charges.lock().unwrap().is_empty());
4346 }
4347
4348 #[tokio::test]
4349 async fn malformed_charge_lookup_key_fails_before_provider_call() {
4350 let executor = MockExecutor::new();
4351
4352 let wf = WorkflowDefinition::new("malformed_charge_lookup_key")
4353 .initial_state("created")
4354 .transition(
4355 "created",
4356 "awaiting_payment",
4357 vec![WorkflowStep::charge(
4358 PaymentKind::Xendit,
4359 "order..total",
4360 "order.id",
4361 Some("charge"),
4362 )],
4363 );
4364
4365 let mut ctx = WorkflowContext::new("wf-malformed-charge-lookup", "created");
4366 ctx.set(
4367 "order",
4368 serde_json::json!({"id": "booking-1", "total": 125_000}),
4369 );
4370 let err = run_workflow(&executor, &wf, &mut ctx)
4371 .await
4372 .expect_err("malformed charge lookup key must fail before provider call");
4373
4374 match err {
4375 WorkflowError::Other(msg) => {
4376 assert!(
4377 msg.contains("empty dot-notation path segments"),
4378 "got: {msg}"
4379 );
4380 }
4381 other => panic!("expected malformed lookup key error, got {other:?}"),
4382 }
4383 assert!(executor.charges.lock().unwrap().is_empty());
4384 }
4385
4386 #[tokio::test]
4387 async fn charge_rejects_missing_optional_string_key_before_provider_call() {
4388 let executor = MockExecutor::new();
4389
4390 let wf = WorkflowDefinition::new("charge_missing_optional_string_key")
4391 .initial_state("created")
4392 .transition(
4393 "created",
4394 "awaiting_payment",
4395 vec![WorkflowStep::charge_with(
4396 PaymentKind::Xendit,
4397 "order.total",
4398 "order.id",
4399 Some("order.description"),
4400 Some("order.payment_method"),
4401 Some("charge"),
4402 )],
4403 );
4404
4405 let mut ctx = WorkflowContext::new("wf-charge-missing-optional-string", "created");
4406 ctx.set(
4407 "order",
4408 serde_json::json!({"id": "booking-1", "total": 125_000}),
4409 );
4410 let err = run_workflow(&executor, &wf, &mut ctx)
4411 .await
4412 .expect_err("missing configured charge string key must fail before provider call");
4413
4414 match err {
4415 WorkflowError::MissingContextKey(key) => {
4416 assert_eq!(key, "order.description (expected string)");
4417 }
4418 other => panic!("expected missing context key error, got {other:?}"),
4419 }
4420 assert!(executor.charges.lock().unwrap().is_empty());
4421 }
4422
4423 #[tokio::test]
4424 async fn charge_rejects_non_string_optional_key_before_provider_call() {
4425 let executor = MockExecutor::new();
4426
4427 let wf = WorkflowDefinition::new("charge_non_string_optional_key")
4428 .initial_state("created")
4429 .transition(
4430 "created",
4431 "awaiting_payment",
4432 vec![WorkflowStep::charge_with(
4433 PaymentKind::Xendit,
4434 "order.total",
4435 "order.id",
4436 None,
4437 Some("order.payment_method"),
4438 Some("charge"),
4439 )],
4440 );
4441
4442 let mut ctx = WorkflowContext::new("wf-charge-non-string-optional", "created");
4443 ctx.set(
4444 "order",
4445 serde_json::json!({
4446 "id": "booking-1",
4447 "total": 125_000,
4448 "payment_method": 100
4449 }),
4450 );
4451 let err = run_workflow(&executor, &wf, &mut ctx)
4452 .await
4453 .expect_err("non-string configured charge key must fail before provider call");
4454
4455 match err {
4456 WorkflowError::MissingContextKey(key) => {
4457 assert_eq!(key, "order.payment_method (expected string)");
4458 }
4459 other => panic!("expected missing context key error, got {other:?}"),
4460 }
4461 assert!(executor.charges.lock().unwrap().is_empty());
4462 }
4463
4464 #[tokio::test]
4465 async fn invalid_timeout_fallback_fails_before_pre_wait_side_effects() {
4466 let executor = MockExecutor::new();
4467
4468 let wf = WorkflowDefinition::new("invalid_timeout_fallback_guard")
4469 .initial_state("active")
4470 .transition(
4471 "active",
4472 "done",
4473 vec![
4474 WorkflowStep::notify(ChannelKind::Email, "before_wait", "customer.email"),
4475 WorkflowStep::wait_or(
4476 "payment.success",
4477 std::time::Duration::from_secs(3600),
4478 vec![WorkflowStep::wait(
4479 "manual_review",
4480 std::time::Duration::from_secs(3600),
4481 )],
4482 ),
4483 WorkflowStep::transition("done"),
4484 ],
4485 );
4486
4487 let mut ctx = WorkflowContext::new("wf-invalid-timeout-fallback", "active");
4488 ctx.set(
4489 "customer",
4490 serde_json::json!({"email": "guest@example.com"}),
4491 );
4492
4493 let err = run_workflow(&executor, &wf, &mut ctx)
4494 .await
4495 .expect_err("unsupported timeout fallback must fail before reaching the Wait");
4496
4497 match err {
4498 WorkflowError::Other(msg) => {
4499 assert!(
4500 msg.contains("Wait steps inside on_timeout fallback are not supported"),
4501 "got: {msg}"
4502 );
4503 }
4504 other => panic!("expected timeout fallback validation error, got {other:?}"),
4505 }
4506 assert!(ctx.cursor.is_none());
4507 assert!(
4508 executor.notifications.lock().unwrap().is_empty(),
4509 "definition validation must run before the pre-wait notification"
4510 );
4511 }
4512
4513 #[tokio::test]
4514 async fn nested_transition_stops_parent_block_before_follow_up_side_effects() {
4515 let executor = MockExecutor::new();
4516
4517 let wf = WorkflowDefinition::new("nested_transition_guard")
4518 .initial_state("active")
4519 .transition(
4520 "active",
4521 "fallback",
4522 vec![
4523 WorkflowStep::branch(
4524 "decision",
4525 vec![("accepted", vec![WorkflowStep::transition("confirmed")])],
4526 vec![],
4527 ),
4528 WorkflowStep::notify(
4529 ChannelKind::WhatsApp,
4530 "old_state_followup",
4531 "vendor.phone",
4532 ),
4533 ],
4534 );
4535
4536 let mut ctx = WorkflowContext::new("wf-nested-transition", "active");
4537 ctx.set("decision", Value::String("accepted".to_string()));
4538 ctx.set("vendor", serde_json::json!({"phone": "+628111"}));
4539
4540 let result = run_workflow(&executor, &wf, &mut ctx)
4541 .await
4542 .expect("nested transition should finish cleanly");
4543
4544 assert_eq!(result, "confirmed");
4545 assert_eq!(ctx.current_state, "confirmed");
4546 assert!(
4547 executor.notifications.lock().unwrap().is_empty(),
4548 "parent block must stop after nested transition changes state"
4549 );
4550 }
4551
4552 #[test]
4553 fn log_message_resolves_nested_context_placeholders() {
4554 let mut ctx = WorkflowContext::new("wf-log", "running");
4555 ctx.set(
4556 "item",
4557 serde_json::json!({
4558 "name": "Captain A",
4559 "phone": "+628111",
4560 }),
4561 );
4562 ctx.set("attempt", serde_json::json!(2));
4563
4564 assert_eq!(
4565 resolve_log_message(
4566 &ctx,
4567 "Processing {item.name} via {item.phone} attempt {attempt}"
4568 ),
4569 "Processing Captain A via +628111 attempt 2"
4570 );
4571 assert_eq!(
4572 resolve_log_message(&ctx, "Missing {item.email} stays"),
4573 "Missing {item.email} stays"
4574 );
4575 }
4576
4577 #[tokio::test]
4578 async fn test_workflow_for_each_notify() {
4579 let executor = MockExecutor::new();
4580
4581 let wf = WorkflowDefinition::new("broadcast")
4582 .initial_state("broadcasting")
4583 .transition(
4584 "broadcasting",
4585 "waiting",
4586 vec![WorkflowStep::for_each(
4587 "operators",
4588 vec![WorkflowStep::notify(
4589 ChannelKind::WhatsApp,
4590 "opportunity",
4591 "item.phone",
4592 )],
4593 )],
4594 );
4595
4596 let mut ctx = WorkflowContext::new("wf-003", "broadcasting");
4597 ctx.set(
4598 "operators",
4599 serde_json::json!([
4600 {"name": "Captain A", "phone": "+628111"},
4601 {"name": "Captain B", "phone": "+628222"},
4602 {"name": "Captain C", "phone": "+628333"},
4603 ]),
4604 );
4605
4606 let result = run_workflow(&executor, &wf, &mut ctx).await.unwrap();
4607 assert_eq!(result, "waiting");
4608
4609 let notifs = executor.notifications.lock().unwrap();
4611 assert_eq!(notifs.len(), 3);
4612 assert_eq!(notifs[0].0, "+628111");
4613 assert_eq!(notifs[1].0, "+628222");
4614 assert_eq!(notifs[2].0, "+628333");
4615 drop(notifs);
4616
4617 let side_effects = executor.completed_side_effects.lock().unwrap();
4618 assert_eq!(side_effects.len(), 3);
4619 for operation_id in side_effects.iter() {
4620 assert!(
4621 !operation_id.contains("+628"),
4622 "side-effect operation id must not leak item payload: {operation_id}"
4623 );
4624 assert!(
4625 !operation_id.contains("Captain"),
4626 "side-effect operation id must not leak item payload: {operation_id}"
4627 );
4628 }
4629 assert_eq!(
4630 side_effects.as_slice(),
4631 &[
4632 side_effect_operation_id(
4633 "wf-003",
4634 "broadcasting",
4635 0,
4636 WorkflowSideEffectKind::Notify,
4637 "steps[0]/for_each[0].steps[0]",
4638 ),
4639 side_effect_operation_id(
4640 "wf-003",
4641 "broadcasting",
4642 0,
4643 WorkflowSideEffectKind::Notify,
4644 "steps[0]/for_each[1].steps[0]",
4645 ),
4646 side_effect_operation_id(
4647 "wf-003",
4648 "broadcasting",
4649 0,
4650 WorkflowSideEffectKind::Notify,
4651 "steps[0]/for_each[2].steps[0]",
4652 ),
4653 ]
4654 );
4655 }
4656
4657 #[test]
4658 fn side_effect_operation_id_does_not_collide_on_delimiters() {
4659 let first = side_effect_operation_id(
4660 "tenant:wf",
4661 "broadcasting",
4662 7,
4663 WorkflowSideEffectKind::Notify,
4664 "steps[0]",
4665 );
4666 let second = side_effect_operation_id(
4667 "tenant",
4668 "wf:broadcasting",
4669 7,
4670 WorkflowSideEffectKind::Notify,
4671 "steps[0]",
4672 );
4673
4674 assert_ne!(
4675 first, second,
4676 "side-effect operation ids must not collide when fields contain delimiters"
4677 );
4678 }
4679
4680 #[tokio::test]
4681 async fn side_effect_operation_id_changes_when_state_is_reentered() {
4682 let executor = MockExecutor::new();
4683
4684 let wf = WorkflowDefinition::new("reentered_state_side_effect")
4685 .initial_state("active")
4686 .transition(
4687 "active",
4688 "idle",
4689 vec![
4690 WorkflowStep::notify(ChannelKind::Email, "active_notice", "customer.email"),
4691 WorkflowStep::transition("idle"),
4692 ],
4693 );
4694
4695 let mut ctx = WorkflowContext::new("wf-reentered-state", "active");
4696 ctx.set(
4697 "customer",
4698 serde_json::json!({"email": "guest@example.com"}),
4699 );
4700
4701 let first = run_workflow(&executor, &wf, &mut ctx).await.unwrap();
4702 assert_eq!(first, "idle");
4703 ctx.transition_to("active", Some("manual requeue".to_string()));
4704 let second = run_workflow(&executor, &wf, &mut ctx).await.unwrap();
4705 assert_eq!(second, "idle");
4706
4707 let notifications = executor.notifications.lock().unwrap();
4708 assert_eq!(
4709 notifications.len(),
4710 2,
4711 "same state re-entry must execute the side effect again"
4712 );
4713 drop(notifications);
4714
4715 let side_effects = executor.completed_side_effects.lock().unwrap();
4716 assert_eq!(
4717 side_effects.as_slice(),
4718 &[
4719 side_effect_operation_id(
4720 "wf-reentered-state",
4721 "active",
4722 0,
4723 WorkflowSideEffectKind::Notify,
4724 "steps[0]",
4725 ),
4726 side_effect_operation_id(
4727 "wf-reentered-state",
4728 "active",
4729 2,
4730 WorkflowSideEffectKind::Notify,
4731 "steps[0]",
4732 ),
4733 ]
4734 );
4735 }
4736
4737 #[tokio::test]
4738 async fn notify_with_missing_payload_key_fails_before_send() {
4739 let executor = MockExecutor::new();
4740
4741 let wf = WorkflowDefinition::new("notify_payload_guard")
4742 .initial_state("active")
4743 .transition(
4744 "active",
4745 "done",
4746 vec![WorkflowStep::notify_with_payload(
4747 ChannelKind::Email,
4748 "booking_confirmed",
4749 "customer.email",
4750 "booking.template_payload",
4751 )],
4752 );
4753
4754 let mut ctx = WorkflowContext::new("wf-notify-payload-guard", "active");
4755 ctx.set(
4756 "customer",
4757 serde_json::json!({"email": "guest@example.com"}),
4758 );
4759
4760 let err = run_workflow(&executor, &wf, &mut ctx)
4761 .await
4762 .expect_err("missing explicit notification payload must fail closed");
4763
4764 match err {
4765 WorkflowError::MissingContextKey(key) => assert_eq!(key, "booking.template_payload"),
4766 other => panic!("expected MissingContextKey, got {other:?}"),
4767 }
4768 assert!(
4769 executor.notifications.lock().unwrap().is_empty(),
4770 "notification provider must not receive an empty payload for a missing explicit payload key"
4771 );
4772 }
4773
4774 #[tokio::test]
4775 async fn test_workflow_pauses_at_wait() {
4776 let executor = MockExecutor::new();
4777
4778 let wf = WorkflowDefinition::new("wait_test")
4779 .initial_state("active")
4780 .transition(
4781 "active",
4782 "resolved",
4783 vec![
4784 WorkflowStep::log("Before wait"),
4785 WorkflowStep::wait("user_response", std::time::Duration::from_secs(3600)),
4786 WorkflowStep::log("After wait"),
4788 ],
4789 );
4790
4791 let mut ctx = WorkflowContext::new("wf-004", "active");
4792 let result = run_workflow(&executor, &wf, &mut ctx).await.unwrap();
4793
4794 assert_eq!(result, "active");
4796 }
4797
4798 #[tokio::test]
4799 async fn test_workflow_pauses_at_nested_branch_wait() {
4800 let executor = MockExecutor::new();
4801
4802 let wf = WorkflowDefinition::new("nested_branch_wait")
4803 .initial_state("active")
4804 .transition(
4805 "active",
4806 "resolved",
4807 vec![
4808 WorkflowStep::branch(
4809 "customer.tier",
4810 vec![(
4811 "vip",
4812 vec![WorkflowStep::wait(
4813 "vip_payment",
4814 std::time::Duration::from_secs(7200),
4815 )],
4816 )],
4817 vec![WorkflowStep::log("Standard customer")],
4818 ),
4819 WorkflowStep::notify(ChannelKind::Email, "booking_confirmed", "customer.email"),
4820 WorkflowStep::transition("resolved"),
4821 ],
4822 );
4823
4824 let mut ctx = WorkflowContext::new("wf-nested-branch-wait", "active");
4825 ctx.set(
4826 "customer",
4827 serde_json::json!({
4828 "tier": "vip",
4829 "email": "guest@example.com"
4830 }),
4831 );
4832
4833 let result = run_workflow(&executor, &wf, &mut ctx).await.unwrap();
4834 assert_eq!(result, "active");
4835 assert_eq!(ctx.current_state, "active");
4836
4837 let notifs = executor.notifications.lock().unwrap();
4838 assert!(
4839 notifs.is_empty(),
4840 "steps after a nested branch Wait must not execute"
4841 );
4842 }
4843
4844 #[tokio::test]
4845 async fn branch_resume_rejects_reordered_branch_definition() {
4846 let executor = MockExecutor::new();
4847
4848 let wf = WorkflowDefinition::new("branch_reorder_source")
4849 .initial_state("active")
4850 .transition(
4851 "active",
4852 "done",
4853 vec![WorkflowStep::branch(
4854 "customer.tier",
4855 vec![
4856 (
4857 "vip",
4858 vec![
4859 WorkflowStep::wait(
4860 "approved",
4861 std::time::Duration::from_secs(3600),
4862 ),
4863 WorkflowStep::notify(
4864 ChannelKind::Email,
4865 "vip_approved",
4866 "customer.email",
4867 ),
4868 ],
4869 ),
4870 (
4871 "standard",
4872 vec![
4873 WorkflowStep::wait(
4874 "approved",
4875 std::time::Duration::from_secs(3600),
4876 ),
4877 WorkflowStep::notify(
4878 ChannelKind::Email,
4879 "standard_approved",
4880 "customer.email",
4881 ),
4882 ],
4883 ),
4884 ],
4885 vec![],
4886 )],
4887 );
4888
4889 let mut ctx = WorkflowContext::new("wf-branch-reorder", "active");
4890 ctx.set(
4891 "customer",
4892 serde_json::json!({
4893 "tier": "vip",
4894 "email": "guest@example.com"
4895 }),
4896 );
4897
4898 let result = run_workflow(&executor, &wf, &mut ctx).await.unwrap();
4899 assert_eq!(result, "active");
4900
4901 let drifted_wf = WorkflowDefinition::new("branch_reorder_source")
4902 .initial_state("active")
4903 .transition(
4904 "active",
4905 "done",
4906 vec![WorkflowStep::branch(
4907 "customer.tier",
4908 vec![
4909 (
4910 "standard",
4911 vec![
4912 WorkflowStep::wait(
4913 "approved",
4914 std::time::Duration::from_secs(3600),
4915 ),
4916 WorkflowStep::notify(
4917 ChannelKind::Email,
4918 "standard_approved",
4919 "customer.email",
4920 ),
4921 ],
4922 ),
4923 (
4924 "vip",
4925 vec![
4926 WorkflowStep::wait(
4927 "approved",
4928 std::time::Duration::from_secs(3600),
4929 ),
4930 WorkflowStep::notify(
4931 ChannelKind::Email,
4932 "vip_approved",
4933 "customer.email",
4934 ),
4935 ],
4936 ),
4937 ],
4938 vec![],
4939 )],
4940 );
4941
4942 let err = resume_workflow(
4943 &executor,
4944 &drifted_wf,
4945 "wf-branch-reorder",
4946 serde_json::json!({"event": "approved"}),
4947 )
4948 .await
4949 .expect_err("branch definition reorder must not silently resume wrong arm");
4950
4951 match err {
4952 WorkflowError::Other(msg) => {
4953 assert!(msg.contains("Invalid workflow resume cursor"));
4954 assert!(msg.contains("standard"));
4955 assert!(msg.contains("vip"));
4956 }
4957 other => panic!("expected invalid cursor error, got {other:?}"),
4958 }
4959 assert!(
4960 executor.notifications.lock().unwrap().is_empty(),
4961 "drifted resume must fail before sending the wrong branch notification"
4962 );
4963 }
4964
4965 #[tokio::test]
4966 async fn branch_resume_rejects_default_branch_drift() {
4967 let executor = MockExecutor::new();
4968
4969 let wf = WorkflowDefinition::new("branch_default_source")
4970 .initial_state("active")
4971 .transition(
4972 "active",
4973 "done",
4974 vec![WorkflowStep::branch(
4975 "customer.tier",
4976 vec![(
4977 "vip",
4978 vec![WorkflowStep::notify(
4979 ChannelKind::Email,
4980 "vip",
4981 "customer.email",
4982 )],
4983 )],
4984 vec![
4985 WorkflowStep::wait("approved", std::time::Duration::from_secs(3600)),
4986 WorkflowStep::notify(ChannelKind::Email, "default", "customer.email"),
4987 ],
4988 )],
4989 );
4990
4991 let mut ctx = WorkflowContext::new("wf-branch-default-drift", "active");
4992 ctx.set(
4993 "customer",
4994 serde_json::json!({
4995 "tier": "gold",
4996 "email": "guest@example.com"
4997 }),
4998 );
4999
5000 let result = run_workflow(&executor, &wf, &mut ctx).await.unwrap();
5001 assert_eq!(result, "active");
5002
5003 let drifted_wf = WorkflowDefinition::new("branch_default_source")
5004 .initial_state("active")
5005 .transition(
5006 "active",
5007 "done",
5008 vec![WorkflowStep::branch(
5009 "customer.tier",
5010 vec![
5011 (
5012 "vip",
5013 vec![WorkflowStep::notify(
5014 ChannelKind::Email,
5015 "vip",
5016 "customer.email",
5017 )],
5018 ),
5019 (
5020 "gold",
5021 vec![WorkflowStep::notify(
5022 ChannelKind::Email,
5023 "gold",
5024 "customer.email",
5025 )],
5026 ),
5027 ],
5028 vec![
5029 WorkflowStep::wait("approved", std::time::Duration::from_secs(3600)),
5030 WorkflowStep::notify(ChannelKind::Email, "default", "customer.email"),
5031 ],
5032 )],
5033 );
5034
5035 let err = resume_workflow(
5036 &executor,
5037 &drifted_wf,
5038 "wf-branch-default-drift",
5039 serde_json::json!({"event": "approved"}),
5040 )
5041 .await
5042 .expect_err("default branch cursor must reject new matching branch arms");
5043
5044 match err {
5045 WorkflowError::Other(msg) => {
5046 assert!(msg.contains("default branch cursor"));
5047 assert!(msg.contains("gold"));
5048 }
5049 other => panic!("expected invalid cursor error, got {other:?}"),
5050 }
5051 assert!(executor.notifications.lock().unwrap().is_empty());
5052 }
5053
5054 #[tokio::test]
5055 async fn test_workflow_pauses_at_nested_for_each_wait() {
5056 let executor = MockExecutor::new();
5057
5058 let wf = WorkflowDefinition::new("nested_for_each_wait")
5059 .initial_state("active")
5060 .transition(
5061 "active",
5062 "resolved",
5063 vec![
5064 WorkflowStep::for_each(
5065 "operators",
5066 vec![
5067 WorkflowStep::wait(
5068 "operator_accept",
5069 std::time::Duration::from_secs(3600),
5070 ),
5071 WorkflowStep::notify(ChannelKind::WhatsApp, "after_wait", "item.phone"),
5072 ],
5073 ),
5074 WorkflowStep::transition("resolved"),
5075 ],
5076 );
5077
5078 let mut ctx = WorkflowContext::new("wf-nested-for-each-wait", "active");
5079 ctx.set(
5080 "operators",
5081 serde_json::json!([
5082 {"name": "Captain A", "phone": "+628111"},
5083 {"name": "Captain B", "phone": "+628222"}
5084 ]),
5085 );
5086
5087 let result = run_workflow(&executor, &wf, &mut ctx).await.unwrap();
5088 assert_eq!(result, "active");
5089 assert_eq!(ctx.current_state, "active");
5090
5091 let notifs = executor.notifications.lock().unwrap();
5092 assert!(
5093 notifs.is_empty(),
5094 "steps after a nested for_each Wait must not execute"
5095 );
5096 }
5097
5098 #[tokio::test]
5099 async fn test_workflow_resumes_for_each_after_wait_without_replaying_items() {
5100 let executor = MockExecutor::new();
5101
5102 let wf = WorkflowDefinition::new("for_each_resume")
5103 .initial_state("active")
5104 .transition(
5105 "active",
5106 "resolved",
5107 vec![
5108 WorkflowStep::for_each(
5109 "operators",
5110 vec![
5111 WorkflowStep::notify(
5112 ChannelKind::WhatsApp,
5113 "before_wait",
5114 "item.phone",
5115 ),
5116 WorkflowStep::wait(
5117 "operator_accept",
5118 std::time::Duration::from_secs(3600),
5119 ),
5120 WorkflowStep::notify(ChannelKind::WhatsApp, "after_wait", "item.phone"),
5121 ],
5122 ),
5123 WorkflowStep::transition("resolved"),
5124 ],
5125 );
5126
5127 let mut ctx = WorkflowContext::new("wf-for-each-resume", "active");
5128 ctx.set(
5129 "operators",
5130 serde_json::json!([
5131 {"name": "Captain A", "phone": "+628111"},
5132 {"name": "Captain B", "phone": "+628222"}
5133 ]),
5134 );
5135
5136 let result = run_workflow(&executor, &wf, &mut ctx).await.unwrap();
5137 assert_eq!(result, "active");
5138 assert!(
5139 ctx.cursor.is_some(),
5140 "first pause should persist a resume cursor"
5141 );
5142
5143 {
5144 let notifs = executor.notifications.lock().unwrap();
5145 assert_eq!(
5146 notifs.as_slice(),
5147 &[("+628111".to_string(), "before_wait".to_string())]
5148 );
5149 }
5150
5151 let result = resume_workflow(
5152 &executor,
5153 &wf,
5154 "wf-for-each-resume",
5155 serde_json::json!({"event": "operator_accept", "accepted": true}),
5156 )
5157 .await
5158 .unwrap();
5159 assert_eq!(result, "active");
5160
5161 {
5162 let notifs = executor.notifications.lock().unwrap();
5163 assert_eq!(
5164 notifs.as_slice(),
5165 &[
5166 ("+628111".to_string(), "before_wait".to_string()),
5167 ("+628111".to_string(), "after_wait".to_string()),
5168 ("+628222".to_string(), "before_wait".to_string()),
5169 ],
5170 "resume must continue within the active item and must not replay item 0"
5171 );
5172 }
5173
5174 let result = resume_workflow(
5175 &executor,
5176 &wf,
5177 "wf-for-each-resume",
5178 serde_json::json!({"event": "operator_accept", "accepted": true}),
5179 )
5180 .await
5181 .unwrap();
5182 assert_eq!(result, "resolved");
5183
5184 let notifs = executor.notifications.lock().unwrap();
5185 assert_eq!(
5186 notifs.as_slice(),
5187 &[
5188 ("+628111".to_string(), "before_wait".to_string()),
5189 ("+628111".to_string(), "after_wait".to_string()),
5190 ("+628222".to_string(), "before_wait".to_string()),
5191 ("+628222".to_string(), "after_wait".to_string()),
5192 ],
5193 "each side effect should run exactly once per operator"
5194 );
5195
5196 let saved = executor.saved_state.lock().unwrap();
5197 let saved = saved.as_ref().expect("final state should be saved");
5198 assert_eq!(saved.current_state, "resolved");
5199 assert!(
5200 saved.cursor.is_none(),
5201 "completed workflow should not retain a resume cursor"
5202 );
5203 }
5204
5205 #[tokio::test]
5206 async fn for_each_resume_rejects_reordered_items_before_notification() {
5207 let executor = MockExecutor::new();
5208
5209 let wf = WorkflowDefinition::new("for_each_reorder_guard")
5210 .initial_state("active")
5211 .transition(
5212 "active",
5213 "resolved",
5214 vec![
5215 WorkflowStep::for_each(
5216 "operators",
5217 vec![
5218 WorkflowStep::wait(
5219 "operator_accept",
5220 std::time::Duration::from_secs(3600),
5221 ),
5222 WorkflowStep::notify(ChannelKind::WhatsApp, "accepted", "item.phone"),
5223 ],
5224 ),
5225 WorkflowStep::transition("resolved"),
5226 ],
5227 );
5228
5229 let mut ctx = WorkflowContext::new("wf-for-each-reorder", "active");
5230 ctx.set(
5231 "operators",
5232 serde_json::json!([
5233 {"name": "Captain A", "phone": "+628111"},
5234 {"name": "Captain B", "phone": "+628222"}
5235 ]),
5236 );
5237
5238 let result = run_workflow(&executor, &wf, &mut ctx).await.unwrap();
5239 assert_eq!(result, "active");
5240
5241 {
5242 let mut saved = executor.saved_state.lock().unwrap();
5243 let saved = saved.as_mut().expect("paused state should be saved");
5244 saved.set(
5245 "operators",
5246 serde_json::json!([
5247 {"name": "Captain B", "phone": "+628222"},
5248 {"name": "Captain A", "phone": "+628111"}
5249 ]),
5250 );
5251 }
5252
5253 let err = resume_workflow(
5254 &executor,
5255 &wf,
5256 "wf-for-each-reorder",
5257 serde_json::json!({"event": "operator_accept"}),
5258 )
5259 .await
5260 .expect_err("reordered operator list must not resume on the wrong item");
5261
5262 match err {
5263 WorkflowError::Other(msg) => {
5264 assert!(msg.contains("Invalid workflow resume cursor"), "got: {msg}");
5265 assert!(msg.contains("for_each cursor item"), "got: {msg}");
5266 assert!(msg.contains("changed"), "got: {msg}");
5267 }
5268 other => panic!("expected invalid cursor error, got {other:?}"),
5269 }
5270 assert!(
5271 executor.notifications.lock().unwrap().is_empty(),
5272 "wrong operator must not receive the post-wait notification"
5273 );
5274 }
5275
5276 #[tokio::test]
5277 async fn legacy_for_each_cursor_is_rejected_before_notification() {
5278 let executor = MockExecutor::new();
5279
5280 let wf = WorkflowDefinition::new("legacy_for_each_cursor_guard")
5281 .initial_state("active")
5282 .transition(
5283 "active",
5284 "resolved",
5285 vec![
5286 WorkflowStep::for_each(
5287 "operators",
5288 vec![
5289 WorkflowStep::wait(
5290 "operator_accept",
5291 std::time::Duration::from_secs(3600),
5292 ),
5293 WorkflowStep::notify(ChannelKind::WhatsApp, "accepted", "item.phone"),
5294 ],
5295 ),
5296 WorkflowStep::transition("resolved"),
5297 ],
5298 );
5299
5300 let mut ctx = WorkflowContext::new("wf-legacy-for-each-cursor", "active");
5301 ctx.set(
5302 "operators",
5303 serde_json::json!([
5304 {"name": "Captain A", "phone": "+628111"},
5305 {"name": "Captain B", "phone": "+628222"}
5306 ]),
5307 );
5308 ctx.set_cursor(WorkflowCursor {
5309 state: "active".to_string(),
5310 frames: vec![
5311 WorkflowCursorFrame::Steps { index: 0 },
5312 WorkflowCursorFrame::ForEach {
5313 item_index: 1,
5314 index: 1,
5315 },
5316 ],
5317 wait: Some(WorkflowPendingWait {
5318 event: "operator_accept".to_string(),
5319 deadline_at: Utc::now() + chrono::Duration::hours(1),
5320 on_timeout: vec![],
5321 }),
5322 });
5323 executor.save_state(&ctx).await.unwrap();
5324
5325 let err = resume_workflow(
5326 &executor,
5327 &wf,
5328 "wf-legacy-for-each-cursor",
5329 serde_json::json!({"event": "operator_accept"}),
5330 )
5331 .await
5332 .expect_err("legacy for_each cursors must fail closed");
5333
5334 match err {
5335 WorkflowError::Other(msg) => {
5336 assert!(msg.contains("Invalid workflow resume cursor"), "got: {msg}");
5337 assert!(msg.contains("legacy for_each cursor"), "got: {msg}");
5338 }
5339 other => panic!("expected invalid cursor error, got {other:?}"),
5340 }
5341 assert!(
5342 executor.notifications.lock().unwrap().is_empty(),
5343 "legacy cursor must not resume onto an unverified loop item"
5344 );
5345 }
5346
5347 #[tokio::test]
5348 async fn test_nested_for_each_restores_outer_item_binding() {
5349 let executor = MockExecutor::new();
5350
5351 let wf = WorkflowDefinition::new("nested_for_each_items")
5352 .initial_state("active")
5353 .transition(
5354 "active",
5355 "done",
5356 vec![
5357 WorkflowStep::for_each(
5358 "operators",
5359 vec![
5360 WorkflowStep::for_each(
5361 "tasks",
5362 vec![WorkflowStep::notify(
5363 ChannelKind::WhatsApp,
5364 "task",
5365 "item.phone",
5366 )],
5367 ),
5368 WorkflowStep::notify(ChannelKind::WhatsApp, "summary", "item.phone"),
5369 ],
5370 ),
5371 WorkflowStep::transition("done"),
5372 ],
5373 );
5374
5375 let mut ctx = WorkflowContext::new("wf-nested-items", "active");
5376 ctx.set(
5377 "operators",
5378 serde_json::json!([
5379 {"name": "Captain A", "phone": "+628111"},
5380 {"name": "Captain B", "phone": "+628222"}
5381 ]),
5382 );
5383 ctx.set(
5384 "tasks",
5385 serde_json::json!([
5386 {"name": "Task 1", "phone": "+629999"}
5387 ]),
5388 );
5389
5390 let result = run_workflow(&executor, &wf, &mut ctx).await.unwrap();
5391 assert_eq!(result, "done");
5392
5393 let notifs = executor.notifications.lock().unwrap();
5394 assert_eq!(
5395 notifs.as_slice(),
5396 &[
5397 ("+629999".to_string(), "task".to_string()),
5398 ("+628111".to_string(), "summary".to_string()),
5399 ("+629999".to_string(), "task".to_string()),
5400 ("+628222".to_string(), "summary".to_string()),
5401 ],
5402 "inner ForEach must not delete the outer item binding"
5403 );
5404 }
5405
5406 #[tokio::test]
5407 async fn test_nested_for_each_wait_restores_item_binding_across_resume() {
5408 let executor = MockExecutor::new();
5409
5410 let wf = WorkflowDefinition::new("nested_for_each_wait_items")
5411 .initial_state("active")
5412 .transition(
5413 "active",
5414 "done",
5415 vec![
5416 WorkflowStep::for_each(
5417 "operators",
5418 vec![
5419 WorkflowStep::for_each(
5420 "tasks",
5421 vec![
5422 WorkflowStep::wait(
5423 "task_done",
5424 std::time::Duration::from_secs(3600),
5425 ),
5426 WorkflowStep::notify(
5427 ChannelKind::WhatsApp,
5428 "task",
5429 "item.phone",
5430 ),
5431 ],
5432 ),
5433 WorkflowStep::notify(ChannelKind::WhatsApp, "summary", "item.phone"),
5434 ],
5435 ),
5436 WorkflowStep::transition("done"),
5437 ],
5438 );
5439
5440 let mut ctx = WorkflowContext::new("wf-nested-wait-items", "active");
5441 ctx.set(
5442 "operators",
5443 serde_json::json!([
5444 {"name": "Captain A", "phone": "+628111"},
5445 {"name": "Captain B", "phone": "+628222"}
5446 ]),
5447 );
5448 ctx.set(
5449 "tasks",
5450 serde_json::json!([
5451 {"name": "Task 1", "phone": "+629999"}
5452 ]),
5453 );
5454
5455 let result = run_workflow(&executor, &wf, &mut ctx).await.unwrap();
5456 assert_eq!(result, "active");
5457 assert!(
5458 ctx.get("item").is_none(),
5459 "paused workflow state must not persist an active loop item"
5460 );
5461
5462 let result = resume_workflow(
5463 &executor,
5464 &wf,
5465 "wf-nested-wait-items",
5466 serde_json::json!({"event": "task_done"}),
5467 )
5468 .await
5469 .unwrap();
5470 assert_eq!(result, "active");
5471 assert!(
5472 executor
5473 .saved_state
5474 .lock()
5475 .unwrap()
5476 .as_ref()
5477 .and_then(|ctx| ctx.get("item"))
5478 .is_none(),
5479 "paused resume state must keep item reconstruction cursor-driven"
5480 );
5481
5482 {
5483 let notifs = executor.notifications.lock().unwrap();
5484 assert_eq!(
5485 notifs.as_slice(),
5486 &[
5487 ("+629999".to_string(), "task".to_string()),
5488 ("+628111".to_string(), "summary".to_string()),
5489 ],
5490 "inner wait resume must restore the outer item before summary"
5491 );
5492 }
5493
5494 let result = resume_workflow(
5495 &executor,
5496 &wf,
5497 "wf-nested-wait-items",
5498 serde_json::json!({"event": "task_done"}),
5499 )
5500 .await
5501 .unwrap();
5502 assert_eq!(result, "done");
5503
5504 let notifs = executor.notifications.lock().unwrap();
5505 assert_eq!(
5506 notifs.as_slice(),
5507 &[
5508 ("+629999".to_string(), "task".to_string()),
5509 ("+628111".to_string(), "summary".to_string()),
5510 ("+629999".to_string(), "task".to_string()),
5511 ("+628222".to_string(), "summary".to_string()),
5512 ],
5513 "nested for_each wait must not leak the inner item into the outer loop"
5514 );
5515 assert!(
5516 executor
5517 .saved_state
5518 .lock()
5519 .unwrap()
5520 .as_ref()
5521 .and_then(|ctx| ctx.get("item"))
5522 .is_none(),
5523 "completed workflow state must not retain a stale loop item"
5524 );
5525 }
5526
5527 #[tokio::test]
5528 async fn test_resume_rejects_unexpected_wait_event() {
5529 let executor = MockExecutor::new();
5530
5531 let wf = WorkflowDefinition::new("wait_event_guard")
5532 .initial_state("active")
5533 .transition(
5534 "active",
5535 "done",
5536 vec![
5537 WorkflowStep::wait("payment.success", std::time::Duration::from_secs(3600)),
5538 WorkflowStep::notify(ChannelKind::Email, "paid", "customer.email"),
5539 WorkflowStep::transition("done"),
5540 ],
5541 );
5542
5543 let mut ctx = WorkflowContext::new("wf-event-guard", "active");
5544 ctx.set(
5545 "customer",
5546 serde_json::json!({"email": "guest@example.com"}),
5547 );
5548 let result = run_workflow(&executor, &wf, &mut ctx).await.unwrap();
5549 assert_eq!(result, "active");
5550
5551 let wait = ctx
5552 .cursor
5553 .as_ref()
5554 .and_then(|cursor| cursor.wait.as_ref())
5555 .expect("wait metadata should be persisted");
5556 assert_eq!(wait.event, "payment.success");
5557
5558 let err = resume_workflow(
5559 &executor,
5560 &wf,
5561 "wf-event-guard",
5562 serde_json::json!({"event": "operator.declined"}),
5563 )
5564 .await
5565 .expect_err("wrong event must not resume the workflow");
5566
5567 match err {
5568 WorkflowError::Other(msg) => {
5569 assert!(msg.contains("payment.success"));
5570 assert!(msg.contains("operator.declined"));
5571 }
5572 other => panic!("expected event mismatch error, got: {other}"),
5573 }
5574 assert!(executor.notifications.lock().unwrap().is_empty());
5575 }
5576
5577 #[tokio::test]
5578 async fn resume_event_rejects_padded_event_name_before_replay() {
5579 let executor = MockExecutor::new();
5580
5581 let wf = WorkflowDefinition::new("wait_event_padding_guard")
5582 .initial_state("active")
5583 .transition(
5584 "active",
5585 "done",
5586 vec![
5587 WorkflowStep::wait("payment.success", std::time::Duration::from_secs(3600)),
5588 WorkflowStep::notify(ChannelKind::Email, "paid", "customer.email"),
5589 WorkflowStep::transition("done"),
5590 ],
5591 );
5592
5593 let mut ctx = WorkflowContext::new("wf-event-padding-guard", "active");
5594 ctx.set(
5595 "customer",
5596 serde_json::json!({"email": "guest@example.com"}),
5597 );
5598 let result = run_workflow(&executor, &wf, &mut ctx).await.unwrap();
5599 assert_eq!(result, "active");
5600
5601 let err = resume_workflow(
5602 &executor,
5603 &wf,
5604 "wf-event-padding-guard",
5605 serde_json::json!({"event": " payment.success "}),
5606 )
5607 .await
5608 .expect_err("padded event name must not resume the workflow");
5609
5610 match err {
5611 WorkflowError::Other(msg) => {
5612 assert!(msg.contains("leading or trailing whitespace"), "got: {msg}");
5613 }
5614 other => panic!("expected padded event error, got {other:?}"),
5615 }
5616 assert!(
5617 executor.notifications.lock().unwrap().is_empty(),
5618 "padded resume event must fail before notification replay"
5619 );
5620 }
5621
5622 #[tokio::test]
5623 async fn resume_event_without_wait_cursor_does_not_run_transition() {
5624 let executor = MockExecutor::new();
5625
5626 let wf = WorkflowDefinition::new("vendor_callback_guard")
5627 .initial_state("awaiting_vendor")
5628 .transition(
5629 "awaiting_vendor",
5630 "notified",
5631 vec![
5632 WorkflowStep::notify(ChannelKind::WhatsApp, "vendor_ready", "vendor.phone"),
5633 WorkflowStep::transition("notified"),
5634 ],
5635 );
5636
5637 let mut ctx = WorkflowContext::new("wf-no-wait-cursor", "awaiting_vendor");
5638 ctx.set("vendor", serde_json::json!({"phone": "+628111"}));
5639 executor.save_state(&ctx).await.unwrap();
5640
5641 let err = resume_workflow(
5642 &executor,
5643 &wf,
5644 "wf-no-wait-cursor",
5645 serde_json::json!({"event": "vendor.ready"}),
5646 )
5647 .await
5648 .expect_err("event resume must not run a workflow with no wait cursor");
5649
5650 match err {
5651 WorkflowError::Other(msg) => {
5652 assert!(
5653 msg.contains("not waiting for an external event"),
5654 "got: {msg}"
5655 );
5656 }
5657 other => panic!("expected not-waiting error, got {other:?}"),
5658 }
5659 assert!(
5660 executor.notifications.lock().unwrap().is_empty(),
5661 "webhook resume without a wait cursor must not send vendor notifications"
5662 );
5663 }
5664
5665 #[tokio::test]
5666 async fn resume_event_rejects_stale_cursor_state_before_transition() {
5667 let executor = MockExecutor::new();
5668
5669 let wf = WorkflowDefinition::new("stale_cursor_guard")
5670 .initial_state("awaiting_vendor")
5671 .transition(
5672 "awaiting_vendor",
5673 "notified",
5674 vec![
5675 WorkflowStep::notify(ChannelKind::WhatsApp, "vendor_ready", "vendor.phone"),
5676 WorkflowStep::transition("notified"),
5677 ],
5678 );
5679
5680 let mut ctx = WorkflowContext::new("wf-stale-cursor", "awaiting_vendor");
5681 ctx.set("vendor", serde_json::json!({"phone": "+628111"}));
5682 ctx.set_cursor(WorkflowCursor {
5683 state: "old_state".to_string(),
5684 frames: vec![WorkflowCursorFrame::Steps { index: 1 }],
5685 wait: Some(WorkflowPendingWait {
5686 event: "vendor.ready".to_string(),
5687 deadline_at: Utc::now() + chrono::Duration::hours(1),
5688 on_timeout: vec![],
5689 }),
5690 });
5691 executor.save_state(&ctx).await.unwrap();
5692
5693 let err = resume_workflow(
5694 &executor,
5695 &wf,
5696 "wf-stale-cursor",
5697 serde_json::json!({"event": "vendor.ready"}),
5698 )
5699 .await
5700 .expect_err("stale cursor state must not resume the current transition");
5701
5702 match err {
5703 WorkflowError::Other(msg) => {
5704 assert!(msg.contains("Invalid workflow resume cursor"), "got: {msg}");
5705 assert!(msg.contains("state does not match"), "got: {msg}");
5706 }
5707 other => panic!("expected invalid cursor error, got {other:?}"),
5708 }
5709 assert!(
5710 executor.notifications.lock().unwrap().is_empty(),
5711 "stale cursor mismatch must fail before side effects"
5712 );
5713 }
5714
5715 #[tokio::test]
5716 async fn normal_resume_rejects_empty_cursor_frames_before_replay() {
5717 let executor = MockExecutor::new();
5718
5719 let wf = WorkflowDefinition::new("empty_cursor_guard")
5720 .initial_state("active")
5721 .transition(
5722 "active",
5723 "done",
5724 vec![
5725 WorkflowStep::notify(ChannelKind::WhatsApp, "vendor_ready", "vendor.phone"),
5726 WorkflowStep::transition("done"),
5727 ],
5728 );
5729
5730 let mut ctx = WorkflowContext::new("wf-empty-cursor", "active");
5731 ctx.set("vendor", serde_json::json!({"phone": "+628111"}));
5732 ctx.set_cursor(WorkflowCursor {
5733 state: "active".to_string(),
5734 frames: vec![],
5735 wait: None,
5736 });
5737
5738 let err = run_workflow(&executor, &wf, &mut ctx)
5739 .await
5740 .expect_err("empty cursor frames must not replay the transition from step zero");
5741
5742 match err {
5743 WorkflowError::Other(msg) => {
5744 assert!(msg.contains("Invalid workflow resume cursor"), "got: {msg}");
5745 assert!(msg.contains("no frames"), "got: {msg}");
5746 }
5747 other => panic!("expected invalid cursor error, got {other:?}"),
5748 }
5749 assert!(
5750 executor.notifications.lock().unwrap().is_empty(),
5751 "empty checkpoint cursor must fail before replaying notifications"
5752 );
5753 }
5754
5755 #[tokio::test]
5756 async fn event_resume_rejects_empty_wait_cursor_frames_before_replay() {
5757 let executor = MockExecutor::new();
5758
5759 let wf = WorkflowDefinition::new("empty_wait_cursor_guard")
5760 .initial_state("active")
5761 .transition(
5762 "active",
5763 "done",
5764 vec![
5765 WorkflowStep::wait("vendor.ready", std::time::Duration::from_secs(3600)),
5766 WorkflowStep::notify(ChannelKind::WhatsApp, "vendor_ready", "vendor.phone"),
5767 WorkflowStep::transition("done"),
5768 ],
5769 );
5770
5771 let mut ctx = WorkflowContext::new("wf-empty-wait-cursor", "active");
5772 ctx.set("vendor", serde_json::json!({"phone": "+628111"}));
5773 ctx.set_cursor(WorkflowCursor {
5774 state: "active".to_string(),
5775 frames: vec![],
5776 wait: Some(WorkflowPendingWait {
5777 event: "vendor.ready".to_string(),
5778 deadline_at: Utc::now() + chrono::Duration::hours(1),
5779 on_timeout: vec![],
5780 }),
5781 });
5782 executor.save_state(&ctx).await.unwrap();
5783
5784 let err = resume_workflow(
5785 &executor,
5786 &wf,
5787 "wf-empty-wait-cursor",
5788 serde_json::json!({"event": "vendor.ready"}),
5789 )
5790 .await
5791 .expect_err("empty wait cursor frames must not resume from step zero");
5792
5793 match err {
5794 WorkflowError::Other(msg) => {
5795 assert!(msg.contains("Invalid workflow resume cursor"), "got: {msg}");
5796 assert!(msg.contains("no frames"), "got: {msg}");
5797 }
5798 other => panic!("expected invalid cursor error, got {other:?}"),
5799 }
5800 assert!(
5801 executor.notifications.lock().unwrap().is_empty(),
5802 "empty wait cursor must fail before replaying notifications"
5803 );
5804 }
5805
5806 #[tokio::test]
5807 async fn wait_with_empty_event_name_is_rejected_before_pause() {
5808 let executor = MockExecutor::new();
5809
5810 let wf = WorkflowDefinition::new("empty_wait_event")
5811 .initial_state("active")
5812 .transition(
5813 "active",
5814 "done",
5815 vec![
5816 WorkflowStep::wait("", std::time::Duration::from_secs(3600)),
5817 WorkflowStep::notify(ChannelKind::Email, "after_wait", "customer.email"),
5818 ],
5819 );
5820
5821 let mut ctx = WorkflowContext::new("wf-empty-wait", "active");
5822 ctx.set(
5823 "customer",
5824 serde_json::json!({"email": "guest@example.com"}),
5825 );
5826
5827 let err = run_workflow(&executor, &wf, &mut ctx)
5828 .await
5829 .expect_err("empty wait event must fail before persisting a pause");
5830
5831 match err {
5832 WorkflowError::Other(msg) => {
5833 assert!(
5834 msg.contains("Wait event name must not be empty"),
5835 "got: {msg}"
5836 );
5837 }
5838 other => panic!("expected empty event error, got {other:?}"),
5839 }
5840 assert!(ctx.cursor.is_none());
5841 assert!(executor.notifications.lock().unwrap().is_empty());
5842 }
5843
5844 #[tokio::test]
5845 async fn wait_with_padded_event_name_is_rejected_before_pause() {
5846 let executor = MockExecutor::new();
5847
5848 let wf = WorkflowDefinition::new("padded_wait_event")
5849 .initial_state("active")
5850 .transition(
5851 "active",
5852 "done",
5853 vec![
5854 WorkflowStep::wait(" payment.success ", std::time::Duration::from_secs(3600)),
5855 WorkflowStep::notify(ChannelKind::Email, "after_wait", "customer.email"),
5856 ],
5857 );
5858
5859 let mut ctx = WorkflowContext::new("wf-padded-wait", "active");
5860 ctx.set(
5861 "customer",
5862 serde_json::json!({"email": "guest@example.com"}),
5863 );
5864
5865 let err = run_workflow(&executor, &wf, &mut ctx)
5866 .await
5867 .expect_err("padded wait event must fail before persisting a pause");
5868
5869 match err {
5870 WorkflowError::Other(msg) => {
5871 assert!(msg.contains("leading or trailing whitespace"), "got: {msg}");
5872 }
5873 other => panic!("expected padded event error, got {other:?}"),
5874 }
5875 assert!(ctx.cursor.is_none());
5876 assert!(executor.notifications.lock().unwrap().is_empty());
5877 }
5878
5879 #[tokio::test]
5880 async fn transition_must_be_final_to_avoid_lost_follow_up_side_effects() {
5881 let executor = MockExecutor::new();
5882
5883 let wf = WorkflowDefinition::new("transition_order_guard")
5884 .initial_state("active")
5885 .transition(
5886 "active",
5887 "done",
5888 vec![
5889 WorkflowStep::transition("done"),
5890 WorkflowStep::notify(ChannelKind::WhatsApp, "after_transition", "vendor.phone"),
5891 ],
5892 );
5893
5894 let mut ctx = WorkflowContext::new("wf-transition-order", "active");
5895 ctx.set("vendor", serde_json::json!({"phone": "+628111"}));
5896
5897 let err = run_workflow(&executor, &wf, &mut ctx)
5898 .await
5899 .expect_err("transition before side effects must fail closed");
5900
5901 match err {
5902 WorkflowError::Other(msg) => {
5903 assert!(
5904 msg.contains("Transition steps must be the final step"),
5905 "got: {msg}"
5906 );
5907 }
5908 other => panic!("expected transition order error, got {other:?}"),
5909 }
5910 assert_eq!(ctx.current_state, "active");
5911 assert!(executor.notifications.lock().unwrap().is_empty());
5912 }
5913
5914 #[tokio::test]
5915 async fn test_timeout_workflow_executes_on_timeout_fallback() {
5916 let executor = MockExecutor::new();
5917
5918 let wf = WorkflowDefinition::new("wait_timeout")
5919 .initial_state("active")
5920 .transition(
5921 "active",
5922 "done",
5923 vec![
5924 WorkflowStep::wait_or(
5925 "payment.success",
5926 std::time::Duration::from_secs(0),
5927 vec![
5928 WorkflowStep::notify(
5929 ChannelKind::Email,
5930 "payment_timeout",
5931 "customer.email",
5932 ),
5933 WorkflowStep::transition("timed_out"),
5934 ],
5935 ),
5936 WorkflowStep::notify(ChannelKind::Email, "paid", "customer.email"),
5937 WorkflowStep::transition("done"),
5938 ],
5939 );
5940
5941 let mut ctx = WorkflowContext::new("wf-timeout", "active");
5942 ctx.set(
5943 "customer",
5944 serde_json::json!({"email": "guest@example.com"}),
5945 );
5946 let result = run_workflow(&executor, &wf, &mut ctx).await.unwrap();
5947 assert_eq!(result, "active");
5948
5949 let result = timeout_workflow(&executor, &wf, "wf-timeout")
5950 .await
5951 .unwrap();
5952 assert_eq!(result, "timed_out");
5953
5954 let notifs = executor.notifications.lock().unwrap();
5955 assert_eq!(
5956 notifs.as_slice(),
5957 &[(
5958 "guest@example.com".to_string(),
5959 "payment_timeout".to_string()
5960 )]
5961 );
5962 }
5963
5964 #[tokio::test]
5965 async fn persisted_timeout_fallback_is_validated_before_side_effects() {
5966 let executor = MockExecutor::new();
5967
5968 let wf = WorkflowDefinition::new("persisted_timeout_fallback_guard")
5969 .initial_state("active")
5970 .transition(
5971 "active",
5972 "done",
5973 vec![WorkflowStep::wait_or(
5974 "payment.success",
5975 std::time::Duration::from_secs(0),
5976 vec![WorkflowStep::transition("timed_out")],
5977 )],
5978 );
5979
5980 let mut ctx = WorkflowContext::new("wf-persisted-timeout-fallback", "active");
5981 ctx.set(
5982 "customer",
5983 serde_json::json!({"email": "guest@example.com"}),
5984 );
5985 let result = run_workflow(&executor, &wf, &mut ctx).await.unwrap();
5986 assert_eq!(result, "active");
5987
5988 {
5989 let mut saved = executor.saved_state.lock().unwrap();
5990 let saved = saved
5991 .as_mut()
5992 .expect("workflow should be saved while waiting for timeout");
5993 let cursor = saved
5994 .cursor
5995 .as_mut()
5996 .expect("saved workflow should have a wait cursor");
5997 let wait = cursor
5998 .wait
5999 .as_mut()
6000 .expect("saved workflow should be waiting for timeout");
6001 wait.on_timeout = vec![
6002 WorkflowStep::notify(ChannelKind::Email, "bad_timeout", "customer.email"),
6003 WorkflowStep::transition(" "),
6004 ];
6005 }
6006
6007 let err = timeout_workflow(&executor, &wf, "wf-persisted-timeout-fallback")
6008 .await
6009 .expect_err("invalid persisted timeout fallback must fail before side effects");
6010
6011 match err {
6012 WorkflowError::Other(msg) => {
6013 assert!(
6014 msg.contains("Transition target state must not be empty"),
6015 "got: {msg}"
6016 );
6017 }
6018 other => panic!("expected timeout fallback validation error, got {other:?}"),
6019 }
6020 assert!(
6021 executor.notifications.lock().unwrap().is_empty(),
6022 "persisted timeout fallback validation must run before notification"
6023 );
6024 }
6025
6026 #[tokio::test]
6027 async fn persisted_timeout_fallback_before_deadline_does_not_execute() {
6028 let executor = MockExecutor::new();
6029
6030 let wf = WorkflowDefinition::new("future_persisted_timeout_fallback")
6031 .initial_state("active")
6032 .transition(
6033 "active",
6034 "done",
6035 vec![WorkflowStep::wait_or(
6036 "payment.success",
6037 std::time::Duration::from_secs(3600),
6038 vec![
6039 WorkflowStep::notify(
6040 ChannelKind::Email,
6041 "payment_timeout",
6042 "customer.email",
6043 ),
6044 WorkflowStep::transition("timed_out"),
6045 ],
6046 )],
6047 );
6048
6049 let mut ctx = WorkflowContext::new("wf-future-persisted-timeout", "active");
6050 ctx.set(
6051 "customer",
6052 serde_json::json!({"email": "guest@example.com"}),
6053 );
6054 let wait = WorkflowPendingWait {
6055 event: "payment.success".to_string(),
6056 deadline_at: Utc::now() + chrono::Duration::hours(1),
6057 on_timeout: vec![
6058 WorkflowStep::notify(ChannelKind::Email, "payment_timeout", "customer.email"),
6059 WorkflowStep::transition("timed_out"),
6060 ],
6061 };
6062 ctx.set_cursor(WorkflowCursor {
6063 state: "active".to_string(),
6064 frames: vec![WorkflowCursorFrame::Steps { index: 1 }],
6065 wait: Some(wait.clone()),
6066 });
6067 set_timeout_fallback(&mut ctx, &wait).unwrap();
6068 executor.save_state(&ctx).await.unwrap();
6069
6070 let err = timeout_workflow(&executor, &wf, "wf-future-persisted-timeout")
6071 .await
6072 .expect_err("persisted timeout fallback must still respect the wait deadline");
6073
6074 match err {
6075 WorkflowError::Other(msg) => {
6076 assert!(msg.contains("has not timed out"), "got: {msg}");
6077 }
6078 other => panic!("expected not-timed-out error, got {other:?}"),
6079 }
6080 assert!(
6081 executor.notifications.lock().unwrap().is_empty(),
6082 "future timeout fallback metadata must fail before notification"
6083 );
6084 }
6085
6086 #[tokio::test]
6087 async fn persisted_timeout_wait_event_is_validated_before_side_effects() {
6088 let executor = MockExecutor::new();
6089
6090 let wf = WorkflowDefinition::new("persisted_timeout_event_guard")
6091 .initial_state("active")
6092 .transition(
6093 "active",
6094 "done",
6095 vec![WorkflowStep::wait_or(
6096 "payment.success",
6097 std::time::Duration::from_secs(0),
6098 vec![
6099 WorkflowStep::notify(ChannelKind::Email, "bad_timeout", "customer.email"),
6100 WorkflowStep::transition("timed_out"),
6101 ],
6102 )],
6103 );
6104
6105 let mut ctx = WorkflowContext::new("wf-persisted-timeout-event", "active");
6106 ctx.set(
6107 "customer",
6108 serde_json::json!({"email": "guest@example.com"}),
6109 );
6110 let result = run_workflow(&executor, &wf, &mut ctx).await.unwrap();
6111 assert_eq!(result, "active");
6112
6113 {
6114 let mut saved = executor.saved_state.lock().unwrap();
6115 let saved = saved
6116 .as_mut()
6117 .expect("workflow should be saved while waiting for timeout");
6118 let cursor = saved
6119 .cursor
6120 .as_mut()
6121 .expect("saved workflow should have a wait cursor");
6122 let wait = cursor
6123 .wait
6124 .as_mut()
6125 .expect("saved workflow should be waiting for timeout");
6126 wait.event = " payment.success ".to_string();
6127 }
6128
6129 let err = timeout_workflow(&executor, &wf, "wf-persisted-timeout-event")
6130 .await
6131 .expect_err("invalid persisted wait event must fail before side effects");
6132
6133 match err {
6134 WorkflowError::Other(msg) => {
6135 assert!(
6136 msg.contains("Wait event name must not have leading or trailing whitespace"),
6137 "got: {msg}"
6138 );
6139 }
6140 other => panic!("expected wait event validation error, got {other:?}"),
6141 }
6142 assert!(
6143 executor.notifications.lock().unwrap().is_empty(),
6144 "persisted timeout wait validation must run before notification"
6145 );
6146 }
6147
6148 #[tokio::test]
6149 async fn timeout_due_workflows_drains_due_waits_with_generated_idempotency() {
6150 let executor = MockExecutor::new();
6151
6152 let wf = WorkflowDefinition::new("wait_timeout_drain")
6153 .initial_state("active")
6154 .transition(
6155 "active",
6156 "done",
6157 vec![WorkflowStep::wait_or(
6158 "payment.success",
6159 std::time::Duration::from_secs(0),
6160 vec![
6161 WorkflowStep::notify(
6162 ChannelKind::Email,
6163 "payment_timeout",
6164 "customer.email",
6165 ),
6166 WorkflowStep::transition("timed_out"),
6167 ],
6168 )],
6169 );
6170
6171 let mut ctx = WorkflowContext::new("wf-timeout-drain", "active");
6172 ctx.set(
6173 "customer",
6174 serde_json::json!({"email": "guest@example.com"}),
6175 );
6176 let result = run_workflow(&executor, &wf, &mut ctx).await.unwrap();
6177 assert_eq!(result, "active");
6178
6179 executor.set_due_timeout_workflow_ids(vec!["wf-timeout-drain"]);
6180 let now = Utc::now();
6181 let outcomes =
6182 timeout_due_workflows(&executor, &wf, now, 10, WorkflowRunOptions::default())
6183 .await
6184 .unwrap();
6185
6186 assert_eq!(
6187 outcomes,
6188 vec![WorkflowTimeoutOutcome {
6189 workflow_id: "wf-timeout-drain".to_string(),
6190 state: Some("timed_out".to_string()),
6191 error: None,
6192 }]
6193 );
6194 assert_eq!(
6195 executor
6196 .completed_workflow_operations
6197 .lock()
6198 .unwrap()
6199 .as_slice(),
6200 &[(
6201 timeout_operation_idempotency_key("wait_timeout_drain", "wf-timeout-drain", now),
6202 "timed_out".to_string(),
6203 )]
6204 );
6205 }
6206
6207 #[tokio::test]
6208 async fn timeout_due_workflows_uses_batch_now_for_timeout_execution() {
6209 let executor = MockExecutor::new();
6210
6211 let wf = WorkflowDefinition::new("future_wait_timeout_drain")
6212 .initial_state("active")
6213 .transition(
6214 "active",
6215 "done",
6216 vec![WorkflowStep::wait_or(
6217 "payment.success",
6218 std::time::Duration::from_secs(3600),
6219 vec![WorkflowStep::transition("timed_out")],
6220 )],
6221 );
6222
6223 let mut ctx = WorkflowContext::new("wf-future-timeout-drain", "active");
6224 let result = run_workflow(&executor, &wf, &mut ctx).await.unwrap();
6225 assert_eq!(result, "active");
6226
6227 executor.set_due_timeout_workflow_ids(vec!["wf-future-timeout-drain"]);
6228 let outcomes = timeout_due_workflows(
6229 &executor,
6230 &wf,
6231 Utc::now() + chrono::Duration::hours(2),
6232 10,
6233 WorkflowRunOptions::default(),
6234 )
6235 .await
6236 .unwrap();
6237
6238 assert_eq!(
6239 outcomes,
6240 vec![WorkflowTimeoutOutcome {
6241 workflow_id: "wf-future-timeout-drain".to_string(),
6242 state: Some("timed_out".to_string()),
6243 error: None,
6244 }]
6245 );
6246 }
6247
6248 #[test]
6249 fn timeout_batch_idempotency_key_is_scoped_per_workflow() {
6250 let wf = WorkflowDefinition::new("wait_timeout_drain")
6251 .initial_state("active")
6252 .transition("active", "done", vec![]);
6253 let options = WorkflowRunOptions::default().with_idempotency_key("batch-20260617");
6254 let now = DateTime::parse_from_rfc3339("2026-06-17T12:00:00Z")
6255 .unwrap()
6256 .with_timezone(&Utc);
6257
6258 let first = timeout_options_for_workflow(&options, &wf, "wf-a", now);
6259 let second = timeout_options_for_workflow(&options, &wf, "wf-b", now);
6260 let first_key = first.idempotency_key.clone();
6261 let second_key = second.idempotency_key.clone();
6262
6263 assert_eq!(
6264 first_key,
6265 Some(timeout_operation_idempotency_key(
6266 "batch-20260617",
6267 "wf-a",
6268 now
6269 ))
6270 );
6271 assert_eq!(
6272 second_key,
6273 Some(timeout_operation_idempotency_key(
6274 "batch-20260617",
6275 "wf-b",
6276 now
6277 ))
6278 );
6279 assert_ne!(first.idempotency_key, second.idempotency_key);
6280 }
6281
6282 #[test]
6283 fn timeout_batch_idempotency_key_does_not_collide_on_delimiters() {
6284 let now = DateTime::parse_from_rfc3339("2026-06-17T12:00:00Z")
6285 .unwrap()
6286 .with_timezone(&Utc);
6287 let first = timeout_operation_idempotency_key("tenant:batch", "wf-a", now);
6288 let second = timeout_operation_idempotency_key("tenant", "batch:wf-a", now);
6289
6290 assert_ne!(
6291 first, second,
6292 "timeout operation idempotency keys must not collide when fields contain delimiters"
6293 );
6294 }
6295
6296 #[test]
6297 fn timeout_batch_idempotency_key_changes_between_scheduler_runs() {
6298 let first_run = DateTime::parse_from_rfc3339("2026-06-17T12:00:00Z")
6299 .unwrap()
6300 .with_timezone(&Utc);
6301 let second_run = DateTime::parse_from_rfc3339("2026-06-17T12:01:00Z")
6302 .unwrap()
6303 .with_timezone(&Utc);
6304
6305 assert_ne!(
6306 timeout_operation_idempotency_key("batch", "wf-a", first_run),
6307 timeout_operation_idempotency_key("batch", "wf-a", second_run),
6308 "later timeout attempts for the same workflow must not collide with an old completed timeout operation"
6309 );
6310 }
6311
6312 #[tokio::test]
6313 async fn test_timeout_fallback_without_transition_does_not_run_success_path() {
6314 let executor = MockExecutor::new();
6315
6316 let wf = WorkflowDefinition::new("wait_timeout_no_transition")
6317 .initial_state("active")
6318 .transition(
6319 "active",
6320 "confirmed",
6321 vec![
6322 WorkflowStep::wait_or(
6323 "payment.success",
6324 std::time::Duration::from_secs(0),
6325 vec![WorkflowStep::notify(
6326 ChannelKind::Email,
6327 "payment_timeout",
6328 "customer.email",
6329 )],
6330 ),
6331 WorkflowStep::notify(ChannelKind::Email, "booking_confirmed", "customer.email"),
6332 WorkflowStep::transition("confirmed"),
6333 ],
6334 );
6335
6336 let mut ctx = WorkflowContext::new("wf-timeout-no-transition", "active");
6337 ctx.set(
6338 "customer",
6339 serde_json::json!({"email": "guest@example.com"}),
6340 );
6341 let result = run_workflow(&executor, &wf, &mut ctx).await.unwrap();
6342 assert_eq!(result, "active");
6343
6344 let result = timeout_workflow(&executor, &wf, "wf-timeout-no-transition")
6345 .await
6346 .unwrap();
6347 assert_eq!(result, "active");
6348
6349 let notifs = executor.notifications.lock().unwrap();
6350 assert_eq!(
6351 notifs.as_slice(),
6352 &[(
6353 "guest@example.com".to_string(),
6354 "payment_timeout".to_string()
6355 )],
6356 "timeout fallback must not fall through into the post-wait success path"
6357 );
6358 }
6359
6360 #[tokio::test]
6361 async fn test_step_checkpoint_prevents_side_effect_replay_after_failure() {
6362 let executor = MockExecutor::new();
6363 let failing_query =
6364 qail_core::wire::encode_cmd_text(&qail_core::Qail::get("bookings").limit(1));
6365 executor.fail_query(failing_query.clone());
6366
6367 let wf = WorkflowDefinition::new("checkpoint_failure")
6368 .initial_state("active")
6369 .transition(
6370 "active",
6371 "done",
6372 vec![
6373 WorkflowStep::notify(ChannelKind::Email, "charged", "customer.email"),
6374 WorkflowStep::Query {
6375 cmd_json: failing_query,
6376 store_as: None,
6377 },
6378 WorkflowStep::transition("done"),
6379 ],
6380 );
6381
6382 let mut ctx = WorkflowContext::new("wf-checkpoint", "active");
6383 ctx.set(
6384 "customer",
6385 serde_json::json!({"email": "guest@example.com"}),
6386 );
6387 let err = run_workflow(&executor, &wf, &mut ctx)
6388 .await
6389 .expect_err("legacy query should fail after notification checkpoint");
6390 assert!(matches!(err, WorkflowError::QueryFailed(_)));
6391 assert_eq!(executor.notifications.lock().unwrap().len(), 1);
6392
6393 let mut restored = executor
6394 .saved_state
6395 .lock()
6396 .unwrap()
6397 .clone()
6398 .expect("checkpoint should be saved after notification");
6399 let cursor = restored.cursor.as_ref().expect("checkpoint cursor");
6400 assert_eq!(cursor.frames, vec![WorkflowCursorFrame::Steps { index: 1 }]);
6401 assert!(cursor.wait.is_none());
6402 assert!(restored.get(TIMEOUT_FALLBACK_KEY).is_none());
6403
6404 let err = run_workflow(&executor, &wf, &mut restored)
6405 .await
6406 .expect_err("query should still fail on retry");
6407 assert!(matches!(err, WorkflowError::QueryFailed(_)));
6408 assert_eq!(
6409 executor.notifications.lock().unwrap().len(),
6410 1,
6411 "retry must resume after the completed notification step"
6412 );
6413 }
6414
6415 #[tokio::test]
6416 async fn failed_query_step_marks_side_effect_failed() {
6417 let executor = MockExecutor::new();
6418 let failing_query =
6419 qail_core::wire::encode_cmd_text(&qail_core::Qail::get("bookings").limit(1));
6420 executor.fail_query(failing_query.clone());
6421
6422 let wf = WorkflowDefinition::new("failed_side_effect")
6423 .initial_state("active")
6424 .transition(
6425 "active",
6426 "done",
6427 vec![
6428 WorkflowStep::Query {
6429 cmd_json: failing_query,
6430 store_as: None,
6431 },
6432 WorkflowStep::transition("done"),
6433 ],
6434 );
6435
6436 let mut ctx = WorkflowContext::new("wf-side-effect-fail", "active");
6437 let err = run_workflow(&executor, &wf, &mut ctx)
6438 .await
6439 .expect_err("query should fail");
6440 assert!(matches!(err, WorkflowError::QueryFailed(_)));
6441
6442 let expected_operation_id = side_effect_operation_id(
6443 "wf-side-effect-fail",
6444 "active",
6445 0,
6446 WorkflowSideEffectKind::Query,
6447 "steps[0]",
6448 );
6449 let failed_side_effects = executor.failed_side_effects.lock().unwrap();
6450 assert_eq!(failed_side_effects.len(), 1);
6451 assert_eq!(failed_side_effects[0].0, expected_operation_id);
6452 assert!(
6453 failed_side_effects[0].1.contains("forced query failure"),
6454 "failure hook must preserve the app/provider error"
6455 );
6456 }
6457
6458 #[tokio::test]
6459 async fn test_timeout_checkpoint_prevents_side_effect_replay_after_failure() {
6460 let executor = MockExecutor::new();
6461 let failing_query =
6462 qail_core::wire::encode_cmd_text(&qail_core::Qail::get("bookings").limit(1));
6463 executor.fail_query(failing_query.clone());
6464
6465 let wf = WorkflowDefinition::new("timeout_checkpoint_failure")
6466 .initial_state("active")
6467 .transition(
6468 "active",
6469 "done",
6470 vec![WorkflowStep::wait_or(
6471 "payment.success",
6472 std::time::Duration::from_secs(0),
6473 vec![
6474 WorkflowStep::notify(
6475 ChannelKind::Email,
6476 "payment_timeout",
6477 "customer.email",
6478 ),
6479 WorkflowStep::Query {
6480 cmd_json: failing_query,
6481 store_as: None,
6482 },
6483 WorkflowStep::transition("timed_out"),
6484 ],
6485 )],
6486 );
6487
6488 let mut ctx = WorkflowContext::new("wf-timeout-checkpoint", "active");
6489 ctx.set(
6490 "customer",
6491 serde_json::json!({"email": "guest@example.com"}),
6492 );
6493 let result = run_workflow(&executor, &wf, &mut ctx).await.unwrap();
6494 assert_eq!(result, "active");
6495
6496 let err = timeout_workflow(&executor, &wf, "wf-timeout-checkpoint")
6497 .await
6498 .expect_err("legacy query should fail after timeout notification checkpoint");
6499 assert!(matches!(err, WorkflowError::QueryFailed(_)));
6500 assert_eq!(executor.notifications.lock().unwrap().len(), 1);
6501
6502 let saved = executor
6503 .saved_state
6504 .lock()
6505 .unwrap()
6506 .clone()
6507 .expect("timeout checkpoint should be saved after notification");
6508 let cursor = saved.cursor.as_ref().expect("timeout checkpoint cursor");
6509 assert_eq!(cursor.frames, vec![WorkflowCursorFrame::Steps { index: 1 }]);
6510 let wait = cursor
6511 .wait
6512 .as_ref()
6513 .expect("timeout fallback checkpoint must remain scheduler-visible");
6514 assert_eq!(wait.event, "payment.success");
6515 assert_eq!(wait.on_timeout.len(), 3);
6516 assert!(
6517 saved.get(TIMEOUT_FALLBACK_KEY).is_some(),
6518 "timeout fallback cursor must retain internal on_timeout metadata"
6519 );
6520
6521 let err = timeout_workflow(&executor, &wf, "wf-timeout-checkpoint")
6522 .await
6523 .expect_err("timeout fallback query should still fail on retry");
6524 assert!(matches!(err, WorkflowError::QueryFailed(_)));
6525 assert_eq!(
6526 executor.notifications.lock().unwrap().len(),
6527 1,
6528 "timeout retry must resume after the completed notification step"
6529 );
6530 }
6531
6532 #[tokio::test]
6533 async fn test_workflow_errors_on_for_each_non_array() {
6534 let executor = MockExecutor::new();
6535
6536 let wf = WorkflowDefinition::new("bad_for_each")
6537 .initial_state("active")
6538 .transition(
6539 "active",
6540 "resolved",
6541 vec![WorkflowStep::for_each(
6542 "operators",
6543 vec![WorkflowStep::notify(
6544 ChannelKind::WhatsApp,
6545 "opportunity",
6546 "item.phone",
6547 )],
6548 )],
6549 );
6550
6551 let mut ctx = WorkflowContext::new("wf-bad-for-each", "active");
6552 ctx.set("operators", serde_json::json!({"phone": "+628111"}));
6553
6554 let err = run_workflow(&executor, &wf, &mut ctx)
6555 .await
6556 .expect_err("ForEach should reject non-array context values");
6557
6558 match err {
6559 WorkflowError::Other(msg) => {
6560 assert!(msg.contains("Expected JSON array for ForEach list 'operators'"));
6561 assert!(msg.contains("object"));
6562 }
6563 other => panic!("expected Other error, got: {other}"),
6564 }
6565 assert!(
6566 executor.notifications.lock().unwrap().is_empty(),
6567 "ForEach type errors must fail before running nested steps"
6568 );
6569 }
6570
6571 #[tokio::test]
6572 async fn test_workflow_with_charge() {
6573 let executor = MockExecutor::new();
6574
6575 let wf = WorkflowDefinition::new("booking_payment")
6576 .initial_state("created")
6577 .transition(
6578 "created",
6579 "awaiting_payment",
6580 vec![
6581 WorkflowStep::charge_with_origin(
6582 PaymentKind::Xendit,
6583 "order.total",
6584 "order.id",
6585 None,
6586 Some("order.payment_method"),
6587 Some("order.origin"),
6588 Some("charge"),
6589 ),
6590 WorkflowStep::wait("payment.success", std::time::Duration::from_secs(3600)),
6591 ],
6592 )
6593 .transition(
6594 "awaiting_payment",
6595 "confirmed",
6596 vec![
6597 WorkflowStep::notify(ChannelKind::Email, "booking_confirmed", "customer.email"),
6598 WorkflowStep::transition("confirmed"),
6599 ],
6600 );
6601
6602 let mut ctx = WorkflowContext::new("wf-payment-001", "created");
6603 ctx.set(
6604 "order",
6605 serde_json::json!({
6606 "id": "booking-789",
6607 "total": 150000,
6608 "payment_method": "QRIS",
6609 "origin": "mcp"
6610 }),
6611 );
6612 ctx.set(
6613 "customer",
6614 serde_json::json!({
6615 "email": "guest@example.com"
6616 }),
6617 );
6618
6619 let result = run_workflow(&executor, &wf, &mut ctx).await.unwrap();
6620
6621 assert_eq!(result, "created");
6623
6624 let charge = ctx.get("charge").expect("charge should be in context");
6626 assert_eq!(
6627 charge.get("charge_id").and_then(|v| v.as_str()),
6628 Some("mock-charge-booking-789")
6629 );
6630 assert_eq!(
6631 charge.get("status").and_then(|v| v.as_str()),
6632 Some("Pending")
6633 );
6634 assert!(charge.get("qr_code").is_some());
6635 assert_eq!(
6636 charge.get("payment_method").and_then(|v| v.as_str()),
6637 Some("QRIS")
6638 );
6639 assert_eq!(
6640 charge.get("order_origin").and_then(|v| v.as_str()),
6641 Some("mcp")
6642 );
6643 assert!(
6644 charge.get("raw").is_none(),
6645 "stored charge context must be safe for chat display"
6646 );
6647
6648 let charges = executor.charges.lock().unwrap();
6649 assert_eq!(charges.len(), 1);
6650 assert_eq!(charges[0].amount, 150000);
6651 assert_eq!(charges[0].order_origin, Some(OrderOrigin::Mcp));
6652 assert!(
6653 charges[0]
6654 .idempotency_key
6655 .as_deref()
6656 .is_some_and(|key| key.contains("qail-workflow-side-effect")),
6657 "provider charge request must carry a stable workflow idempotency key"
6658 );
6659 }
6660
6661 #[tokio::test]
6662 async fn charge_rejects_invalid_origin_before_provider_call() {
6663 let executor = MockExecutor::new();
6664
6665 let wf = WorkflowDefinition::new("charge_invalid_origin")
6666 .initial_state("created")
6667 .transition(
6668 "created",
6669 "awaiting_payment",
6670 vec![WorkflowStep::charge_with_origin(
6671 PaymentKind::Xendit,
6672 "order.total",
6673 "order.id",
6674 None,
6675 None,
6676 Some("order.origin"),
6677 Some("charge"),
6678 )],
6679 );
6680
6681 let mut ctx = WorkflowContext::new("wf-charge-invalid-origin", "created");
6682 ctx.set(
6683 "order",
6684 serde_json::json!({
6685 "id": "booking-invalid-origin",
6686 "total": 150000,
6687 "origin": "telegram"
6688 }),
6689 );
6690
6691 let err = run_workflow(&executor, &wf, &mut ctx)
6692 .await
6693 .expect_err("invalid order origin must fail before provider call");
6694
6695 match err {
6696 WorkflowError::Other(msg) => {
6697 assert!(
6698 msg.contains("Charge order_origin_key 'order.origin'"),
6699 "got: {msg}"
6700 );
6701 }
6702 other => panic!("expected invalid order origin error, got {other:?}"),
6703 }
6704 assert!(executor.charges.lock().unwrap().is_empty());
6705 }
6706
6707 #[tokio::test]
6708 async fn charge_rejects_reserved_store_key_before_provider_call() {
6709 let executor = MockExecutor::new();
6710
6711 let wf = WorkflowDefinition::new("charge_reserved_key")
6712 .initial_state("created")
6713 .transition(
6714 "created",
6715 "awaiting_payment",
6716 vec![WorkflowStep::charge(
6717 PaymentKind::Xendit,
6718 "order.total",
6719 "order.id",
6720 Some("item"),
6721 )],
6722 );
6723
6724 let mut ctx = WorkflowContext::new("wf-charge-reserved-key", "created");
6725 ctx.set(
6726 "order",
6727 serde_json::json!({
6728 "id": "booking-reserved",
6729 "total": 150000
6730 }),
6731 );
6732
6733 let err = run_workflow(&executor, &wf, &mut ctx)
6734 .await
6735 .expect_err("reserved charge output key must fail before provider call");
6736
6737 match err {
6738 WorkflowError::Other(msg) => {
6739 assert!(msg.contains("reserved context key 'item'"), "got: {msg}");
6740 }
6741 other => panic!("expected reserved context key error, got {other:?}"),
6742 }
6743 assert!(
6744 executor.charges.lock().unwrap().is_empty(),
6745 "charge provider must not be called after reserved key validation fails"
6746 );
6747 }
6748
6749 #[tokio::test]
6750 async fn charge_rejects_dotted_store_key_before_provider_call() {
6751 let executor = MockExecutor::new();
6752
6753 let wf = WorkflowDefinition::new("charge_dotted_key")
6754 .initial_state("created")
6755 .transition(
6756 "created",
6757 "awaiting_payment",
6758 vec![WorkflowStep::charge(
6759 PaymentKind::Xendit,
6760 "order.total",
6761 "order.id",
6762 Some("payment.charge"),
6763 )],
6764 );
6765
6766 let mut ctx = WorkflowContext::new("wf-charge-dotted-key", "created");
6767 ctx.set(
6768 "order",
6769 serde_json::json!({
6770 "id": "booking-dotted",
6771 "total": 150000
6772 }),
6773 );
6774
6775 let err = run_workflow(&executor, &wf, &mut ctx)
6776 .await
6777 .expect_err("dotted charge output key must fail before provider call");
6778
6779 match err {
6780 WorkflowError::Other(msg) => {
6781 assert!(msg.contains("top-level context key"), "got: {msg}");
6782 }
6783 other => panic!("expected dotted context key error, got {other:?}"),
6784 }
6785 assert!(
6786 executor.charges.lock().unwrap().is_empty(),
6787 "charge provider must not be called after dotted key validation fails"
6788 );
6789 }
6790
6791 #[tokio::test]
6792 async fn charge_accepts_safe_integer_float_amount() {
6793 let executor = MockExecutor::new();
6794 let wf = charge_only_workflow("booking_payment_float");
6795
6796 let mut ctx = WorkflowContext::new("wf-payment-float", "created");
6797 ctx.set(
6798 "order",
6799 serde_json::json!({
6800 "id": "booking-float",
6801 "total": 150000.0
6802 }),
6803 );
6804
6805 let result = run_workflow(&executor, &wf, &mut ctx).await.unwrap();
6806
6807 assert_eq!(result, "awaiting_payment");
6808 let charges = executor.charges.lock().unwrap();
6809 assert_eq!(charges.len(), 1);
6810 assert_eq!(charges[0].amount, 150000);
6811 }
6812
6813 #[tokio::test]
6814 async fn charge_rejects_fractional_amount_before_provider_call() {
6815 let executor = MockExecutor::new();
6816 let wf = charge_only_workflow("booking_payment_fractional");
6817
6818 let mut ctx = WorkflowContext::new("wf-payment-fractional", "created");
6819 ctx.set(
6820 "order",
6821 serde_json::json!({
6822 "id": "booking-fractional",
6823 "total": 150000.75
6824 }),
6825 );
6826
6827 let err = run_workflow(&executor, &wf, &mut ctx)
6828 .await
6829 .expect_err("fractional charge amount must fail before provider call");
6830
6831 match err {
6832 WorkflowError::Other(msg) => {
6833 assert!(msg.contains("Invalid charge amount"));
6834 assert!(msg.contains("smallest currency unit"));
6835 }
6836 other => panic!("expected invalid amount error, got {other:?}"),
6837 }
6838 assert!(executor.charges.lock().unwrap().is_empty());
6839 }
6840
6841 #[tokio::test]
6842 async fn charge_rejects_non_positive_amount_before_provider_call() {
6843 let executor = MockExecutor::new();
6844 let wf = charge_only_workflow("booking_payment_negative");
6845
6846 let mut ctx = WorkflowContext::new("wf-payment-negative", "created");
6847 ctx.set(
6848 "order",
6849 serde_json::json!({
6850 "id": "booking-negative",
6851 "total": -1
6852 }),
6853 );
6854
6855 let err = run_workflow(&executor, &wf, &mut ctx)
6856 .await
6857 .expect_err("non-positive charge amount must fail before provider call");
6858
6859 match err {
6860 WorkflowError::Other(msg) => {
6861 assert!(msg.contains("Invalid charge amount"));
6862 assert!(msg.contains("greater than zero"));
6863 }
6864 other => panic!("expected invalid amount error, got {other:?}"),
6865 }
6866 assert!(executor.charges.lock().unwrap().is_empty());
6867 }
6868
6869 #[tokio::test]
6870 async fn charge_rejects_oversized_unsigned_amount_before_provider_call() {
6871 let executor = MockExecutor::new();
6872 let wf = charge_only_workflow("booking_payment_oversized");
6873
6874 let mut ctx = WorkflowContext::new("wf-payment-oversized", "created");
6875 ctx.set(
6876 "order",
6877 serde_json::json!({
6878 "id": "booking-oversized",
6879 "total": u64::MAX
6880 }),
6881 );
6882
6883 let err = run_workflow(&executor, &wf, &mut ctx)
6884 .await
6885 .expect_err("oversized charge amount must fail before provider call");
6886
6887 match err {
6888 WorkflowError::Other(msg) => {
6889 assert!(msg.contains("Invalid charge amount"));
6890 assert!(msg.contains("signed 64-bit"));
6891 }
6892 other => panic!("expected invalid amount error, got {other:?}"),
6893 }
6894 assert!(executor.charges.lock().unwrap().is_empty());
6895 }
6896
6897 #[tokio::test]
6898 async fn test_query_step_rejects_legacy_non_wire_payload() {
6899 let executor = MockExecutor::new();
6900
6901 let wf = WorkflowDefinition::new("legacy_query")
6902 .initial_state("start")
6903 .transition(
6904 "start",
6905 "done",
6906 vec![
6907 WorkflowStep::Query {
6908 cmd_json: "get users limit 1".to_string(),
6909 store_as: Some("rows".to_string()),
6910 },
6911 WorkflowStep::transition("done"),
6912 ],
6913 );
6914
6915 let mut ctx = WorkflowContext::new("wf-legacy-query-001", "start");
6916 let err = run_workflow(&executor, &wf, &mut ctx)
6917 .await
6918 .expect_err("legacy non-wire query payload must fail");
6919
6920 match err {
6921 WorkflowError::QueryFailed(msg) => {
6922 assert!(
6923 msg.contains("QAIL-CMD/1"),
6924 "error should mention required wire magic"
6925 );
6926 assert!(
6927 msg.contains("purge/restart pending workflows"),
6928 "error should include cutover guidance"
6929 );
6930 }
6931 other => panic!("expected QueryFailed, got: {other}"),
6932 }
6933
6934 assert!(
6935 executor.queries.lock().unwrap().is_empty(),
6936 "legacy payload must fail before executor query is invoked"
6937 );
6938 }
6939
6940 #[tokio::test]
6941 async fn future_legacy_query_payload_fails_before_current_side_effects() {
6942 let executor = MockExecutor::new();
6943
6944 let wf = WorkflowDefinition::new("future_legacy_query")
6945 .initial_state("active")
6946 .transition(
6947 "active",
6948 "query_state",
6949 vec![
6950 WorkflowStep::notify(ChannelKind::Email, "before_bad_query", "customer.email"),
6951 WorkflowStep::transition("query_state"),
6952 ],
6953 )
6954 .transition(
6955 "query_state",
6956 "done",
6957 vec![
6958 WorkflowStep::Query {
6959 cmd_json: "get users limit 1".to_string(),
6960 store_as: Some("rows".to_string()),
6961 },
6962 WorkflowStep::transition("done"),
6963 ],
6964 );
6965
6966 let mut ctx = WorkflowContext::new("wf-future-legacy-query", "active");
6967 ctx.set(
6968 "customer",
6969 serde_json::json!({"email": "guest@example.com"}),
6970 );
6971
6972 let err = run_workflow(&executor, &wf, &mut ctx)
6973 .await
6974 .expect_err("future legacy query payload must fail before current side effects");
6975
6976 match err {
6977 WorkflowError::QueryFailed(msg) => {
6978 assert!(
6979 msg.contains("QAIL-CMD/1"),
6980 "error should mention required wire magic"
6981 );
6982 }
6983 other => panic!("expected QueryFailed, got: {other}"),
6984 }
6985 assert_eq!(ctx.current_state, "active");
6986 assert!(
6987 executor.notifications.lock().unwrap().is_empty(),
6988 "definition validation must run before the current-state notification"
6989 );
6990 assert!(
6991 executor.queries.lock().unwrap().is_empty(),
6992 "legacy payload must fail before executor query is invoked"
6993 );
6994 }
6995
6996 #[tokio::test]
6997 async fn test_query_step_accepts_wire_payload_and_executes() {
6998 let executor = MockExecutor::new();
6999 let cmd = qail_core::Qail::get("users").columns(["id"]).limit(1);
7000 let wire = qail_core::wire::encode_cmd_text(&cmd);
7001
7002 let wf = WorkflowDefinition::new("wire_query")
7003 .initial_state("start")
7004 .transition(
7005 "start",
7006 "done",
7007 vec![
7008 WorkflowStep::Query {
7009 cmd_json: wire.clone(),
7010 store_as: Some("rows".to_string()),
7011 },
7012 WorkflowStep::transition("done"),
7013 ],
7014 );
7015
7016 let mut ctx = WorkflowContext::new("wf-wire-query-001", "start");
7017 let result = run_workflow(&executor, &wf, &mut ctx)
7018 .await
7019 .expect("wire payload should execute");
7020 assert_eq!(result, "done");
7021
7022 let queries = executor.queries.lock().unwrap();
7023 assert_eq!(queries.len(), 1);
7024 assert_eq!(
7025 queries[0], wire,
7026 "executor should receive canonical wire payload"
7027 );
7028 }
7029
7030 #[tokio::test]
7031 async fn query_rejects_reserved_store_key_before_execution() {
7032 let executor = MockExecutor::new();
7033 let cmd = qail_core::Qail::get("users").columns(["id"]).limit(1);
7034 let wire = qail_core::wire::encode_cmd_text(&cmd);
7035
7036 let wf = WorkflowDefinition::new("query_reserved_key")
7037 .initial_state("start")
7038 .transition(
7039 "start",
7040 "done",
7041 vec![
7042 WorkflowStep::Query {
7043 cmd_json: wire,
7044 store_as: Some("__qail_timeout_fallback".to_string()),
7045 },
7046 WorkflowStep::transition("done"),
7047 ],
7048 );
7049
7050 let mut ctx = WorkflowContext::new("wf-query-reserved-key", "start");
7051 let err = run_workflow(&executor, &wf, &mut ctx)
7052 .await
7053 .expect_err("reserved query output key must fail before query execution");
7054
7055 match err {
7056 WorkflowError::Other(msg) => {
7057 assert!(
7058 msg.contains("reserved context key '__qail_timeout_fallback'"),
7059 "got: {msg}"
7060 );
7061 }
7062 other => panic!("expected reserved context key error, got {other:?}"),
7063 }
7064 assert!(
7065 executor.queries.lock().unwrap().is_empty(),
7066 "query executor must not be called after reserved key validation fails"
7067 );
7068 }
7069
7070 #[tokio::test]
7071 async fn query_rejects_dotted_store_key_before_execution() {
7072 let executor = MockExecutor::new();
7073 let cmd = qail_core::Qail::get("users").columns(["id"]).limit(1);
7074 let wire = qail_core::wire::encode_cmd_text(&cmd);
7075
7076 let wf = WorkflowDefinition::new("query_dotted_key")
7077 .initial_state("start")
7078 .transition(
7079 "start",
7080 "done",
7081 vec![
7082 WorkflowStep::Query {
7083 cmd_json: wire,
7084 store_as: Some("query.rows".to_string()),
7085 },
7086 WorkflowStep::transition("done"),
7087 ],
7088 );
7089
7090 let mut ctx = WorkflowContext::new("wf-query-dotted-key", "start");
7091 let err = run_workflow(&executor, &wf, &mut ctx)
7092 .await
7093 .expect_err("dotted query output key must fail before query execution");
7094
7095 match err {
7096 WorkflowError::Other(msg) => {
7097 assert!(msg.contains("top-level context key"), "got: {msg}");
7098 }
7099 other => panic!("expected dotted context key error, got {other:?}"),
7100 }
7101 assert!(
7102 executor.queries.lock().unwrap().is_empty(),
7103 "query executor must not be called after dotted key validation fails"
7104 );
7105 }
7106
7107 #[test]
7108 fn test_collect_legacy_query_payload_issues_reports_nested_paths() {
7109 let wire_cmd = qail_core::wire::encode_cmd_text(&qail_core::Qail::get("users").limit(1));
7110 let wf = WorkflowDefinition::new("legacy_audit")
7111 .initial_state("start")
7112 .transition(
7113 "start",
7114 "done",
7115 vec![
7116 WorkflowStep::Query {
7117 cmd_json: "get users limit 1".to_string(),
7118 store_as: Some("rows".to_string()),
7119 },
7120 WorkflowStep::Wait {
7121 event: "timeout".to_string(),
7122 timeout: std::time::Duration::from_secs(10),
7123 on_timeout: vec![WorkflowStep::Query {
7124 cmd_json: "{\"legacy\":true}".to_string(),
7125 store_as: None,
7126 }],
7127 },
7128 WorkflowStep::branch(
7129 "kind",
7130 vec![(
7131 "wire_ok",
7132 vec![WorkflowStep::Query {
7133 cmd_json: wire_cmd,
7134 store_as: None,
7135 }],
7136 )],
7137 vec![WorkflowStep::ForEach {
7138 list_key: "items".to_string(),
7139 steps: vec![WorkflowStep::Query {
7140 cmd_json: "select * from legacy".to_string(),
7141 store_as: None,
7142 }],
7143 }],
7144 ),
7145 ],
7146 );
7147
7148 let issues = collect_legacy_query_payload_issues(&wf);
7149 assert_eq!(
7150 issues.len(),
7151 3,
7152 "expected all legacy payloads to be detected"
7153 );
7154
7155 let paths: Vec<String> = issues.iter().map(|i| i.step_path.clone()).collect();
7156 assert!(
7157 paths.iter().any(|p| p == "steps[0]"),
7158 "top-level legacy query should be reported"
7159 );
7160 assert!(
7161 paths.iter().any(|p| p == "steps[1].on_timeout.steps[0]"),
7162 "nested wait/on_timeout legacy query should be reported"
7163 );
7164 assert!(
7165 paths
7166 .iter()
7167 .any(|p| p == "steps[2].default.steps[0].for_each.steps[0]"),
7168 "nested branch/default/foreach legacy query should be reported"
7169 );
7170 }
7171}