1use async_trait::async_trait;
7use std::cmp::Ordering;
8
9use chrono::{DateTime, SecondsFormat, Utc};
10use serde_json::Value;
11
12use crate::channel::ChannelKind;
13use crate::context::{
14 WorkflowBranchCursorSelection, WorkflowContext, WorkflowCursor, WorkflowCursorFrame,
15 WorkflowPendingWait,
16};
17use crate::payment::{ChargeRequest, ChargeResponse, Currency, OrderOrigin, PaymentKind};
18use crate::registry::{StateTransition, WorkflowDefinition};
19use crate::runtime::{
20 WorkflowLease, WorkflowOperation, WorkflowOperationKind, WorkflowOperationStatus,
21 WorkflowRunOptions, WorkflowSideEffect, WorkflowSideEffectKind, WorkflowSideEffectStatus,
22 WorkflowTimeoutOutcome,
23};
24use crate::step::{WorkflowBranchCondition, WorkflowStep};
25
26const WORKFLOW_QUERY_WIRE_MAGIC: &str = "QAIL-CMD/1\n";
27const WORKFLOW_QUERY_AST_WIRE_MAGIC: &str = "QAIL-CMD/2\n";
28
29#[derive(Debug, Clone, PartialEq, Eq)]
31pub struct LegacyQueryPayloadIssue {
32 pub transition_from: String,
34 pub transition_to: String,
36 pub step_path: String,
38 pub payload_preview: String,
40}
41
42pub fn collect_legacy_query_payload_issues(
50 definition: &WorkflowDefinition,
51) -> Vec<LegacyQueryPayloadIssue> {
52 let mut out = Vec::new();
53 for transition in &definition.transitions {
54 collect_legacy_query_payload_issues_in_steps(
55 &transition.from,
56 &transition.to,
57 &transition.steps,
58 "",
59 &mut out,
60 );
61 }
62 out
63}
64
65fn collect_legacy_query_payload_issues_in_steps(
66 transition_from: &str,
67 transition_to: &str,
68 steps: &[WorkflowStep],
69 path_prefix: &str,
70 out: &mut Vec<LegacyQueryPayloadIssue>,
71) {
72 for (idx, step) in steps.iter().enumerate() {
73 let step_path = if path_prefix.is_empty() {
74 format!("steps[{idx}]")
75 } else {
76 format!("{path_prefix}.steps[{idx}]")
77 };
78
79 match step {
80 WorkflowStep::Query { cmd_json, .. } => {
81 if !is_current_workflow_query_wire(cmd_json) {
82 out.push(LegacyQueryPayloadIssue {
83 transition_from: transition_from.to_string(),
84 transition_to: transition_to.to_string(),
85 step_path,
86 payload_preview: summarize_payload_preview(cmd_json),
87 });
88 }
89 }
90 WorkflowStep::Wait { on_timeout, .. } => {
91 collect_legacy_query_payload_issues_in_steps(
92 transition_from,
93 transition_to,
94 on_timeout,
95 &format!("{step_path}.on_timeout"),
96 out,
97 );
98 }
99 WorkflowStep::Branch {
100 branches, default, ..
101 } => {
102 for (branch_idx, (branch_value, branch_steps)) in branches.iter().enumerate() {
103 collect_legacy_query_payload_issues_in_steps(
104 transition_from,
105 transition_to,
106 branch_steps,
107 &format!("{step_path}.branches[{branch_idx}:{branch_value}]"),
108 out,
109 );
110 }
111 collect_legacy_query_payload_issues_in_steps(
112 transition_from,
113 transition_to,
114 default,
115 &format!("{step_path}.default"),
116 out,
117 );
118 }
119 WorkflowStep::BranchWhen {
120 branches, default, ..
121 } => {
122 for (branch_idx, (condition, branch_steps)) in branches.iter().enumerate() {
123 collect_legacy_query_payload_issues_in_steps(
124 transition_from,
125 transition_to,
126 branch_steps,
127 &format!("{step_path}.branch_when[{branch_idx}:{condition:?}]"),
128 out,
129 );
130 }
131 collect_legacy_query_payload_issues_in_steps(
132 transition_from,
133 transition_to,
134 default,
135 &format!("{step_path}.default"),
136 out,
137 );
138 }
139 WorkflowStep::ForEach { steps: nested, .. } => {
140 collect_legacy_query_payload_issues_in_steps(
141 transition_from,
142 transition_to,
143 nested,
144 &format!("{step_path}.for_each"),
145 out,
146 );
147 }
148 WorkflowStep::Notify { .. }
149 | WorkflowStep::Transition { .. }
150 | WorkflowStep::Log { .. }
151 | WorkflowStep::Charge { .. } => {}
152 }
153 }
154}
155
156fn summarize_payload_preview(cmd_json: &str) -> String {
157 const MAX_CHARS: usize = 64;
158 let escaped = cmd_json.replace('\n', "\\n");
159 let mut preview = escaped.chars().take(MAX_CHARS).collect::<String>();
160 if escaped.chars().count() > MAX_CHARS {
161 preview.push_str("...");
162 }
163 preview
164}
165
166fn is_current_workflow_query_wire(cmd_json: &str) -> bool {
167 cmd_json.starts_with(WORKFLOW_QUERY_WIRE_MAGIC)
168 || cmd_json.starts_with(WORKFLOW_QUERY_AST_WIRE_MAGIC)
169}
170
171#[derive(Debug)]
173pub enum WorkflowError {
174 QueryFailed(String),
176 NotifyFailed(String),
178 NoTransition {
180 state: String,
182 workflow: String,
184 },
185 MissingContextKey(String),
187 AlreadyTerminal(String),
189 PersistenceFailed(String),
191 Timeout {
193 event: String,
195 },
196 ChargeFailed(String),
198 Other(String),
200}
201
202impl std::fmt::Display for WorkflowError {
203 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
204 match self {
205 WorkflowError::QueryFailed(e) => write!(f, "Query failed: {}", e),
206 WorkflowError::NotifyFailed(e) => write!(f, "Notification failed: {}", e),
207 WorkflowError::NoTransition { state, workflow } => {
208 write!(
209 f,
210 "No transition from '{}' in workflow '{}'",
211 state, workflow
212 )
213 }
214 WorkflowError::MissingContextKey(k) => write!(f, "Context key not found: {}", k),
215 WorkflowError::AlreadyTerminal(s) => write!(f, "Workflow already terminal: {}", s),
216 WorkflowError::PersistenceFailed(e) => write!(f, "Persistence failed: {}", e),
217 WorkflowError::Timeout { event } => write!(f, "Timeout waiting for: {}", event),
218 WorkflowError::ChargeFailed(e) => write!(f, "Charge failed: {}", e),
219 WorkflowError::Other(e) => write!(f, "{}", e),
220 }
221 }
222}
223
224impl std::error::Error for WorkflowError {}
225
226#[allow(clippy::double_must_use)]
264#[async_trait]
265pub trait WorkflowExecutor: Send + Sync {
266 async fn execute_query(&self, cmd_json: &str) -> Result<Value, WorkflowError>;
268
269 async fn send_notification(
271 &self,
272 channel: &ChannelKind,
273 recipient: &str,
274 template: &str,
275 params: &Value,
276 ) -> Result<(), WorkflowError>;
277
278 async fn save_state(&self, ctx: &WorkflowContext) -> Result<(), WorkflowError>;
280
281 async fn load_state(&self, workflow_id: &str)
283 -> Result<Option<WorkflowContext>, WorkflowError>;
284
285 async fn create_charge(
292 &self,
293 provider: &PaymentKind,
294 request: ChargeRequest,
295 ) -> Result<ChargeResponse, WorkflowError>;
296
297 async fn acquire_workflow_lease(&self, _lease: &WorkflowLease) -> Result<bool, WorkflowError> {
302 Ok(true)
303 }
304
305 async fn release_workflow_lease(&self, _lease: &WorkflowLease) -> Result<(), WorkflowError> {
307 Ok(())
308 }
309
310 async fn begin_workflow_operation(
314 &self,
315 _operation: &WorkflowOperation,
316 ) -> Result<WorkflowOperationStatus, WorkflowError> {
317 Ok(WorkflowOperationStatus::Started)
318 }
319
320 async fn complete_workflow_operation(
322 &self,
323 _operation: &WorkflowOperation,
324 _state: &str,
325 ) -> Result<(), WorkflowError> {
326 Ok(())
327 }
328
329 async fn fail_workflow_operation(
331 &self,
332 _operation: &WorkflowOperation,
333 _error: &str,
334 ) -> Result<(), WorkflowError> {
335 Ok(())
336 }
337
338 async fn load_due_workflow_timeouts(
344 &self,
345 _workflow_name: &str,
346 _now: DateTime<Utc>,
347 _limit: usize,
348 ) -> Result<Vec<String>, WorkflowError> {
349 Ok(Vec::new())
350 }
351
352 async fn begin_workflow_side_effect(
354 &self,
355 _operation: &WorkflowSideEffect,
356 ) -> Result<WorkflowSideEffectStatus, WorkflowError> {
357 Ok(WorkflowSideEffectStatus::Execute)
358 }
359
360 async fn complete_workflow_side_effect(
362 &self,
363 _operation: &WorkflowSideEffect,
364 _result: Option<&Value>,
365 ) -> Result<(), WorkflowError> {
366 Ok(())
367 }
368
369 async fn fail_workflow_side_effect(
376 &self,
377 _operation: &WorkflowSideEffect,
378 _error: &str,
379 ) -> Result<(), WorkflowError> {
380 Ok(())
381 }
382
383 async fn execute_query_once(
385 &self,
386 operation: &WorkflowSideEffect,
387 cmd_json: &str,
388 ) -> Result<Value, WorkflowError> {
389 match self.begin_workflow_side_effect(operation).await? {
390 WorkflowSideEffectStatus::Execute => {
391 let result = match self.execute_query(cmd_json).await {
392 Ok(result) => result,
393 Err(err) => {
394 let _ = self
395 .fail_workflow_side_effect(operation, &err.to_string())
396 .await;
397 return Err(err);
398 }
399 };
400 self.complete_workflow_side_effect(operation, Some(&result))
401 .await?;
402 Ok(result)
403 }
404 WorkflowSideEffectStatus::AlreadyCompleted { result } => result.ok_or_else(|| {
405 WorkflowError::Other(format!(
406 "Workflow side effect '{}' completed without a stored query result",
407 operation.operation_id
408 ))
409 }),
410 }
411 }
412
413 async fn send_notification_once(
415 &self,
416 operation: &WorkflowSideEffect,
417 channel: &ChannelKind,
418 recipient: &str,
419 template: &str,
420 params: &Value,
421 ) -> Result<(), WorkflowError> {
422 match self.begin_workflow_side_effect(operation).await? {
423 WorkflowSideEffectStatus::Execute => {
424 if let Err(err) = self
425 .send_notification(channel, recipient, template, params)
426 .await
427 {
428 let _ = self
429 .fail_workflow_side_effect(operation, &err.to_string())
430 .await;
431 return Err(err);
432 }
433 self.complete_workflow_side_effect(operation, None).await
434 }
435 WorkflowSideEffectStatus::AlreadyCompleted { .. } => Ok(()),
436 }
437 }
438
439 async fn create_charge_once(
441 &self,
442 operation: &WorkflowSideEffect,
443 provider: &PaymentKind,
444 mut request: ChargeRequest,
445 ) -> Result<ChargeResponse, WorkflowError> {
446 match self.begin_workflow_side_effect(operation).await? {
447 WorkflowSideEffectStatus::Execute => {
448 if request.idempotency_key.is_none() {
449 request.idempotency_key = Some(operation.operation_id.clone());
450 }
451 let response = match self.create_charge(provider, request).await {
452 Ok(response) => response,
453 Err(err) => {
454 let _ = self
455 .fail_workflow_side_effect(operation, &err.to_string())
456 .await;
457 return Err(err);
458 }
459 };
460 let response_json = serde_json::to_value(&response)
461 .map_err(|e| WorkflowError::Other(e.to_string()))?;
462 self.complete_workflow_side_effect(operation, Some(&response_json))
463 .await?;
464 Ok(response)
465 }
466 WorkflowSideEffectStatus::AlreadyCompleted {
467 result: Some(result),
468 } => serde_json::from_value(result).map_err(|e| {
469 WorkflowError::Other(format!(
470 "Workflow side effect '{}' stored an invalid charge result: {e}",
471 operation.operation_id
472 ))
473 }),
474 WorkflowSideEffectStatus::AlreadyCompleted { result: None } => {
475 Err(WorkflowError::Other(format!(
476 "Workflow side effect '{}' completed without a stored charge result",
477 operation.operation_id
478 )))
479 }
480 }
481 }
482}
483
484#[derive(Debug, Clone, PartialEq, Eq)]
485enum StepOutcome {
486 Continue,
487 Paused(StepPause),
488}
489
490#[derive(Debug, Clone, PartialEq, Eq)]
491struct StepPause {
492 frames: Vec<WorkflowCursorFrame>,
493 wait: WorkflowPendingWait,
494}
495
496struct RuntimeGuard {
497 lease: Option<WorkflowLease>,
498 operation: Option<WorkflowOperation>,
499}
500
501enum RuntimeEntry {
502 Active(RuntimeGuard),
503 Completed(String),
504}
505
506#[derive(Debug, Clone)]
507enum StepListCursorKind {
508 Steps,
509 Branch {
510 selection: WorkflowBranchCursorSelection,
511 },
512 ForEach {
513 item_index: usize,
514 item: Value,
515 },
516}
517
518#[derive(Debug, Clone, Copy, PartialEq, Eq)]
519enum RunMode<'a> {
520 Normal,
521 EventResume { event: &'a str },
522}
523
524const FOR_EACH_ITEM_KEY: &str = "item";
525const RESUME_EVENT_KEY: &str = "event";
526const TIMEOUT_FALLBACK_KEY: &str = "__qail_timeout_fallback";
527const RESERVED_CONTEXT_KEYS: &[&str] = &[FOR_EACH_ITEM_KEY, RESUME_EVENT_KEY, TIMEOUT_FALLBACK_KEY];
528
529#[derive(Debug, Clone)]
530struct StepExecutionScope<'a> {
531 state: &'a str,
532 path_prefix: Vec<WorkflowCursorFrame>,
533 list_kind: StepListCursorKind,
534 checkpoint_steps: bool,
535 checkpoint_timeout: bool,
536}
537
538impl<'a> StepExecutionScope<'a> {
539 fn new(state: &'a str, list_kind: StepListCursorKind, checkpoint_steps: bool) -> Self {
540 Self {
541 state,
542 path_prefix: Vec::new(),
543 list_kind,
544 checkpoint_steps,
545 checkpoint_timeout: false,
546 }
547 }
548
549 fn with_timeout_checkpoint(mut self) -> Self {
550 self.checkpoint_timeout = true;
551 self
552 }
553
554 fn child(&self, step_index: usize, list_kind: StepListCursorKind) -> Self {
555 Self {
556 state: self.state,
557 path_prefix: cursor_frames_for_index(&self.path_prefix, &self.list_kind, step_index),
558 list_kind,
559 checkpoint_steps: self.checkpoint_steps,
560 checkpoint_timeout: self.checkpoint_timeout,
561 }
562 }
563}
564
565impl RuntimeGuard {
566 async fn enter<E: WorkflowExecutor>(
567 executor: &E,
568 workflow_name: &str,
569 workflow_id: &str,
570 kind: WorkflowOperationKind,
571 options: WorkflowRunOptions,
572 ) -> Result<RuntimeEntry, WorkflowError> {
573 let lease_options = match options.lease {
574 Some(lease_options) => {
575 ensure_definition_text(&lease_options.owner, "Workflow lease owner")?;
576 if lease_options.ttl.is_zero() {
577 return Err(WorkflowError::Other(
578 "Workflow lease TTL must be greater than zero".to_string(),
579 ));
580 }
581 Some(lease_options)
582 }
583 None => None,
584 };
585
586 let operation = match options.idempotency_key {
587 Some(idempotency_key) => {
588 ensure_definition_text(&idempotency_key, "Workflow idempotency key")?;
589 let operation = WorkflowOperation {
590 workflow_name: workflow_name.to_string(),
591 workflow_id: workflow_id.to_string(),
592 idempotency_key,
593 kind: kind.clone(),
594 };
595 let operation_status = match executor.begin_workflow_operation(&operation).await {
596 Ok(status) => status,
597 Err(err) => return Err(err),
598 };
599 match operation_status {
600 WorkflowOperationStatus::Started => Some(operation),
601 WorkflowOperationStatus::InProgress => {
602 return Err(WorkflowError::Other(format!(
603 "Workflow operation '{}' is already in progress",
604 operation.idempotency_key
605 )));
606 }
607 WorkflowOperationStatus::Completed { state } => {
608 ensure_definition_text(&state, "Workflow completed operation state")?;
609 return Ok(RuntimeEntry::Completed(state));
610 }
611 }
612 }
613 None => None,
614 };
615
616 let lease = lease_options.map(|lease_options| WorkflowLease {
617 workflow_id: workflow_id.to_string(),
618 owner: runtime_lease_owner(
619 &lease_options.owner,
620 workflow_name,
621 workflow_id,
622 &kind,
623 operation
624 .as_ref()
625 .map(|operation| operation.idempotency_key.as_str()),
626 Utc::now(),
627 ),
628 ttl: lease_options.ttl,
629 });
630
631 let lease = match lease {
632 Some(lease) => match executor.acquire_workflow_lease(&lease).await {
633 Ok(true) => Some(lease),
634 Ok(false) => {
635 let error = format!("Workflow '{}' is already locked", workflow_id);
636 if let Some(operation) = &operation {
637 let _ = executor.fail_workflow_operation(operation, &error).await;
638 }
639 return Err(WorkflowError::Other(error));
640 }
641 Err(err) => {
642 if let Some(operation) = &operation {
643 let _ = executor
644 .fail_workflow_operation(operation, &err.to_string())
645 .await;
646 }
647 return Err(err);
648 }
649 },
650 None => None,
651 };
652
653 Ok(RuntimeEntry::Active(Self { lease, operation }))
654 }
655
656 async fn finish<E: WorkflowExecutor>(
657 self,
658 executor: &E,
659 result: &Result<String, WorkflowError>,
660 ) -> Result<(), WorkflowError> {
661 let operation_result = match &self.operation {
662 Some(operation) => match result {
663 Ok(state) => executor.complete_workflow_operation(operation, state).await,
664 Err(err) => {
665 executor
666 .fail_workflow_operation(operation, &err.to_string())
667 .await
668 }
669 },
670 None => Ok(()),
671 };
672
673 let release_result = match &self.lease {
674 Some(lease) => executor.release_workflow_lease(lease).await,
675 None => Ok(()),
676 };
677
678 match (operation_result, release_result) {
679 (Err(err), _) => Err(err),
680 (Ok(()), Err(err)) => Err(err),
681 (Ok(()), Ok(())) => Ok(()),
682 }
683 }
684}
685
686fn runtime_lease_owner(
687 owner: &str,
688 workflow_name: &str,
689 workflow_id: &str,
690 kind: &WorkflowOperationKind,
691 idempotency_key: Option<&str>,
692 acquired_at: DateTime<Utc>,
693) -> String {
694 serde_json::json!([
695 "qail-workflow-lease",
696 1,
697 owner,
698 workflow_name,
699 workflow_id,
700 operation_kind_lease_key(kind),
701 idempotency_key,
702 acquired_at.to_rfc3339_opts(SecondsFormat::Micros, true)
703 ])
704 .to_string()
705}
706
707fn operation_kind_lease_key(kind: &WorkflowOperationKind) -> serde_json::Value {
708 match kind {
709 WorkflowOperationKind::Run => serde_json::json!(["run"]),
710 WorkflowOperationKind::Resume { event } => serde_json::json!(["resume", event]),
711 WorkflowOperationKind::Timeout => serde_json::json!(["timeout"]),
712 }
713}
714
715async fn finish_runtime_operation<E: WorkflowExecutor>(
716 executor: &E,
717 guard: RuntimeGuard,
718 result: Result<String, WorkflowError>,
719) -> Result<String, WorkflowError> {
720 let finish_result = guard.finish(executor, &result).await;
721 match (result, finish_result) {
722 (Ok(state), Ok(())) => Ok(state),
723 (Ok(_), Err(err)) => Err(err),
724 (Err(err), Ok(())) | (Err(err), Err(_)) => Err(err),
725 }
726}
727
728fn invalid_cursor(message: impl Into<String>) -> WorkflowError {
729 WorkflowError::Other(format!(
730 "Invalid workflow resume cursor: {}",
731 message.into()
732 ))
733}
734
735fn set_timeout_fallback(
736 ctx: &mut WorkflowContext,
737 wait: &WorkflowPendingWait,
738) -> Result<(), WorkflowError> {
739 let value = serde_json::to_value(wait)
740 .map_err(|e| WorkflowError::Other(format!("Failed to serialize timeout cursor: {e}")))?;
741 ctx.set(TIMEOUT_FALLBACK_KEY, value);
742 Ok(())
743}
744
745fn timeout_fallback_from_context(
746 ctx: &WorkflowContext,
747) -> Result<Option<WorkflowPendingWait>, WorkflowError> {
748 let Some(value) = ctx.get(TIMEOUT_FALLBACK_KEY) else {
749 return Ok(None);
750 };
751 serde_json::from_value(value.clone())
752 .map(Some)
753 .map_err(|e| invalid_cursor(format!("timeout fallback metadata is invalid: {e}")))
754}
755
756fn clear_timeout_fallback(ctx: &mut WorkflowContext) {
757 if ctx.data.remove(TIMEOUT_FALLBACK_KEY).is_some() {
758 ctx.updated_at = Utc::now();
759 }
760}
761
762fn json_type_name(value: &Value) -> &'static str {
763 match value {
764 Value::Null => "null",
765 Value::Bool(_) => "boolean",
766 Value::Number(_) => "number",
767 Value::String(_) => "string",
768 Value::Array(_) => "array",
769 Value::Object(_) => "object",
770 }
771}
772
773fn ensure_no_child_cursor(
774 step_name: &str,
775 cursor_frames: &[WorkflowCursorFrame],
776) -> Result<(), WorkflowError> {
777 if cursor_frames.is_empty() {
778 Ok(())
779 } else {
780 Err(invalid_cursor(format!(
781 "{step_name} cannot resume nested cursor frames"
782 )))
783 }
784}
785
786fn ensure_step_position(
787 step: &WorkflowStep,
788 idx: usize,
789 step_count: usize,
790) -> Result<(), WorkflowError> {
791 if matches!(step, WorkflowStep::Transition { .. }) && idx + 1 < step_count {
792 return Err(WorkflowError::Other(
793 "Transition steps must be the final step in their block; move follow-up work before the Transition or into the target state's transition".to_string(),
794 ));
795 }
796 Ok(())
797}
798
799fn ensure_wait_event_name(event: &str) -> Result<(), WorkflowError> {
800 if event.trim().is_empty() {
801 return Err(WorkflowError::Other(
802 "Wait event name must not be empty".to_string(),
803 ));
804 }
805 if event.trim() != event {
806 return Err(WorkflowError::Other(
807 "Wait event name must not have leading or trailing whitespace".to_string(),
808 ));
809 }
810 Ok(())
811}
812
813fn ensure_definition_text(value: &str, usage: &str) -> Result<(), WorkflowError> {
814 if value.trim().is_empty() {
815 return Err(WorkflowError::Other(format!("{usage} must not be empty")));
816 }
817 if value.trim() != value {
818 return Err(WorkflowError::Other(format!(
819 "{usage} must not have leading or trailing whitespace"
820 )));
821 }
822 Ok(())
823}
824
825fn validate_workflow_context_identity(ctx: &WorkflowContext) -> Result<(), WorkflowError> {
826 ensure_definition_text(&ctx.workflow_id, "Workflow id")?;
827 ensure_definition_text(&ctx.current_state, "Workflow current state")?;
828 Ok(())
829}
830
831fn ensure_user_context_key(key: &str, usage: &str) -> Result<(), WorkflowError> {
832 ensure_definition_text(key, usage)?;
833 if key.contains('.') {
834 return Err(WorkflowError::Other(format!(
835 "Workflow {usage} must be a top-level context key; dot notation is only supported for lookups"
836 )));
837 }
838 if RESERVED_CONTEXT_KEYS.contains(&key) {
839 return Err(WorkflowError::Other(format!(
840 "Workflow {usage} uses reserved context key '{key}'"
841 )));
842 }
843 Ok(())
844}
845
846fn ensure_context_lookup_key(key: &str, usage: &str) -> Result<(), WorkflowError> {
847 ensure_definition_text(key, usage)?;
848 if key.split('.').any(str::is_empty) {
849 return Err(WorkflowError::Other(format!(
850 "{usage} must not contain empty dot-notation path segments"
851 )));
852 }
853 Ok(())
854}
855
856fn ensure_optional_user_context_key(
857 key: Option<&String>,
858 usage: &str,
859) -> Result<(), WorkflowError> {
860 if let Some(key) = key {
861 ensure_user_context_key(key, usage)?;
862 }
863 Ok(())
864}
865
866fn ensure_cursor_has_frames(cursor: &WorkflowCursor) -> Result<(), WorkflowError> {
867 if cursor.frames.is_empty() {
868 return Err(invalid_cursor("resume cursor has no frames"));
869 }
870 Ok(())
871}
872
873fn ensure_unique_branch_values(
874 condition_key: &str,
875 branches: &[(String, Vec<WorkflowStep>)],
876) -> Result<(), WorkflowError> {
877 let mut seen = std::collections::HashSet::new();
878 for (branch_value, _) in branches {
879 if !seen.insert(branch_value.as_str()) {
880 return Err(WorkflowError::Other(format!(
881 "Ambiguous workflow branch for '{condition_key}': duplicate value '{branch_value}'"
882 )));
883 }
884 }
885 Ok(())
886}
887
888fn ensure_unique_branch_conditions(
889 condition_key: &str,
890 branches: &[(WorkflowBranchCondition, Vec<WorkflowStep>)],
891) -> Result<(), WorkflowError> {
892 let mut seen = std::collections::HashSet::new();
893 for (condition, _) in branches {
894 validate_branch_condition(condition_key, condition)?;
895 if !seen.insert(condition) {
896 return Err(WorkflowError::Other(format!(
897 "Ambiguous workflow branch for '{condition_key}': duplicate condition '{condition:?}'"
898 )));
899 }
900 }
901 Ok(())
902}
903
904fn validate_branch_condition(
905 condition_key: &str,
906 condition: &WorkflowBranchCondition,
907) -> Result<(), WorkflowError> {
908 match condition {
909 WorkflowBranchCondition::OneOf(expected_values) if expected_values.is_empty() => {
910 Err(WorkflowError::Other(format!(
911 "Ambiguous workflow branch for '{condition_key}': OneOf condition must include at least one value"
912 )))
913 }
914 WorkflowBranchCondition::StringContains(needle) if needle.is_empty() => {
915 Err(WorkflowError::Other(format!(
916 "Ambiguous workflow branch for '{condition_key}': StringContains condition must not be empty"
917 )))
918 }
919 _ => Ok(()),
920 }
921}
922
923fn branch_condition_matches(condition: &WorkflowBranchCondition, value: Option<&Value>) -> bool {
924 match condition {
925 WorkflowBranchCondition::Exists => value.is_some(),
926 WorkflowBranchCondition::Missing => value.is_none(),
927 WorkflowBranchCondition::Null => matches!(value, Some(Value::Null)),
928 WorkflowBranchCondition::Bool(expected) => {
929 value.and_then(Value::as_bool) == Some(*expected)
930 }
931 WorkflowBranchCondition::Equals(expected) => {
932 value.map(branch_condition_text).as_deref() == Some(expected.as_str())
933 }
934 WorkflowBranchCondition::NotEquals(expected) => value
935 .map(branch_condition_text)
936 .is_some_and(|actual| actual != *expected),
937 WorkflowBranchCondition::OneOf(expected_values) => value
938 .map(branch_condition_text)
939 .is_some_and(|actual| expected_values.iter().any(|expected| expected == &actual)),
940 WorkflowBranchCondition::NumberGt(expected) => value
941 .and_then(|value| branch_condition_number_cmp(value, *expected))
942 .is_some_and(|ordering| ordering == Ordering::Greater),
943 WorkflowBranchCondition::NumberGte(expected) => value
944 .and_then(|value| branch_condition_number_cmp(value, *expected))
945 .is_some_and(|ordering| ordering != Ordering::Less),
946 WorkflowBranchCondition::NumberLt(expected) => value
947 .and_then(|value| branch_condition_number_cmp(value, *expected))
948 .is_some_and(|ordering| ordering == Ordering::Less),
949 WorkflowBranchCondition::NumberLte(expected) => value
950 .and_then(|value| branch_condition_number_cmp(value, *expected))
951 .is_some_and(|ordering| ordering != Ordering::Greater),
952 WorkflowBranchCondition::StringContains(needle) => value
953 .map(branch_condition_text)
954 .is_some_and(|actual| actual.contains(needle)),
955 }
956}
957
958fn branch_condition_text(value: &Value) -> String {
959 match value {
960 Value::String(value) => value.clone(),
961 _ => value.to_string(),
962 }
963}
964
965fn branch_condition_number_cmp(value: &Value, expected: i64) -> Option<Ordering> {
966 let Value::Number(number) = value else {
967 return None;
968 };
969
970 if let Some(actual) = number.as_i64() {
971 return Some(actual.cmp(&expected));
972 }
973
974 if let Some(actual) = number.as_u64() {
975 return Some(if expected < 0 {
976 Ordering::Greater
977 } else {
978 actual.cmp(&(expected as u64))
979 });
980 }
981
982 let actual = number.as_f64()?;
983 actual.partial_cmp(&(expected as f64))
984}
985
986fn find_single_transition<'a>(
987 definition: &'a WorkflowDefinition,
988 state: &str,
989) -> Result<Option<&'a StateTransition>, WorkflowError> {
990 let mut matches = definition.transitions.iter().filter(|t| t.from == state);
991 let Some(first) = matches.next() else {
992 return Ok(None);
993 };
994 if matches.next().is_some() {
995 let count = definition.transitions_from(state).len();
996 return Err(WorkflowError::Other(format!(
997 "Ambiguous workflow definition '{}': state '{state}' has {count} outgoing transitions; use a Branch step or split states explicitly",
998 definition.name
999 )));
1000 }
1001 Ok(Some(first))
1002}
1003
1004fn validate_workflow_definition(definition: &WorkflowDefinition) -> Result<(), WorkflowError> {
1005 ensure_definition_text(&definition.name, "Workflow definition name")?;
1006 ensure_definition_text(&definition.initial_state, "Workflow initial state")?;
1007 if let Some(version) = &definition.version {
1008 ensure_definition_text(version, "Workflow definition version")?;
1009 }
1010
1011 let mut seen_states = std::collections::HashMap::<&str, usize>::new();
1012 for transition in &definition.transitions {
1013 ensure_definition_text(&transition.from, "Workflow transition source state")?;
1014 ensure_definition_text(&transition.to, "Workflow transition target state")?;
1015 *seen_states.entry(&transition.from).or_default() += 1;
1016 validate_workflow_steps(&transition.steps)?;
1017 }
1018
1019 if let Some((state, count)) = seen_states.into_iter().find(|(_, count)| *count > 1) {
1020 return Err(WorkflowError::Other(format!(
1021 "Ambiguous workflow definition '{}': state '{state}' has {count} outgoing transitions; use a Branch step or split states explicitly",
1022 definition.name
1023 )));
1024 }
1025
1026 Ok(())
1027}
1028
1029fn ensure_context_definition(
1030 ctx: &mut WorkflowContext,
1031 definition: &WorkflowDefinition,
1032) -> Result<(), WorkflowError> {
1033 if let Some(context_name) = &ctx.definition_name
1034 && context_name != &definition.name
1035 {
1036 return Err(WorkflowError::Other(format!(
1037 "Workflow context '{}' belongs to definition '{context_name}', not '{}'",
1038 ctx.workflow_id, definition.name
1039 )));
1040 }
1041
1042 if let Some(context_version) = &ctx.definition_version
1043 && Some(context_version) != definition.version.as_ref()
1044 {
1045 return Err(WorkflowError::Other(format!(
1046 "Workflow context '{}' uses definition version '{}', not '{}'",
1047 ctx.workflow_id,
1048 context_version,
1049 definition.version.as_deref().unwrap_or("<unversioned>")
1050 )));
1051 }
1052
1053 if ctx.definition_name.is_none() {
1054 ctx.definition_name = Some(definition.name.clone());
1055 ctx.updated_at = Utc::now();
1056 }
1057 if ctx.definition_version.is_none()
1058 && let Some(version) = &definition.version
1059 {
1060 ctx.definition_version = Some(version.clone());
1061 ctx.updated_at = Utc::now();
1062 }
1063
1064 Ok(())
1065}
1066
1067fn validate_workflow_steps(steps: &[WorkflowStep]) -> Result<(), WorkflowError> {
1068 for (idx, step) in steps.iter().enumerate() {
1069 ensure_step_position(step, idx, steps.len())?;
1070 match step {
1071 WorkflowStep::Query { cmd_json, store_as } => {
1072 normalize_query_wire_for_execution(cmd_json)?;
1073 ensure_optional_user_context_key(store_as.as_ref(), "Query store_as")?;
1074 }
1075 WorkflowStep::Wait {
1076 event, on_timeout, ..
1077 } => {
1078 ensure_wait_event_name(event)?;
1079 if steps_contain_wait(on_timeout) {
1080 return Err(WorkflowError::Other(
1081 "Wait steps inside on_timeout fallback are not supported".to_string(),
1082 ));
1083 }
1084 validate_workflow_steps(on_timeout)?;
1085 }
1086 WorkflowStep::Branch {
1087 condition_key,
1088 branches,
1089 default,
1090 } => {
1091 ensure_context_lookup_key(condition_key, "Branch condition_key")?;
1092 ensure_unique_branch_values(condition_key, branches)?;
1093 for (_, branch_steps) in branches {
1094 validate_workflow_steps(branch_steps)?;
1095 }
1096 validate_workflow_steps(default)?;
1097 }
1098 WorkflowStep::BranchWhen {
1099 condition_key,
1100 branches,
1101 default,
1102 } => {
1103 ensure_context_lookup_key(condition_key, "BranchWhen condition_key")?;
1104 ensure_unique_branch_conditions(condition_key, branches)?;
1105 for (_, branch_steps) in branches {
1106 validate_workflow_steps(branch_steps)?;
1107 }
1108 validate_workflow_steps(default)?;
1109 }
1110 WorkflowStep::ForEach { list_key, steps } => {
1111 ensure_context_lookup_key(list_key, "ForEach list_key")?;
1112 validate_workflow_steps(steps)?;
1113 }
1114 WorkflowStep::Charge {
1115 amount_key,
1116 reference_key,
1117 description_key,
1118 payment_method_key,
1119 order_origin_key,
1120 store_as,
1121 ..
1122 } => {
1123 ensure_context_lookup_key(amount_key, "Charge amount_key")?;
1124 ensure_context_lookup_key(reference_key, "Charge reference_key")?;
1125 if let Some(description_key) = description_key {
1126 ensure_context_lookup_key(description_key, "Charge description_key")?;
1127 }
1128 if let Some(payment_method_key) = payment_method_key {
1129 ensure_context_lookup_key(payment_method_key, "Charge payment_method_key")?;
1130 }
1131 if let Some(order_origin_key) = order_origin_key {
1132 ensure_context_lookup_key(order_origin_key, "Charge order_origin_key")?;
1133 }
1134 ensure_optional_user_context_key(store_as.as_ref(), "Charge store_as")?;
1135 }
1136 WorkflowStep::Notify {
1137 template,
1138 recipient_key,
1139 payload_key,
1140 ..
1141 } => {
1142 ensure_definition_text(template, "Notify template")?;
1143 ensure_context_lookup_key(recipient_key, "Notify recipient_key")?;
1144 if let Some(payload_key) = payload_key {
1145 ensure_context_lookup_key(payload_key, "Notify payload_key")?;
1146 }
1147 }
1148 WorkflowStep::Transition { to } => {
1149 ensure_definition_text(to, "Transition target state")?;
1150 }
1151 WorkflowStep::Log { .. } => {}
1152 }
1153 }
1154
1155 Ok(())
1156}
1157
1158fn deadline_from_timeout(timeout: &std::time::Duration) -> Result<DateTime<Utc>, WorkflowError> {
1159 let timeout = chrono::Duration::from_std(*timeout).map_err(|_| {
1160 WorkflowError::Other("Wait timeout duration is too large to represent".to_string())
1161 })?;
1162 Utc::now()
1163 .checked_add_signed(timeout)
1164 .ok_or_else(|| WorkflowError::Other("Wait timeout deadline overflowed".to_string()))
1165}
1166
1167fn cursor_frame_for_index(kind: &StepListCursorKind, index: usize) -> WorkflowCursorFrame {
1168 match kind {
1169 StepListCursorKind::Steps => WorkflowCursorFrame::Steps { index },
1170 StepListCursorKind::Branch { selection } => WorkflowCursorFrame::Branch {
1171 selection: selection.clone(),
1172 index,
1173 },
1174 StepListCursorKind::ForEach { item_index, item } => WorkflowCursorFrame::ForEachItem {
1175 item_index: *item_index,
1176 index,
1177 item: item.clone(),
1178 },
1179 }
1180}
1181
1182fn cursor_frames_for_index(
1183 path_prefix: &[WorkflowCursorFrame],
1184 kind: &StepListCursorKind,
1185 index: usize,
1186) -> Vec<WorkflowCursorFrame> {
1187 let mut frames = Vec::with_capacity(path_prefix.len() + 1);
1188 frames.extend_from_slice(path_prefix);
1189 frames.push(cursor_frame_for_index(kind, index));
1190 frames
1191}
1192
1193async fn checkpoint_cursor<E: WorkflowExecutor>(
1194 executor: &E,
1195 ctx: &mut WorkflowContext,
1196 state: &str,
1197 frames: Vec<WorkflowCursorFrame>,
1198 wait: Option<WorkflowPendingWait>,
1199 timeout_fallback: bool,
1200) -> Result<(), WorkflowError> {
1201 if !timeout_fallback {
1202 clear_timeout_fallback(ctx);
1203 }
1204 let wait = if timeout_fallback && wait.is_none() {
1205 timeout_fallback_from_context(ctx)?
1206 } else {
1207 wait
1208 };
1209 if ctx.current_state == state {
1210 ctx.set_cursor(WorkflowCursor {
1211 state: state.to_string(),
1212 frames,
1213 wait,
1214 });
1215 } else {
1216 ctx.clear_cursor();
1217 if timeout_fallback {
1218 clear_timeout_fallback(ctx);
1219 }
1220 }
1221 executor.save_state(ctx).await
1222}
1223
1224async fn checkpoint_completed_step<E: WorkflowExecutor>(
1225 executor: &E,
1226 ctx: &mut WorkflowContext,
1227 state: &str,
1228 path_prefix: &[WorkflowCursorFrame],
1229 kind: &StepListCursorKind,
1230 next_index: usize,
1231 timeout_fallback: bool,
1232) -> Result<(), WorkflowError> {
1233 let frames = cursor_frames_for_index(path_prefix, kind, next_index);
1234 checkpoint_cursor(executor, ctx, state, frames, None, timeout_fallback).await
1235}
1236
1237fn restore_for_each_item(ctx: &mut WorkflowContext, previous_item: Option<Value>) {
1238 match previous_item {
1239 Some(item) => {
1240 ctx.data.insert(FOR_EACH_ITEM_KEY.to_string(), item);
1241 }
1242 None => {
1243 ctx.data.remove(FOR_EACH_ITEM_KEY);
1244 }
1245 }
1246 ctx.updated_at = Utc::now();
1247}
1248
1249fn side_effect_operation(
1250 ctx: &WorkflowContext,
1251 scope: &StepExecutionScope<'_>,
1252 step_index: usize,
1253 kind: WorkflowSideEffectKind,
1254) -> WorkflowSideEffect {
1255 let frames = cursor_frames_for_index(&scope.path_prefix, &scope.list_kind, step_index);
1256 let step_path = render_cursor_path(&frames);
1257 WorkflowSideEffect {
1258 workflow_id: ctx.workflow_id.clone(),
1259 state: scope.state.to_string(),
1260 step_path: step_path.clone(),
1261 kind,
1262 operation_id: side_effect_operation_id(
1263 &ctx.workflow_id,
1264 scope.state,
1265 ctx.transition_count(),
1266 kind,
1267 &step_path,
1268 ),
1269 }
1270}
1271
1272fn side_effect_operation_id(
1273 workflow_id: &str,
1274 state: &str,
1275 state_generation: usize,
1276 kind: WorkflowSideEffectKind,
1277 step_path: &str,
1278) -> String {
1279 serde_json::json!([
1280 "qail-workflow-side-effect",
1281 2,
1282 workflow_id,
1283 state,
1284 state_generation,
1285 kind.as_str(),
1286 step_path
1287 ])
1288 .to_string()
1289}
1290
1291fn render_cursor_path(frames: &[WorkflowCursorFrame]) -> String {
1292 frames
1293 .iter()
1294 .map(render_cursor_frame)
1295 .collect::<Vec<_>>()
1296 .join("/")
1297}
1298
1299fn render_cursor_frame(frame: &WorkflowCursorFrame) -> String {
1300 match frame {
1301 WorkflowCursorFrame::Steps { index } => format!("steps[{index}]"),
1302 WorkflowCursorFrame::Branch { selection, index } => {
1303 format!(
1304 "branch[{}].steps[{index}]",
1305 render_branch_selection(selection)
1306 )
1307 }
1308 WorkflowCursorFrame::ForEach { item_index, index } => {
1309 format!("for_each[{item_index}].steps[{index}]")
1310 }
1311 WorkflowCursorFrame::ForEachItem {
1312 item_index, index, ..
1313 } => format!("for_each[{item_index}].steps[{index}]"),
1314 }
1315}
1316
1317fn render_branch_selection(selection: &WorkflowBranchCursorSelection) -> String {
1318 match selection {
1319 WorkflowBranchCursorSelection::Branch(index) => index.to_string(),
1320 WorkflowBranchCursorSelection::Default => "default".to_string(),
1321 }
1322}
1323
1324fn log_value_to_string(value: &Value) -> String {
1325 match value {
1326 Value::String(value) => value.clone(),
1327 _ => value.to_string(),
1328 }
1329}
1330
1331fn resolve_log_message(ctx: &WorkflowContext, message: &str) -> String {
1332 let mut resolved = String::with_capacity(message.len());
1333 let mut rest = message;
1334
1335 while let Some(start) = rest.find('{') {
1336 resolved.push_str(&rest[..start]);
1337 let after_open = &rest[start + 1..];
1338 let Some(end) = after_open.find('}') else {
1339 resolved.push_str(&rest[start..]);
1340 return resolved;
1341 };
1342
1343 let key = &after_open[..end];
1344 if !key.is_empty()
1345 && let Some(value) = ctx.get(key)
1346 {
1347 resolved.push_str(&log_value_to_string(value));
1348 } else {
1349 resolved.push('{');
1350 resolved.push_str(key);
1351 resolved.push('}');
1352 }
1353 rest = &after_open[end + 1..];
1354 }
1355
1356 resolved.push_str(rest);
1357 resolved
1358}
1359
1360fn execute_steps<'a, E: WorkflowExecutor>(
1361 executor: &'a E,
1362 steps: &'a [WorkflowStep],
1363 ctx: &'a mut WorkflowContext,
1364 start_index: usize,
1365 cursor_frames: &'a [WorkflowCursorFrame],
1366 scope: StepExecutionScope<'a>,
1367) -> std::pin::Pin<
1368 Box<dyn std::future::Future<Output = Result<Option<StepPause>, WorkflowError>> + Send + 'a>,
1369> {
1370 Box::pin(async move {
1371 if start_index > steps.len() {
1372 return Err(invalid_cursor(format!(
1373 "step index {start_index} is past step count {}",
1374 steps.len()
1375 )));
1376 }
1377 if start_index == steps.len() && !cursor_frames.is_empty() {
1378 return Err(invalid_cursor(
1379 "cursor contains child frames after the step list ended",
1380 ));
1381 }
1382
1383 for (idx, step) in steps.iter().enumerate().skip(start_index) {
1384 ensure_step_position(step, idx, steps.len())?;
1385 let step_cursor = if idx == start_index {
1386 cursor_frames
1387 } else {
1388 &[]
1389 };
1390 match execute_step(executor, step, ctx, idx, step_cursor, &scope).await? {
1391 StepOutcome::Continue => {
1392 if scope.checkpoint_steps {
1393 checkpoint_completed_step(
1394 executor,
1395 ctx,
1396 scope.state,
1397 &scope.path_prefix,
1398 &scope.list_kind,
1399 idx + 1,
1400 scope.checkpoint_timeout,
1401 )
1402 .await?;
1403 }
1404 if ctx.current_state != scope.state {
1405 return Ok(None);
1406 }
1407 }
1408 StepOutcome::Paused(pause) => return Ok(Some(pause)),
1409 }
1410 }
1411
1412 Ok(None)
1413 })
1414}
1415
1416fn selected_branch_steps<'a>(
1417 branches: &'a [(String, Vec<WorkflowStep>)],
1418 default: &'a [WorkflowStep],
1419 selection: &WorkflowBranchCursorSelection,
1420) -> Result<&'a [WorkflowStep], WorkflowError> {
1421 match selection {
1422 WorkflowBranchCursorSelection::Branch(idx) => branches
1423 .get(*idx)
1424 .map(|(_, steps)| steps.as_slice())
1425 .ok_or_else(|| invalid_cursor(format!("branch index {idx} no longer exists"))),
1426 WorkflowBranchCursorSelection::Default => Ok(default),
1427 }
1428}
1429
1430fn selected_branch_when_steps<'a>(
1431 branches: &'a [(WorkflowBranchCondition, Vec<WorkflowStep>)],
1432 default: &'a [WorkflowStep],
1433 selection: &WorkflowBranchCursorSelection,
1434) -> Result<&'a [WorkflowStep], WorkflowError> {
1435 match selection {
1436 WorkflowBranchCursorSelection::Branch(idx) => branches
1437 .get(*idx)
1438 .map(|(_, steps)| steps.as_slice())
1439 .ok_or_else(|| invalid_cursor(format!("branch index {idx} no longer exists"))),
1440 WorkflowBranchCursorSelection::Default => Ok(default),
1441 }
1442}
1443
1444fn validate_branch_resume_selection(
1445 branches: &[(String, Vec<WorkflowStep>)],
1446 selection: &WorkflowBranchCursorSelection,
1447 condition_key: &str,
1448 condition_value: &str,
1449) -> Result<(), WorkflowError> {
1450 match selection {
1451 WorkflowBranchCursorSelection::Branch(idx) => {
1452 let Some((branch_value, _)) = branches.get(*idx) else {
1453 return Err(invalid_cursor(format!(
1454 "branch index {idx} no longer exists"
1455 )));
1456 };
1457 if branch_value != condition_value {
1458 return Err(invalid_cursor(format!(
1459 "branch cursor selected index {idx} for value '{branch_value}', \
1460 but current condition '{condition_key}' is '{condition_value}'"
1461 )));
1462 }
1463 }
1464 WorkflowBranchCursorSelection::Default => {
1465 if branches
1466 .iter()
1467 .any(|(branch_value, _)| branch_value == condition_value)
1468 {
1469 return Err(invalid_cursor(format!(
1470 "default branch cursor no longer matches condition '{condition_key}' \
1471 value '{condition_value}'"
1472 )));
1473 }
1474 }
1475 }
1476 Ok(())
1477}
1478
1479fn validate_branch_when_resume_selection(
1480 branches: &[(WorkflowBranchCondition, Vec<WorkflowStep>)],
1481 selection: &WorkflowBranchCursorSelection,
1482 condition_key: &str,
1483 condition_value: Option<&Value>,
1484) -> Result<(), WorkflowError> {
1485 match selection {
1486 WorkflowBranchCursorSelection::Branch(idx) => {
1487 let Some((condition, _)) = branches.get(*idx) else {
1488 return Err(invalid_cursor(format!(
1489 "branch index {idx} no longer exists"
1490 )));
1491 };
1492 if !branch_condition_matches(condition, condition_value) {
1493 return Err(invalid_cursor(format!(
1494 "branch cursor selected index {idx} for condition '{condition:?}', \
1495 but current condition '{condition_key}' no longer matches"
1496 )));
1497 }
1498 }
1499 WorkflowBranchCursorSelection::Default => {
1500 if branches
1501 .iter()
1502 .any(|(condition, _)| branch_condition_matches(condition, condition_value))
1503 {
1504 return Err(invalid_cursor(format!(
1505 "default branch cursor no longer matches condition '{condition_key}'"
1506 )));
1507 }
1508 }
1509 }
1510 Ok(())
1511}
1512
1513fn execute_step<'a, E: WorkflowExecutor>(
1515 executor: &'a E,
1516 step: &'a WorkflowStep,
1517 ctx: &'a mut WorkflowContext,
1518 step_index: usize,
1519 cursor_frames: &'a [WorkflowCursorFrame],
1520 scope: &'a StepExecutionScope<'a>,
1521) -> std::pin::Pin<
1522 Box<dyn std::future::Future<Output = Result<StepOutcome, WorkflowError>> + Send + 'a>,
1523> {
1524 Box::pin(async move {
1525 match step {
1526 WorkflowStep::Query { cmd_json, store_as } => {
1527 ensure_no_child_cursor("Query", cursor_frames)?;
1528 ensure_optional_user_context_key(store_as.as_ref(), "Query store_as")?;
1529 let cmd_wire = normalize_query_wire_for_execution(cmd_json)?;
1530 let operation =
1531 side_effect_operation(ctx, scope, step_index, WorkflowSideEffectKind::Query);
1532 let result = executor.execute_query_once(&operation, &cmd_wire).await?;
1533 if let Some(key) = store_as {
1534 ctx.set(key, result);
1535 }
1536 }
1537
1538 WorkflowStep::Notify {
1539 channel,
1540 template,
1541 recipient_key,
1542 payload_key,
1543 } => {
1544 ensure_no_child_cursor("Notify", cursor_frames)?;
1545 let recipient = ctx
1546 .get_str(recipient_key)
1547 .ok_or_else(|| WorkflowError::MissingContextKey(recipient_key.clone()))?
1548 .to_string();
1549
1550 let params = payload_key
1551 .as_ref()
1552 .map(|key| {
1553 ctx.get(key)
1554 .cloned()
1555 .ok_or_else(|| WorkflowError::MissingContextKey(key.clone()))
1556 })
1557 .transpose()?
1558 .unwrap_or(Value::Object(serde_json::Map::new()));
1559
1560 let operation =
1561 side_effect_operation(ctx, scope, step_index, WorkflowSideEffectKind::Notify);
1562 executor
1563 .send_notification_once(&operation, channel, &recipient, template, ¶ms)
1564 .await?;
1565 }
1566
1567 WorkflowStep::Wait {
1568 event,
1569 timeout,
1570 on_timeout,
1571 } => {
1572 ensure_no_child_cursor("Wait", cursor_frames)?;
1573 ensure_wait_event_name(event)?;
1574 return Ok(StepOutcome::Paused(StepPause {
1577 frames: cursor_frames_for_index(
1578 &scope.path_prefix,
1579 &scope.list_kind,
1580 step_index + 1,
1581 ),
1582 wait: WorkflowPendingWait {
1583 event: event.clone(),
1584 deadline_at: deadline_from_timeout(timeout)?,
1585 on_timeout: on_timeout.clone(),
1586 },
1587 }));
1588 }
1589
1590 WorkflowStep::Branch {
1591 condition_key,
1592 branches,
1593 default,
1594 } => {
1595 ensure_unique_branch_values(condition_key, branches)?;
1596 let (selection, selected_steps, start_index, nested_cursor) = match cursor_frames
1597 .first()
1598 {
1599 Some(WorkflowCursorFrame::Branch { selection, index }) => {
1600 let condition_value = ctx
1601 .get_str(condition_key)
1602 .ok_or_else(|| WorkflowError::MissingContextKey(condition_key.clone()))?
1603 .to_string();
1604 validate_branch_resume_selection(
1605 branches,
1606 selection,
1607 condition_key,
1608 &condition_value,
1609 )?;
1610 (
1611 selection.clone(),
1612 selected_branch_steps(branches, default, selection)?,
1613 *index,
1614 &cursor_frames[1..],
1615 )
1616 }
1617 Some(_) => {
1618 return Err(invalid_cursor(
1619 "expected Branch frame for nested branch resume",
1620 ));
1621 }
1622 None => {
1623 let value = ctx
1624 .get_str(condition_key)
1625 .ok_or_else(|| WorkflowError::MissingContextKey(condition_key.clone()))?
1626 .to_string();
1627 match branches.iter().enumerate().find(|(_, (k, _))| k == &value) {
1628 Some((idx, (_, steps))) => (
1629 WorkflowBranchCursorSelection::Branch(idx),
1630 steps.as_slice(),
1631 0,
1632 &[][..],
1633 ),
1634 None => (
1635 WorkflowBranchCursorSelection::Default,
1636 default.as_slice(),
1637 0,
1638 &[][..],
1639 ),
1640 }
1641 }
1642 };
1643
1644 if let Some(pause) = execute_steps(
1645 executor,
1646 selected_steps,
1647 ctx,
1648 start_index,
1649 nested_cursor,
1650 scope.child(step_index, StepListCursorKind::Branch { selection }),
1651 )
1652 .await?
1653 {
1654 return Ok(StepOutcome::Paused(pause));
1655 }
1656 }
1657
1658 WorkflowStep::BranchWhen {
1659 condition_key,
1660 branches,
1661 default,
1662 } => {
1663 ensure_unique_branch_conditions(condition_key, branches)?;
1664 let condition_value = ctx.get(condition_key).cloned();
1665 let (selection, selected_steps, start_index, nested_cursor) =
1666 match cursor_frames.first() {
1667 Some(WorkflowCursorFrame::Branch { selection, index }) => {
1668 validate_branch_when_resume_selection(
1669 branches,
1670 selection,
1671 condition_key,
1672 condition_value.as_ref(),
1673 )?;
1674 (
1675 selection.clone(),
1676 selected_branch_when_steps(branches, default, selection)?,
1677 *index,
1678 &cursor_frames[1..],
1679 )
1680 }
1681 Some(_) => {
1682 return Err(invalid_cursor(
1683 "expected Branch frame for nested branch resume",
1684 ));
1685 }
1686 None => match branches.iter().enumerate().find(|(_, (condition, _))| {
1687 branch_condition_matches(condition, condition_value.as_ref())
1688 }) {
1689 Some((idx, (_, steps))) => (
1690 WorkflowBranchCursorSelection::Branch(idx),
1691 steps.as_slice(),
1692 0,
1693 &[][..],
1694 ),
1695 None => (
1696 WorkflowBranchCursorSelection::Default,
1697 default.as_slice(),
1698 0,
1699 &[][..],
1700 ),
1701 },
1702 };
1703
1704 if let Some(pause) = execute_steps(
1705 executor,
1706 selected_steps,
1707 ctx,
1708 start_index,
1709 nested_cursor,
1710 scope.child(step_index, StepListCursorKind::Branch { selection }),
1711 )
1712 .await?
1713 {
1714 return Ok(StepOutcome::Paused(pause));
1715 }
1716 }
1717
1718 WorkflowStep::ForEach { list_key, steps } => {
1719 let list = ctx
1720 .get(list_key)
1721 .cloned()
1722 .ok_or_else(|| WorkflowError::MissingContextKey(list_key.clone()))?;
1723
1724 let Value::Array(items) = list else {
1725 return Err(WorkflowError::Other(format!(
1726 "Expected JSON array for ForEach list '{}', got {}",
1727 list_key,
1728 json_type_name(&list)
1729 )));
1730 };
1731
1732 let (start_item_index, start_step_index, nested_cursor) = match cursor_frames
1733 .first()
1734 {
1735 Some(WorkflowCursorFrame::ForEachItem {
1736 item_index,
1737 index,
1738 item,
1739 }) => {
1740 if *item_index >= items.len() {
1741 return Err(invalid_cursor(format!(
1742 "for_each item index {item_index} is past item count {}",
1743 items.len()
1744 )));
1745 }
1746 if items.get(*item_index) != Some(item) {
1747 return Err(invalid_cursor(format!(
1748 "for_each cursor item at index {item_index} changed while the workflow was paused"
1749 )));
1750 }
1751 (*item_index, *index, &cursor_frames[1..])
1752 }
1753 Some(WorkflowCursorFrame::ForEach { .. }) => {
1754 return Err(invalid_cursor(
1755 "legacy for_each cursor without item snapshot cannot be resumed safely",
1756 ));
1757 }
1758 Some(_) => {
1759 return Err(invalid_cursor(
1760 "expected ForEach frame for nested loop resume",
1761 ));
1762 }
1763 None => (0, 0, &[][..]),
1764 };
1765
1766 for (item_index, item) in items.into_iter().enumerate().skip(start_item_index) {
1767 let item_snapshot = item.clone();
1768 let previous_item = ctx.data.insert(FOR_EACH_ITEM_KEY.to_string(), item);
1769 ctx.updated_at = Utc::now();
1770 let item_step_start = if item_index == start_item_index {
1771 start_step_index
1772 } else {
1773 0
1774 };
1775 let item_cursor = if item_index == start_item_index {
1776 nested_cursor
1777 } else {
1778 &[]
1779 };
1780
1781 match execute_steps(
1782 executor,
1783 steps,
1784 ctx,
1785 item_step_start,
1786 item_cursor,
1787 scope.child(
1788 step_index,
1789 StepListCursorKind::ForEach {
1790 item_index,
1791 item: item_snapshot,
1792 },
1793 ),
1794 )
1795 .await
1796 {
1797 Ok(Some(pause)) => {
1798 restore_for_each_item(ctx, previous_item);
1799 return Ok(StepOutcome::Paused(pause));
1800 }
1801 Ok(None) => {
1802 restore_for_each_item(ctx, previous_item);
1803 }
1804 Err(err) => {
1805 restore_for_each_item(ctx, previous_item);
1806 return Err(err);
1807 }
1808 }
1809 }
1810 }
1811
1812 WorkflowStep::Transition { to } => {
1813 ensure_no_child_cursor("Transition", cursor_frames)?;
1814 ctx.transition_to(to, None);
1815 }
1816
1817 WorkflowStep::Log { message } => {
1818 ensure_no_child_cursor("Log", cursor_frames)?;
1819 let resolved = resolve_log_message(ctx, message);
1820 eprintln!("[workflow:{}] {}", ctx.workflow_id, resolved);
1822 }
1823
1824 WorkflowStep::Charge {
1825 provider,
1826 amount_key,
1827 reference_key,
1828 description_key,
1829 payment_method_key,
1830 order_origin_key,
1831 store_as,
1832 } => {
1833 ensure_no_child_cursor("Charge", cursor_frames)?;
1834 ensure_optional_user_context_key(store_as.as_ref(), "Charge store_as")?;
1835 let amount = resolve_charge_amount(ctx, amount_key)?;
1836
1837 let reference_id = ctx
1838 .get_str(reference_key)
1839 .ok_or_else(|| WorkflowError::MissingContextKey(reference_key.clone()))?
1840 .to_string();
1841
1842 let description = resolve_optional_charge_string(ctx, description_key.as_ref())?;
1843 let payment_method =
1844 resolve_optional_charge_string(ctx, payment_method_key.as_ref())?;
1845 let order_origin = resolve_optional_order_origin(ctx, order_origin_key.as_ref())?;
1846
1847 let request = ChargeRequest {
1848 amount,
1849 currency: Currency::default(),
1850 reference_id,
1851 description,
1852 payment_method,
1853 order_origin,
1854 idempotency_key: None,
1855 return_url: None,
1856 metadata: None,
1857 };
1858
1859 let operation =
1860 side_effect_operation(ctx, scope, step_index, WorkflowSideEffectKind::Charge);
1861 let response = executor
1862 .create_charge_once(&operation, provider, request.clone())
1863 .await?;
1864
1865 if let Some(key) = store_as {
1866 let response_json = serde_json::to_value(response.display_for(&request))
1867 .map_err(|e| WorkflowError::Other(e.to_string()))?;
1868 ctx.set(key, response_json);
1869 }
1870 }
1871 }
1872
1873 Ok(StepOutcome::Continue)
1874 })
1875}
1876
1877fn resolve_charge_amount(ctx: &WorkflowContext, amount_key: &str) -> Result<i64, WorkflowError> {
1878 let value = ctx.get(amount_key).ok_or_else(|| {
1879 WorkflowError::MissingContextKey(format!("{amount_key} (expected numeric amount)"))
1880 })?;
1881
1882 let amount = match value {
1883 Value::Number(number) => charge_amount_from_number(number, amount_key)?,
1884 _ => {
1885 return Err(WorkflowError::MissingContextKey(format!(
1886 "{amount_key} (expected numeric amount)"
1887 )));
1888 }
1889 };
1890
1891 if amount <= 0 {
1892 return Err(invalid_charge_amount(
1893 amount_key,
1894 "amount must be greater than zero",
1895 ));
1896 }
1897
1898 Ok(amount)
1899}
1900
1901fn resolve_optional_charge_string(
1902 ctx: &WorkflowContext,
1903 key: Option<&String>,
1904) -> Result<Option<String>, WorkflowError> {
1905 let Some(key) = key else {
1906 return Ok(None);
1907 };
1908 ctx.get_str(key)
1909 .map(|value| Some(value.to_string()))
1910 .ok_or_else(|| WorkflowError::MissingContextKey(format!("{key} (expected string)")))
1911}
1912
1913fn resolve_optional_order_origin(
1914 ctx: &WorkflowContext,
1915 key: Option<&String>,
1916) -> Result<Option<OrderOrigin>, WorkflowError> {
1917 let Some(key) = key else {
1918 return Ok(None);
1919 };
1920 let value = ctx.get_str(key).ok_or_else(|| {
1921 WorkflowError::MissingContextKey(format!("{key} (expected order origin)"))
1922 })?;
1923
1924 OrderOrigin::parse(value).map(Some).ok_or_else(|| {
1925 WorkflowError::Other(format!(
1926 "Charge order_origin_key '{key}' must resolve to one of: whatsapp, mcp, web, ios_app, android_app, api"
1927 ))
1928 })
1929}
1930
1931fn charge_amount_from_number(
1932 number: &serde_json::Number,
1933 amount_key: &str,
1934) -> Result<i64, WorkflowError> {
1935 if let Some(value) = number.as_i64() {
1936 return Ok(value);
1937 }
1938
1939 if let Some(value) = number.as_u64() {
1940 return i64::try_from(value).map_err(|_| {
1941 invalid_charge_amount(amount_key, "integer amount must fit in signed 64-bit range")
1942 });
1943 }
1944
1945 if let Some(value) = number.as_f64() {
1946 const MAX_SAFE_JSON_INTEGER: f64 = 9_007_199_254_740_991.0;
1947
1948 if !value.is_finite() {
1949 return Err(invalid_charge_amount(
1950 amount_key,
1951 "float amount must be finite",
1952 ));
1953 }
1954
1955 if value.fract() != 0.0 {
1956 return Err(invalid_charge_amount(
1957 amount_key,
1958 "float amount must be an integer in the smallest currency unit",
1959 ));
1960 }
1961
1962 if !(-MAX_SAFE_JSON_INTEGER..=MAX_SAFE_JSON_INTEGER).contains(&value) {
1963 return Err(invalid_charge_amount(
1964 amount_key,
1965 "float amount exceeds JSON safe integer range",
1966 ));
1967 }
1968
1969 return Ok(value as i64);
1970 }
1971
1972 Err(invalid_charge_amount(
1973 amount_key,
1974 "amount number is not representable",
1975 ))
1976}
1977
1978fn invalid_charge_amount(amount_key: &str, message: &str) -> WorkflowError {
1979 WorkflowError::Other(format!(
1980 "Invalid charge amount at '{amount_key}': {message}"
1981 ))
1982}
1983
1984fn normalize_query_wire_for_execution(cmd_json: &str) -> Result<String, WorkflowError> {
1985 if !is_current_workflow_query_wire(cmd_json) {
1986 return Err(WorkflowError::QueryFailed(
1987 "Legacy workflow query payload detected: cmd_json must use QAIL wire text \
1988 (QAIL-CMD/1 or QAIL-CMD/2). Migrate persisted workflow rows to wire text or purge/restart pending workflows."
1989 .to_string(),
1990 ));
1991 }
1992
1993 let cmd = qail_core::wire::decode_cmd_text(cmd_json).map_err(|e| {
1994 WorkflowError::QueryFailed(format!(
1995 "Invalid workflow query wire payload (expected QAIL-CMD/1 or QAIL-CMD/2): {}",
1996 e
1997 ))
1998 })?;
1999
2000 Ok(qail_core::wire::encode_cmd_text(&cmd))
2002}
2003
2004pub async fn run_workflow<E: WorkflowExecutor>(
2025 executor: &E,
2026 definition: &WorkflowDefinition,
2027 ctx: &mut WorkflowContext,
2028) -> Result<String, WorkflowError> {
2029 run_workflow_with_options(executor, definition, ctx, WorkflowRunOptions::default()).await
2030}
2031
2032pub async fn run_workflow_with_options<E: WorkflowExecutor>(
2034 executor: &E,
2035 definition: &WorkflowDefinition,
2036 ctx: &mut WorkflowContext,
2037 options: WorkflowRunOptions,
2038) -> Result<String, WorkflowError> {
2039 validate_workflow_definition(definition)?;
2040 validate_workflow_context_identity(ctx)?;
2041 let guard = match RuntimeGuard::enter(
2042 executor,
2043 &definition.name,
2044 &ctx.workflow_id,
2045 WorkflowOperationKind::Run,
2046 options,
2047 )
2048 .await?
2049 {
2050 RuntimeEntry::Active(guard) => guard,
2051 RuntimeEntry::Completed(state) => return Ok(state),
2052 };
2053
2054 let result = run_workflow_inner(executor, definition, ctx, RunMode::Normal).await;
2055 finish_runtime_operation(executor, guard, result).await
2056}
2057
2058async fn run_workflow_inner<'a, E: WorkflowExecutor>(
2059 executor: &E,
2060 definition: &WorkflowDefinition,
2061 ctx: &mut WorkflowContext,
2062 mode: RunMode<'a>,
2063) -> Result<String, WorkflowError> {
2064 validate_workflow_definition(definition)?;
2065 validate_workflow_context_identity(ctx)?;
2066 ensure_context_definition(ctx, definition)?;
2067 let run_start_transition_count = ctx.transition_count();
2068 let mut pending_cursor_frames = match ctx.cursor.clone() {
2069 Some(cursor) if cursor.state == ctx.current_state => {
2070 ensure_cursor_has_frames(&cursor)?;
2071 if timeout_fallback_from_context(ctx)?.is_some() {
2072 return Err(WorkflowError::Other(
2073 "Workflow is executing a timeout fallback; resume with timeout_workflow"
2074 .to_string(),
2075 ));
2076 }
2077 match (&cursor.wait, mode) {
2078 (Some(wait), RunMode::Normal) => {
2079 return Err(WorkflowError::Other(format!(
2080 "Workflow is paused awaiting event '{}'; resume with a matching event",
2081 wait.event
2082 )));
2083 }
2084 (Some(wait), RunMode::EventResume { event }) => {
2085 if wait.event != event {
2086 return Err(WorkflowError::Other(format!(
2087 "Workflow is waiting for event '{}', received '{}'",
2088 wait.event, event
2089 )));
2090 }
2091 if Utc::now() > wait.deadline_at {
2092 return Err(WorkflowError::Timeout {
2093 event: wait.event.clone(),
2094 });
2095 }
2096 ctx.clear_cursor();
2097 Some(cursor.frames)
2098 }
2099 (None, RunMode::EventResume { .. }) => {
2100 return Err(WorkflowError::Other(
2101 "Workflow is not waiting for an external event".to_string(),
2102 ));
2103 }
2104 (None, RunMode::Normal) => {
2105 ctx.clear_cursor();
2106 Some(cursor.frames)
2107 }
2108 }
2109 }
2110 Some(_) => match mode {
2111 RunMode::Normal => {
2112 ctx.clear_cursor();
2113 None
2114 }
2115 RunMode::EventResume { .. } => {
2116 return Err(invalid_cursor(
2117 "resume cursor state does not match current workflow state",
2118 ));
2119 }
2120 },
2121 None => match mode {
2122 RunMode::Normal => None,
2123 RunMode::EventResume { .. } => {
2124 return Err(WorkflowError::Other(
2125 "Workflow is not waiting for an external event".to_string(),
2126 ));
2127 }
2128 },
2129 };
2130
2131 loop {
2132 let transition = find_single_transition(definition, &ctx.current_state)?;
2134
2135 let transition = match transition {
2136 Some(t) => t,
2137 None => {
2138 executor.save_state(ctx).await?;
2140 return Ok(ctx.current_state.clone());
2141 }
2142 };
2143
2144 if ctx
2146 .transition_count()
2147 .saturating_sub(run_start_transition_count)
2148 >= 50
2149 {
2150 return Err(WorkflowError::Other(
2151 "Maximum transition count exceeded (50)".into(),
2152 ));
2153 }
2154
2155 let cursor_frames = pending_cursor_frames.take().unwrap_or_default();
2156 let (start_index, nested_cursor) = match cursor_frames.first() {
2157 Some(WorkflowCursorFrame::Steps { index }) => (*index, &cursor_frames[1..]),
2158 Some(_) => {
2159 return Err(invalid_cursor(
2160 "top-level transition resume must start with a Steps frame",
2161 ));
2162 }
2163 None => (0, &[][..]),
2164 };
2165
2166 if let Some(pause) = execute_steps(
2167 executor,
2168 &transition.steps,
2169 ctx,
2170 start_index,
2171 nested_cursor,
2172 StepExecutionScope::new(&transition.from, StepListCursorKind::Steps, true),
2173 )
2174 .await?
2175 {
2176 clear_timeout_fallback(ctx);
2177 ctx.set_cursor(WorkflowCursor {
2178 state: transition.from.clone(),
2179 frames: pause.frames,
2180 wait: Some(pause.wait),
2181 });
2182 executor.save_state(ctx).await?;
2183 return Ok(ctx.current_state.clone());
2184 }
2185
2186 if ctx.current_state == transition.from {
2190 ctx.transition_to(&transition.to, None);
2192 }
2193
2194 executor.save_state(ctx).await?;
2196 }
2197}
2198
2199pub async fn resume_workflow<E: WorkflowExecutor>(
2204 executor: &E,
2205 definition: &WorkflowDefinition,
2206 workflow_id: &str,
2207 event_data: Value,
2208) -> Result<String, WorkflowError> {
2209 resume_workflow_with_options(
2210 executor,
2211 definition,
2212 workflow_id,
2213 event_data,
2214 WorkflowRunOptions::default(),
2215 )
2216 .await
2217}
2218
2219pub async fn resume_workflow_with_options<E: WorkflowExecutor>(
2221 executor: &E,
2222 definition: &WorkflowDefinition,
2223 workflow_id: &str,
2224 event_data: Value,
2225 options: WorkflowRunOptions,
2226) -> Result<String, WorkflowError> {
2227 let event_name = extract_resume_event_name(&event_data)?;
2228 resume_workflow_with_event_and_options(
2229 executor,
2230 definition,
2231 workflow_id,
2232 &event_name,
2233 event_data,
2234 options,
2235 )
2236 .await
2237}
2238
2239pub async fn resume_workflow_with_event<E: WorkflowExecutor>(
2241 executor: &E,
2242 definition: &WorkflowDefinition,
2243 workflow_id: &str,
2244 event_name: &str,
2245 event_data: Value,
2246) -> Result<String, WorkflowError> {
2247 resume_workflow_with_event_and_options(
2248 executor,
2249 definition,
2250 workflow_id,
2251 event_name,
2252 event_data,
2253 WorkflowRunOptions::default(),
2254 )
2255 .await
2256}
2257
2258pub async fn resume_workflow_with_event_and_options<E: WorkflowExecutor>(
2260 executor: &E,
2261 definition: &WorkflowDefinition,
2262 workflow_id: &str,
2263 event_name: &str,
2264 event_data: Value,
2265 options: WorkflowRunOptions,
2266) -> Result<String, WorkflowError> {
2267 validate_workflow_definition(definition)?;
2268 ensure_definition_text(workflow_id, "Workflow id")?;
2269 ensure_wait_event_name(event_name)?;
2270
2271 let guard = match RuntimeGuard::enter(
2272 executor,
2273 &definition.name,
2274 workflow_id,
2275 WorkflowOperationKind::Resume {
2276 event: event_name.to_string(),
2277 },
2278 options,
2279 )
2280 .await?
2281 {
2282 RuntimeEntry::Active(guard) => guard,
2283 RuntimeEntry::Completed(state) => return Ok(state),
2284 };
2285
2286 let mut ctx = executor
2288 .load_state(workflow_id)
2289 .await?
2290 .ok_or_else(|| WorkflowError::Other(format!("Workflow not found: {}", workflow_id)))?;
2291
2292 ctx.set(RESUME_EVENT_KEY, event_data);
2294
2295 let result = run_workflow_inner(
2297 executor,
2298 definition,
2299 &mut ctx,
2300 RunMode::EventResume { event: event_name },
2301 )
2302 .await;
2303 finish_runtime_operation(executor, guard, result).await
2304}
2305
2306pub async fn timeout_workflow<E: WorkflowExecutor>(
2308 executor: &E,
2309 definition: &WorkflowDefinition,
2310 workflow_id: &str,
2311) -> Result<String, WorkflowError> {
2312 timeout_workflow_with_options(
2313 executor,
2314 definition,
2315 workflow_id,
2316 WorkflowRunOptions::default(),
2317 )
2318 .await
2319}
2320
2321pub async fn timeout_workflow_with_options<E: WorkflowExecutor>(
2323 executor: &E,
2324 definition: &WorkflowDefinition,
2325 workflow_id: &str,
2326 options: WorkflowRunOptions,
2327) -> Result<String, WorkflowError> {
2328 timeout_workflow_with_options_at(executor, definition, workflow_id, options, Utc::now()).await
2329}
2330
2331async fn timeout_workflow_with_options_at<E: WorkflowExecutor>(
2332 executor: &E,
2333 definition: &WorkflowDefinition,
2334 workflow_id: &str,
2335 options: WorkflowRunOptions,
2336 now: DateTime<Utc>,
2337) -> Result<String, WorkflowError> {
2338 validate_workflow_definition(definition)?;
2339 ensure_definition_text(workflow_id, "Workflow id")?;
2340 let guard = match RuntimeGuard::enter(
2341 executor,
2342 &definition.name,
2343 workflow_id,
2344 WorkflowOperationKind::Timeout,
2345 options,
2346 )
2347 .await?
2348 {
2349 RuntimeEntry::Active(guard) => guard,
2350 RuntimeEntry::Completed(state) => return Ok(state),
2351 };
2352
2353 let result = timeout_workflow_inner(executor, definition, workflow_id, now).await;
2354 finish_runtime_operation(executor, guard, result).await
2355}
2356
2357pub async fn timeout_due_workflows<E: WorkflowExecutor>(
2363 executor: &E,
2364 definition: &WorkflowDefinition,
2365 now: DateTime<Utc>,
2366 limit: usize,
2367 options: WorkflowRunOptions,
2368) -> Result<Vec<WorkflowTimeoutOutcome>, WorkflowError> {
2369 if limit == 0 {
2370 return Ok(Vec::new());
2371 }
2372 validate_workflow_definition(definition)?;
2373
2374 let workflow_ids = executor
2375 .load_due_workflow_timeouts(&definition.name, now, limit)
2376 .await?;
2377 let mut outcomes = Vec::with_capacity(workflow_ids.len().min(limit));
2378
2379 for workflow_id in workflow_ids.into_iter().take(limit) {
2380 let run_options = timeout_options_for_workflow(&options, definition, &workflow_id, now);
2381 let outcome = match timeout_workflow_with_options_at(
2382 executor,
2383 definition,
2384 &workflow_id,
2385 run_options,
2386 now,
2387 )
2388 .await
2389 {
2390 Ok(state) => WorkflowTimeoutOutcome {
2391 workflow_id,
2392 state: Some(state),
2393 error: None,
2394 },
2395 Err(err) => WorkflowTimeoutOutcome {
2396 workflow_id,
2397 state: None,
2398 error: Some(err.to_string()),
2399 },
2400 };
2401 outcomes.push(outcome);
2402 }
2403
2404 Ok(outcomes)
2405}
2406
2407fn timeout_options_for_workflow(
2408 options: &WorkflowRunOptions,
2409 definition: &WorkflowDefinition,
2410 workflow_id: &str,
2411 now: DateTime<Utc>,
2412) -> WorkflowRunOptions {
2413 let mut run_options = options.clone();
2414 run_options.idempotency_key = match &options.idempotency_key {
2415 Some(prefix) if prefix.trim().is_empty() => Some(prefix.clone()),
2416 Some(prefix) => Some(timeout_operation_idempotency_key(prefix, workflow_id, now)),
2417 None => Some(timeout_operation_idempotency_key(
2418 &definition.name,
2419 workflow_id,
2420 now,
2421 )),
2422 };
2423 run_options
2424}
2425
2426fn timeout_operation_idempotency_key(scope: &str, workflow_id: &str, now: DateTime<Utc>) -> String {
2427 serde_json::json!([
2428 "qail-workflow-timeout",
2429 2,
2430 scope,
2431 workflow_id,
2432 now.to_rfc3339_opts(SecondsFormat::Micros, true)
2433 ])
2434 .to_string()
2435}
2436
2437async fn timeout_workflow_inner<E: WorkflowExecutor>(
2438 executor: &E,
2439 definition: &WorkflowDefinition,
2440 workflow_id: &str,
2441 now: DateTime<Utc>,
2442) -> Result<String, WorkflowError> {
2443 let mut ctx = executor
2444 .load_state(workflow_id)
2445 .await?
2446 .ok_or_else(|| WorkflowError::Other(format!("Workflow not found: {}", workflow_id)))?;
2447 validate_workflow_context_identity(&ctx)?;
2448 ensure_context_definition(&mut ctx, definition)?;
2449
2450 let cursor = ctx
2451 .cursor
2452 .clone()
2453 .ok_or_else(|| WorkflowError::Other("Workflow is not paused at a Wait step".to_string()))?;
2454 ensure_cursor_has_frames(&cursor)?;
2455 if cursor.state != ctx.current_state {
2456 return Err(invalid_cursor(
2457 "timeout cursor state does not match current workflow state",
2458 ));
2459 }
2460 let timeout_fallback = timeout_fallback_from_context(&ctx)?;
2461 if let (Some(cursor_wait), Some(timeout)) = (&cursor.wait, &timeout_fallback)
2462 && cursor_wait != timeout
2463 {
2464 return Err(invalid_cursor(
2465 "timeout cursor wait metadata does not match timeout fallback metadata",
2466 ));
2467 }
2468
2469 let (wait, cursor_frames) = if let Some(timeout) = timeout_fallback {
2470 (timeout, cursor.frames.clone())
2471 } else {
2472 let wait = cursor.wait.clone().ok_or_else(|| {
2473 WorkflowError::Other("Workflow is not waiting for a timeout".to_string())
2474 })?;
2475 (wait, Vec::new())
2476 };
2477
2478 ensure_wait_event_name(&wait.event)?;
2479 if now < wait.deadline_at {
2480 return Err(WorkflowError::Other(format!(
2481 "Workflow wait for event '{}' has not timed out",
2482 wait.event
2483 )));
2484 }
2485 if wait.on_timeout.is_empty() {
2486 return Err(WorkflowError::Timeout { event: wait.event });
2487 }
2488 if steps_contain_wait(&wait.on_timeout) {
2489 return Err(WorkflowError::Other(
2490 "Wait steps inside on_timeout fallback are not supported".to_string(),
2491 ));
2492 }
2493 validate_workflow_steps(&wait.on_timeout)?;
2494
2495 set_timeout_fallback(&mut ctx, &wait)?;
2496 ctx.clear_cursor();
2497 ctx.set(
2498 RESUME_EVENT_KEY,
2499 serde_json::json!({
2500 "event": wait.event.clone(),
2501 "timeout": true,
2502 }),
2503 );
2504
2505 let (start_index, nested_cursor) = match cursor_frames.first() {
2506 Some(WorkflowCursorFrame::Steps { index }) => (*index, &cursor_frames[1..]),
2507 Some(_) => {
2508 return Err(invalid_cursor(
2509 "timeout fallback resume must start with a Steps frame",
2510 ));
2511 }
2512 None => (0, &[][..]),
2513 };
2514
2515 if execute_steps(
2516 executor,
2517 &wait.on_timeout,
2518 &mut ctx,
2519 start_index,
2520 nested_cursor,
2521 StepExecutionScope::new(&cursor.state, StepListCursorKind::Steps, true)
2522 .with_timeout_checkpoint(),
2523 )
2524 .await?
2525 .is_some()
2526 {
2527 return Err(WorkflowError::Other(
2528 "on_timeout fallback paused unexpectedly".to_string(),
2529 ));
2530 }
2531
2532 clear_timeout_fallback(&mut ctx);
2533 ctx.clear_cursor();
2534 if ctx.current_state == cursor.state {
2535 executor.save_state(&ctx).await?;
2536 return Ok(ctx.current_state.clone());
2537 }
2538
2539 run_workflow_inner(executor, definition, &mut ctx, RunMode::Normal).await
2540}
2541
2542fn extract_resume_event_name(event_data: &Value) -> Result<String, WorkflowError> {
2543 let event = ["event", "event_name", "type"]
2544 .iter()
2545 .find_map(|key| event_data.get(*key).and_then(Value::as_str))
2546 .ok_or_else(|| {
2547 WorkflowError::Other(
2548 "Resume event data must include a string 'event' field".to_string(),
2549 )
2550 })?;
2551 ensure_wait_event_name(event)?;
2552 Ok(event.to_string())
2553}
2554
2555fn steps_contain_wait(steps: &[WorkflowStep]) -> bool {
2556 steps.iter().any(|step| match step {
2557 WorkflowStep::Wait { .. } => true,
2558 WorkflowStep::Branch {
2559 branches, default, ..
2560 } => {
2561 branches
2562 .iter()
2563 .any(|(_, branch_steps)| steps_contain_wait(branch_steps))
2564 || steps_contain_wait(default)
2565 }
2566 WorkflowStep::BranchWhen {
2567 branches, default, ..
2568 } => {
2569 branches
2570 .iter()
2571 .any(|(_, branch_steps)| steps_contain_wait(branch_steps))
2572 || steps_contain_wait(default)
2573 }
2574 WorkflowStep::ForEach { steps, .. } => steps_contain_wait(steps),
2575 WorkflowStep::Query { .. }
2576 | WorkflowStep::Notify { .. }
2577 | WorkflowStep::Transition { .. }
2578 | WorkflowStep::Log { .. }
2579 | WorkflowStep::Charge { .. } => false,
2580 })
2581}
2582
2583#[cfg(test)]
2584mod tests {
2585 use super::*;
2586 use crate::context::StateChange;
2587 use crate::payment::{ChargeResponse, ChargeStatus, PaymentKind};
2588
2589 struct MockExecutor {
2590 queries: std::sync::Mutex<Vec<String>>,
2591 query_failures: std::sync::Mutex<std::collections::HashSet<String>>,
2592 notifications: std::sync::Mutex<Vec<(String, String)>>,
2593 charges: std::sync::Mutex<Vec<ChargeRequest>>,
2594 saved_state: std::sync::Mutex<Option<WorkflowContext>>,
2595 reject_leases: std::sync::Mutex<bool>,
2596 acquired_leases: std::sync::Mutex<Vec<WorkflowLease>>,
2597 released_leases: std::sync::Mutex<Vec<WorkflowLease>>,
2598 workflow_operations:
2599 std::sync::Mutex<std::collections::HashMap<String, WorkflowOperationStatus>>,
2600 workflow_operation_begin_failures: std::sync::Mutex<std::collections::HashSet<String>>,
2601 workflow_operation_complete_failures: std::sync::Mutex<std::collections::HashSet<String>>,
2602 workflow_operation_fail_failures: std::sync::Mutex<std::collections::HashSet<String>>,
2603 completed_workflow_operations: std::sync::Mutex<Vec<(String, String)>>,
2604 failed_workflow_operations: std::sync::Mutex<Vec<(String, String)>>,
2605 due_timeout_workflow_ids: std::sync::Mutex<Vec<String>>,
2606 skip_notify_side_effects: std::sync::Mutex<bool>,
2607 completed_side_effects: std::sync::Mutex<Vec<String>>,
2608 failed_side_effects: std::sync::Mutex<Vec<(String, String)>>,
2609 }
2610
2611 impl MockExecutor {
2612 fn new() -> Self {
2613 Self {
2614 queries: std::sync::Mutex::new(Vec::new()),
2615 query_failures: std::sync::Mutex::new(std::collections::HashSet::new()),
2616 notifications: std::sync::Mutex::new(Vec::new()),
2617 charges: std::sync::Mutex::new(Vec::new()),
2618 saved_state: std::sync::Mutex::new(None),
2619 reject_leases: std::sync::Mutex::new(false),
2620 acquired_leases: std::sync::Mutex::new(Vec::new()),
2621 released_leases: std::sync::Mutex::new(Vec::new()),
2622 workflow_operations: std::sync::Mutex::new(std::collections::HashMap::new()),
2623 workflow_operation_begin_failures: std::sync::Mutex::new(
2624 std::collections::HashSet::new(),
2625 ),
2626 workflow_operation_complete_failures: std::sync::Mutex::new(
2627 std::collections::HashSet::new(),
2628 ),
2629 workflow_operation_fail_failures: std::sync::Mutex::new(
2630 std::collections::HashSet::new(),
2631 ),
2632 completed_workflow_operations: std::sync::Mutex::new(Vec::new()),
2633 failed_workflow_operations: std::sync::Mutex::new(Vec::new()),
2634 due_timeout_workflow_ids: std::sync::Mutex::new(Vec::new()),
2635 skip_notify_side_effects: std::sync::Mutex::new(false),
2636 completed_side_effects: std::sync::Mutex::new(Vec::new()),
2637 failed_side_effects: std::sync::Mutex::new(Vec::new()),
2638 }
2639 }
2640
2641 fn fail_query(&self, cmd_wire: impl Into<String>) {
2642 self.query_failures.lock().unwrap().insert(cmd_wire.into());
2643 }
2644
2645 fn reject_workflow_leases(&self) {
2646 *self.reject_leases.lock().unwrap() = true;
2647 }
2648
2649 fn set_workflow_operation_status(
2650 &self,
2651 idempotency_key: impl Into<String>,
2652 status: WorkflowOperationStatus,
2653 ) {
2654 self.workflow_operations
2655 .lock()
2656 .unwrap()
2657 .insert(idempotency_key.into(), status);
2658 }
2659
2660 fn fail_workflow_operation_begin(&self, idempotency_key: impl Into<String>) {
2661 self.workflow_operation_begin_failures
2662 .lock()
2663 .unwrap()
2664 .insert(idempotency_key.into());
2665 }
2666
2667 fn fail_workflow_operation_complete(&self, idempotency_key: impl Into<String>) {
2668 self.workflow_operation_complete_failures
2669 .lock()
2670 .unwrap()
2671 .insert(idempotency_key.into());
2672 }
2673
2674 fn fail_workflow_operation_fail(&self, idempotency_key: impl Into<String>) {
2675 self.workflow_operation_fail_failures
2676 .lock()
2677 .unwrap()
2678 .insert(idempotency_key.into());
2679 }
2680
2681 fn skip_notify_side_effects(&self) {
2682 *self.skip_notify_side_effects.lock().unwrap() = true;
2683 }
2684
2685 fn set_due_timeout_workflow_ids(&self, workflow_ids: Vec<&str>) {
2686 *self.due_timeout_workflow_ids.lock().unwrap() =
2687 workflow_ids.into_iter().map(ToString::to_string).collect();
2688 }
2689 }
2690
2691 fn charge_only_workflow(name: &str) -> WorkflowDefinition {
2692 WorkflowDefinition::new(name)
2693 .initial_state("created")
2694 .transition(
2695 "created",
2696 "awaiting_payment",
2697 vec![WorkflowStep::charge(
2698 PaymentKind::Xendit,
2699 "order.total",
2700 "order.id",
2701 Some("charge"),
2702 )],
2703 )
2704 }
2705
2706 #[async_trait]
2707 impl WorkflowExecutor for MockExecutor {
2708 async fn execute_query(&self, cmd_json: &str) -> Result<Value, WorkflowError> {
2709 self.queries.lock().unwrap().push(cmd_json.to_string());
2710 if self.query_failures.lock().unwrap().contains(cmd_json) {
2711 return Err(WorkflowError::QueryFailed(
2712 "forced query failure".to_string(),
2713 ));
2714 }
2715 Ok(serde_json::json!([
2717 {"id": "op-1", "phone": "+628111"},
2718 {"id": "op-2", "phone": "+628222"},
2719 ]))
2720 }
2721
2722 async fn send_notification(
2723 &self,
2724 _channel: &ChannelKind,
2725 recipient: &str,
2726 template: &str,
2727 _params: &Value,
2728 ) -> Result<(), WorkflowError> {
2729 self.notifications
2730 .lock()
2731 .unwrap()
2732 .push((recipient.to_string(), template.to_string()));
2733 Ok(())
2734 }
2735
2736 async fn save_state(&self, ctx: &WorkflowContext) -> Result<(), WorkflowError> {
2737 *self.saved_state.lock().unwrap() = Some(ctx.clone());
2738 Ok(())
2739 }
2740
2741 async fn load_state(
2742 &self,
2743 workflow_id: &str,
2744 ) -> Result<Option<WorkflowContext>, WorkflowError> {
2745 Ok(self
2746 .saved_state
2747 .lock()
2748 .unwrap()
2749 .as_ref()
2750 .filter(|ctx| ctx.workflow_id == workflow_id)
2751 .cloned())
2752 }
2753
2754 async fn create_charge(
2755 &self,
2756 _provider: &PaymentKind,
2757 request: ChargeRequest,
2758 ) -> Result<ChargeResponse, WorkflowError> {
2759 self.charges.lock().unwrap().push(request.clone());
2760 Ok(ChargeResponse {
2761 charge_id: format!("mock-charge-{}", request.reference_id),
2762 status: ChargeStatus::Pending,
2763 redirect_url: None,
2764 qr_code: Some("00020101021226610014ID.CO.MOCK".into()),
2765 payment_code: None,
2766 expires_at: Some("2026-02-13T16:00:00Z".into()),
2767 raw: Some(serde_json::json!({"provider_secret": "hidden"})),
2768 })
2769 }
2770
2771 async fn acquire_workflow_lease(
2772 &self,
2773 lease: &WorkflowLease,
2774 ) -> Result<bool, WorkflowError> {
2775 if *self.reject_leases.lock().unwrap() {
2776 return Ok(false);
2777 }
2778 self.acquired_leases.lock().unwrap().push(lease.clone());
2779 Ok(true)
2780 }
2781
2782 async fn release_workflow_lease(&self, lease: &WorkflowLease) -> Result<(), WorkflowError> {
2783 self.released_leases.lock().unwrap().push(lease.clone());
2784 Ok(())
2785 }
2786
2787 async fn begin_workflow_operation(
2788 &self,
2789 operation: &WorkflowOperation,
2790 ) -> Result<WorkflowOperationStatus, WorkflowError> {
2791 if self
2792 .workflow_operation_begin_failures
2793 .lock()
2794 .unwrap()
2795 .contains(&operation.idempotency_key)
2796 {
2797 return Err(WorkflowError::Other(
2798 "forced operation begin failure".to_string(),
2799 ));
2800 }
2801 Ok(self
2802 .workflow_operations
2803 .lock()
2804 .unwrap()
2805 .get(&operation.idempotency_key)
2806 .cloned()
2807 .unwrap_or(WorkflowOperationStatus::Started))
2808 }
2809
2810 async fn complete_workflow_operation(
2811 &self,
2812 operation: &WorkflowOperation,
2813 state: &str,
2814 ) -> Result<(), WorkflowError> {
2815 if self
2816 .workflow_operation_complete_failures
2817 .lock()
2818 .unwrap()
2819 .contains(&operation.idempotency_key)
2820 {
2821 return Err(WorkflowError::Other(
2822 "forced operation complete failure".to_string(),
2823 ));
2824 }
2825 self.completed_workflow_operations
2826 .lock()
2827 .unwrap()
2828 .push((operation.idempotency_key.clone(), state.to_string()));
2829 Ok(())
2830 }
2831
2832 async fn fail_workflow_operation(
2833 &self,
2834 operation: &WorkflowOperation,
2835 error: &str,
2836 ) -> Result<(), WorkflowError> {
2837 if self
2838 .workflow_operation_fail_failures
2839 .lock()
2840 .unwrap()
2841 .contains(&operation.idempotency_key)
2842 {
2843 return Err(WorkflowError::Other(
2844 "forced operation fail failure".to_string(),
2845 ));
2846 }
2847 self.failed_workflow_operations
2848 .lock()
2849 .unwrap()
2850 .push((operation.idempotency_key.clone(), error.to_string()));
2851 Ok(())
2852 }
2853
2854 async fn load_due_workflow_timeouts(
2855 &self,
2856 _workflow_name: &str,
2857 _now: DateTime<Utc>,
2858 limit: usize,
2859 ) -> Result<Vec<String>, WorkflowError> {
2860 Ok(self
2861 .due_timeout_workflow_ids
2862 .lock()
2863 .unwrap()
2864 .iter()
2865 .take(limit)
2866 .cloned()
2867 .collect())
2868 }
2869
2870 async fn begin_workflow_side_effect(
2871 &self,
2872 operation: &WorkflowSideEffect,
2873 ) -> Result<WorkflowSideEffectStatus, WorkflowError> {
2874 if operation.kind == WorkflowSideEffectKind::Notify
2875 && *self.skip_notify_side_effects.lock().unwrap()
2876 {
2877 return Ok(WorkflowSideEffectStatus::AlreadyCompleted { result: None });
2878 }
2879 Ok(WorkflowSideEffectStatus::Execute)
2880 }
2881
2882 async fn complete_workflow_side_effect(
2883 &self,
2884 operation: &WorkflowSideEffect,
2885 _result: Option<&Value>,
2886 ) -> Result<(), WorkflowError> {
2887 self.completed_side_effects
2888 .lock()
2889 .unwrap()
2890 .push(operation.operation_id.clone());
2891 Ok(())
2892 }
2893
2894 async fn fail_workflow_side_effect(
2895 &self,
2896 operation: &WorkflowSideEffect,
2897 error: &str,
2898 ) -> Result<(), WorkflowError> {
2899 self.failed_side_effects
2900 .lock()
2901 .unwrap()
2902 .push((operation.operation_id.clone(), error.to_string()));
2903 Ok(())
2904 }
2905 }
2906
2907 #[tokio::test]
2908 async fn test_simple_workflow() {
2909 let executor = MockExecutor::new();
2910
2911 let wf = WorkflowDefinition::new("test")
2912 .initial_state("start")
2913 .transition(
2914 "start",
2915 "done",
2916 vec![
2917 WorkflowStep::log("Starting workflow"),
2918 WorkflowStep::transition("done"),
2919 ],
2920 );
2921
2922 let mut ctx = WorkflowContext::new("wf-001", "start");
2923 let result = run_workflow(&executor, &wf, &mut ctx).await.unwrap();
2924 assert_eq!(result, "done");
2925 assert_eq!(ctx.transition_count(), 1);
2926 }
2927
2928 #[tokio::test]
2929 async fn versioned_workflow_stamps_context_metadata() {
2930 let executor = MockExecutor::new();
2931
2932 let wf = WorkflowDefinition::new("versioned_booking")
2933 .version("v1")
2934 .initial_state("start")
2935 .transition("start", "done", vec![WorkflowStep::transition("done")]);
2936
2937 let mut ctx = WorkflowContext::new("wf-versioned", "start");
2938 let result = run_workflow(&executor, &wf, &mut ctx).await.unwrap();
2939
2940 assert_eq!(result, "done");
2941 assert_eq!(ctx.definition_name.as_deref(), Some("versioned_booking"));
2942 assert_eq!(ctx.definition_version.as_deref(), Some("v1"));
2943 }
2944
2945 #[tokio::test]
2946 async fn versioned_resume_rejects_definition_version_drift() {
2947 let executor = MockExecutor::new();
2948
2949 let wf_v1 = WorkflowDefinition::new("versioned_resume")
2950 .version("v1")
2951 .initial_state("active")
2952 .transition(
2953 "active",
2954 "done",
2955 vec![
2956 WorkflowStep::wait("operator.accepted", std::time::Duration::from_secs(3600)),
2957 WorkflowStep::notify(ChannelKind::Email, "accepted", "customer.email"),
2958 WorkflowStep::transition("done"),
2959 ],
2960 );
2961 let wf_v2 = WorkflowDefinition::new("versioned_resume")
2962 .version("v2")
2963 .initial_state("active")
2964 .transition(
2965 "active",
2966 "done",
2967 vec![
2968 WorkflowStep::wait("operator.accepted", std::time::Duration::from_secs(3600)),
2969 WorkflowStep::notify(ChannelKind::Email, "accepted", "customer.email"),
2970 WorkflowStep::transition("done"),
2971 ],
2972 );
2973
2974 let mut ctx = WorkflowContext::new("wf-version-drift", "active");
2975 ctx.set(
2976 "customer",
2977 serde_json::json!({"email": "guest@example.com"}),
2978 );
2979 let result = run_workflow(&executor, &wf_v1, &mut ctx).await.unwrap();
2980 assert_eq!(result, "active");
2981
2982 let err = resume_workflow(
2983 &executor,
2984 &wf_v2,
2985 "wf-version-drift",
2986 serde_json::json!({"event": "operator.accepted"}),
2987 )
2988 .await
2989 .expect_err("version drift must fail before replay");
2990
2991 match err {
2992 WorkflowError::Other(msg) => {
2993 assert!(msg.contains("definition version 'v1'"), "got: {msg}");
2994 assert!(msg.contains("not 'v2'"), "got: {msg}");
2995 }
2996 other => panic!("expected version drift error, got {other:?}"),
2997 }
2998 assert!(executor.notifications.lock().unwrap().is_empty());
2999 }
3000
3001 #[tokio::test]
3002 async fn run_with_options_acquires_lease_and_completes_operation() {
3003 let executor = MockExecutor::new();
3004
3005 let wf = WorkflowDefinition::new("runtime_guarded_run")
3006 .initial_state("start")
3007 .transition("start", "done", vec![WorkflowStep::transition("done")]);
3008
3009 let mut ctx = WorkflowContext::new("wf-runtime-run", "start");
3010 let options = WorkflowRunOptions::default()
3011 .with_lease("worker-a", std::time::Duration::from_secs(30))
3012 .with_idempotency_key("run-1");
3013
3014 let result = run_workflow_with_options(&executor, &wf, &mut ctx, options)
3015 .await
3016 .unwrap();
3017
3018 assert_eq!(result, "done");
3019 let acquired_leases = executor.acquired_leases.lock().unwrap();
3020 let released_leases = executor.released_leases.lock().unwrap();
3021 assert_eq!(acquired_leases.len(), 1);
3022 assert_eq!(released_leases.len(), 1);
3023 assert_eq!(released_leases[0].owner, acquired_leases[0].owner);
3024 assert_ne!(
3025 acquired_leases[0].owner, "worker-a",
3026 "runtime lease owner must include a fencing token"
3027 );
3028 let lease_owner: serde_json::Value =
3029 serde_json::from_str(&acquired_leases[0].owner).unwrap();
3030 let lease_owner = lease_owner.as_array().unwrap();
3031 assert_eq!(lease_owner[0], "qail-workflow-lease");
3032 assert_eq!(lease_owner[2], "worker-a");
3033 assert_eq!(lease_owner[3], "runtime_guarded_run");
3034 assert_eq!(lease_owner[4], "wf-runtime-run");
3035 assert_eq!(lease_owner[6], "run-1");
3036 drop(acquired_leases);
3037 drop(released_leases);
3038 assert_eq!(
3039 executor
3040 .completed_workflow_operations
3041 .lock()
3042 .unwrap()
3043 .as_slice(),
3044 &[("run-1".to_string(), "done".to_string())]
3045 );
3046 }
3047
3048 #[test]
3049 fn runtime_lease_owner_changes_per_acquire_time() {
3050 let first_at = DateTime::parse_from_rfc3339("2026-06-17T12:00:00Z")
3051 .unwrap()
3052 .with_timezone(&Utc);
3053 let second_at = DateTime::parse_from_rfc3339("2026-06-17T12:00:01Z")
3054 .unwrap()
3055 .with_timezone(&Utc);
3056
3057 let first = runtime_lease_owner(
3058 "worker-a",
3059 "booking",
3060 "wf-1",
3061 &WorkflowOperationKind::Resume {
3062 event: "vendor.accepted".to_string(),
3063 },
3064 Some("event-1"),
3065 first_at,
3066 );
3067 let second = runtime_lease_owner(
3068 "worker-a",
3069 "booking",
3070 "wf-1",
3071 &WorkflowOperationKind::Resume {
3072 event: "vendor.accepted".to_string(),
3073 },
3074 Some("event-1"),
3075 second_at,
3076 );
3077
3078 assert_ne!(
3079 first, second,
3080 "stale release calls must not share the same lease owner as later acquisitions"
3081 );
3082 }
3083
3084 #[tokio::test]
3085 async fn lease_rejection_stops_workflow_before_side_effects() {
3086 let executor = MockExecutor::new();
3087 executor.reject_workflow_leases();
3088
3089 let wf = WorkflowDefinition::new("lease_guard")
3090 .initial_state("active")
3091 .transition(
3092 "active",
3093 "done",
3094 vec![
3095 WorkflowStep::notify(ChannelKind::Email, "should_not_send", "customer.email"),
3096 WorkflowStep::transition("done"),
3097 ],
3098 );
3099
3100 let mut ctx = WorkflowContext::new("wf-lease-guard", "active");
3101 ctx.set(
3102 "customer",
3103 serde_json::json!({"email": "guest@example.com"}),
3104 );
3105 let options = WorkflowRunOptions::default()
3106 .with_lease("worker-a", std::time::Duration::from_secs(30));
3107
3108 let err = run_workflow_with_options(&executor, &wf, &mut ctx, options)
3109 .await
3110 .expect_err("lease rejection must fail before executing steps");
3111
3112 match err {
3113 WorkflowError::Other(msg) => {
3114 assert!(msg.contains("already locked"), "got: {msg}");
3115 }
3116 other => panic!("expected lock error, got {other:?}"),
3117 }
3118 assert_eq!(ctx.current_state, "active");
3119 assert!(executor.notifications.lock().unwrap().is_empty());
3120 }
3121
3122 #[tokio::test]
3123 async fn resume_idempotency_completion_short_circuits_before_load_and_side_effects() {
3124 let executor = MockExecutor::new();
3125 executor.set_workflow_operation_status(
3126 "event-1",
3127 WorkflowOperationStatus::Completed {
3128 state: "done".to_string(),
3129 },
3130 );
3131
3132 let wf = WorkflowDefinition::new("resume_idempotency_guard")
3133 .initial_state("active")
3134 .transition(
3135 "active",
3136 "done",
3137 vec![
3138 WorkflowStep::wait("payment.success", std::time::Duration::from_secs(3600)),
3139 WorkflowStep::notify(ChannelKind::Email, "paid", "customer.email"),
3140 WorkflowStep::transition("done"),
3141 ],
3142 );
3143
3144 let result = resume_workflow_with_options(
3145 &executor,
3146 &wf,
3147 "missing-workflow",
3148 serde_json::json!({"event": "payment.success"}),
3149 WorkflowRunOptions::default().with_idempotency_key("event-1"),
3150 )
3151 .await
3152 .unwrap();
3153
3154 assert_eq!(result, "done");
3155 assert!(executor.notifications.lock().unwrap().is_empty());
3156 assert!(
3157 executor
3158 .completed_workflow_operations
3159 .lock()
3160 .unwrap()
3161 .is_empty()
3162 );
3163 }
3164
3165 #[tokio::test]
3166 async fn completed_operation_short_circuits_before_rejected_lease() {
3167 let executor = MockExecutor::new();
3168 executor.reject_workflow_leases();
3169 executor.set_workflow_operation_status(
3170 "run-replay-1",
3171 WorkflowOperationStatus::Completed {
3172 state: "done".to_string(),
3173 },
3174 );
3175
3176 let wf = WorkflowDefinition::new("completed_before_lease")
3177 .initial_state("active")
3178 .transition(
3179 "active",
3180 "done",
3181 vec![
3182 WorkflowStep::notify(ChannelKind::Email, "should_not_send", "customer.email"),
3183 WorkflowStep::transition("done"),
3184 ],
3185 );
3186
3187 let mut ctx = WorkflowContext::new("wf-completed-before-lease", "active");
3188 ctx.set(
3189 "customer",
3190 serde_json::json!({"email": "guest@example.com"}),
3191 );
3192
3193 let result = run_workflow_with_options(
3194 &executor,
3195 &wf,
3196 &mut ctx,
3197 WorkflowRunOptions::default()
3198 .with_idempotency_key("run-replay-1")
3199 .with_lease("worker-a", std::time::Duration::from_secs(30)),
3200 )
3201 .await
3202 .unwrap();
3203
3204 assert_eq!(result, "done");
3205 assert!(executor.acquired_leases.lock().unwrap().is_empty());
3206 assert!(executor.released_leases.lock().unwrap().is_empty());
3207 assert!(executor.notifications.lock().unwrap().is_empty());
3208 assert!(
3209 executor
3210 .completed_workflow_operations
3211 .lock()
3212 .unwrap()
3213 .is_empty()
3214 );
3215 }
3216
3217 #[tokio::test]
3218 async fn started_operation_is_failed_when_lease_is_rejected() {
3219 let executor = MockExecutor::new();
3220 executor.reject_workflow_leases();
3221
3222 let wf = WorkflowDefinition::new("started_then_lease_rejected")
3223 .initial_state("active")
3224 .transition(
3225 "active",
3226 "done",
3227 vec![
3228 WorkflowStep::notify(ChannelKind::Email, "should_not_send", "customer.email"),
3229 WorkflowStep::transition("done"),
3230 ],
3231 );
3232
3233 let mut ctx = WorkflowContext::new("wf-started-lease-rejected", "active");
3234 ctx.set(
3235 "customer",
3236 serde_json::json!({"email": "guest@example.com"}),
3237 );
3238
3239 let err = run_workflow_with_options(
3240 &executor,
3241 &wf,
3242 &mut ctx,
3243 WorkflowRunOptions::default()
3244 .with_idempotency_key("run-lock-1")
3245 .with_lease("worker-a", std::time::Duration::from_secs(30)),
3246 )
3247 .await
3248 .expect_err("new idempotent operation must not execute without its lease");
3249
3250 match err {
3251 WorkflowError::Other(msg) => {
3252 assert!(msg.contains("already locked"), "got: {msg}");
3253 }
3254 other => panic!("expected lock error, got {other:?}"),
3255 }
3256 assert_eq!(ctx.current_state, "active");
3257 assert!(executor.acquired_leases.lock().unwrap().is_empty());
3258 assert!(executor.notifications.lock().unwrap().is_empty());
3259 assert_eq!(
3260 executor
3261 .failed_workflow_operations
3262 .lock()
3263 .unwrap()
3264 .as_slice(),
3265 &[(
3266 "run-lock-1".to_string(),
3267 "Workflow 'wf-started-lease-rejected' is already locked".to_string()
3268 )]
3269 );
3270 }
3271
3272 #[tokio::test]
3273 async fn empty_idempotency_key_does_not_release_unacquired_lease() {
3274 let executor = MockExecutor::new();
3275
3276 let wf = WorkflowDefinition::new("empty_idempotency_key")
3277 .initial_state("active")
3278 .transition("active", "done", vec![WorkflowStep::transition("done")]);
3279
3280 let mut ctx = WorkflowContext::new("wf-empty-idempotency", "active");
3281 let err = run_workflow_with_options(
3282 &executor,
3283 &wf,
3284 &mut ctx,
3285 WorkflowRunOptions::default()
3286 .with_idempotency_key(" ")
3287 .with_lease("worker-a", std::time::Duration::from_secs(30)),
3288 )
3289 .await
3290 .expect_err("empty idempotency key must fail before lease acquisition");
3291
3292 match err {
3293 WorkflowError::Other(msg) => {
3294 assert!(
3295 msg.contains("idempotency key must not be empty"),
3296 "got: {msg}"
3297 );
3298 }
3299 other => panic!("expected idempotency validation error, got {other:?}"),
3300 }
3301 assert!(executor.acquired_leases.lock().unwrap().is_empty());
3302 assert!(executor.released_leases.lock().unwrap().is_empty());
3303 assert_eq!(ctx.current_state, "active");
3304 }
3305
3306 #[tokio::test]
3307 async fn padded_runtime_guard_identity_fails_before_side_effects() {
3308 let executor = MockExecutor::new();
3309
3310 let wf = WorkflowDefinition::new("padded_runtime_guard")
3311 .initial_state("active")
3312 .transition(
3313 "active",
3314 "done",
3315 vec![
3316 WorkflowStep::notify(ChannelKind::Email, "should_not_send", "customer.email"),
3317 WorkflowStep::transition("done"),
3318 ],
3319 );
3320
3321 let mut padded_key_ctx = WorkflowContext::new("wf-padded-idempotency", "active");
3322 padded_key_ctx.set(
3323 "customer",
3324 serde_json::json!({"email": "guest@example.com"}),
3325 );
3326 let err = run_workflow_with_options(
3327 &executor,
3328 &wf,
3329 &mut padded_key_ctx,
3330 WorkflowRunOptions::default()
3331 .with_idempotency_key(" run-1 ")
3332 .with_lease("worker-a", std::time::Duration::from_secs(30)),
3333 )
3334 .await
3335 .expect_err("padded idempotency key must fail before side effects");
3336 match err {
3337 WorkflowError::Other(msg) => {
3338 assert!(
3339 msg.contains("Workflow idempotency key must not have leading"),
3340 "got: {msg}"
3341 );
3342 }
3343 other => panic!("expected idempotency key validation error, got {other:?}"),
3344 }
3345
3346 let mut padded_owner_ctx = WorkflowContext::new("wf-padded-owner", "active");
3347 padded_owner_ctx.set(
3348 "customer",
3349 serde_json::json!({"email": "guest@example.com"}),
3350 );
3351 let err = run_workflow_with_options(
3352 &executor,
3353 &wf,
3354 &mut padded_owner_ctx,
3355 WorkflowRunOptions::default()
3356 .with_idempotency_key("run-2")
3357 .with_lease(" worker-a ", std::time::Duration::from_secs(30)),
3358 )
3359 .await
3360 .expect_err("padded lease owner must fail before side effects");
3361 match err {
3362 WorkflowError::Other(msg) => {
3363 assert!(
3364 msg.contains("Workflow lease owner must not have leading"),
3365 "got: {msg}"
3366 );
3367 }
3368 other => panic!("expected lease owner validation error, got {other:?}"),
3369 }
3370
3371 assert!(executor.acquired_leases.lock().unwrap().is_empty());
3372 assert!(executor.released_leases.lock().unwrap().is_empty());
3373 assert!(executor.notifications.lock().unwrap().is_empty());
3374 }
3375
3376 #[tokio::test]
3377 async fn operation_begin_failure_does_not_release_unacquired_lease() {
3378 let executor = MockExecutor::new();
3379 executor.fail_workflow_operation_begin("begin-fails-1");
3380
3381 let wf = WorkflowDefinition::new("operation_begin_failure")
3382 .initial_state("active")
3383 .transition("active", "done", vec![WorkflowStep::transition("done")]);
3384
3385 let mut ctx = WorkflowContext::new("wf-begin-failure", "active");
3386 let err = run_workflow_with_options(
3387 &executor,
3388 &wf,
3389 &mut ctx,
3390 WorkflowRunOptions::default()
3391 .with_idempotency_key("begin-fails-1")
3392 .with_lease("worker-a", std::time::Duration::from_secs(30)),
3393 )
3394 .await
3395 .expect_err("operation begin failure must happen before lease acquisition");
3396
3397 match err {
3398 WorkflowError::Other(msg) => {
3399 assert!(msg.contains("forced operation begin failure"), "got: {msg}");
3400 }
3401 other => panic!("expected operation begin error, got {other:?}"),
3402 }
3403 assert!(executor.acquired_leases.lock().unwrap().is_empty());
3404 assert!(executor.released_leases.lock().unwrap().is_empty());
3405 assert_eq!(ctx.current_state, "active");
3406 }
3407
3408 #[tokio::test]
3409 async fn completed_operation_with_invalid_state_is_rejected_before_lease() {
3410 let executor = MockExecutor::new();
3411 executor.set_workflow_operation_status(
3412 "completed-invalid-state-1",
3413 WorkflowOperationStatus::Completed {
3414 state: " ".to_string(),
3415 },
3416 );
3417
3418 let wf = WorkflowDefinition::new("completed_invalid_state")
3419 .initial_state("active")
3420 .transition("active", "done", vec![WorkflowStep::transition("done")]);
3421
3422 let mut ctx = WorkflowContext::new("wf-completed-invalid-state", "active");
3423 let err = run_workflow_with_options(
3424 &executor,
3425 &wf,
3426 &mut ctx,
3427 WorkflowRunOptions::default()
3428 .with_idempotency_key("completed-invalid-state-1")
3429 .with_lease("worker-a", std::time::Duration::from_secs(30)),
3430 )
3431 .await
3432 .expect_err("invalid completed replay state must be rejected");
3433
3434 match err {
3435 WorkflowError::Other(msg) => {
3436 assert!(
3437 msg.contains("Workflow completed operation state must not be empty"),
3438 "got: {msg}"
3439 );
3440 }
3441 other => panic!("expected completed state validation error, got {other:?}"),
3442 }
3443 assert!(executor.acquired_leases.lock().unwrap().is_empty());
3444 assert!(executor.released_leases.lock().unwrap().is_empty());
3445 assert_eq!(ctx.current_state, "active");
3446 }
3447
3448 #[tokio::test]
3449 async fn invalid_definition_fails_before_runtime_guards() {
3450 let executor = MockExecutor::new();
3451
3452 let wf = WorkflowDefinition::new("invalid_definition")
3453 .version(" v2")
3454 .initial_state("active")
3455 .transition(
3456 "active",
3457 "done",
3458 vec![
3459 WorkflowStep::notify(ChannelKind::Email, "should_not_send", "customer.email"),
3460 WorkflowStep::transition("done"),
3461 ],
3462 );
3463
3464 let mut ctx = WorkflowContext::new("wf-invalid-definition", "active");
3465 ctx.set(
3466 "customer",
3467 serde_json::json!({"email": "guest@example.com"}),
3468 );
3469
3470 let err = run_workflow_with_options(
3471 &executor,
3472 &wf,
3473 &mut ctx,
3474 WorkflowRunOptions::default()
3475 .with_idempotency_key("invalid-run-1")
3476 .with_lease("worker-a", std::time::Duration::from_secs(30)),
3477 )
3478 .await
3479 .expect_err("invalid definition must fail before runtime guards");
3480
3481 match err {
3482 WorkflowError::Other(msg) => {
3483 assert!(
3484 msg.contains("Workflow definition version must not have leading"),
3485 "got: {msg}"
3486 );
3487 }
3488 other => panic!("expected definition validation error, got {other:?}"),
3489 }
3490 assert!(executor.acquired_leases.lock().unwrap().is_empty());
3491 assert!(executor.released_leases.lock().unwrap().is_empty());
3492 assert!(
3493 executor
3494 .completed_workflow_operations
3495 .lock()
3496 .unwrap()
3497 .is_empty()
3498 );
3499 assert!(
3500 executor
3501 .failed_workflow_operations
3502 .lock()
3503 .unwrap()
3504 .is_empty()
3505 );
3506 assert!(executor.notifications.lock().unwrap().is_empty());
3507 }
3508
3509 #[tokio::test]
3510 async fn empty_workflow_id_fails_before_runtime_guards() {
3511 let executor = MockExecutor::new();
3512
3513 let wf = WorkflowDefinition::new("empty_workflow_id")
3514 .initial_state("active")
3515 .transition("active", "done", vec![WorkflowStep::transition("done")]);
3516
3517 let mut ctx = WorkflowContext::new(" ", "active");
3518 let err = run_workflow_with_options(
3519 &executor,
3520 &wf,
3521 &mut ctx,
3522 WorkflowRunOptions::default()
3523 .with_idempotency_key("empty-id-1")
3524 .with_lease("worker-a", std::time::Duration::from_secs(30)),
3525 )
3526 .await
3527 .expect_err("empty workflow id must fail before runtime guards");
3528
3529 match err {
3530 WorkflowError::Other(msg) => {
3531 assert!(msg.contains("Workflow id must not be empty"), "got: {msg}");
3532 }
3533 other => panic!("expected workflow id validation error, got {other:?}"),
3534 }
3535 assert!(executor.acquired_leases.lock().unwrap().is_empty());
3536 assert!(executor.released_leases.lock().unwrap().is_empty());
3537 assert!(
3538 executor
3539 .completed_workflow_operations
3540 .lock()
3541 .unwrap()
3542 .is_empty()
3543 );
3544 assert!(
3545 executor
3546 .failed_workflow_operations
3547 .lock()
3548 .unwrap()
3549 .is_empty()
3550 );
3551 }
3552
3553 #[tokio::test]
3554 async fn empty_resume_workflow_id_fails_before_runtime_guards() {
3555 let executor = MockExecutor::new();
3556
3557 let wf = WorkflowDefinition::new("empty_resume_id")
3558 .initial_state("active")
3559 .transition(
3560 "active",
3561 "done",
3562 vec![
3563 WorkflowStep::wait("vendor.ready", std::time::Duration::from_secs(3600)),
3564 WorkflowStep::transition("done"),
3565 ],
3566 );
3567
3568 let err = resume_workflow_with_options(
3569 &executor,
3570 &wf,
3571 " ",
3572 serde_json::json!({"event": "vendor.ready"}),
3573 WorkflowRunOptions::default()
3574 .with_idempotency_key("empty-resume-id-1")
3575 .with_lease("worker-a", std::time::Duration::from_secs(30)),
3576 )
3577 .await
3578 .expect_err("empty workflow id must fail before runtime guards");
3579
3580 match err {
3581 WorkflowError::Other(msg) => {
3582 assert!(msg.contains("Workflow id must not be empty"), "got: {msg}");
3583 }
3584 other => panic!("expected workflow id validation error, got {other:?}"),
3585 }
3586 assert!(executor.acquired_leases.lock().unwrap().is_empty());
3587 assert!(executor.released_leases.lock().unwrap().is_empty());
3588 assert!(
3589 executor
3590 .completed_workflow_operations
3591 .lock()
3592 .unwrap()
3593 .is_empty()
3594 );
3595 assert!(
3596 executor
3597 .failed_workflow_operations
3598 .lock()
3599 .unwrap()
3600 .is_empty()
3601 );
3602 }
3603
3604 #[tokio::test]
3605 async fn operation_complete_failure_still_releases_lease() {
3606 let executor = MockExecutor::new();
3607 executor.fail_workflow_operation_complete("complete-fails-1");
3608
3609 let wf = WorkflowDefinition::new("complete_failure_release")
3610 .initial_state("active")
3611 .transition("active", "done", vec![WorkflowStep::transition("done")]);
3612
3613 let mut ctx = WorkflowContext::new("wf-complete-failure-release", "active");
3614 let err = run_workflow_with_options(
3615 &executor,
3616 &wf,
3617 &mut ctx,
3618 WorkflowRunOptions::default()
3619 .with_idempotency_key("complete-fails-1")
3620 .with_lease("worker-a", std::time::Duration::from_secs(30)),
3621 )
3622 .await
3623 .expect_err("operation completion failure should be returned");
3624
3625 match err {
3626 WorkflowError::Other(msg) => {
3627 assert!(
3628 msg.contains("forced operation complete failure"),
3629 "got: {msg}"
3630 );
3631 }
3632 other => panic!("expected completion failure, got {other:?}"),
3633 }
3634 assert_eq!(ctx.current_state, "done");
3635 assert_eq!(executor.acquired_leases.lock().unwrap().len(), 1);
3636 assert_eq!(executor.released_leases.lock().unwrap().len(), 1);
3637 }
3638
3639 #[tokio::test]
3640 async fn operation_fail_failure_still_releases_lease() {
3641 let executor = MockExecutor::new();
3642 executor.fail_workflow_operation_fail("fail-fails-1");
3643
3644 let wf = WorkflowDefinition::new("fail_failure_release")
3645 .initial_state("active")
3646 .transition(
3647 "active",
3648 "done",
3649 vec![
3650 WorkflowStep::notify(ChannelKind::Email, "missing_recipient", "customer.email"),
3651 WorkflowStep::transition("done"),
3652 ],
3653 );
3654
3655 let mut ctx = WorkflowContext::new("wf-fail-failure-release", "active");
3656 let err = run_workflow_with_options(
3657 &executor,
3658 &wf,
3659 &mut ctx,
3660 WorkflowRunOptions::default()
3661 .with_idempotency_key("fail-fails-1")
3662 .with_lease("worker-a", std::time::Duration::from_secs(30)),
3663 )
3664 .await
3665 .expect_err("original workflow error should be returned");
3666
3667 match err {
3668 WorkflowError::MissingContextKey(key) => {
3669 assert_eq!(key, "customer.email");
3670 }
3671 other => panic!("expected original workflow error, got {other:?}"),
3672 }
3673 assert_eq!(ctx.current_state, "active");
3674 assert_eq!(executor.acquired_leases.lock().unwrap().len(), 1);
3675 assert_eq!(executor.released_leases.lock().unwrap().len(), 1);
3676 }
3677
3678 #[tokio::test]
3679 async fn completed_notify_side_effect_skips_provider_call() {
3680 let executor = MockExecutor::new();
3681 executor.skip_notify_side_effects();
3682
3683 let wf = WorkflowDefinition::new("notify_side_effect_replay")
3684 .initial_state("active")
3685 .transition(
3686 "active",
3687 "done",
3688 vec![
3689 WorkflowStep::notify(ChannelKind::Email, "booking_confirmed", "customer.email"),
3690 WorkflowStep::transition("done"),
3691 ],
3692 );
3693
3694 let mut ctx = WorkflowContext::new("wf-notify-side-effect", "active");
3695 ctx.set(
3696 "customer",
3697 serde_json::json!({"email": "guest@example.com"}),
3698 );
3699
3700 let result = run_workflow(&executor, &wf, &mut ctx).await.unwrap();
3701
3702 assert_eq!(result, "done");
3703 assert!(
3704 executor.notifications.lock().unwrap().is_empty(),
3705 "completed notify side effect must not call the provider again"
3706 );
3707 }
3708
3709 #[tokio::test]
3710 async fn transition_limit_counts_only_current_run() {
3711 let executor = MockExecutor::new();
3712
3713 let wf = WorkflowDefinition::new("aged_workflow")
3714 .initial_state("active")
3715 .transition("active", "done", vec![WorkflowStep::transition("done")]);
3716
3717 let mut ctx = WorkflowContext::new("wf-aged-history", "active");
3718 for idx in 0..51 {
3719 ctx.history.push(StateChange {
3720 from: format!("past_{idx}"),
3721 to: format!("past_{}", idx + 1),
3722 at: Utc::now(),
3723 reason: None,
3724 });
3725 }
3726
3727 let result = run_workflow(&executor, &wf, &mut ctx)
3728 .await
3729 .expect("old workflow history must not trip per-run loop guard");
3730
3731 assert_eq!(result, "done");
3732 assert_eq!(ctx.transition_count(), 52);
3733 }
3734
3735 #[tokio::test]
3736 async fn transition_limit_blocks_next_transition_before_side_effects() {
3737 let executor = MockExecutor::new();
3738
3739 let mut wf = WorkflowDefinition::new("runaway_workflow").initial_state("s0");
3740 for idx in 0..50 {
3741 wf = wf.transition(
3742 format!("s{idx}"),
3743 format!("s{}", idx + 1),
3744 vec![WorkflowStep::transition(&format!("s{}", idx + 1))],
3745 );
3746 }
3747 wf = wf.transition(
3748 "s50",
3749 "s51",
3750 vec![
3751 WorkflowStep::notify(ChannelKind::Email, "runaway_side_effect", "ops.email"),
3752 WorkflowStep::transition("s51"),
3753 ],
3754 );
3755
3756 let mut ctx = WorkflowContext::new("wf-runaway", "s0");
3757 ctx.set("ops", serde_json::json!({"email": "ops@example.com"}));
3758
3759 let err = run_workflow(&executor, &wf, &mut ctx)
3760 .await
3761 .expect_err("the 51st transition must be blocked before side effects");
3762
3763 match err {
3764 WorkflowError::Other(msg) => {
3765 assert!(
3766 msg.contains("Maximum transition count exceeded"),
3767 "got: {msg}"
3768 );
3769 }
3770 other => panic!("expected transition limit error, got {other:?}"),
3771 }
3772 assert_eq!(ctx.current_state, "s50");
3773 assert_eq!(ctx.transition_count(), 50);
3774 assert!(
3775 executor.notifications.lock().unwrap().is_empty(),
3776 "blocked transition side effects must not execute"
3777 );
3778 let saved = executor
3779 .saved_state
3780 .lock()
3781 .unwrap()
3782 .clone()
3783 .expect("state after the last allowed transition should be saved");
3784 assert_eq!(saved.current_state, "s50");
3785 }
3786
3787 #[tokio::test]
3788 async fn test_workflow_with_branch() {
3789 let executor = MockExecutor::new();
3790
3791 let wf = WorkflowDefinition::new("branching")
3792 .initial_state("pending")
3793 .transition(
3794 "pending",
3795 "resolved",
3796 vec![
3797 WorkflowStep::branch(
3798 "reason",
3799 vec![
3800 ("full", vec![WorkflowStep::log("Fully booked")]),
3801 ("maintenance", vec![WorkflowStep::log("Under maintenance")]),
3802 ],
3803 vec![WorkflowStep::log("Unknown reason")],
3804 ),
3805 WorkflowStep::transition("resolved"),
3806 ],
3807 );
3808
3809 let mut ctx = WorkflowContext::new("wf-002", "pending");
3810 ctx.set("reason", Value::String("maintenance".into()));
3811
3812 let result = run_workflow(&executor, &wf, &mut ctx).await.unwrap();
3813 assert_eq!(result, "resolved");
3814 }
3815
3816 #[tokio::test]
3817 async fn branch_when_routes_by_typed_numeric_condition() {
3818 let executor = MockExecutor::new();
3819
3820 let wf = WorkflowDefinition::new("typed_branching")
3821 .initial_state("pending")
3822 .transition(
3823 "pending",
3824 "resolved",
3825 vec![WorkflowStep::branch_when(
3826 "payment.attempts",
3827 vec![(
3828 WorkflowBranchCondition::NumberGte(3),
3829 vec![WorkflowStep::transition("manual_review")],
3830 )],
3831 vec![WorkflowStep::transition("retry")],
3832 )],
3833 );
3834
3835 let mut ctx = WorkflowContext::new("wf-typed-branch", "pending");
3836 ctx.set("payment", serde_json::json!({"attempts": 3}));
3837
3838 let result = run_workflow(&executor, &wf, &mut ctx).await.unwrap();
3839
3840 assert_eq!(result, "manual_review");
3841 assert_eq!(ctx.current_state, "manual_review");
3842 }
3843
3844 #[tokio::test]
3845 async fn branch_when_routes_by_decimal_numeric_condition() {
3846 let executor = MockExecutor::new();
3847
3848 let wf = WorkflowDefinition::new("decimal_branching")
3849 .initial_state("pending")
3850 .transition(
3851 "pending",
3852 "resolved",
3853 vec![WorkflowStep::branch_when(
3854 "payment.risk_score",
3855 vec![(
3856 WorkflowBranchCondition::NumberGt(10),
3857 vec![WorkflowStep::transition("manual_review")],
3858 )],
3859 vec![WorkflowStep::transition("retry")],
3860 )],
3861 );
3862
3863 let mut ctx = WorkflowContext::new("wf-decimal-branch", "pending");
3864 ctx.set("payment", serde_json::json!({"risk_score": 10.5}));
3865
3866 let result = run_workflow(&executor, &wf, &mut ctx).await.unwrap();
3867
3868 assert_eq!(result, "manual_review");
3869 assert_eq!(ctx.current_state, "manual_review");
3870 }
3871
3872 #[tokio::test]
3873 async fn branch_when_resume_rejects_condition_drift_before_side_effects() {
3874 let executor = MockExecutor::new();
3875
3876 let wf = WorkflowDefinition::new("branch_when_drift_source")
3877 .initial_state("active")
3878 .transition(
3879 "active",
3880 "done",
3881 vec![WorkflowStep::branch_when(
3882 "payment.attempts",
3883 vec![(
3884 WorkflowBranchCondition::NumberGte(3),
3885 vec![
3886 WorkflowStep::wait(
3887 "manual.approved",
3888 std::time::Duration::from_secs(3600),
3889 ),
3890 WorkflowStep::notify(
3891 ChannelKind::Email,
3892 "manual_review_approved",
3893 "customer.email",
3894 ),
3895 ],
3896 )],
3897 vec![],
3898 )],
3899 );
3900
3901 let mut ctx = WorkflowContext::new("wf-branch-when-drift", "active");
3902 ctx.set(
3903 "payment",
3904 serde_json::json!({
3905 "attempts": 3
3906 }),
3907 );
3908 ctx.set(
3909 "customer",
3910 serde_json::json!({"email": "guest@example.com"}),
3911 );
3912
3913 let result = run_workflow(&executor, &wf, &mut ctx).await.unwrap();
3914 assert_eq!(result, "active");
3915
3916 {
3917 let mut saved = executor.saved_state.lock().unwrap();
3918 let saved = saved.as_mut().expect("paused state should be saved");
3919 saved.set(
3920 "payment",
3921 serde_json::json!({
3922 "attempts": 1
3923 }),
3924 );
3925 }
3926
3927 let err = resume_workflow(
3928 &executor,
3929 &wf,
3930 "wf-branch-when-drift",
3931 serde_json::json!({"event": "manual.approved"}),
3932 )
3933 .await
3934 .expect_err("branch predicate drift must reject resume");
3935
3936 match err {
3937 WorkflowError::Other(msg) => {
3938 assert!(msg.contains("Invalid workflow resume cursor"), "got: {msg}");
3939 assert!(msg.contains("no longer matches"), "got: {msg}");
3940 }
3941 other => panic!("expected invalid cursor error, got {other:?}"),
3942 }
3943 assert!(executor.notifications.lock().unwrap().is_empty());
3944 }
3945
3946 #[tokio::test]
3947 async fn branch_missing_context_key_errors_instead_of_taking_default() {
3948 let executor = MockExecutor::new();
3949
3950 let wf = WorkflowDefinition::new("branching_missing_key")
3951 .initial_state("pending")
3952 .transition(
3953 "pending",
3954 "resolved",
3955 vec![WorkflowStep::branch(
3956 "payment.status",
3957 vec![("paid", vec![WorkflowStep::transition("resolved")])],
3958 vec![WorkflowStep::transition("fallback")],
3959 )],
3960 );
3961
3962 let mut ctx = WorkflowContext::new("wf-missing-branch-key", "pending");
3963 let err = run_workflow(&executor, &wf, &mut ctx)
3964 .await
3965 .expect_err("missing branch context must not route to default");
3966
3967 match err {
3968 WorkflowError::MissingContextKey(key) => assert_eq!(key, "payment.status"),
3969 other => panic!("expected MissingContextKey, got {other:?}"),
3970 }
3971 assert_eq!(ctx.current_state, "pending");
3972 }
3973
3974 #[tokio::test]
3975 async fn duplicate_transitions_from_same_state_fail_before_side_effects() {
3976 let executor = MockExecutor::new();
3977
3978 let wf = WorkflowDefinition::new("duplicate_transition_guard")
3979 .initial_state("active")
3980 .transition(
3981 "active",
3982 "vendor_a",
3983 vec![WorkflowStep::notify(
3984 ChannelKind::WhatsApp,
3985 "vendor_a",
3986 "vendor.phone",
3987 )],
3988 )
3989 .transition(
3990 "active",
3991 "vendor_b",
3992 vec![WorkflowStep::notify(
3993 ChannelKind::WhatsApp,
3994 "vendor_b",
3995 "vendor.phone",
3996 )],
3997 );
3998
3999 let mut ctx = WorkflowContext::new("wf-duplicate-transition", "active");
4000 ctx.set("vendor", serde_json::json!({"phone": "+628111"}));
4001
4002 let err = run_workflow(&executor, &wf, &mut ctx)
4003 .await
4004 .expect_err("duplicate transitions must fail closed");
4005
4006 match err {
4007 WorkflowError::Other(msg) => {
4008 assert!(msg.contains("Ambiguous workflow definition"), "got: {msg}");
4009 assert!(msg.contains("active"), "got: {msg}");
4010 assert!(msg.contains("2 outgoing transitions"), "got: {msg}");
4011 }
4012 other => panic!("expected ambiguous transition error, got {other:?}"),
4013 }
4014 assert!(executor.notifications.lock().unwrap().is_empty());
4015 }
4016
4017 #[tokio::test]
4018 async fn duplicate_future_transitions_fail_before_current_side_effects() {
4019 let executor = MockExecutor::new();
4020
4021 let wf = WorkflowDefinition::new("future_duplicate_transition_guard")
4022 .initial_state("active")
4023 .transition(
4024 "active",
4025 "routing",
4026 vec![
4027 WorkflowStep::notify(ChannelKind::WhatsApp, "before_bad_state", "vendor.phone"),
4028 WorkflowStep::transition("routing"),
4029 ],
4030 )
4031 .transition(
4032 "routing",
4033 "vendor_a",
4034 vec![WorkflowStep::notify(
4035 ChannelKind::WhatsApp,
4036 "vendor_a",
4037 "vendor.phone",
4038 )],
4039 )
4040 .transition(
4041 "routing",
4042 "vendor_b",
4043 vec![WorkflowStep::notify(
4044 ChannelKind::WhatsApp,
4045 "vendor_b",
4046 "vendor.phone",
4047 )],
4048 );
4049
4050 let mut ctx = WorkflowContext::new("wf-future-duplicate-transition", "active");
4051 ctx.set("vendor", serde_json::json!({"phone": "+628111"}));
4052
4053 let err = run_workflow(&executor, &wf, &mut ctx)
4054 .await
4055 .expect_err("future duplicate transitions must fail before current side effects");
4056
4057 match err {
4058 WorkflowError::Other(msg) => {
4059 assert!(msg.contains("Ambiguous workflow definition"), "got: {msg}");
4060 assert!(msg.contains("routing"), "got: {msg}");
4061 }
4062 other => panic!("expected ambiguous transition error, got {other:?}"),
4063 }
4064 assert_eq!(ctx.current_state, "active");
4065 assert!(
4066 executor.notifications.lock().unwrap().is_empty(),
4067 "definition validation must run before the initial notification"
4068 );
4069 }
4070
4071 #[tokio::test]
4072 async fn duplicate_branch_values_fail_before_side_effects() {
4073 let executor = MockExecutor::new();
4074
4075 let wf = WorkflowDefinition::new("duplicate_branch_guard")
4076 .initial_state("active")
4077 .transition(
4078 "active",
4079 "done",
4080 vec![WorkflowStep::branch(
4081 "decision",
4082 vec![
4083 (
4084 "accepted",
4085 vec![WorkflowStep::notify(
4086 ChannelKind::WhatsApp,
4087 "accepted_a",
4088 "vendor.phone",
4089 )],
4090 ),
4091 (
4092 "accepted",
4093 vec![WorkflowStep::notify(
4094 ChannelKind::WhatsApp,
4095 "accepted_b",
4096 "vendor.phone",
4097 )],
4098 ),
4099 ],
4100 vec![],
4101 )],
4102 );
4103
4104 let mut ctx = WorkflowContext::new("wf-duplicate-branch", "active");
4105 ctx.set("decision", Value::String("accepted".to_string()));
4106 ctx.set("vendor", serde_json::json!({"phone": "+628111"}));
4107
4108 let err = run_workflow(&executor, &wf, &mut ctx)
4109 .await
4110 .expect_err("duplicate branch values must fail closed");
4111
4112 match err {
4113 WorkflowError::Other(msg) => {
4114 assert!(msg.contains("Ambiguous workflow branch"), "got: {msg}");
4115 assert!(msg.contains("accepted"), "got: {msg}");
4116 }
4117 other => panic!("expected ambiguous branch error, got {other:?}"),
4118 }
4119 assert!(executor.notifications.lock().unwrap().is_empty());
4120 }
4121
4122 #[tokio::test]
4123 async fn branch_when_rejects_empty_string_contains_before_side_effects() {
4124 let executor = MockExecutor::new();
4125
4126 let wf = WorkflowDefinition::new("empty_string_contains_guard")
4127 .initial_state("active")
4128 .transition(
4129 "active",
4130 "done",
4131 vec![
4132 WorkflowStep::notify(ChannelKind::Email, "before_bad_branch", "ops.email"),
4133 WorkflowStep::branch_when(
4134 "decision",
4135 vec![(
4136 WorkflowBranchCondition::StringContains(String::new()),
4137 vec![WorkflowStep::transition("catch_all")],
4138 )],
4139 vec![WorkflowStep::transition("done")],
4140 ),
4141 ],
4142 );
4143
4144 let mut ctx = WorkflowContext::new("wf-empty-string-contains", "active");
4145 ctx.set("ops", serde_json::json!({"email": "ops@example.com"}));
4146 ctx.set("decision", Value::String("accepted".to_string()));
4147
4148 let err = run_workflow(&executor, &wf, &mut ctx)
4149 .await
4150 .expect_err("empty StringContains predicate must fail before notification");
4151
4152 match err {
4153 WorkflowError::Other(msg) => {
4154 assert!(
4155 msg.contains("StringContains condition must not be empty"),
4156 "got: {msg}"
4157 );
4158 }
4159 other => panic!("expected branch condition validation error, got {other:?}"),
4160 }
4161 assert!(executor.notifications.lock().unwrap().is_empty());
4162 }
4163
4164 #[tokio::test]
4165 async fn branch_when_rejects_empty_one_of_before_side_effects() {
4166 let executor = MockExecutor::new();
4167
4168 let wf = WorkflowDefinition::new("empty_one_of_guard")
4169 .initial_state("active")
4170 .transition(
4171 "active",
4172 "done",
4173 vec![
4174 WorkflowStep::notify(ChannelKind::Email, "before_bad_branch", "ops.email"),
4175 WorkflowStep::branch_when(
4176 "decision",
4177 vec![(
4178 WorkflowBranchCondition::OneOf(Vec::new()),
4179 vec![WorkflowStep::transition("impossible")],
4180 )],
4181 vec![WorkflowStep::transition("done")],
4182 ),
4183 ],
4184 );
4185
4186 let mut ctx = WorkflowContext::new("wf-empty-one-of", "active");
4187 ctx.set("ops", serde_json::json!({"email": "ops@example.com"}));
4188 ctx.set("decision", Value::String("accepted".to_string()));
4189
4190 let err = run_workflow(&executor, &wf, &mut ctx)
4191 .await
4192 .expect_err("empty OneOf predicate must fail before notification");
4193
4194 match err {
4195 WorkflowError::Other(msg) => {
4196 assert!(
4197 msg.contains("OneOf condition must include at least one value"),
4198 "got: {msg}"
4199 );
4200 }
4201 other => panic!("expected branch condition validation error, got {other:?}"),
4202 }
4203 assert!(executor.notifications.lock().unwrap().is_empty());
4204 }
4205
4206 #[tokio::test]
4207 async fn invalid_transition_target_fails_before_side_effects() {
4208 let executor = MockExecutor::new();
4209
4210 let wf = WorkflowDefinition::new("invalid_transition_target")
4211 .initial_state("active")
4212 .transition(
4213 "active",
4214 "done",
4215 vec![
4216 WorkflowStep::notify(ChannelKind::Email, "before_bad_transition", "ops.email"),
4217 WorkflowStep::transition(" "),
4218 ],
4219 );
4220
4221 let mut ctx = WorkflowContext::new("wf-invalid-transition-target", "active");
4222 ctx.set("ops", serde_json::json!({"email": "ops@example.com"}));
4223
4224 let err = run_workflow(&executor, &wf, &mut ctx)
4225 .await
4226 .expect_err("invalid transition target must fail during definition validation");
4227
4228 match err {
4229 WorkflowError::Other(msg) => {
4230 assert!(
4231 msg.contains("Transition target state must not be empty"),
4232 "got: {msg}"
4233 );
4234 }
4235 other => panic!("expected transition target validation error, got {other:?}"),
4236 }
4237 assert!(executor.notifications.lock().unwrap().is_empty());
4238 }
4239
4240 #[tokio::test]
4241 async fn invalid_notify_recipient_key_fails_before_side_effects() {
4242 let executor = MockExecutor::new();
4243
4244 let wf = WorkflowDefinition::new("invalid_notify_recipient")
4245 .initial_state("active")
4246 .transition(
4247 "active",
4248 "done",
4249 vec![
4250 WorkflowStep::notify(ChannelKind::Email, "bad_notify", " "),
4251 WorkflowStep::transition("done"),
4252 ],
4253 );
4254
4255 let mut ctx = WorkflowContext::new("wf-invalid-notify-recipient", "active");
4256 let err = run_workflow(&executor, &wf, &mut ctx)
4257 .await
4258 .expect_err("invalid notify recipient key must fail before send");
4259
4260 match err {
4261 WorkflowError::Other(msg) => {
4262 assert!(
4263 msg.contains("Notify recipient_key must not be empty"),
4264 "got: {msg}"
4265 );
4266 }
4267 other => panic!("expected notify recipient validation error, got {other:?}"),
4268 }
4269 assert!(executor.notifications.lock().unwrap().is_empty());
4270 }
4271
4272 #[tokio::test]
4273 async fn malformed_lookup_key_fails_before_side_effects() {
4274 let executor = MockExecutor::new();
4275
4276 let wf = WorkflowDefinition::new("malformed_lookup_key_guard")
4277 .initial_state("active")
4278 .transition(
4279 "active",
4280 "done",
4281 vec![
4282 WorkflowStep::notify(ChannelKind::Email, "before_bad_lookup", "ops.email"),
4283 WorkflowStep::notify(ChannelKind::Email, "bad_lookup", "customer."),
4284 WorkflowStep::transition("done"),
4285 ],
4286 );
4287
4288 let mut ctx = WorkflowContext::new("wf-malformed-lookup-key", "active");
4289 ctx.set("ops", serde_json::json!({"email": "ops@example.com"}));
4290 ctx.set(
4291 "customer",
4292 serde_json::json!({"email": "guest@example.com"}),
4293 );
4294
4295 let err = run_workflow(&executor, &wf, &mut ctx)
4296 .await
4297 .expect_err("malformed lookup key must fail before first notification");
4298
4299 match err {
4300 WorkflowError::Other(msg) => {
4301 assert!(
4302 msg.contains("empty dot-notation path segments"),
4303 "got: {msg}"
4304 );
4305 }
4306 other => panic!("expected malformed lookup key error, got {other:?}"),
4307 }
4308 assert!(
4309 executor.notifications.lock().unwrap().is_empty(),
4310 "definition validation must fail before earlier notification side effects"
4311 );
4312 }
4313
4314 #[tokio::test]
4315 async fn invalid_charge_amount_key_fails_before_provider_call() {
4316 let executor = MockExecutor::new();
4317
4318 let wf = WorkflowDefinition::new("invalid_charge_amount_key")
4319 .initial_state("created")
4320 .transition(
4321 "created",
4322 "awaiting_payment",
4323 vec![WorkflowStep::charge(
4324 PaymentKind::Xendit,
4325 " ",
4326 "order.id",
4327 Some("charge"),
4328 )],
4329 );
4330
4331 let mut ctx = WorkflowContext::new("wf-invalid-charge-amount", "created");
4332 ctx.set(
4333 "order",
4334 serde_json::json!({"id": "booking-1", "total": 125_000}),
4335 );
4336 let err = run_workflow(&executor, &wf, &mut ctx)
4337 .await
4338 .expect_err("invalid charge amount key must fail before provider call");
4339
4340 match err {
4341 WorkflowError::Other(msg) => {
4342 assert!(
4343 msg.contains("Charge amount_key must not be empty"),
4344 "got: {msg}"
4345 );
4346 }
4347 other => panic!("expected charge amount key validation error, got {other:?}"),
4348 }
4349 assert!(executor.charges.lock().unwrap().is_empty());
4350 }
4351
4352 #[tokio::test]
4353 async fn malformed_charge_lookup_key_fails_before_provider_call() {
4354 let executor = MockExecutor::new();
4355
4356 let wf = WorkflowDefinition::new("malformed_charge_lookup_key")
4357 .initial_state("created")
4358 .transition(
4359 "created",
4360 "awaiting_payment",
4361 vec![WorkflowStep::charge(
4362 PaymentKind::Xendit,
4363 "order..total",
4364 "order.id",
4365 Some("charge"),
4366 )],
4367 );
4368
4369 let mut ctx = WorkflowContext::new("wf-malformed-charge-lookup", "created");
4370 ctx.set(
4371 "order",
4372 serde_json::json!({"id": "booking-1", "total": 125_000}),
4373 );
4374 let err = run_workflow(&executor, &wf, &mut ctx)
4375 .await
4376 .expect_err("malformed charge lookup key must fail before provider call");
4377
4378 match err {
4379 WorkflowError::Other(msg) => {
4380 assert!(
4381 msg.contains("empty dot-notation path segments"),
4382 "got: {msg}"
4383 );
4384 }
4385 other => panic!("expected malformed lookup key error, got {other:?}"),
4386 }
4387 assert!(executor.charges.lock().unwrap().is_empty());
4388 }
4389
4390 #[tokio::test]
4391 async fn charge_rejects_missing_optional_string_key_before_provider_call() {
4392 let executor = MockExecutor::new();
4393
4394 let wf = WorkflowDefinition::new("charge_missing_optional_string_key")
4395 .initial_state("created")
4396 .transition(
4397 "created",
4398 "awaiting_payment",
4399 vec![WorkflowStep::charge_with(
4400 PaymentKind::Xendit,
4401 "order.total",
4402 "order.id",
4403 Some("order.description"),
4404 Some("order.payment_method"),
4405 Some("charge"),
4406 )],
4407 );
4408
4409 let mut ctx = WorkflowContext::new("wf-charge-missing-optional-string", "created");
4410 ctx.set(
4411 "order",
4412 serde_json::json!({"id": "booking-1", "total": 125_000}),
4413 );
4414 let err = run_workflow(&executor, &wf, &mut ctx)
4415 .await
4416 .expect_err("missing configured charge string key must fail before provider call");
4417
4418 match err {
4419 WorkflowError::MissingContextKey(key) => {
4420 assert_eq!(key, "order.description (expected string)");
4421 }
4422 other => panic!("expected missing context key error, got {other:?}"),
4423 }
4424 assert!(executor.charges.lock().unwrap().is_empty());
4425 }
4426
4427 #[tokio::test]
4428 async fn charge_rejects_non_string_optional_key_before_provider_call() {
4429 let executor = MockExecutor::new();
4430
4431 let wf = WorkflowDefinition::new("charge_non_string_optional_key")
4432 .initial_state("created")
4433 .transition(
4434 "created",
4435 "awaiting_payment",
4436 vec![WorkflowStep::charge_with(
4437 PaymentKind::Xendit,
4438 "order.total",
4439 "order.id",
4440 None,
4441 Some("order.payment_method"),
4442 Some("charge"),
4443 )],
4444 );
4445
4446 let mut ctx = WorkflowContext::new("wf-charge-non-string-optional", "created");
4447 ctx.set(
4448 "order",
4449 serde_json::json!({
4450 "id": "booking-1",
4451 "total": 125_000,
4452 "payment_method": 100
4453 }),
4454 );
4455 let err = run_workflow(&executor, &wf, &mut ctx)
4456 .await
4457 .expect_err("non-string configured charge key must fail before provider call");
4458
4459 match err {
4460 WorkflowError::MissingContextKey(key) => {
4461 assert_eq!(key, "order.payment_method (expected string)");
4462 }
4463 other => panic!("expected missing context key error, got {other:?}"),
4464 }
4465 assert!(executor.charges.lock().unwrap().is_empty());
4466 }
4467
4468 #[tokio::test]
4469 async fn invalid_timeout_fallback_fails_before_pre_wait_side_effects() {
4470 let executor = MockExecutor::new();
4471
4472 let wf = WorkflowDefinition::new("invalid_timeout_fallback_guard")
4473 .initial_state("active")
4474 .transition(
4475 "active",
4476 "done",
4477 vec![
4478 WorkflowStep::notify(ChannelKind::Email, "before_wait", "customer.email"),
4479 WorkflowStep::wait_or(
4480 "payment.success",
4481 std::time::Duration::from_secs(3600),
4482 vec![WorkflowStep::wait(
4483 "manual_review",
4484 std::time::Duration::from_secs(3600),
4485 )],
4486 ),
4487 WorkflowStep::transition("done"),
4488 ],
4489 );
4490
4491 let mut ctx = WorkflowContext::new("wf-invalid-timeout-fallback", "active");
4492 ctx.set(
4493 "customer",
4494 serde_json::json!({"email": "guest@example.com"}),
4495 );
4496
4497 let err = run_workflow(&executor, &wf, &mut ctx)
4498 .await
4499 .expect_err("unsupported timeout fallback must fail before reaching the Wait");
4500
4501 match err {
4502 WorkflowError::Other(msg) => {
4503 assert!(
4504 msg.contains("Wait steps inside on_timeout fallback are not supported"),
4505 "got: {msg}"
4506 );
4507 }
4508 other => panic!("expected timeout fallback validation error, got {other:?}"),
4509 }
4510 assert!(ctx.cursor.is_none());
4511 assert!(
4512 executor.notifications.lock().unwrap().is_empty(),
4513 "definition validation must run before the pre-wait notification"
4514 );
4515 }
4516
4517 #[tokio::test]
4518 async fn nested_transition_stops_parent_block_before_follow_up_side_effects() {
4519 let executor = MockExecutor::new();
4520
4521 let wf = WorkflowDefinition::new("nested_transition_guard")
4522 .initial_state("active")
4523 .transition(
4524 "active",
4525 "fallback",
4526 vec![
4527 WorkflowStep::branch(
4528 "decision",
4529 vec![("accepted", vec![WorkflowStep::transition("confirmed")])],
4530 vec![],
4531 ),
4532 WorkflowStep::notify(
4533 ChannelKind::WhatsApp,
4534 "old_state_followup",
4535 "vendor.phone",
4536 ),
4537 ],
4538 );
4539
4540 let mut ctx = WorkflowContext::new("wf-nested-transition", "active");
4541 ctx.set("decision", Value::String("accepted".to_string()));
4542 ctx.set("vendor", serde_json::json!({"phone": "+628111"}));
4543
4544 let result = run_workflow(&executor, &wf, &mut ctx)
4545 .await
4546 .expect("nested transition should finish cleanly");
4547
4548 assert_eq!(result, "confirmed");
4549 assert_eq!(ctx.current_state, "confirmed");
4550 assert!(
4551 executor.notifications.lock().unwrap().is_empty(),
4552 "parent block must stop after nested transition changes state"
4553 );
4554 }
4555
4556 #[test]
4557 fn log_message_resolves_nested_context_placeholders() {
4558 let mut ctx = WorkflowContext::new("wf-log", "running");
4559 ctx.set(
4560 "item",
4561 serde_json::json!({
4562 "name": "Captain A",
4563 "phone": "+628111",
4564 }),
4565 );
4566 ctx.set("attempt", serde_json::json!(2));
4567
4568 assert_eq!(
4569 resolve_log_message(
4570 &ctx,
4571 "Processing {item.name} via {item.phone} attempt {attempt}"
4572 ),
4573 "Processing Captain A via +628111 attempt 2"
4574 );
4575 assert_eq!(
4576 resolve_log_message(&ctx, "Missing {item.email} stays"),
4577 "Missing {item.email} stays"
4578 );
4579 }
4580
4581 #[tokio::test]
4582 async fn test_workflow_for_each_notify() {
4583 let executor = MockExecutor::new();
4584
4585 let wf = WorkflowDefinition::new("broadcast")
4586 .initial_state("broadcasting")
4587 .transition(
4588 "broadcasting",
4589 "waiting",
4590 vec![WorkflowStep::for_each(
4591 "operators",
4592 vec![WorkflowStep::notify(
4593 ChannelKind::WhatsApp,
4594 "opportunity",
4595 "item.phone",
4596 )],
4597 )],
4598 );
4599
4600 let mut ctx = WorkflowContext::new("wf-003", "broadcasting");
4601 ctx.set(
4602 "operators",
4603 serde_json::json!([
4604 {"name": "Captain A", "phone": "+628111"},
4605 {"name": "Captain B", "phone": "+628222"},
4606 {"name": "Captain C", "phone": "+628333"},
4607 ]),
4608 );
4609
4610 let result = run_workflow(&executor, &wf, &mut ctx).await.unwrap();
4611 assert_eq!(result, "waiting");
4612
4613 let notifs = executor.notifications.lock().unwrap();
4615 assert_eq!(notifs.len(), 3);
4616 assert_eq!(notifs[0].0, "+628111");
4617 assert_eq!(notifs[1].0, "+628222");
4618 assert_eq!(notifs[2].0, "+628333");
4619 drop(notifs);
4620
4621 let side_effects = executor.completed_side_effects.lock().unwrap();
4622 assert_eq!(side_effects.len(), 3);
4623 for operation_id in side_effects.iter() {
4624 assert!(
4625 !operation_id.contains("+628"),
4626 "side-effect operation id must not leak item payload: {operation_id}"
4627 );
4628 assert!(
4629 !operation_id.contains("Captain"),
4630 "side-effect operation id must not leak item payload: {operation_id}"
4631 );
4632 }
4633 assert_eq!(
4634 side_effects.as_slice(),
4635 &[
4636 side_effect_operation_id(
4637 "wf-003",
4638 "broadcasting",
4639 0,
4640 WorkflowSideEffectKind::Notify,
4641 "steps[0]/for_each[0].steps[0]",
4642 ),
4643 side_effect_operation_id(
4644 "wf-003",
4645 "broadcasting",
4646 0,
4647 WorkflowSideEffectKind::Notify,
4648 "steps[0]/for_each[1].steps[0]",
4649 ),
4650 side_effect_operation_id(
4651 "wf-003",
4652 "broadcasting",
4653 0,
4654 WorkflowSideEffectKind::Notify,
4655 "steps[0]/for_each[2].steps[0]",
4656 ),
4657 ]
4658 );
4659 }
4660
4661 #[test]
4662 fn side_effect_operation_id_does_not_collide_on_delimiters() {
4663 let first = side_effect_operation_id(
4664 "tenant:wf",
4665 "broadcasting",
4666 7,
4667 WorkflowSideEffectKind::Notify,
4668 "steps[0]",
4669 );
4670 let second = side_effect_operation_id(
4671 "tenant",
4672 "wf:broadcasting",
4673 7,
4674 WorkflowSideEffectKind::Notify,
4675 "steps[0]",
4676 );
4677
4678 assert_ne!(
4679 first, second,
4680 "side-effect operation ids must not collide when fields contain delimiters"
4681 );
4682 }
4683
4684 #[tokio::test]
4685 async fn side_effect_operation_id_changes_when_state_is_reentered() {
4686 let executor = MockExecutor::new();
4687
4688 let wf = WorkflowDefinition::new("reentered_state_side_effect")
4689 .initial_state("active")
4690 .transition(
4691 "active",
4692 "idle",
4693 vec![
4694 WorkflowStep::notify(ChannelKind::Email, "active_notice", "customer.email"),
4695 WorkflowStep::transition("idle"),
4696 ],
4697 );
4698
4699 let mut ctx = WorkflowContext::new("wf-reentered-state", "active");
4700 ctx.set(
4701 "customer",
4702 serde_json::json!({"email": "guest@example.com"}),
4703 );
4704
4705 let first = run_workflow(&executor, &wf, &mut ctx).await.unwrap();
4706 assert_eq!(first, "idle");
4707 ctx.transition_to("active", Some("manual requeue".to_string()));
4708 let second = run_workflow(&executor, &wf, &mut ctx).await.unwrap();
4709 assert_eq!(second, "idle");
4710
4711 let notifications = executor.notifications.lock().unwrap();
4712 assert_eq!(
4713 notifications.len(),
4714 2,
4715 "same state re-entry must execute the side effect again"
4716 );
4717 drop(notifications);
4718
4719 let side_effects = executor.completed_side_effects.lock().unwrap();
4720 assert_eq!(
4721 side_effects.as_slice(),
4722 &[
4723 side_effect_operation_id(
4724 "wf-reentered-state",
4725 "active",
4726 0,
4727 WorkflowSideEffectKind::Notify,
4728 "steps[0]",
4729 ),
4730 side_effect_operation_id(
4731 "wf-reentered-state",
4732 "active",
4733 2,
4734 WorkflowSideEffectKind::Notify,
4735 "steps[0]",
4736 ),
4737 ]
4738 );
4739 }
4740
4741 #[tokio::test]
4742 async fn notify_with_missing_payload_key_fails_before_send() {
4743 let executor = MockExecutor::new();
4744
4745 let wf = WorkflowDefinition::new("notify_payload_guard")
4746 .initial_state("active")
4747 .transition(
4748 "active",
4749 "done",
4750 vec![WorkflowStep::notify_with_payload(
4751 ChannelKind::Email,
4752 "booking_confirmed",
4753 "customer.email",
4754 "booking.template_payload",
4755 )],
4756 );
4757
4758 let mut ctx = WorkflowContext::new("wf-notify-payload-guard", "active");
4759 ctx.set(
4760 "customer",
4761 serde_json::json!({"email": "guest@example.com"}),
4762 );
4763
4764 let err = run_workflow(&executor, &wf, &mut ctx)
4765 .await
4766 .expect_err("missing explicit notification payload must fail closed");
4767
4768 match err {
4769 WorkflowError::MissingContextKey(key) => assert_eq!(key, "booking.template_payload"),
4770 other => panic!("expected MissingContextKey, got {other:?}"),
4771 }
4772 assert!(
4773 executor.notifications.lock().unwrap().is_empty(),
4774 "notification provider must not receive an empty payload for a missing explicit payload key"
4775 );
4776 }
4777
4778 #[tokio::test]
4779 async fn test_workflow_pauses_at_wait() {
4780 let executor = MockExecutor::new();
4781
4782 let wf = WorkflowDefinition::new("wait_test")
4783 .initial_state("active")
4784 .transition(
4785 "active",
4786 "resolved",
4787 vec![
4788 WorkflowStep::log("Before wait"),
4789 WorkflowStep::wait("user_response", std::time::Duration::from_secs(3600)),
4790 WorkflowStep::log("After wait"),
4792 ],
4793 );
4794
4795 let mut ctx = WorkflowContext::new("wf-004", "active");
4796 let result = run_workflow(&executor, &wf, &mut ctx).await.unwrap();
4797
4798 assert_eq!(result, "active");
4800 }
4801
4802 #[tokio::test]
4803 async fn test_workflow_pauses_at_nested_branch_wait() {
4804 let executor = MockExecutor::new();
4805
4806 let wf = WorkflowDefinition::new("nested_branch_wait")
4807 .initial_state("active")
4808 .transition(
4809 "active",
4810 "resolved",
4811 vec![
4812 WorkflowStep::branch(
4813 "customer.tier",
4814 vec![(
4815 "vip",
4816 vec![WorkflowStep::wait(
4817 "vip_payment",
4818 std::time::Duration::from_secs(7200),
4819 )],
4820 )],
4821 vec![WorkflowStep::log("Standard customer")],
4822 ),
4823 WorkflowStep::notify(ChannelKind::Email, "booking_confirmed", "customer.email"),
4824 WorkflowStep::transition("resolved"),
4825 ],
4826 );
4827
4828 let mut ctx = WorkflowContext::new("wf-nested-branch-wait", "active");
4829 ctx.set(
4830 "customer",
4831 serde_json::json!({
4832 "tier": "vip",
4833 "email": "guest@example.com"
4834 }),
4835 );
4836
4837 let result = run_workflow(&executor, &wf, &mut ctx).await.unwrap();
4838 assert_eq!(result, "active");
4839 assert_eq!(ctx.current_state, "active");
4840
4841 let notifs = executor.notifications.lock().unwrap();
4842 assert!(
4843 notifs.is_empty(),
4844 "steps after a nested branch Wait must not execute"
4845 );
4846 }
4847
4848 #[tokio::test]
4849 async fn branch_resume_rejects_reordered_branch_definition() {
4850 let executor = MockExecutor::new();
4851
4852 let wf = WorkflowDefinition::new("branch_reorder_source")
4853 .initial_state("active")
4854 .transition(
4855 "active",
4856 "done",
4857 vec![WorkflowStep::branch(
4858 "customer.tier",
4859 vec![
4860 (
4861 "vip",
4862 vec![
4863 WorkflowStep::wait(
4864 "approved",
4865 std::time::Duration::from_secs(3600),
4866 ),
4867 WorkflowStep::notify(
4868 ChannelKind::Email,
4869 "vip_approved",
4870 "customer.email",
4871 ),
4872 ],
4873 ),
4874 (
4875 "standard",
4876 vec![
4877 WorkflowStep::wait(
4878 "approved",
4879 std::time::Duration::from_secs(3600),
4880 ),
4881 WorkflowStep::notify(
4882 ChannelKind::Email,
4883 "standard_approved",
4884 "customer.email",
4885 ),
4886 ],
4887 ),
4888 ],
4889 vec![],
4890 )],
4891 );
4892
4893 let mut ctx = WorkflowContext::new("wf-branch-reorder", "active");
4894 ctx.set(
4895 "customer",
4896 serde_json::json!({
4897 "tier": "vip",
4898 "email": "guest@example.com"
4899 }),
4900 );
4901
4902 let result = run_workflow(&executor, &wf, &mut ctx).await.unwrap();
4903 assert_eq!(result, "active");
4904
4905 let drifted_wf = WorkflowDefinition::new("branch_reorder_source")
4906 .initial_state("active")
4907 .transition(
4908 "active",
4909 "done",
4910 vec![WorkflowStep::branch(
4911 "customer.tier",
4912 vec![
4913 (
4914 "standard",
4915 vec![
4916 WorkflowStep::wait(
4917 "approved",
4918 std::time::Duration::from_secs(3600),
4919 ),
4920 WorkflowStep::notify(
4921 ChannelKind::Email,
4922 "standard_approved",
4923 "customer.email",
4924 ),
4925 ],
4926 ),
4927 (
4928 "vip",
4929 vec![
4930 WorkflowStep::wait(
4931 "approved",
4932 std::time::Duration::from_secs(3600),
4933 ),
4934 WorkflowStep::notify(
4935 ChannelKind::Email,
4936 "vip_approved",
4937 "customer.email",
4938 ),
4939 ],
4940 ),
4941 ],
4942 vec![],
4943 )],
4944 );
4945
4946 let err = resume_workflow(
4947 &executor,
4948 &drifted_wf,
4949 "wf-branch-reorder",
4950 serde_json::json!({"event": "approved"}),
4951 )
4952 .await
4953 .expect_err("branch definition reorder must not silently resume wrong arm");
4954
4955 match err {
4956 WorkflowError::Other(msg) => {
4957 assert!(msg.contains("Invalid workflow resume cursor"));
4958 assert!(msg.contains("standard"));
4959 assert!(msg.contains("vip"));
4960 }
4961 other => panic!("expected invalid cursor error, got {other:?}"),
4962 }
4963 assert!(
4964 executor.notifications.lock().unwrap().is_empty(),
4965 "drifted resume must fail before sending the wrong branch notification"
4966 );
4967 }
4968
4969 #[tokio::test]
4970 async fn branch_resume_rejects_default_branch_drift() {
4971 let executor = MockExecutor::new();
4972
4973 let wf = WorkflowDefinition::new("branch_default_source")
4974 .initial_state("active")
4975 .transition(
4976 "active",
4977 "done",
4978 vec![WorkflowStep::branch(
4979 "customer.tier",
4980 vec![(
4981 "vip",
4982 vec![WorkflowStep::notify(
4983 ChannelKind::Email,
4984 "vip",
4985 "customer.email",
4986 )],
4987 )],
4988 vec![
4989 WorkflowStep::wait("approved", std::time::Duration::from_secs(3600)),
4990 WorkflowStep::notify(ChannelKind::Email, "default", "customer.email"),
4991 ],
4992 )],
4993 );
4994
4995 let mut ctx = WorkflowContext::new("wf-branch-default-drift", "active");
4996 ctx.set(
4997 "customer",
4998 serde_json::json!({
4999 "tier": "gold",
5000 "email": "guest@example.com"
5001 }),
5002 );
5003
5004 let result = run_workflow(&executor, &wf, &mut ctx).await.unwrap();
5005 assert_eq!(result, "active");
5006
5007 let drifted_wf = WorkflowDefinition::new("branch_default_source")
5008 .initial_state("active")
5009 .transition(
5010 "active",
5011 "done",
5012 vec![WorkflowStep::branch(
5013 "customer.tier",
5014 vec![
5015 (
5016 "vip",
5017 vec![WorkflowStep::notify(
5018 ChannelKind::Email,
5019 "vip",
5020 "customer.email",
5021 )],
5022 ),
5023 (
5024 "gold",
5025 vec![WorkflowStep::notify(
5026 ChannelKind::Email,
5027 "gold",
5028 "customer.email",
5029 )],
5030 ),
5031 ],
5032 vec![
5033 WorkflowStep::wait("approved", std::time::Duration::from_secs(3600)),
5034 WorkflowStep::notify(ChannelKind::Email, "default", "customer.email"),
5035 ],
5036 )],
5037 );
5038
5039 let err = resume_workflow(
5040 &executor,
5041 &drifted_wf,
5042 "wf-branch-default-drift",
5043 serde_json::json!({"event": "approved"}),
5044 )
5045 .await
5046 .expect_err("default branch cursor must reject new matching branch arms");
5047
5048 match err {
5049 WorkflowError::Other(msg) => {
5050 assert!(msg.contains("default branch cursor"));
5051 assert!(msg.contains("gold"));
5052 }
5053 other => panic!("expected invalid cursor error, got {other:?}"),
5054 }
5055 assert!(executor.notifications.lock().unwrap().is_empty());
5056 }
5057
5058 #[tokio::test]
5059 async fn test_workflow_pauses_at_nested_for_each_wait() {
5060 let executor = MockExecutor::new();
5061
5062 let wf = WorkflowDefinition::new("nested_for_each_wait")
5063 .initial_state("active")
5064 .transition(
5065 "active",
5066 "resolved",
5067 vec![
5068 WorkflowStep::for_each(
5069 "operators",
5070 vec![
5071 WorkflowStep::wait(
5072 "operator_accept",
5073 std::time::Duration::from_secs(3600),
5074 ),
5075 WorkflowStep::notify(ChannelKind::WhatsApp, "after_wait", "item.phone"),
5076 ],
5077 ),
5078 WorkflowStep::transition("resolved"),
5079 ],
5080 );
5081
5082 let mut ctx = WorkflowContext::new("wf-nested-for-each-wait", "active");
5083 ctx.set(
5084 "operators",
5085 serde_json::json!([
5086 {"name": "Captain A", "phone": "+628111"},
5087 {"name": "Captain B", "phone": "+628222"}
5088 ]),
5089 );
5090
5091 let result = run_workflow(&executor, &wf, &mut ctx).await.unwrap();
5092 assert_eq!(result, "active");
5093 assert_eq!(ctx.current_state, "active");
5094
5095 let notifs = executor.notifications.lock().unwrap();
5096 assert!(
5097 notifs.is_empty(),
5098 "steps after a nested for_each Wait must not execute"
5099 );
5100 }
5101
5102 #[tokio::test]
5103 async fn test_workflow_resumes_for_each_after_wait_without_replaying_items() {
5104 let executor = MockExecutor::new();
5105
5106 let wf = WorkflowDefinition::new("for_each_resume")
5107 .initial_state("active")
5108 .transition(
5109 "active",
5110 "resolved",
5111 vec![
5112 WorkflowStep::for_each(
5113 "operators",
5114 vec![
5115 WorkflowStep::notify(
5116 ChannelKind::WhatsApp,
5117 "before_wait",
5118 "item.phone",
5119 ),
5120 WorkflowStep::wait(
5121 "operator_accept",
5122 std::time::Duration::from_secs(3600),
5123 ),
5124 WorkflowStep::notify(ChannelKind::WhatsApp, "after_wait", "item.phone"),
5125 ],
5126 ),
5127 WorkflowStep::transition("resolved"),
5128 ],
5129 );
5130
5131 let mut ctx = WorkflowContext::new("wf-for-each-resume", "active");
5132 ctx.set(
5133 "operators",
5134 serde_json::json!([
5135 {"name": "Captain A", "phone": "+628111"},
5136 {"name": "Captain B", "phone": "+628222"}
5137 ]),
5138 );
5139
5140 let result = run_workflow(&executor, &wf, &mut ctx).await.unwrap();
5141 assert_eq!(result, "active");
5142 assert!(
5143 ctx.cursor.is_some(),
5144 "first pause should persist a resume cursor"
5145 );
5146
5147 {
5148 let notifs = executor.notifications.lock().unwrap();
5149 assert_eq!(
5150 notifs.as_slice(),
5151 &[("+628111".to_string(), "before_wait".to_string())]
5152 );
5153 }
5154
5155 let result = resume_workflow(
5156 &executor,
5157 &wf,
5158 "wf-for-each-resume",
5159 serde_json::json!({"event": "operator_accept", "accepted": true}),
5160 )
5161 .await
5162 .unwrap();
5163 assert_eq!(result, "active");
5164
5165 {
5166 let notifs = executor.notifications.lock().unwrap();
5167 assert_eq!(
5168 notifs.as_slice(),
5169 &[
5170 ("+628111".to_string(), "before_wait".to_string()),
5171 ("+628111".to_string(), "after_wait".to_string()),
5172 ("+628222".to_string(), "before_wait".to_string()),
5173 ],
5174 "resume must continue within the active item and must not replay item 0"
5175 );
5176 }
5177
5178 let result = resume_workflow(
5179 &executor,
5180 &wf,
5181 "wf-for-each-resume",
5182 serde_json::json!({"event": "operator_accept", "accepted": true}),
5183 )
5184 .await
5185 .unwrap();
5186 assert_eq!(result, "resolved");
5187
5188 let notifs = executor.notifications.lock().unwrap();
5189 assert_eq!(
5190 notifs.as_slice(),
5191 &[
5192 ("+628111".to_string(), "before_wait".to_string()),
5193 ("+628111".to_string(), "after_wait".to_string()),
5194 ("+628222".to_string(), "before_wait".to_string()),
5195 ("+628222".to_string(), "after_wait".to_string()),
5196 ],
5197 "each side effect should run exactly once per operator"
5198 );
5199
5200 let saved = executor.saved_state.lock().unwrap();
5201 let saved = saved.as_ref().expect("final state should be saved");
5202 assert_eq!(saved.current_state, "resolved");
5203 assert!(
5204 saved.cursor.is_none(),
5205 "completed workflow should not retain a resume cursor"
5206 );
5207 }
5208
5209 #[tokio::test]
5210 async fn for_each_resume_rejects_reordered_items_before_notification() {
5211 let executor = MockExecutor::new();
5212
5213 let wf = WorkflowDefinition::new("for_each_reorder_guard")
5214 .initial_state("active")
5215 .transition(
5216 "active",
5217 "resolved",
5218 vec![
5219 WorkflowStep::for_each(
5220 "operators",
5221 vec![
5222 WorkflowStep::wait(
5223 "operator_accept",
5224 std::time::Duration::from_secs(3600),
5225 ),
5226 WorkflowStep::notify(ChannelKind::WhatsApp, "accepted", "item.phone"),
5227 ],
5228 ),
5229 WorkflowStep::transition("resolved"),
5230 ],
5231 );
5232
5233 let mut ctx = WorkflowContext::new("wf-for-each-reorder", "active");
5234 ctx.set(
5235 "operators",
5236 serde_json::json!([
5237 {"name": "Captain A", "phone": "+628111"},
5238 {"name": "Captain B", "phone": "+628222"}
5239 ]),
5240 );
5241
5242 let result = run_workflow(&executor, &wf, &mut ctx).await.unwrap();
5243 assert_eq!(result, "active");
5244
5245 {
5246 let mut saved = executor.saved_state.lock().unwrap();
5247 let saved = saved.as_mut().expect("paused state should be saved");
5248 saved.set(
5249 "operators",
5250 serde_json::json!([
5251 {"name": "Captain B", "phone": "+628222"},
5252 {"name": "Captain A", "phone": "+628111"}
5253 ]),
5254 );
5255 }
5256
5257 let err = resume_workflow(
5258 &executor,
5259 &wf,
5260 "wf-for-each-reorder",
5261 serde_json::json!({"event": "operator_accept"}),
5262 )
5263 .await
5264 .expect_err("reordered operator list must not resume on the wrong item");
5265
5266 match err {
5267 WorkflowError::Other(msg) => {
5268 assert!(msg.contains("Invalid workflow resume cursor"), "got: {msg}");
5269 assert!(msg.contains("for_each cursor item"), "got: {msg}");
5270 assert!(msg.contains("changed"), "got: {msg}");
5271 }
5272 other => panic!("expected invalid cursor error, got {other:?}"),
5273 }
5274 assert!(
5275 executor.notifications.lock().unwrap().is_empty(),
5276 "wrong operator must not receive the post-wait notification"
5277 );
5278 }
5279
5280 #[tokio::test]
5281 async fn legacy_for_each_cursor_is_rejected_before_notification() {
5282 let executor = MockExecutor::new();
5283
5284 let wf = WorkflowDefinition::new("legacy_for_each_cursor_guard")
5285 .initial_state("active")
5286 .transition(
5287 "active",
5288 "resolved",
5289 vec![
5290 WorkflowStep::for_each(
5291 "operators",
5292 vec![
5293 WorkflowStep::wait(
5294 "operator_accept",
5295 std::time::Duration::from_secs(3600),
5296 ),
5297 WorkflowStep::notify(ChannelKind::WhatsApp, "accepted", "item.phone"),
5298 ],
5299 ),
5300 WorkflowStep::transition("resolved"),
5301 ],
5302 );
5303
5304 let mut ctx = WorkflowContext::new("wf-legacy-for-each-cursor", "active");
5305 ctx.set(
5306 "operators",
5307 serde_json::json!([
5308 {"name": "Captain A", "phone": "+628111"},
5309 {"name": "Captain B", "phone": "+628222"}
5310 ]),
5311 );
5312 ctx.set_cursor(WorkflowCursor {
5313 state: "active".to_string(),
5314 frames: vec![
5315 WorkflowCursorFrame::Steps { index: 0 },
5316 WorkflowCursorFrame::ForEach {
5317 item_index: 1,
5318 index: 1,
5319 },
5320 ],
5321 wait: Some(WorkflowPendingWait {
5322 event: "operator_accept".to_string(),
5323 deadline_at: Utc::now() + chrono::Duration::hours(1),
5324 on_timeout: vec![],
5325 }),
5326 });
5327 executor.save_state(&ctx).await.unwrap();
5328
5329 let err = resume_workflow(
5330 &executor,
5331 &wf,
5332 "wf-legacy-for-each-cursor",
5333 serde_json::json!({"event": "operator_accept"}),
5334 )
5335 .await
5336 .expect_err("legacy for_each cursors must fail closed");
5337
5338 match err {
5339 WorkflowError::Other(msg) => {
5340 assert!(msg.contains("Invalid workflow resume cursor"), "got: {msg}");
5341 assert!(msg.contains("legacy for_each cursor"), "got: {msg}");
5342 }
5343 other => panic!("expected invalid cursor error, got {other:?}"),
5344 }
5345 assert!(
5346 executor.notifications.lock().unwrap().is_empty(),
5347 "legacy cursor must not resume onto an unverified loop item"
5348 );
5349 }
5350
5351 #[tokio::test]
5352 async fn test_nested_for_each_restores_outer_item_binding() {
5353 let executor = MockExecutor::new();
5354
5355 let wf = WorkflowDefinition::new("nested_for_each_items")
5356 .initial_state("active")
5357 .transition(
5358 "active",
5359 "done",
5360 vec![
5361 WorkflowStep::for_each(
5362 "operators",
5363 vec![
5364 WorkflowStep::for_each(
5365 "tasks",
5366 vec![WorkflowStep::notify(
5367 ChannelKind::WhatsApp,
5368 "task",
5369 "item.phone",
5370 )],
5371 ),
5372 WorkflowStep::notify(ChannelKind::WhatsApp, "summary", "item.phone"),
5373 ],
5374 ),
5375 WorkflowStep::transition("done"),
5376 ],
5377 );
5378
5379 let mut ctx = WorkflowContext::new("wf-nested-items", "active");
5380 ctx.set(
5381 "operators",
5382 serde_json::json!([
5383 {"name": "Captain A", "phone": "+628111"},
5384 {"name": "Captain B", "phone": "+628222"}
5385 ]),
5386 );
5387 ctx.set(
5388 "tasks",
5389 serde_json::json!([
5390 {"name": "Task 1", "phone": "+629999"}
5391 ]),
5392 );
5393
5394 let result = run_workflow(&executor, &wf, &mut ctx).await.unwrap();
5395 assert_eq!(result, "done");
5396
5397 let notifs = executor.notifications.lock().unwrap();
5398 assert_eq!(
5399 notifs.as_slice(),
5400 &[
5401 ("+629999".to_string(), "task".to_string()),
5402 ("+628111".to_string(), "summary".to_string()),
5403 ("+629999".to_string(), "task".to_string()),
5404 ("+628222".to_string(), "summary".to_string()),
5405 ],
5406 "inner ForEach must not delete the outer item binding"
5407 );
5408 }
5409
5410 #[tokio::test]
5411 async fn test_nested_for_each_wait_restores_item_binding_across_resume() {
5412 let executor = MockExecutor::new();
5413
5414 let wf = WorkflowDefinition::new("nested_for_each_wait_items")
5415 .initial_state("active")
5416 .transition(
5417 "active",
5418 "done",
5419 vec![
5420 WorkflowStep::for_each(
5421 "operators",
5422 vec![
5423 WorkflowStep::for_each(
5424 "tasks",
5425 vec![
5426 WorkflowStep::wait(
5427 "task_done",
5428 std::time::Duration::from_secs(3600),
5429 ),
5430 WorkflowStep::notify(
5431 ChannelKind::WhatsApp,
5432 "task",
5433 "item.phone",
5434 ),
5435 ],
5436 ),
5437 WorkflowStep::notify(ChannelKind::WhatsApp, "summary", "item.phone"),
5438 ],
5439 ),
5440 WorkflowStep::transition("done"),
5441 ],
5442 );
5443
5444 let mut ctx = WorkflowContext::new("wf-nested-wait-items", "active");
5445 ctx.set(
5446 "operators",
5447 serde_json::json!([
5448 {"name": "Captain A", "phone": "+628111"},
5449 {"name": "Captain B", "phone": "+628222"}
5450 ]),
5451 );
5452 ctx.set(
5453 "tasks",
5454 serde_json::json!([
5455 {"name": "Task 1", "phone": "+629999"}
5456 ]),
5457 );
5458
5459 let result = run_workflow(&executor, &wf, &mut ctx).await.unwrap();
5460 assert_eq!(result, "active");
5461 assert!(
5462 ctx.get("item").is_none(),
5463 "paused workflow state must not persist an active loop item"
5464 );
5465
5466 let result = resume_workflow(
5467 &executor,
5468 &wf,
5469 "wf-nested-wait-items",
5470 serde_json::json!({"event": "task_done"}),
5471 )
5472 .await
5473 .unwrap();
5474 assert_eq!(result, "active");
5475 assert!(
5476 executor
5477 .saved_state
5478 .lock()
5479 .unwrap()
5480 .as_ref()
5481 .and_then(|ctx| ctx.get("item"))
5482 .is_none(),
5483 "paused resume state must keep item reconstruction cursor-driven"
5484 );
5485
5486 {
5487 let notifs = executor.notifications.lock().unwrap();
5488 assert_eq!(
5489 notifs.as_slice(),
5490 &[
5491 ("+629999".to_string(), "task".to_string()),
5492 ("+628111".to_string(), "summary".to_string()),
5493 ],
5494 "inner wait resume must restore the outer item before summary"
5495 );
5496 }
5497
5498 let result = resume_workflow(
5499 &executor,
5500 &wf,
5501 "wf-nested-wait-items",
5502 serde_json::json!({"event": "task_done"}),
5503 )
5504 .await
5505 .unwrap();
5506 assert_eq!(result, "done");
5507
5508 let notifs = executor.notifications.lock().unwrap();
5509 assert_eq!(
5510 notifs.as_slice(),
5511 &[
5512 ("+629999".to_string(), "task".to_string()),
5513 ("+628111".to_string(), "summary".to_string()),
5514 ("+629999".to_string(), "task".to_string()),
5515 ("+628222".to_string(), "summary".to_string()),
5516 ],
5517 "nested for_each wait must not leak the inner item into the outer loop"
5518 );
5519 assert!(
5520 executor
5521 .saved_state
5522 .lock()
5523 .unwrap()
5524 .as_ref()
5525 .and_then(|ctx| ctx.get("item"))
5526 .is_none(),
5527 "completed workflow state must not retain a stale loop item"
5528 );
5529 }
5530
5531 #[tokio::test]
5532 async fn test_resume_rejects_unexpected_wait_event() {
5533 let executor = MockExecutor::new();
5534
5535 let wf = WorkflowDefinition::new("wait_event_guard")
5536 .initial_state("active")
5537 .transition(
5538 "active",
5539 "done",
5540 vec![
5541 WorkflowStep::wait("payment.success", std::time::Duration::from_secs(3600)),
5542 WorkflowStep::notify(ChannelKind::Email, "paid", "customer.email"),
5543 WorkflowStep::transition("done"),
5544 ],
5545 );
5546
5547 let mut ctx = WorkflowContext::new("wf-event-guard", "active");
5548 ctx.set(
5549 "customer",
5550 serde_json::json!({"email": "guest@example.com"}),
5551 );
5552 let result = run_workflow(&executor, &wf, &mut ctx).await.unwrap();
5553 assert_eq!(result, "active");
5554
5555 let wait = ctx
5556 .cursor
5557 .as_ref()
5558 .and_then(|cursor| cursor.wait.as_ref())
5559 .expect("wait metadata should be persisted");
5560 assert_eq!(wait.event, "payment.success");
5561
5562 let err = resume_workflow(
5563 &executor,
5564 &wf,
5565 "wf-event-guard",
5566 serde_json::json!({"event": "operator.declined"}),
5567 )
5568 .await
5569 .expect_err("wrong event must not resume the workflow");
5570
5571 match err {
5572 WorkflowError::Other(msg) => {
5573 assert!(msg.contains("payment.success"));
5574 assert!(msg.contains("operator.declined"));
5575 }
5576 other => panic!("expected event mismatch error, got: {other}"),
5577 }
5578 assert!(executor.notifications.lock().unwrap().is_empty());
5579 }
5580
5581 #[tokio::test]
5582 async fn resume_event_rejects_padded_event_name_before_replay() {
5583 let executor = MockExecutor::new();
5584
5585 let wf = WorkflowDefinition::new("wait_event_padding_guard")
5586 .initial_state("active")
5587 .transition(
5588 "active",
5589 "done",
5590 vec![
5591 WorkflowStep::wait("payment.success", std::time::Duration::from_secs(3600)),
5592 WorkflowStep::notify(ChannelKind::Email, "paid", "customer.email"),
5593 WorkflowStep::transition("done"),
5594 ],
5595 );
5596
5597 let mut ctx = WorkflowContext::new("wf-event-padding-guard", "active");
5598 ctx.set(
5599 "customer",
5600 serde_json::json!({"email": "guest@example.com"}),
5601 );
5602 let result = run_workflow(&executor, &wf, &mut ctx).await.unwrap();
5603 assert_eq!(result, "active");
5604
5605 let err = resume_workflow(
5606 &executor,
5607 &wf,
5608 "wf-event-padding-guard",
5609 serde_json::json!({"event": " payment.success "}),
5610 )
5611 .await
5612 .expect_err("padded event name must not resume the workflow");
5613
5614 match err {
5615 WorkflowError::Other(msg) => {
5616 assert!(msg.contains("leading or trailing whitespace"), "got: {msg}");
5617 }
5618 other => panic!("expected padded event error, got {other:?}"),
5619 }
5620 assert!(
5621 executor.notifications.lock().unwrap().is_empty(),
5622 "padded resume event must fail before notification replay"
5623 );
5624 }
5625
5626 #[tokio::test]
5627 async fn resume_event_without_wait_cursor_does_not_run_transition() {
5628 let executor = MockExecutor::new();
5629
5630 let wf = WorkflowDefinition::new("vendor_callback_guard")
5631 .initial_state("awaiting_vendor")
5632 .transition(
5633 "awaiting_vendor",
5634 "notified",
5635 vec![
5636 WorkflowStep::notify(ChannelKind::WhatsApp, "vendor_ready", "vendor.phone"),
5637 WorkflowStep::transition("notified"),
5638 ],
5639 );
5640
5641 let mut ctx = WorkflowContext::new("wf-no-wait-cursor", "awaiting_vendor");
5642 ctx.set("vendor", serde_json::json!({"phone": "+628111"}));
5643 executor.save_state(&ctx).await.unwrap();
5644
5645 let err = resume_workflow(
5646 &executor,
5647 &wf,
5648 "wf-no-wait-cursor",
5649 serde_json::json!({"event": "vendor.ready"}),
5650 )
5651 .await
5652 .expect_err("event resume must not run a workflow with no wait cursor");
5653
5654 match err {
5655 WorkflowError::Other(msg) => {
5656 assert!(
5657 msg.contains("not waiting for an external event"),
5658 "got: {msg}"
5659 );
5660 }
5661 other => panic!("expected not-waiting error, got {other:?}"),
5662 }
5663 assert!(
5664 executor.notifications.lock().unwrap().is_empty(),
5665 "webhook resume without a wait cursor must not send vendor notifications"
5666 );
5667 }
5668
5669 #[tokio::test]
5670 async fn resume_event_rejects_stale_cursor_state_before_transition() {
5671 let executor = MockExecutor::new();
5672
5673 let wf = WorkflowDefinition::new("stale_cursor_guard")
5674 .initial_state("awaiting_vendor")
5675 .transition(
5676 "awaiting_vendor",
5677 "notified",
5678 vec![
5679 WorkflowStep::notify(ChannelKind::WhatsApp, "vendor_ready", "vendor.phone"),
5680 WorkflowStep::transition("notified"),
5681 ],
5682 );
5683
5684 let mut ctx = WorkflowContext::new("wf-stale-cursor", "awaiting_vendor");
5685 ctx.set("vendor", serde_json::json!({"phone": "+628111"}));
5686 ctx.set_cursor(WorkflowCursor {
5687 state: "old_state".to_string(),
5688 frames: vec![WorkflowCursorFrame::Steps { index: 1 }],
5689 wait: Some(WorkflowPendingWait {
5690 event: "vendor.ready".to_string(),
5691 deadline_at: Utc::now() + chrono::Duration::hours(1),
5692 on_timeout: vec![],
5693 }),
5694 });
5695 executor.save_state(&ctx).await.unwrap();
5696
5697 let err = resume_workflow(
5698 &executor,
5699 &wf,
5700 "wf-stale-cursor",
5701 serde_json::json!({"event": "vendor.ready"}),
5702 )
5703 .await
5704 .expect_err("stale cursor state must not resume the current transition");
5705
5706 match err {
5707 WorkflowError::Other(msg) => {
5708 assert!(msg.contains("Invalid workflow resume cursor"), "got: {msg}");
5709 assert!(msg.contains("state does not match"), "got: {msg}");
5710 }
5711 other => panic!("expected invalid cursor error, got {other:?}"),
5712 }
5713 assert!(
5714 executor.notifications.lock().unwrap().is_empty(),
5715 "stale cursor mismatch must fail before side effects"
5716 );
5717 }
5718
5719 #[tokio::test]
5720 async fn normal_resume_rejects_empty_cursor_frames_before_replay() {
5721 let executor = MockExecutor::new();
5722
5723 let wf = WorkflowDefinition::new("empty_cursor_guard")
5724 .initial_state("active")
5725 .transition(
5726 "active",
5727 "done",
5728 vec![
5729 WorkflowStep::notify(ChannelKind::WhatsApp, "vendor_ready", "vendor.phone"),
5730 WorkflowStep::transition("done"),
5731 ],
5732 );
5733
5734 let mut ctx = WorkflowContext::new("wf-empty-cursor", "active");
5735 ctx.set("vendor", serde_json::json!({"phone": "+628111"}));
5736 ctx.set_cursor(WorkflowCursor {
5737 state: "active".to_string(),
5738 frames: vec![],
5739 wait: None,
5740 });
5741
5742 let err = run_workflow(&executor, &wf, &mut ctx)
5743 .await
5744 .expect_err("empty cursor frames must not replay the transition from step zero");
5745
5746 match err {
5747 WorkflowError::Other(msg) => {
5748 assert!(msg.contains("Invalid workflow resume cursor"), "got: {msg}");
5749 assert!(msg.contains("no frames"), "got: {msg}");
5750 }
5751 other => panic!("expected invalid cursor error, got {other:?}"),
5752 }
5753 assert!(
5754 executor.notifications.lock().unwrap().is_empty(),
5755 "empty checkpoint cursor must fail before replaying notifications"
5756 );
5757 }
5758
5759 #[tokio::test]
5760 async fn event_resume_rejects_empty_wait_cursor_frames_before_replay() {
5761 let executor = MockExecutor::new();
5762
5763 let wf = WorkflowDefinition::new("empty_wait_cursor_guard")
5764 .initial_state("active")
5765 .transition(
5766 "active",
5767 "done",
5768 vec![
5769 WorkflowStep::wait("vendor.ready", std::time::Duration::from_secs(3600)),
5770 WorkflowStep::notify(ChannelKind::WhatsApp, "vendor_ready", "vendor.phone"),
5771 WorkflowStep::transition("done"),
5772 ],
5773 );
5774
5775 let mut ctx = WorkflowContext::new("wf-empty-wait-cursor", "active");
5776 ctx.set("vendor", serde_json::json!({"phone": "+628111"}));
5777 ctx.set_cursor(WorkflowCursor {
5778 state: "active".to_string(),
5779 frames: vec![],
5780 wait: Some(WorkflowPendingWait {
5781 event: "vendor.ready".to_string(),
5782 deadline_at: Utc::now() + chrono::Duration::hours(1),
5783 on_timeout: vec![],
5784 }),
5785 });
5786 executor.save_state(&ctx).await.unwrap();
5787
5788 let err = resume_workflow(
5789 &executor,
5790 &wf,
5791 "wf-empty-wait-cursor",
5792 serde_json::json!({"event": "vendor.ready"}),
5793 )
5794 .await
5795 .expect_err("empty wait cursor frames must not resume from step zero");
5796
5797 match err {
5798 WorkflowError::Other(msg) => {
5799 assert!(msg.contains("Invalid workflow resume cursor"), "got: {msg}");
5800 assert!(msg.contains("no frames"), "got: {msg}");
5801 }
5802 other => panic!("expected invalid cursor error, got {other:?}"),
5803 }
5804 assert!(
5805 executor.notifications.lock().unwrap().is_empty(),
5806 "empty wait cursor must fail before replaying notifications"
5807 );
5808 }
5809
5810 #[tokio::test]
5811 async fn wait_with_empty_event_name_is_rejected_before_pause() {
5812 let executor = MockExecutor::new();
5813
5814 let wf = WorkflowDefinition::new("empty_wait_event")
5815 .initial_state("active")
5816 .transition(
5817 "active",
5818 "done",
5819 vec![
5820 WorkflowStep::wait("", std::time::Duration::from_secs(3600)),
5821 WorkflowStep::notify(ChannelKind::Email, "after_wait", "customer.email"),
5822 ],
5823 );
5824
5825 let mut ctx = WorkflowContext::new("wf-empty-wait", "active");
5826 ctx.set(
5827 "customer",
5828 serde_json::json!({"email": "guest@example.com"}),
5829 );
5830
5831 let err = run_workflow(&executor, &wf, &mut ctx)
5832 .await
5833 .expect_err("empty wait event must fail before persisting a pause");
5834
5835 match err {
5836 WorkflowError::Other(msg) => {
5837 assert!(
5838 msg.contains("Wait event name must not be empty"),
5839 "got: {msg}"
5840 );
5841 }
5842 other => panic!("expected empty event error, got {other:?}"),
5843 }
5844 assert!(ctx.cursor.is_none());
5845 assert!(executor.notifications.lock().unwrap().is_empty());
5846 }
5847
5848 #[tokio::test]
5849 async fn wait_with_padded_event_name_is_rejected_before_pause() {
5850 let executor = MockExecutor::new();
5851
5852 let wf = WorkflowDefinition::new("padded_wait_event")
5853 .initial_state("active")
5854 .transition(
5855 "active",
5856 "done",
5857 vec![
5858 WorkflowStep::wait(" payment.success ", std::time::Duration::from_secs(3600)),
5859 WorkflowStep::notify(ChannelKind::Email, "after_wait", "customer.email"),
5860 ],
5861 );
5862
5863 let mut ctx = WorkflowContext::new("wf-padded-wait", "active");
5864 ctx.set(
5865 "customer",
5866 serde_json::json!({"email": "guest@example.com"}),
5867 );
5868
5869 let err = run_workflow(&executor, &wf, &mut ctx)
5870 .await
5871 .expect_err("padded wait event must fail before persisting a pause");
5872
5873 match err {
5874 WorkflowError::Other(msg) => {
5875 assert!(msg.contains("leading or trailing whitespace"), "got: {msg}");
5876 }
5877 other => panic!("expected padded event error, got {other:?}"),
5878 }
5879 assert!(ctx.cursor.is_none());
5880 assert!(executor.notifications.lock().unwrap().is_empty());
5881 }
5882
5883 #[tokio::test]
5884 async fn transition_must_be_final_to_avoid_lost_follow_up_side_effects() {
5885 let executor = MockExecutor::new();
5886
5887 let wf = WorkflowDefinition::new("transition_order_guard")
5888 .initial_state("active")
5889 .transition(
5890 "active",
5891 "done",
5892 vec![
5893 WorkflowStep::transition("done"),
5894 WorkflowStep::notify(ChannelKind::WhatsApp, "after_transition", "vendor.phone"),
5895 ],
5896 );
5897
5898 let mut ctx = WorkflowContext::new("wf-transition-order", "active");
5899 ctx.set("vendor", serde_json::json!({"phone": "+628111"}));
5900
5901 let err = run_workflow(&executor, &wf, &mut ctx)
5902 .await
5903 .expect_err("transition before side effects must fail closed");
5904
5905 match err {
5906 WorkflowError::Other(msg) => {
5907 assert!(
5908 msg.contains("Transition steps must be the final step"),
5909 "got: {msg}"
5910 );
5911 }
5912 other => panic!("expected transition order error, got {other:?}"),
5913 }
5914 assert_eq!(ctx.current_state, "active");
5915 assert!(executor.notifications.lock().unwrap().is_empty());
5916 }
5917
5918 #[tokio::test]
5919 async fn test_timeout_workflow_executes_on_timeout_fallback() {
5920 let executor = MockExecutor::new();
5921
5922 let wf = WorkflowDefinition::new("wait_timeout")
5923 .initial_state("active")
5924 .transition(
5925 "active",
5926 "done",
5927 vec![
5928 WorkflowStep::wait_or(
5929 "payment.success",
5930 std::time::Duration::from_secs(0),
5931 vec![
5932 WorkflowStep::notify(
5933 ChannelKind::Email,
5934 "payment_timeout",
5935 "customer.email",
5936 ),
5937 WorkflowStep::transition("timed_out"),
5938 ],
5939 ),
5940 WorkflowStep::notify(ChannelKind::Email, "paid", "customer.email"),
5941 WorkflowStep::transition("done"),
5942 ],
5943 );
5944
5945 let mut ctx = WorkflowContext::new("wf-timeout", "active");
5946 ctx.set(
5947 "customer",
5948 serde_json::json!({"email": "guest@example.com"}),
5949 );
5950 let result = run_workflow(&executor, &wf, &mut ctx).await.unwrap();
5951 assert_eq!(result, "active");
5952
5953 let result = timeout_workflow(&executor, &wf, "wf-timeout")
5954 .await
5955 .unwrap();
5956 assert_eq!(result, "timed_out");
5957
5958 let notifs = executor.notifications.lock().unwrap();
5959 assert_eq!(
5960 notifs.as_slice(),
5961 &[(
5962 "guest@example.com".to_string(),
5963 "payment_timeout".to_string()
5964 )]
5965 );
5966 }
5967
5968 #[tokio::test]
5969 async fn persisted_timeout_fallback_is_validated_before_side_effects() {
5970 let executor = MockExecutor::new();
5971
5972 let wf = WorkflowDefinition::new("persisted_timeout_fallback_guard")
5973 .initial_state("active")
5974 .transition(
5975 "active",
5976 "done",
5977 vec![WorkflowStep::wait_or(
5978 "payment.success",
5979 std::time::Duration::from_secs(0),
5980 vec![WorkflowStep::transition("timed_out")],
5981 )],
5982 );
5983
5984 let mut ctx = WorkflowContext::new("wf-persisted-timeout-fallback", "active");
5985 ctx.set(
5986 "customer",
5987 serde_json::json!({"email": "guest@example.com"}),
5988 );
5989 let result = run_workflow(&executor, &wf, &mut ctx).await.unwrap();
5990 assert_eq!(result, "active");
5991
5992 {
5993 let mut saved = executor.saved_state.lock().unwrap();
5994 let saved = saved
5995 .as_mut()
5996 .expect("workflow should be saved while waiting for timeout");
5997 let cursor = saved
5998 .cursor
5999 .as_mut()
6000 .expect("saved workflow should have a wait cursor");
6001 let wait = cursor
6002 .wait
6003 .as_mut()
6004 .expect("saved workflow should be waiting for timeout");
6005 wait.on_timeout = vec![
6006 WorkflowStep::notify(ChannelKind::Email, "bad_timeout", "customer.email"),
6007 WorkflowStep::transition(" "),
6008 ];
6009 }
6010
6011 let err = timeout_workflow(&executor, &wf, "wf-persisted-timeout-fallback")
6012 .await
6013 .expect_err("invalid persisted timeout fallback must fail before side effects");
6014
6015 match err {
6016 WorkflowError::Other(msg) => {
6017 assert!(
6018 msg.contains("Transition target state must not be empty"),
6019 "got: {msg}"
6020 );
6021 }
6022 other => panic!("expected timeout fallback validation error, got {other:?}"),
6023 }
6024 assert!(
6025 executor.notifications.lock().unwrap().is_empty(),
6026 "persisted timeout fallback validation must run before notification"
6027 );
6028 }
6029
6030 #[tokio::test]
6031 async fn persisted_timeout_fallback_before_deadline_does_not_execute() {
6032 let executor = MockExecutor::new();
6033
6034 let wf = WorkflowDefinition::new("future_persisted_timeout_fallback")
6035 .initial_state("active")
6036 .transition(
6037 "active",
6038 "done",
6039 vec![WorkflowStep::wait_or(
6040 "payment.success",
6041 std::time::Duration::from_secs(3600),
6042 vec![
6043 WorkflowStep::notify(
6044 ChannelKind::Email,
6045 "payment_timeout",
6046 "customer.email",
6047 ),
6048 WorkflowStep::transition("timed_out"),
6049 ],
6050 )],
6051 );
6052
6053 let mut ctx = WorkflowContext::new("wf-future-persisted-timeout", "active");
6054 ctx.set(
6055 "customer",
6056 serde_json::json!({"email": "guest@example.com"}),
6057 );
6058 let wait = WorkflowPendingWait {
6059 event: "payment.success".to_string(),
6060 deadline_at: Utc::now() + chrono::Duration::hours(1),
6061 on_timeout: vec![
6062 WorkflowStep::notify(ChannelKind::Email, "payment_timeout", "customer.email"),
6063 WorkflowStep::transition("timed_out"),
6064 ],
6065 };
6066 ctx.set_cursor(WorkflowCursor {
6067 state: "active".to_string(),
6068 frames: vec![WorkflowCursorFrame::Steps { index: 1 }],
6069 wait: Some(wait.clone()),
6070 });
6071 set_timeout_fallback(&mut ctx, &wait).unwrap();
6072 executor.save_state(&ctx).await.unwrap();
6073
6074 let err = timeout_workflow(&executor, &wf, "wf-future-persisted-timeout")
6075 .await
6076 .expect_err("persisted timeout fallback must still respect the wait deadline");
6077
6078 match err {
6079 WorkflowError::Other(msg) => {
6080 assert!(msg.contains("has not timed out"), "got: {msg}");
6081 }
6082 other => panic!("expected not-timed-out error, got {other:?}"),
6083 }
6084 assert!(
6085 executor.notifications.lock().unwrap().is_empty(),
6086 "future timeout fallback metadata must fail before notification"
6087 );
6088 }
6089
6090 #[tokio::test]
6091 async fn persisted_timeout_wait_event_is_validated_before_side_effects() {
6092 let executor = MockExecutor::new();
6093
6094 let wf = WorkflowDefinition::new("persisted_timeout_event_guard")
6095 .initial_state("active")
6096 .transition(
6097 "active",
6098 "done",
6099 vec![WorkflowStep::wait_or(
6100 "payment.success",
6101 std::time::Duration::from_secs(0),
6102 vec![
6103 WorkflowStep::notify(ChannelKind::Email, "bad_timeout", "customer.email"),
6104 WorkflowStep::transition("timed_out"),
6105 ],
6106 )],
6107 );
6108
6109 let mut ctx = WorkflowContext::new("wf-persisted-timeout-event", "active");
6110 ctx.set(
6111 "customer",
6112 serde_json::json!({"email": "guest@example.com"}),
6113 );
6114 let result = run_workflow(&executor, &wf, &mut ctx).await.unwrap();
6115 assert_eq!(result, "active");
6116
6117 {
6118 let mut saved = executor.saved_state.lock().unwrap();
6119 let saved = saved
6120 .as_mut()
6121 .expect("workflow should be saved while waiting for timeout");
6122 let cursor = saved
6123 .cursor
6124 .as_mut()
6125 .expect("saved workflow should have a wait cursor");
6126 let wait = cursor
6127 .wait
6128 .as_mut()
6129 .expect("saved workflow should be waiting for timeout");
6130 wait.event = " payment.success ".to_string();
6131 }
6132
6133 let err = timeout_workflow(&executor, &wf, "wf-persisted-timeout-event")
6134 .await
6135 .expect_err("invalid persisted wait event must fail before side effects");
6136
6137 match err {
6138 WorkflowError::Other(msg) => {
6139 assert!(
6140 msg.contains("Wait event name must not have leading or trailing whitespace"),
6141 "got: {msg}"
6142 );
6143 }
6144 other => panic!("expected wait event validation error, got {other:?}"),
6145 }
6146 assert!(
6147 executor.notifications.lock().unwrap().is_empty(),
6148 "persisted timeout wait validation must run before notification"
6149 );
6150 }
6151
6152 #[tokio::test]
6153 async fn timeout_due_workflows_drains_due_waits_with_generated_idempotency() {
6154 let executor = MockExecutor::new();
6155
6156 let wf = WorkflowDefinition::new("wait_timeout_drain")
6157 .initial_state("active")
6158 .transition(
6159 "active",
6160 "done",
6161 vec![WorkflowStep::wait_or(
6162 "payment.success",
6163 std::time::Duration::from_secs(0),
6164 vec![
6165 WorkflowStep::notify(
6166 ChannelKind::Email,
6167 "payment_timeout",
6168 "customer.email",
6169 ),
6170 WorkflowStep::transition("timed_out"),
6171 ],
6172 )],
6173 );
6174
6175 let mut ctx = WorkflowContext::new("wf-timeout-drain", "active");
6176 ctx.set(
6177 "customer",
6178 serde_json::json!({"email": "guest@example.com"}),
6179 );
6180 let result = run_workflow(&executor, &wf, &mut ctx).await.unwrap();
6181 assert_eq!(result, "active");
6182
6183 executor.set_due_timeout_workflow_ids(vec!["wf-timeout-drain"]);
6184 let now = Utc::now();
6185 let outcomes =
6186 timeout_due_workflows(&executor, &wf, now, 10, WorkflowRunOptions::default())
6187 .await
6188 .unwrap();
6189
6190 assert_eq!(
6191 outcomes,
6192 vec![WorkflowTimeoutOutcome {
6193 workflow_id: "wf-timeout-drain".to_string(),
6194 state: Some("timed_out".to_string()),
6195 error: None,
6196 }]
6197 );
6198 assert_eq!(
6199 executor
6200 .completed_workflow_operations
6201 .lock()
6202 .unwrap()
6203 .as_slice(),
6204 &[(
6205 timeout_operation_idempotency_key("wait_timeout_drain", "wf-timeout-drain", now),
6206 "timed_out".to_string(),
6207 )]
6208 );
6209 }
6210
6211 #[tokio::test]
6212 async fn timeout_due_workflows_uses_batch_now_for_timeout_execution() {
6213 let executor = MockExecutor::new();
6214
6215 let wf = WorkflowDefinition::new("future_wait_timeout_drain")
6216 .initial_state("active")
6217 .transition(
6218 "active",
6219 "done",
6220 vec![WorkflowStep::wait_or(
6221 "payment.success",
6222 std::time::Duration::from_secs(3600),
6223 vec![WorkflowStep::transition("timed_out")],
6224 )],
6225 );
6226
6227 let mut ctx = WorkflowContext::new("wf-future-timeout-drain", "active");
6228 let result = run_workflow(&executor, &wf, &mut ctx).await.unwrap();
6229 assert_eq!(result, "active");
6230
6231 executor.set_due_timeout_workflow_ids(vec!["wf-future-timeout-drain"]);
6232 let outcomes = timeout_due_workflows(
6233 &executor,
6234 &wf,
6235 Utc::now() + chrono::Duration::hours(2),
6236 10,
6237 WorkflowRunOptions::default(),
6238 )
6239 .await
6240 .unwrap();
6241
6242 assert_eq!(
6243 outcomes,
6244 vec![WorkflowTimeoutOutcome {
6245 workflow_id: "wf-future-timeout-drain".to_string(),
6246 state: Some("timed_out".to_string()),
6247 error: None,
6248 }]
6249 );
6250 }
6251
6252 #[test]
6253 fn timeout_batch_idempotency_key_is_scoped_per_workflow() {
6254 let wf = WorkflowDefinition::new("wait_timeout_drain")
6255 .initial_state("active")
6256 .transition("active", "done", vec![]);
6257 let options = WorkflowRunOptions::default().with_idempotency_key("batch-20260617");
6258 let now = DateTime::parse_from_rfc3339("2026-06-17T12:00:00Z")
6259 .unwrap()
6260 .with_timezone(&Utc);
6261
6262 let first = timeout_options_for_workflow(&options, &wf, "wf-a", now);
6263 let second = timeout_options_for_workflow(&options, &wf, "wf-b", now);
6264 let first_key = first.idempotency_key.clone();
6265 let second_key = second.idempotency_key.clone();
6266
6267 assert_eq!(
6268 first_key,
6269 Some(timeout_operation_idempotency_key(
6270 "batch-20260617",
6271 "wf-a",
6272 now
6273 ))
6274 );
6275 assert_eq!(
6276 second_key,
6277 Some(timeout_operation_idempotency_key(
6278 "batch-20260617",
6279 "wf-b",
6280 now
6281 ))
6282 );
6283 assert_ne!(first.idempotency_key, second.idempotency_key);
6284 }
6285
6286 #[test]
6287 fn timeout_batch_idempotency_key_does_not_collide_on_delimiters() {
6288 let now = DateTime::parse_from_rfc3339("2026-06-17T12:00:00Z")
6289 .unwrap()
6290 .with_timezone(&Utc);
6291 let first = timeout_operation_idempotency_key("tenant:batch", "wf-a", now);
6292 let second = timeout_operation_idempotency_key("tenant", "batch:wf-a", now);
6293
6294 assert_ne!(
6295 first, second,
6296 "timeout operation idempotency keys must not collide when fields contain delimiters"
6297 );
6298 }
6299
6300 #[test]
6301 fn timeout_batch_idempotency_key_changes_between_scheduler_runs() {
6302 let first_run = DateTime::parse_from_rfc3339("2026-06-17T12:00:00Z")
6303 .unwrap()
6304 .with_timezone(&Utc);
6305 let second_run = DateTime::parse_from_rfc3339("2026-06-17T12:01:00Z")
6306 .unwrap()
6307 .with_timezone(&Utc);
6308
6309 assert_ne!(
6310 timeout_operation_idempotency_key("batch", "wf-a", first_run),
6311 timeout_operation_idempotency_key("batch", "wf-a", second_run),
6312 "later timeout attempts for the same workflow must not collide with an old completed timeout operation"
6313 );
6314 }
6315
6316 #[tokio::test]
6317 async fn test_timeout_fallback_without_transition_does_not_run_success_path() {
6318 let executor = MockExecutor::new();
6319
6320 let wf = WorkflowDefinition::new("wait_timeout_no_transition")
6321 .initial_state("active")
6322 .transition(
6323 "active",
6324 "confirmed",
6325 vec![
6326 WorkflowStep::wait_or(
6327 "payment.success",
6328 std::time::Duration::from_secs(0),
6329 vec![WorkflowStep::notify(
6330 ChannelKind::Email,
6331 "payment_timeout",
6332 "customer.email",
6333 )],
6334 ),
6335 WorkflowStep::notify(ChannelKind::Email, "booking_confirmed", "customer.email"),
6336 WorkflowStep::transition("confirmed"),
6337 ],
6338 );
6339
6340 let mut ctx = WorkflowContext::new("wf-timeout-no-transition", "active");
6341 ctx.set(
6342 "customer",
6343 serde_json::json!({"email": "guest@example.com"}),
6344 );
6345 let result = run_workflow(&executor, &wf, &mut ctx).await.unwrap();
6346 assert_eq!(result, "active");
6347
6348 let result = timeout_workflow(&executor, &wf, "wf-timeout-no-transition")
6349 .await
6350 .unwrap();
6351 assert_eq!(result, "active");
6352
6353 let notifs = executor.notifications.lock().unwrap();
6354 assert_eq!(
6355 notifs.as_slice(),
6356 &[(
6357 "guest@example.com".to_string(),
6358 "payment_timeout".to_string()
6359 )],
6360 "timeout fallback must not fall through into the post-wait success path"
6361 );
6362 }
6363
6364 #[tokio::test]
6365 async fn test_step_checkpoint_prevents_side_effect_replay_after_failure() {
6366 let executor = MockExecutor::new();
6367 let failing_query =
6368 qail_core::wire::encode_cmd_text(&qail_core::Qail::get("bookings").limit(1));
6369 executor.fail_query(failing_query.clone());
6370
6371 let wf = WorkflowDefinition::new("checkpoint_failure")
6372 .initial_state("active")
6373 .transition(
6374 "active",
6375 "done",
6376 vec![
6377 WorkflowStep::notify(ChannelKind::Email, "charged", "customer.email"),
6378 WorkflowStep::Query {
6379 cmd_json: failing_query,
6380 store_as: None,
6381 },
6382 WorkflowStep::transition("done"),
6383 ],
6384 );
6385
6386 let mut ctx = WorkflowContext::new("wf-checkpoint", "active");
6387 ctx.set(
6388 "customer",
6389 serde_json::json!({"email": "guest@example.com"}),
6390 );
6391 let err = run_workflow(&executor, &wf, &mut ctx)
6392 .await
6393 .expect_err("legacy query should fail after notification checkpoint");
6394 assert!(matches!(err, WorkflowError::QueryFailed(_)));
6395 assert_eq!(executor.notifications.lock().unwrap().len(), 1);
6396
6397 let mut restored = executor
6398 .saved_state
6399 .lock()
6400 .unwrap()
6401 .clone()
6402 .expect("checkpoint should be saved after notification");
6403 let cursor = restored.cursor.as_ref().expect("checkpoint cursor");
6404 assert_eq!(cursor.frames, vec![WorkflowCursorFrame::Steps { index: 1 }]);
6405 assert!(cursor.wait.is_none());
6406 assert!(restored.get(TIMEOUT_FALLBACK_KEY).is_none());
6407
6408 let err = run_workflow(&executor, &wf, &mut restored)
6409 .await
6410 .expect_err("query should still fail on retry");
6411 assert!(matches!(err, WorkflowError::QueryFailed(_)));
6412 assert_eq!(
6413 executor.notifications.lock().unwrap().len(),
6414 1,
6415 "retry must resume after the completed notification step"
6416 );
6417 }
6418
6419 #[tokio::test]
6420 async fn failed_query_step_marks_side_effect_failed() {
6421 let executor = MockExecutor::new();
6422 let failing_query =
6423 qail_core::wire::encode_cmd_text(&qail_core::Qail::get("bookings").limit(1));
6424 executor.fail_query(failing_query.clone());
6425
6426 let wf = WorkflowDefinition::new("failed_side_effect")
6427 .initial_state("active")
6428 .transition(
6429 "active",
6430 "done",
6431 vec![
6432 WorkflowStep::Query {
6433 cmd_json: failing_query,
6434 store_as: None,
6435 },
6436 WorkflowStep::transition("done"),
6437 ],
6438 );
6439
6440 let mut ctx = WorkflowContext::new("wf-side-effect-fail", "active");
6441 let err = run_workflow(&executor, &wf, &mut ctx)
6442 .await
6443 .expect_err("query should fail");
6444 assert!(matches!(err, WorkflowError::QueryFailed(_)));
6445
6446 let expected_operation_id = side_effect_operation_id(
6447 "wf-side-effect-fail",
6448 "active",
6449 0,
6450 WorkflowSideEffectKind::Query,
6451 "steps[0]",
6452 );
6453 let failed_side_effects = executor.failed_side_effects.lock().unwrap();
6454 assert_eq!(failed_side_effects.len(), 1);
6455 assert_eq!(failed_side_effects[0].0, expected_operation_id);
6456 assert!(
6457 failed_side_effects[0].1.contains("forced query failure"),
6458 "failure hook must preserve the app/provider error"
6459 );
6460 }
6461
6462 #[tokio::test]
6463 async fn test_timeout_checkpoint_prevents_side_effect_replay_after_failure() {
6464 let executor = MockExecutor::new();
6465 let failing_query =
6466 qail_core::wire::encode_cmd_text(&qail_core::Qail::get("bookings").limit(1));
6467 executor.fail_query(failing_query.clone());
6468
6469 let wf = WorkflowDefinition::new("timeout_checkpoint_failure")
6470 .initial_state("active")
6471 .transition(
6472 "active",
6473 "done",
6474 vec![WorkflowStep::wait_or(
6475 "payment.success",
6476 std::time::Duration::from_secs(0),
6477 vec![
6478 WorkflowStep::notify(
6479 ChannelKind::Email,
6480 "payment_timeout",
6481 "customer.email",
6482 ),
6483 WorkflowStep::Query {
6484 cmd_json: failing_query,
6485 store_as: None,
6486 },
6487 WorkflowStep::transition("timed_out"),
6488 ],
6489 )],
6490 );
6491
6492 let mut ctx = WorkflowContext::new("wf-timeout-checkpoint", "active");
6493 ctx.set(
6494 "customer",
6495 serde_json::json!({"email": "guest@example.com"}),
6496 );
6497 let result = run_workflow(&executor, &wf, &mut ctx).await.unwrap();
6498 assert_eq!(result, "active");
6499
6500 let err = timeout_workflow(&executor, &wf, "wf-timeout-checkpoint")
6501 .await
6502 .expect_err("legacy query should fail after timeout notification checkpoint");
6503 assert!(matches!(err, WorkflowError::QueryFailed(_)));
6504 assert_eq!(executor.notifications.lock().unwrap().len(), 1);
6505
6506 let saved = executor
6507 .saved_state
6508 .lock()
6509 .unwrap()
6510 .clone()
6511 .expect("timeout checkpoint should be saved after notification");
6512 let cursor = saved.cursor.as_ref().expect("timeout checkpoint cursor");
6513 assert_eq!(cursor.frames, vec![WorkflowCursorFrame::Steps { index: 1 }]);
6514 let wait = cursor
6515 .wait
6516 .as_ref()
6517 .expect("timeout fallback checkpoint must remain scheduler-visible");
6518 assert_eq!(wait.event, "payment.success");
6519 assert_eq!(wait.on_timeout.len(), 3);
6520 assert!(
6521 saved.get(TIMEOUT_FALLBACK_KEY).is_some(),
6522 "timeout fallback cursor must retain internal on_timeout metadata"
6523 );
6524
6525 let err = timeout_workflow(&executor, &wf, "wf-timeout-checkpoint")
6526 .await
6527 .expect_err("timeout fallback query should still fail on retry");
6528 assert!(matches!(err, WorkflowError::QueryFailed(_)));
6529 assert_eq!(
6530 executor.notifications.lock().unwrap().len(),
6531 1,
6532 "timeout retry must resume after the completed notification step"
6533 );
6534 }
6535
6536 #[tokio::test]
6537 async fn test_workflow_errors_on_for_each_non_array() {
6538 let executor = MockExecutor::new();
6539
6540 let wf = WorkflowDefinition::new("bad_for_each")
6541 .initial_state("active")
6542 .transition(
6543 "active",
6544 "resolved",
6545 vec![WorkflowStep::for_each(
6546 "operators",
6547 vec![WorkflowStep::notify(
6548 ChannelKind::WhatsApp,
6549 "opportunity",
6550 "item.phone",
6551 )],
6552 )],
6553 );
6554
6555 let mut ctx = WorkflowContext::new("wf-bad-for-each", "active");
6556 ctx.set("operators", serde_json::json!({"phone": "+628111"}));
6557
6558 let err = run_workflow(&executor, &wf, &mut ctx)
6559 .await
6560 .expect_err("ForEach should reject non-array context values");
6561
6562 match err {
6563 WorkflowError::Other(msg) => {
6564 assert!(msg.contains("Expected JSON array for ForEach list 'operators'"));
6565 assert!(msg.contains("object"));
6566 }
6567 other => panic!("expected Other error, got: {other}"),
6568 }
6569 assert!(
6570 executor.notifications.lock().unwrap().is_empty(),
6571 "ForEach type errors must fail before running nested steps"
6572 );
6573 }
6574
6575 #[tokio::test]
6576 async fn test_workflow_with_charge() {
6577 let executor = MockExecutor::new();
6578
6579 let wf = WorkflowDefinition::new("booking_payment")
6580 .initial_state("created")
6581 .transition(
6582 "created",
6583 "awaiting_payment",
6584 vec![
6585 WorkflowStep::charge_with_origin(
6586 PaymentKind::Xendit,
6587 "order.total",
6588 "order.id",
6589 None,
6590 Some("order.payment_method"),
6591 Some("order.origin"),
6592 Some("charge"),
6593 ),
6594 WorkflowStep::wait("payment.success", std::time::Duration::from_secs(3600)),
6595 ],
6596 )
6597 .transition(
6598 "awaiting_payment",
6599 "confirmed",
6600 vec![
6601 WorkflowStep::notify(ChannelKind::Email, "booking_confirmed", "customer.email"),
6602 WorkflowStep::transition("confirmed"),
6603 ],
6604 );
6605
6606 let mut ctx = WorkflowContext::new("wf-payment-001", "created");
6607 ctx.set(
6608 "order",
6609 serde_json::json!({
6610 "id": "booking-789",
6611 "total": 150000,
6612 "payment_method": "QRIS",
6613 "origin": "mcp"
6614 }),
6615 );
6616 ctx.set(
6617 "customer",
6618 serde_json::json!({
6619 "email": "guest@example.com"
6620 }),
6621 );
6622
6623 let result = run_workflow(&executor, &wf, &mut ctx).await.unwrap();
6624
6625 assert_eq!(result, "created");
6627
6628 let charge = ctx.get("charge").expect("charge should be in context");
6630 assert_eq!(
6631 charge.get("charge_id").and_then(|v| v.as_str()),
6632 Some("mock-charge-booking-789")
6633 );
6634 assert_eq!(
6635 charge.get("status").and_then(|v| v.as_str()),
6636 Some("Pending")
6637 );
6638 assert!(charge.get("qr_code").is_some());
6639 assert_eq!(
6640 charge.get("payment_method").and_then(|v| v.as_str()),
6641 Some("QRIS")
6642 );
6643 assert_eq!(
6644 charge.get("order_origin").and_then(|v| v.as_str()),
6645 Some("mcp")
6646 );
6647 assert!(
6648 charge.get("raw").is_none(),
6649 "stored charge context must be safe for chat display"
6650 );
6651
6652 let charges = executor.charges.lock().unwrap();
6653 assert_eq!(charges.len(), 1);
6654 assert_eq!(charges[0].amount, 150000);
6655 assert_eq!(charges[0].order_origin, Some(OrderOrigin::Mcp));
6656 assert!(
6657 charges[0]
6658 .idempotency_key
6659 .as_deref()
6660 .is_some_and(|key| key.contains("qail-workflow-side-effect")),
6661 "provider charge request must carry a stable workflow idempotency key"
6662 );
6663 }
6664
6665 #[tokio::test]
6666 async fn charge_rejects_invalid_origin_before_provider_call() {
6667 let executor = MockExecutor::new();
6668
6669 let wf = WorkflowDefinition::new("charge_invalid_origin")
6670 .initial_state("created")
6671 .transition(
6672 "created",
6673 "awaiting_payment",
6674 vec![WorkflowStep::charge_with_origin(
6675 PaymentKind::Xendit,
6676 "order.total",
6677 "order.id",
6678 None,
6679 None,
6680 Some("order.origin"),
6681 Some("charge"),
6682 )],
6683 );
6684
6685 let mut ctx = WorkflowContext::new("wf-charge-invalid-origin", "created");
6686 ctx.set(
6687 "order",
6688 serde_json::json!({
6689 "id": "booking-invalid-origin",
6690 "total": 150000,
6691 "origin": "telegram"
6692 }),
6693 );
6694
6695 let err = run_workflow(&executor, &wf, &mut ctx)
6696 .await
6697 .expect_err("invalid order origin must fail before provider call");
6698
6699 match err {
6700 WorkflowError::Other(msg) => {
6701 assert!(
6702 msg.contains("Charge order_origin_key 'order.origin'"),
6703 "got: {msg}"
6704 );
6705 }
6706 other => panic!("expected invalid order origin error, got {other:?}"),
6707 }
6708 assert!(executor.charges.lock().unwrap().is_empty());
6709 }
6710
6711 #[tokio::test]
6712 async fn charge_rejects_reserved_store_key_before_provider_call() {
6713 let executor = MockExecutor::new();
6714
6715 let wf = WorkflowDefinition::new("charge_reserved_key")
6716 .initial_state("created")
6717 .transition(
6718 "created",
6719 "awaiting_payment",
6720 vec![WorkflowStep::charge(
6721 PaymentKind::Xendit,
6722 "order.total",
6723 "order.id",
6724 Some("item"),
6725 )],
6726 );
6727
6728 let mut ctx = WorkflowContext::new("wf-charge-reserved-key", "created");
6729 ctx.set(
6730 "order",
6731 serde_json::json!({
6732 "id": "booking-reserved",
6733 "total": 150000
6734 }),
6735 );
6736
6737 let err = run_workflow(&executor, &wf, &mut ctx)
6738 .await
6739 .expect_err("reserved charge output key must fail before provider call");
6740
6741 match err {
6742 WorkflowError::Other(msg) => {
6743 assert!(msg.contains("reserved context key 'item'"), "got: {msg}");
6744 }
6745 other => panic!("expected reserved context key error, got {other:?}"),
6746 }
6747 assert!(
6748 executor.charges.lock().unwrap().is_empty(),
6749 "charge provider must not be called after reserved key validation fails"
6750 );
6751 }
6752
6753 #[tokio::test]
6754 async fn charge_rejects_dotted_store_key_before_provider_call() {
6755 let executor = MockExecutor::new();
6756
6757 let wf = WorkflowDefinition::new("charge_dotted_key")
6758 .initial_state("created")
6759 .transition(
6760 "created",
6761 "awaiting_payment",
6762 vec![WorkflowStep::charge(
6763 PaymentKind::Xendit,
6764 "order.total",
6765 "order.id",
6766 Some("payment.charge"),
6767 )],
6768 );
6769
6770 let mut ctx = WorkflowContext::new("wf-charge-dotted-key", "created");
6771 ctx.set(
6772 "order",
6773 serde_json::json!({
6774 "id": "booking-dotted",
6775 "total": 150000
6776 }),
6777 );
6778
6779 let err = run_workflow(&executor, &wf, &mut ctx)
6780 .await
6781 .expect_err("dotted charge output key must fail before provider call");
6782
6783 match err {
6784 WorkflowError::Other(msg) => {
6785 assert!(msg.contains("top-level context key"), "got: {msg}");
6786 }
6787 other => panic!("expected dotted context key error, got {other:?}"),
6788 }
6789 assert!(
6790 executor.charges.lock().unwrap().is_empty(),
6791 "charge provider must not be called after dotted key validation fails"
6792 );
6793 }
6794
6795 #[tokio::test]
6796 async fn charge_accepts_safe_integer_float_amount() {
6797 let executor = MockExecutor::new();
6798 let wf = charge_only_workflow("booking_payment_float");
6799
6800 let mut ctx = WorkflowContext::new("wf-payment-float", "created");
6801 ctx.set(
6802 "order",
6803 serde_json::json!({
6804 "id": "booking-float",
6805 "total": 150000.0
6806 }),
6807 );
6808
6809 let result = run_workflow(&executor, &wf, &mut ctx).await.unwrap();
6810
6811 assert_eq!(result, "awaiting_payment");
6812 let charges = executor.charges.lock().unwrap();
6813 assert_eq!(charges.len(), 1);
6814 assert_eq!(charges[0].amount, 150000);
6815 }
6816
6817 #[tokio::test]
6818 async fn charge_rejects_fractional_amount_before_provider_call() {
6819 let executor = MockExecutor::new();
6820 let wf = charge_only_workflow("booking_payment_fractional");
6821
6822 let mut ctx = WorkflowContext::new("wf-payment-fractional", "created");
6823 ctx.set(
6824 "order",
6825 serde_json::json!({
6826 "id": "booking-fractional",
6827 "total": 150000.75
6828 }),
6829 );
6830
6831 let err = run_workflow(&executor, &wf, &mut ctx)
6832 .await
6833 .expect_err("fractional charge amount must fail before provider call");
6834
6835 match err {
6836 WorkflowError::Other(msg) => {
6837 assert!(msg.contains("Invalid charge amount"));
6838 assert!(msg.contains("smallest currency unit"));
6839 }
6840 other => panic!("expected invalid amount error, got {other:?}"),
6841 }
6842 assert!(executor.charges.lock().unwrap().is_empty());
6843 }
6844
6845 #[tokio::test]
6846 async fn charge_rejects_non_positive_amount_before_provider_call() {
6847 let executor = MockExecutor::new();
6848 let wf = charge_only_workflow("booking_payment_negative");
6849
6850 let mut ctx = WorkflowContext::new("wf-payment-negative", "created");
6851 ctx.set(
6852 "order",
6853 serde_json::json!({
6854 "id": "booking-negative",
6855 "total": -1
6856 }),
6857 );
6858
6859 let err = run_workflow(&executor, &wf, &mut ctx)
6860 .await
6861 .expect_err("non-positive charge amount must fail before provider call");
6862
6863 match err {
6864 WorkflowError::Other(msg) => {
6865 assert!(msg.contains("Invalid charge amount"));
6866 assert!(msg.contains("greater than zero"));
6867 }
6868 other => panic!("expected invalid amount error, got {other:?}"),
6869 }
6870 assert!(executor.charges.lock().unwrap().is_empty());
6871 }
6872
6873 #[tokio::test]
6874 async fn charge_rejects_oversized_unsigned_amount_before_provider_call() {
6875 let executor = MockExecutor::new();
6876 let wf = charge_only_workflow("booking_payment_oversized");
6877
6878 let mut ctx = WorkflowContext::new("wf-payment-oversized", "created");
6879 ctx.set(
6880 "order",
6881 serde_json::json!({
6882 "id": "booking-oversized",
6883 "total": u64::MAX
6884 }),
6885 );
6886
6887 let err = run_workflow(&executor, &wf, &mut ctx)
6888 .await
6889 .expect_err("oversized charge amount must fail before provider call");
6890
6891 match err {
6892 WorkflowError::Other(msg) => {
6893 assert!(msg.contains("Invalid charge amount"));
6894 assert!(msg.contains("signed 64-bit"));
6895 }
6896 other => panic!("expected invalid amount error, got {other:?}"),
6897 }
6898 assert!(executor.charges.lock().unwrap().is_empty());
6899 }
6900
6901 #[tokio::test]
6902 async fn test_query_step_rejects_legacy_non_wire_payload() {
6903 let executor = MockExecutor::new();
6904
6905 let wf = WorkflowDefinition::new("legacy_query")
6906 .initial_state("start")
6907 .transition(
6908 "start",
6909 "done",
6910 vec![
6911 WorkflowStep::Query {
6912 cmd_json: "get users limit 1".to_string(),
6913 store_as: Some("rows".to_string()),
6914 },
6915 WorkflowStep::transition("done"),
6916 ],
6917 );
6918
6919 let mut ctx = WorkflowContext::new("wf-legacy-query-001", "start");
6920 let err = run_workflow(&executor, &wf, &mut ctx)
6921 .await
6922 .expect_err("legacy non-wire query payload must fail");
6923
6924 match err {
6925 WorkflowError::QueryFailed(msg) => {
6926 assert!(
6927 msg.contains("QAIL-CMD/1"),
6928 "error should mention required wire magic"
6929 );
6930 assert!(
6931 msg.contains("purge/restart pending workflows"),
6932 "error should include cutover guidance"
6933 );
6934 }
6935 other => panic!("expected QueryFailed, got: {other}"),
6936 }
6937
6938 assert!(
6939 executor.queries.lock().unwrap().is_empty(),
6940 "legacy payload must fail before executor query is invoked"
6941 );
6942 }
6943
6944 #[tokio::test]
6945 async fn future_legacy_query_payload_fails_before_current_side_effects() {
6946 let executor = MockExecutor::new();
6947
6948 let wf = WorkflowDefinition::new("future_legacy_query")
6949 .initial_state("active")
6950 .transition(
6951 "active",
6952 "query_state",
6953 vec![
6954 WorkflowStep::notify(ChannelKind::Email, "before_bad_query", "customer.email"),
6955 WorkflowStep::transition("query_state"),
6956 ],
6957 )
6958 .transition(
6959 "query_state",
6960 "done",
6961 vec![
6962 WorkflowStep::Query {
6963 cmd_json: "get users limit 1".to_string(),
6964 store_as: Some("rows".to_string()),
6965 },
6966 WorkflowStep::transition("done"),
6967 ],
6968 );
6969
6970 let mut ctx = WorkflowContext::new("wf-future-legacy-query", "active");
6971 ctx.set(
6972 "customer",
6973 serde_json::json!({"email": "guest@example.com"}),
6974 );
6975
6976 let err = run_workflow(&executor, &wf, &mut ctx)
6977 .await
6978 .expect_err("future legacy query payload must fail before current side effects");
6979
6980 match err {
6981 WorkflowError::QueryFailed(msg) => {
6982 assert!(
6983 msg.contains("QAIL-CMD/1"),
6984 "error should mention required wire magic"
6985 );
6986 }
6987 other => panic!("expected QueryFailed, got: {other}"),
6988 }
6989 assert_eq!(ctx.current_state, "active");
6990 assert!(
6991 executor.notifications.lock().unwrap().is_empty(),
6992 "definition validation must run before the current-state notification"
6993 );
6994 assert!(
6995 executor.queries.lock().unwrap().is_empty(),
6996 "legacy payload must fail before executor query is invoked"
6997 );
6998 }
6999
7000 #[tokio::test]
7001 async fn test_query_step_accepts_wire_payload_and_executes() {
7002 let executor = MockExecutor::new();
7003 let cmd = qail_core::Qail::get("users").columns(["id"]).limit(1);
7004 let wire = qail_core::wire::encode_cmd_text(&cmd);
7005
7006 let wf = WorkflowDefinition::new("wire_query")
7007 .initial_state("start")
7008 .transition(
7009 "start",
7010 "done",
7011 vec![
7012 WorkflowStep::Query {
7013 cmd_json: wire.clone(),
7014 store_as: Some("rows".to_string()),
7015 },
7016 WorkflowStep::transition("done"),
7017 ],
7018 );
7019
7020 let mut ctx = WorkflowContext::new("wf-wire-query-001", "start");
7021 let result = run_workflow(&executor, &wf, &mut ctx)
7022 .await
7023 .expect("wire payload should execute");
7024 assert_eq!(result, "done");
7025
7026 let queries = executor.queries.lock().unwrap();
7027 assert_eq!(queries.len(), 1);
7028 assert_eq!(
7029 queries[0], wire,
7030 "executor should receive canonical wire payload"
7031 );
7032 }
7033
7034 #[tokio::test]
7035 async fn query_step_executes_the_exact_command_it_persisted() {
7036 let executor = MockExecutor::new();
7037 let insert = qail_core::Qail::add("rows")
7040 .set_value("id", 1)
7041 .set_value("status", "a");
7042 let mut locked = qail_core::Qail::get("orders")
7043 .columns(["id"])
7044 .where_eq("id", 1)
7045 .for_update();
7046 locked.distinct = true;
7047
7048 let steps = vec![
7049 WorkflowStep::query(&insert, None),
7050 WorkflowStep::query(&locked, Some("rows")),
7051 WorkflowStep::transition("done"),
7052 ];
7053 let persisted: Vec<WorkflowStep> =
7054 serde_json::from_str(&serde_json::to_string(&steps).unwrap()).unwrap();
7055 let wf = WorkflowDefinition::new("exact_query")
7056 .initial_state("start")
7057 .transition("start", "done", persisted);
7058
7059 let mut ctx = WorkflowContext::new("wf-exact-query-001", "start");
7060 let result = run_workflow(&executor, &wf, &mut ctx)
7061 .await
7062 .expect("exact payloads should execute");
7063 assert_eq!(result, "done");
7064
7065 let queries = executor.queries.lock().unwrap();
7066 let decoded: Vec<_> = queries
7067 .iter()
7068 .map(|wire| qail_core::wire::decode_cmd_text(wire).unwrap())
7069 .collect();
7070 assert_eq!(decoded, vec![insert, locked]);
7071 }
7072
7073 #[test]
7074 fn query_scope_wire_normalization_retains_pending_scope() {
7075 let mut cmd = qail_core::Qail::add("scope_transport").set_value("id", 1);
7076 cmd.conflict_update_scope.push(qail_core::ast::builders::eq(
7077 "scope_transport.tenant_id",
7078 "a",
7079 ));
7080 let wire = qail_core::wire::encode_cmd_text(&cmd);
7081 assert!(wire.starts_with("QAIL-CMD/2\n"));
7082 assert!(is_current_workflow_query_wire(&wire));
7083 let normalized = normalize_query_wire_for_execution(&wire).unwrap();
7084 assert_eq!(qail_core::wire::decode_cmd_text(&normalized).unwrap(), cmd);
7085 assert!(normalize_query_wire_for_execution("QAIL-CMD/2\n2\n{}").is_err());
7086 assert!(normalize_query_wire_for_execution("QAIL-CMD/3\n0\n").is_err());
7087 }
7088
7089 #[tokio::test]
7090 async fn query_rejects_reserved_store_key_before_execution() {
7091 let executor = MockExecutor::new();
7092 let cmd = qail_core::Qail::get("users").columns(["id"]).limit(1);
7093 let wire = qail_core::wire::encode_cmd_text(&cmd);
7094
7095 let wf = WorkflowDefinition::new("query_reserved_key")
7096 .initial_state("start")
7097 .transition(
7098 "start",
7099 "done",
7100 vec![
7101 WorkflowStep::Query {
7102 cmd_json: wire,
7103 store_as: Some("__qail_timeout_fallback".to_string()),
7104 },
7105 WorkflowStep::transition("done"),
7106 ],
7107 );
7108
7109 let mut ctx = WorkflowContext::new("wf-query-reserved-key", "start");
7110 let err = run_workflow(&executor, &wf, &mut ctx)
7111 .await
7112 .expect_err("reserved query output key must fail before query execution");
7113
7114 match err {
7115 WorkflowError::Other(msg) => {
7116 assert!(
7117 msg.contains("reserved context key '__qail_timeout_fallback'"),
7118 "got: {msg}"
7119 );
7120 }
7121 other => panic!("expected reserved context key error, got {other:?}"),
7122 }
7123 assert!(
7124 executor.queries.lock().unwrap().is_empty(),
7125 "query executor must not be called after reserved key validation fails"
7126 );
7127 }
7128
7129 #[tokio::test]
7130 async fn query_rejects_dotted_store_key_before_execution() {
7131 let executor = MockExecutor::new();
7132 let cmd = qail_core::Qail::get("users").columns(["id"]).limit(1);
7133 let wire = qail_core::wire::encode_cmd_text(&cmd);
7134
7135 let wf = WorkflowDefinition::new("query_dotted_key")
7136 .initial_state("start")
7137 .transition(
7138 "start",
7139 "done",
7140 vec![
7141 WorkflowStep::Query {
7142 cmd_json: wire,
7143 store_as: Some("query.rows".to_string()),
7144 },
7145 WorkflowStep::transition("done"),
7146 ],
7147 );
7148
7149 let mut ctx = WorkflowContext::new("wf-query-dotted-key", "start");
7150 let err = run_workflow(&executor, &wf, &mut ctx)
7151 .await
7152 .expect_err("dotted query output key must fail before query execution");
7153
7154 match err {
7155 WorkflowError::Other(msg) => {
7156 assert!(msg.contains("top-level context key"), "got: {msg}");
7157 }
7158 other => panic!("expected dotted context key error, got {other:?}"),
7159 }
7160 assert!(
7161 executor.queries.lock().unwrap().is_empty(),
7162 "query executor must not be called after dotted key validation fails"
7163 );
7164 }
7165
7166 #[test]
7167 fn test_collect_legacy_query_payload_issues_reports_nested_paths() {
7168 let wire_cmd = qail_core::wire::encode_cmd_text(&qail_core::Qail::get("users").limit(1));
7169 let wf = WorkflowDefinition::new("legacy_audit")
7170 .initial_state("start")
7171 .transition(
7172 "start",
7173 "done",
7174 vec![
7175 WorkflowStep::Query {
7176 cmd_json: "get users limit 1".to_string(),
7177 store_as: Some("rows".to_string()),
7178 },
7179 WorkflowStep::Wait {
7180 event: "timeout".to_string(),
7181 timeout: std::time::Duration::from_secs(10),
7182 on_timeout: vec![WorkflowStep::Query {
7183 cmd_json: "{\"legacy\":true}".to_string(),
7184 store_as: None,
7185 }],
7186 },
7187 WorkflowStep::branch(
7188 "kind",
7189 vec![(
7190 "wire_ok",
7191 vec![WorkflowStep::Query {
7192 cmd_json: wire_cmd,
7193 store_as: None,
7194 }],
7195 )],
7196 vec![WorkflowStep::ForEach {
7197 list_key: "items".to_string(),
7198 steps: vec![WorkflowStep::Query {
7199 cmd_json: "select * from legacy".to_string(),
7200 store_as: None,
7201 }],
7202 }],
7203 ),
7204 ],
7205 );
7206
7207 let issues = collect_legacy_query_payload_issues(&wf);
7208 assert_eq!(
7209 issues.len(),
7210 3,
7211 "expected all legacy payloads to be detected"
7212 );
7213
7214 let paths: Vec<String> = issues.iter().map(|i| i.step_path.clone()).collect();
7215 assert!(
7216 paths.iter().any(|p| p == "steps[0]"),
7217 "top-level legacy query should be reported"
7218 );
7219 assert!(
7220 paths.iter().any(|p| p == "steps[1].on_timeout.steps[0]"),
7221 "nested wait/on_timeout legacy query should be reported"
7222 );
7223 assert!(
7224 paths
7225 .iter()
7226 .any(|p| p == "steps[2].default.steps[0].for_each.steps[0]"),
7227 "nested branch/default/foreach legacy query should be reported"
7228 );
7229 }
7230}