Skip to main content

qail_workflow/
engine.rs

1//! Workflow engine — the executor trait and step runner.
2//!
3//! Consumers implement `WorkflowExecutor` to connect the engine
4//! to their database driver and notification channels.
5
6use async_trait::async_trait;
7use std::cmp::Ordering;
8
9use chrono::{DateTime, SecondsFormat, Utc};
10use serde_json::Value;
11
12use crate::channel::ChannelKind;
13use crate::context::{
14    WorkflowBranchCursorSelection, WorkflowContext, WorkflowCursor, WorkflowCursorFrame,
15    WorkflowPendingWait,
16};
17use crate::payment::{ChargeRequest, ChargeResponse, Currency, OrderOrigin, PaymentKind};
18use crate::registry::{StateTransition, WorkflowDefinition};
19use crate::runtime::{
20    WorkflowLease, WorkflowOperation, WorkflowOperationKind, WorkflowOperationStatus,
21    WorkflowRunOptions, WorkflowSideEffect, WorkflowSideEffectKind, WorkflowSideEffectStatus,
22    WorkflowTimeoutOutcome,
23};
24use crate::step::{WorkflowBranchCondition, WorkflowStep};
25
26const WORKFLOW_QUERY_WIRE_MAGIC: &str = "QAIL-CMD/1\n";
27const WORKFLOW_QUERY_AST_WIRE_MAGIC: &str = "QAIL-CMD/2\n";
28
29/// A single legacy query payload detected in a workflow definition.
30#[derive(Debug, Clone, PartialEq, Eq)]
31pub struct LegacyQueryPayloadIssue {
32    /// Transition source state containing the step.
33    pub transition_from: String,
34    /// Transition destination state containing the step.
35    pub transition_to: String,
36    /// Path to the offending step inside the transition.
37    pub step_path: String,
38    /// Short preview of the legacy payload (escaped, truncated).
39    pub payload_preview: String,
40}
41
42/// Return legacy query payload issues found in a workflow definition.
43///
44/// A query payload is considered legacy when `cmd_json` does not start with
45/// a supported QAIL wire text magic (`QAIL-CMD/1\n` or `QAIL-CMD/2\n`).
46///
47/// This helper is intended for cutover audits before loading persisted workflows
48/// into runtime execution.
49pub fn collect_legacy_query_payload_issues(
50    definition: &WorkflowDefinition,
51) -> Vec<LegacyQueryPayloadIssue> {
52    let mut out = Vec::new();
53    for transition in &definition.transitions {
54        collect_legacy_query_payload_issues_in_steps(
55            &transition.from,
56            &transition.to,
57            &transition.steps,
58            "",
59            &mut out,
60        );
61    }
62    out
63}
64
65fn collect_legacy_query_payload_issues_in_steps(
66    transition_from: &str,
67    transition_to: &str,
68    steps: &[WorkflowStep],
69    path_prefix: &str,
70    out: &mut Vec<LegacyQueryPayloadIssue>,
71) {
72    for (idx, step) in steps.iter().enumerate() {
73        let step_path = if path_prefix.is_empty() {
74            format!("steps[{idx}]")
75        } else {
76            format!("{path_prefix}.steps[{idx}]")
77        };
78
79        match step {
80            WorkflowStep::Query { cmd_json, .. } => {
81                if !is_current_workflow_query_wire(cmd_json) {
82                    out.push(LegacyQueryPayloadIssue {
83                        transition_from: transition_from.to_string(),
84                        transition_to: transition_to.to_string(),
85                        step_path,
86                        payload_preview: summarize_payload_preview(cmd_json),
87                    });
88                }
89            }
90            WorkflowStep::Wait { on_timeout, .. } => {
91                collect_legacy_query_payload_issues_in_steps(
92                    transition_from,
93                    transition_to,
94                    on_timeout,
95                    &format!("{step_path}.on_timeout"),
96                    out,
97                );
98            }
99            WorkflowStep::Branch {
100                branches, default, ..
101            } => {
102                for (branch_idx, (branch_value, branch_steps)) in branches.iter().enumerate() {
103                    collect_legacy_query_payload_issues_in_steps(
104                        transition_from,
105                        transition_to,
106                        branch_steps,
107                        &format!("{step_path}.branches[{branch_idx}:{branch_value}]"),
108                        out,
109                    );
110                }
111                collect_legacy_query_payload_issues_in_steps(
112                    transition_from,
113                    transition_to,
114                    default,
115                    &format!("{step_path}.default"),
116                    out,
117                );
118            }
119            WorkflowStep::BranchWhen {
120                branches, default, ..
121            } => {
122                for (branch_idx, (condition, branch_steps)) in branches.iter().enumerate() {
123                    collect_legacy_query_payload_issues_in_steps(
124                        transition_from,
125                        transition_to,
126                        branch_steps,
127                        &format!("{step_path}.branch_when[{branch_idx}:{condition:?}]"),
128                        out,
129                    );
130                }
131                collect_legacy_query_payload_issues_in_steps(
132                    transition_from,
133                    transition_to,
134                    default,
135                    &format!("{step_path}.default"),
136                    out,
137                );
138            }
139            WorkflowStep::ForEach { steps: nested, .. } => {
140                collect_legacy_query_payload_issues_in_steps(
141                    transition_from,
142                    transition_to,
143                    nested,
144                    &format!("{step_path}.for_each"),
145                    out,
146                );
147            }
148            WorkflowStep::Notify { .. }
149            | WorkflowStep::Transition { .. }
150            | WorkflowStep::Log { .. }
151            | WorkflowStep::Charge { .. } => {}
152        }
153    }
154}
155
156fn summarize_payload_preview(cmd_json: &str) -> String {
157    const MAX_CHARS: usize = 64;
158    let escaped = cmd_json.replace('\n', "\\n");
159    let mut preview = escaped.chars().take(MAX_CHARS).collect::<String>();
160    if escaped.chars().count() > MAX_CHARS {
161        preview.push_str("...");
162    }
163    preview
164}
165
166fn is_current_workflow_query_wire(cmd_json: &str) -> bool {
167    cmd_json.starts_with(WORKFLOW_QUERY_WIRE_MAGIC)
168        || cmd_json.starts_with(WORKFLOW_QUERY_AST_WIRE_MAGIC)
169}
170
171/// Errors that can occur during workflow execution.
172#[derive(Debug)]
173pub enum WorkflowError {
174    /// Query execution failed
175    QueryFailed(String),
176    /// Notification delivery failed
177    NotifyFailed(String),
178    /// No transition found for current state
179    NoTransition {
180        /// Current state that has no outgoing transition.
181        state: String,
182        /// Workflow definition name.
183        workflow: String,
184    },
185    /// Context key not found
186    MissingContextKey(String),
187    /// Workflow has reached a terminal state
188    AlreadyTerminal(String),
189    /// State persistence failed
190    PersistenceFailed(String),
191    /// Timeout reached while waiting
192    Timeout {
193        /// Name of the event that timed out.
194        event: String,
195    },
196    /// Payment charge creation failed
197    ChargeFailed(String),
198    /// Generic error
199    Other(String),
200}
201
202impl std::fmt::Display for WorkflowError {
203    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
204        match self {
205            WorkflowError::QueryFailed(e) => write!(f, "Query failed: {}", e),
206            WorkflowError::NotifyFailed(e) => write!(f, "Notification failed: {}", e),
207            WorkflowError::NoTransition { state, workflow } => {
208                write!(
209                    f,
210                    "No transition from '{}' in workflow '{}'",
211                    state, workflow
212                )
213            }
214            WorkflowError::MissingContextKey(k) => write!(f, "Context key not found: {}", k),
215            WorkflowError::AlreadyTerminal(s) => write!(f, "Workflow already terminal: {}", s),
216            WorkflowError::PersistenceFailed(e) => write!(f, "Persistence failed: {}", e),
217            WorkflowError::Timeout { event } => write!(f, "Timeout waiting for: {}", event),
218            WorkflowError::ChargeFailed(e) => write!(f, "Charge failed: {}", e),
219            WorkflowError::Other(e) => write!(f, "{}", e),
220        }
221    }
222}
223
224impl std::error::Error for WorkflowError {}
225
226/// Executor trait — consumers implement this to connect workflows to their infrastructure.
227///
228/// This is the bridge between `qail-workflow` (domain-agnostic) and your app
229/// (domain-specific). Your app provides the database driver, notification channels,
230/// and state persistence.
231///
232/// # Example
233///
234/// ```rust,ignore
235/// struct MyExecutor { pg: PgDriver, waba: WabaClient }
236///
237/// #[async_trait]
238/// impl WorkflowExecutor for MyExecutor {
239///     async fn execute_query(&self, cmd_json: &str) -> Result<Value, WorkflowError> {
240///         let cmd: Qail = qail_core::wire::decode_cmd_text(cmd_json)
241///             .map_err(WorkflowError::QueryFailed)?;
242///         let rows = self.pg.fetch_all(&cmd).await?;
243///         Ok(rows_to_json(rows))
244///     }
245///
246///     async fn send_notification(
247///         &self, channel: &ChannelKind, recipient: &str,
248///         template: &str, params: &Value,
249///     ) -> Result<(), WorkflowError> {
250///         self.waba.send_template(recipient, template, params).await
251///     }
252///
253///     async fn save_state(&self, ctx: &WorkflowContext) -> Result<(), WorkflowError> {
254///         // Persist to _qail_workflow_states table
255///     }
256///
257///     async fn load_state(&self, workflow_id: &str) -> Result<Option<WorkflowContext>, WorkflowError> {
258///         // Load from _qail_workflow_states table
259///     }
260/// }
261/// ```
262// See NotifyChannel: async_trait's #[must_use] and clippy 1.99.
263#[allow(clippy::double_must_use)]
264#[async_trait]
265pub trait WorkflowExecutor: Send + Sync {
266    /// Execute a QAIL query (QAIL wire text) and return results.
267    async fn execute_query(&self, cmd_json: &str) -> Result<Value, WorkflowError>;
268
269    /// Send a notification through a channel.
270    async fn send_notification(
271        &self,
272        channel: &ChannelKind,
273        recipient: &str,
274        template: &str,
275        params: &Value,
276    ) -> Result<(), WorkflowError>;
277
278    /// Persist the workflow state (for crash recovery).
279    async fn save_state(&self, ctx: &WorkflowContext) -> Result<(), WorkflowError>;
280
281    /// Load a previously persisted workflow state.
282    async fn load_state(&self, workflow_id: &str)
283    -> Result<Option<WorkflowContext>, WorkflowError>;
284
285    /// Create a payment charge via the appropriate provider.
286    ///
287    /// The engine resolves the charge parameters from context and
288    /// delegates to the provider matching `provider_kind`.
289    /// Implementations should look up the registered `PaymentProvider`
290    /// and call `create_charge()` on it.
291    async fn create_charge(
292        &self,
293        provider: &PaymentKind,
294        request: ChargeRequest,
295    ) -> Result<ChargeResponse, WorkflowError>;
296
297    /// Acquire a per-workflow lease before executing a workflow operation.
298    ///
299    /// The default implementation allows execution. Production executors should
300    /// back this with a DB/advisory lock or lease row keyed by `workflow_id`.
301    async fn acquire_workflow_lease(&self, _lease: &WorkflowLease) -> Result<bool, WorkflowError> {
302        Ok(true)
303    }
304
305    /// Release a per-workflow lease after a workflow operation finishes.
306    async fn release_workflow_lease(&self, _lease: &WorkflowLease) -> Result<(), WorkflowError> {
307        Ok(())
308    }
309
310    /// Begin or replay-detect an idempotent workflow operation.
311    ///
312    /// Production executors should key this by `(workflow_id, idempotency_key)`.
313    async fn begin_workflow_operation(
314        &self,
315        _operation: &WorkflowOperation,
316    ) -> Result<WorkflowOperationStatus, WorkflowError> {
317        Ok(WorkflowOperationStatus::Started)
318    }
319
320    /// Mark an idempotent workflow operation as completed.
321    async fn complete_workflow_operation(
322        &self,
323        _operation: &WorkflowOperation,
324        _state: &str,
325    ) -> Result<(), WorkflowError> {
326        Ok(())
327    }
328
329    /// Mark an idempotent workflow operation as failed.
330    async fn fail_workflow_operation(
331        &self,
332        _operation: &WorkflowOperation,
333        _error: &str,
334    ) -> Result<(), WorkflowError> {
335        Ok(())
336    }
337
338    /// Return workflow ids whose wait deadlines are due for timeout handling.
339    ///
340    /// The workflow crate does not spawn a background worker. Apps can expose
341    /// their own scheduler by implementing this hook and calling
342    /// [`timeout_due_workflows`].
343    async fn load_due_workflow_timeouts(
344        &self,
345        _workflow_name: &str,
346        _now: DateTime<Utc>,
347        _limit: usize,
348    ) -> Result<Vec<String>, WorkflowError> {
349        Ok(Vec::new())
350    }
351
352    /// Begin or replay-detect an idempotent side-effect operation.
353    async fn begin_workflow_side_effect(
354        &self,
355        _operation: &WorkflowSideEffect,
356    ) -> Result<WorkflowSideEffectStatus, WorkflowError> {
357        Ok(WorkflowSideEffectStatus::Execute)
358    }
359
360    /// Mark a side effect as completed, optionally storing its result.
361    async fn complete_workflow_side_effect(
362        &self,
363        _operation: &WorkflowSideEffect,
364        _result: Option<&Value>,
365    ) -> Result<(), WorkflowError> {
366        Ok(())
367    }
368
369    /// Mark a side effect as failed so a later retry can make a fresh attempt.
370    ///
371    /// Backends that retry failed side effects should only use this for errors
372    /// returned by the app/provider before the workflow engine records a
373    /// completed side-effect result. External providers should still receive
374    /// stable idempotency keys where duplicate delivery/charge risk matters.
375    async fn fail_workflow_side_effect(
376        &self,
377        _operation: &WorkflowSideEffect,
378        _error: &str,
379    ) -> Result<(), WorkflowError> {
380        Ok(())
381    }
382
383    /// Execute a QAIL query with a stable workflow side-effect id.
384    async fn execute_query_once(
385        &self,
386        operation: &WorkflowSideEffect,
387        cmd_json: &str,
388    ) -> Result<Value, WorkflowError> {
389        match self.begin_workflow_side_effect(operation).await? {
390            WorkflowSideEffectStatus::Execute => {
391                let result = match self.execute_query(cmd_json).await {
392                    Ok(result) => result,
393                    Err(err) => {
394                        let _ = self
395                            .fail_workflow_side_effect(operation, &err.to_string())
396                            .await;
397                        return Err(err);
398                    }
399                };
400                self.complete_workflow_side_effect(operation, Some(&result))
401                    .await?;
402                Ok(result)
403            }
404            WorkflowSideEffectStatus::AlreadyCompleted { result } => result.ok_or_else(|| {
405                WorkflowError::Other(format!(
406                    "Workflow side effect '{}' completed without a stored query result",
407                    operation.operation_id
408                ))
409            }),
410        }
411    }
412
413    /// Send a notification with a stable workflow side-effect id.
414    async fn send_notification_once(
415        &self,
416        operation: &WorkflowSideEffect,
417        channel: &ChannelKind,
418        recipient: &str,
419        template: &str,
420        params: &Value,
421    ) -> Result<(), WorkflowError> {
422        match self.begin_workflow_side_effect(operation).await? {
423            WorkflowSideEffectStatus::Execute => {
424                if let Err(err) = self
425                    .send_notification(channel, recipient, template, params)
426                    .await
427                {
428                    let _ = self
429                        .fail_workflow_side_effect(operation, &err.to_string())
430                        .await;
431                    return Err(err);
432                }
433                self.complete_workflow_side_effect(operation, None).await
434            }
435            WorkflowSideEffectStatus::AlreadyCompleted { .. } => Ok(()),
436        }
437    }
438
439    /// Create a payment charge with a stable workflow side-effect id.
440    async fn create_charge_once(
441        &self,
442        operation: &WorkflowSideEffect,
443        provider: &PaymentKind,
444        mut request: ChargeRequest,
445    ) -> Result<ChargeResponse, WorkflowError> {
446        match self.begin_workflow_side_effect(operation).await? {
447            WorkflowSideEffectStatus::Execute => {
448                if request.idempotency_key.is_none() {
449                    request.idempotency_key = Some(operation.operation_id.clone());
450                }
451                let response = match self.create_charge(provider, request).await {
452                    Ok(response) => response,
453                    Err(err) => {
454                        let _ = self
455                            .fail_workflow_side_effect(operation, &err.to_string())
456                            .await;
457                        return Err(err);
458                    }
459                };
460                let response_json = serde_json::to_value(&response)
461                    .map_err(|e| WorkflowError::Other(e.to_string()))?;
462                self.complete_workflow_side_effect(operation, Some(&response_json))
463                    .await?;
464                Ok(response)
465            }
466            WorkflowSideEffectStatus::AlreadyCompleted {
467                result: Some(result),
468            } => serde_json::from_value(result).map_err(|e| {
469                WorkflowError::Other(format!(
470                    "Workflow side effect '{}' stored an invalid charge result: {e}",
471                    operation.operation_id
472                ))
473            }),
474            WorkflowSideEffectStatus::AlreadyCompleted { result: None } => {
475                Err(WorkflowError::Other(format!(
476                    "Workflow side effect '{}' completed without a stored charge result",
477                    operation.operation_id
478                )))
479            }
480        }
481    }
482}
483
484#[derive(Debug, Clone, PartialEq, Eq)]
485enum StepOutcome {
486    Continue,
487    Paused(StepPause),
488}
489
490#[derive(Debug, Clone, PartialEq, Eq)]
491struct StepPause {
492    frames: Vec<WorkflowCursorFrame>,
493    wait: WorkflowPendingWait,
494}
495
496struct RuntimeGuard {
497    lease: Option<WorkflowLease>,
498    operation: Option<WorkflowOperation>,
499}
500
501enum RuntimeEntry {
502    Active(RuntimeGuard),
503    Completed(String),
504}
505
506#[derive(Debug, Clone)]
507enum StepListCursorKind {
508    Steps,
509    Branch {
510        selection: WorkflowBranchCursorSelection,
511    },
512    ForEach {
513        item_index: usize,
514        item: Value,
515    },
516}
517
518#[derive(Debug, Clone, Copy, PartialEq, Eq)]
519enum RunMode<'a> {
520    Normal,
521    EventResume { event: &'a str },
522}
523
524const FOR_EACH_ITEM_KEY: &str = "item";
525const RESUME_EVENT_KEY: &str = "event";
526const TIMEOUT_FALLBACK_KEY: &str = "__qail_timeout_fallback";
527const RESERVED_CONTEXT_KEYS: &[&str] = &[FOR_EACH_ITEM_KEY, RESUME_EVENT_KEY, TIMEOUT_FALLBACK_KEY];
528
529#[derive(Debug, Clone)]
530struct StepExecutionScope<'a> {
531    state: &'a str,
532    path_prefix: Vec<WorkflowCursorFrame>,
533    list_kind: StepListCursorKind,
534    checkpoint_steps: bool,
535    checkpoint_timeout: bool,
536}
537
538impl<'a> StepExecutionScope<'a> {
539    fn new(state: &'a str, list_kind: StepListCursorKind, checkpoint_steps: bool) -> Self {
540        Self {
541            state,
542            path_prefix: Vec::new(),
543            list_kind,
544            checkpoint_steps,
545            checkpoint_timeout: false,
546        }
547    }
548
549    fn with_timeout_checkpoint(mut self) -> Self {
550        self.checkpoint_timeout = true;
551        self
552    }
553
554    fn child(&self, step_index: usize, list_kind: StepListCursorKind) -> Self {
555        Self {
556            state: self.state,
557            path_prefix: cursor_frames_for_index(&self.path_prefix, &self.list_kind, step_index),
558            list_kind,
559            checkpoint_steps: self.checkpoint_steps,
560            checkpoint_timeout: self.checkpoint_timeout,
561        }
562    }
563}
564
565impl RuntimeGuard {
566    async fn enter<E: WorkflowExecutor>(
567        executor: &E,
568        workflow_name: &str,
569        workflow_id: &str,
570        kind: WorkflowOperationKind,
571        options: WorkflowRunOptions,
572    ) -> Result<RuntimeEntry, WorkflowError> {
573        let lease_options = match options.lease {
574            Some(lease_options) => {
575                ensure_definition_text(&lease_options.owner, "Workflow lease owner")?;
576                if lease_options.ttl.is_zero() {
577                    return Err(WorkflowError::Other(
578                        "Workflow lease TTL must be greater than zero".to_string(),
579                    ));
580                }
581                Some(lease_options)
582            }
583            None => None,
584        };
585
586        let operation = match options.idempotency_key {
587            Some(idempotency_key) => {
588                ensure_definition_text(&idempotency_key, "Workflow idempotency key")?;
589                let operation = WorkflowOperation {
590                    workflow_name: workflow_name.to_string(),
591                    workflow_id: workflow_id.to_string(),
592                    idempotency_key,
593                    kind: kind.clone(),
594                };
595                let operation_status = match executor.begin_workflow_operation(&operation).await {
596                    Ok(status) => status,
597                    Err(err) => return Err(err),
598                };
599                match operation_status {
600                    WorkflowOperationStatus::Started => Some(operation),
601                    WorkflowOperationStatus::InProgress => {
602                        return Err(WorkflowError::Other(format!(
603                            "Workflow operation '{}' is already in progress",
604                            operation.idempotency_key
605                        )));
606                    }
607                    WorkflowOperationStatus::Completed { state } => {
608                        ensure_definition_text(&state, "Workflow completed operation state")?;
609                        return Ok(RuntimeEntry::Completed(state));
610                    }
611                }
612            }
613            None => None,
614        };
615
616        let lease = lease_options.map(|lease_options| WorkflowLease {
617            workflow_id: workflow_id.to_string(),
618            owner: runtime_lease_owner(
619                &lease_options.owner,
620                workflow_name,
621                workflow_id,
622                &kind,
623                operation
624                    .as_ref()
625                    .map(|operation| operation.idempotency_key.as_str()),
626                Utc::now(),
627            ),
628            ttl: lease_options.ttl,
629        });
630
631        let lease = match lease {
632            Some(lease) => match executor.acquire_workflow_lease(&lease).await {
633                Ok(true) => Some(lease),
634                Ok(false) => {
635                    let error = format!("Workflow '{}' is already locked", workflow_id);
636                    if let Some(operation) = &operation {
637                        let _ = executor.fail_workflow_operation(operation, &error).await;
638                    }
639                    return Err(WorkflowError::Other(error));
640                }
641                Err(err) => {
642                    if let Some(operation) = &operation {
643                        let _ = executor
644                            .fail_workflow_operation(operation, &err.to_string())
645                            .await;
646                    }
647                    return Err(err);
648                }
649            },
650            None => None,
651        };
652
653        Ok(RuntimeEntry::Active(Self { lease, operation }))
654    }
655
656    async fn finish<E: WorkflowExecutor>(
657        self,
658        executor: &E,
659        result: &Result<String, WorkflowError>,
660    ) -> Result<(), WorkflowError> {
661        let operation_result = match &self.operation {
662            Some(operation) => match result {
663                Ok(state) => executor.complete_workflow_operation(operation, state).await,
664                Err(err) => {
665                    executor
666                        .fail_workflow_operation(operation, &err.to_string())
667                        .await
668                }
669            },
670            None => Ok(()),
671        };
672
673        let release_result = match &self.lease {
674            Some(lease) => executor.release_workflow_lease(lease).await,
675            None => Ok(()),
676        };
677
678        match (operation_result, release_result) {
679            (Err(err), _) => Err(err),
680            (Ok(()), Err(err)) => Err(err),
681            (Ok(()), Ok(())) => Ok(()),
682        }
683    }
684}
685
686fn runtime_lease_owner(
687    owner: &str,
688    workflow_name: &str,
689    workflow_id: &str,
690    kind: &WorkflowOperationKind,
691    idempotency_key: Option<&str>,
692    acquired_at: DateTime<Utc>,
693) -> String {
694    serde_json::json!([
695        "qail-workflow-lease",
696        1,
697        owner,
698        workflow_name,
699        workflow_id,
700        operation_kind_lease_key(kind),
701        idempotency_key,
702        acquired_at.to_rfc3339_opts(SecondsFormat::Micros, true)
703    ])
704    .to_string()
705}
706
707fn operation_kind_lease_key(kind: &WorkflowOperationKind) -> serde_json::Value {
708    match kind {
709        WorkflowOperationKind::Run => serde_json::json!(["run"]),
710        WorkflowOperationKind::Resume { event } => serde_json::json!(["resume", event]),
711        WorkflowOperationKind::Timeout => serde_json::json!(["timeout"]),
712    }
713}
714
715async fn finish_runtime_operation<E: WorkflowExecutor>(
716    executor: &E,
717    guard: RuntimeGuard,
718    result: Result<String, WorkflowError>,
719) -> Result<String, WorkflowError> {
720    let finish_result = guard.finish(executor, &result).await;
721    match (result, finish_result) {
722        (Ok(state), Ok(())) => Ok(state),
723        (Ok(_), Err(err)) => Err(err),
724        (Err(err), Ok(())) | (Err(err), Err(_)) => Err(err),
725    }
726}
727
728fn invalid_cursor(message: impl Into<String>) -> WorkflowError {
729    WorkflowError::Other(format!(
730        "Invalid workflow resume cursor: {}",
731        message.into()
732    ))
733}
734
735fn set_timeout_fallback(
736    ctx: &mut WorkflowContext,
737    wait: &WorkflowPendingWait,
738) -> Result<(), WorkflowError> {
739    let value = serde_json::to_value(wait)
740        .map_err(|e| WorkflowError::Other(format!("Failed to serialize timeout cursor: {e}")))?;
741    ctx.set(TIMEOUT_FALLBACK_KEY, value);
742    Ok(())
743}
744
745fn timeout_fallback_from_context(
746    ctx: &WorkflowContext,
747) -> Result<Option<WorkflowPendingWait>, WorkflowError> {
748    let Some(value) = ctx.get(TIMEOUT_FALLBACK_KEY) else {
749        return Ok(None);
750    };
751    serde_json::from_value(value.clone())
752        .map(Some)
753        .map_err(|e| invalid_cursor(format!("timeout fallback metadata is invalid: {e}")))
754}
755
756fn clear_timeout_fallback(ctx: &mut WorkflowContext) {
757    if ctx.data.remove(TIMEOUT_FALLBACK_KEY).is_some() {
758        ctx.updated_at = Utc::now();
759    }
760}
761
762fn json_type_name(value: &Value) -> &'static str {
763    match value {
764        Value::Null => "null",
765        Value::Bool(_) => "boolean",
766        Value::Number(_) => "number",
767        Value::String(_) => "string",
768        Value::Array(_) => "array",
769        Value::Object(_) => "object",
770    }
771}
772
773fn ensure_no_child_cursor(
774    step_name: &str,
775    cursor_frames: &[WorkflowCursorFrame],
776) -> Result<(), WorkflowError> {
777    if cursor_frames.is_empty() {
778        Ok(())
779    } else {
780        Err(invalid_cursor(format!(
781            "{step_name} cannot resume nested cursor frames"
782        )))
783    }
784}
785
786fn ensure_step_position(
787    step: &WorkflowStep,
788    idx: usize,
789    step_count: usize,
790) -> Result<(), WorkflowError> {
791    if matches!(step, WorkflowStep::Transition { .. }) && idx + 1 < step_count {
792        return Err(WorkflowError::Other(
793            "Transition steps must be the final step in their block; move follow-up work before the Transition or into the target state's transition".to_string(),
794        ));
795    }
796    Ok(())
797}
798
799fn ensure_wait_event_name(event: &str) -> Result<(), WorkflowError> {
800    if event.trim().is_empty() {
801        return Err(WorkflowError::Other(
802            "Wait event name must not be empty".to_string(),
803        ));
804    }
805    if event.trim() != event {
806        return Err(WorkflowError::Other(
807            "Wait event name must not have leading or trailing whitespace".to_string(),
808        ));
809    }
810    Ok(())
811}
812
813fn ensure_definition_text(value: &str, usage: &str) -> Result<(), WorkflowError> {
814    if value.trim().is_empty() {
815        return Err(WorkflowError::Other(format!("{usage} must not be empty")));
816    }
817    if value.trim() != value {
818        return Err(WorkflowError::Other(format!(
819            "{usage} must not have leading or trailing whitespace"
820        )));
821    }
822    Ok(())
823}
824
825fn validate_workflow_context_identity(ctx: &WorkflowContext) -> Result<(), WorkflowError> {
826    ensure_definition_text(&ctx.workflow_id, "Workflow id")?;
827    ensure_definition_text(&ctx.current_state, "Workflow current state")?;
828    Ok(())
829}
830
831fn ensure_user_context_key(key: &str, usage: &str) -> Result<(), WorkflowError> {
832    ensure_definition_text(key, usage)?;
833    if key.contains('.') {
834        return Err(WorkflowError::Other(format!(
835            "Workflow {usage} must be a top-level context key; dot notation is only supported for lookups"
836        )));
837    }
838    if RESERVED_CONTEXT_KEYS.contains(&key) {
839        return Err(WorkflowError::Other(format!(
840            "Workflow {usage} uses reserved context key '{key}'"
841        )));
842    }
843    Ok(())
844}
845
846fn ensure_context_lookup_key(key: &str, usage: &str) -> Result<(), WorkflowError> {
847    ensure_definition_text(key, usage)?;
848    if key.split('.').any(str::is_empty) {
849        return Err(WorkflowError::Other(format!(
850            "{usage} must not contain empty dot-notation path segments"
851        )));
852    }
853    Ok(())
854}
855
856fn ensure_optional_user_context_key(
857    key: Option<&String>,
858    usage: &str,
859) -> Result<(), WorkflowError> {
860    if let Some(key) = key {
861        ensure_user_context_key(key, usage)?;
862    }
863    Ok(())
864}
865
866fn ensure_cursor_has_frames(cursor: &WorkflowCursor) -> Result<(), WorkflowError> {
867    if cursor.frames.is_empty() {
868        return Err(invalid_cursor("resume cursor has no frames"));
869    }
870    Ok(())
871}
872
873fn ensure_unique_branch_values(
874    condition_key: &str,
875    branches: &[(String, Vec<WorkflowStep>)],
876) -> Result<(), WorkflowError> {
877    let mut seen = std::collections::HashSet::new();
878    for (branch_value, _) in branches {
879        if !seen.insert(branch_value.as_str()) {
880            return Err(WorkflowError::Other(format!(
881                "Ambiguous workflow branch for '{condition_key}': duplicate value '{branch_value}'"
882            )));
883        }
884    }
885    Ok(())
886}
887
888fn ensure_unique_branch_conditions(
889    condition_key: &str,
890    branches: &[(WorkflowBranchCondition, Vec<WorkflowStep>)],
891) -> Result<(), WorkflowError> {
892    let mut seen = std::collections::HashSet::new();
893    for (condition, _) in branches {
894        validate_branch_condition(condition_key, condition)?;
895        if !seen.insert(condition) {
896            return Err(WorkflowError::Other(format!(
897                "Ambiguous workflow branch for '{condition_key}': duplicate condition '{condition:?}'"
898            )));
899        }
900    }
901    Ok(())
902}
903
904fn validate_branch_condition(
905    condition_key: &str,
906    condition: &WorkflowBranchCondition,
907) -> Result<(), WorkflowError> {
908    match condition {
909        WorkflowBranchCondition::OneOf(expected_values) if expected_values.is_empty() => {
910            Err(WorkflowError::Other(format!(
911                "Ambiguous workflow branch for '{condition_key}': OneOf condition must include at least one value"
912            )))
913        }
914        WorkflowBranchCondition::StringContains(needle) if needle.is_empty() => {
915            Err(WorkflowError::Other(format!(
916                "Ambiguous workflow branch for '{condition_key}': StringContains condition must not be empty"
917            )))
918        }
919        _ => Ok(()),
920    }
921}
922
923fn branch_condition_matches(condition: &WorkflowBranchCondition, value: Option<&Value>) -> bool {
924    match condition {
925        WorkflowBranchCondition::Exists => value.is_some(),
926        WorkflowBranchCondition::Missing => value.is_none(),
927        WorkflowBranchCondition::Null => matches!(value, Some(Value::Null)),
928        WorkflowBranchCondition::Bool(expected) => {
929            value.and_then(Value::as_bool) == Some(*expected)
930        }
931        WorkflowBranchCondition::Equals(expected) => {
932            value.map(branch_condition_text).as_deref() == Some(expected.as_str())
933        }
934        WorkflowBranchCondition::NotEquals(expected) => value
935            .map(branch_condition_text)
936            .is_some_and(|actual| actual != *expected),
937        WorkflowBranchCondition::OneOf(expected_values) => value
938            .map(branch_condition_text)
939            .is_some_and(|actual| expected_values.iter().any(|expected| expected == &actual)),
940        WorkflowBranchCondition::NumberGt(expected) => value
941            .and_then(|value| branch_condition_number_cmp(value, *expected))
942            .is_some_and(|ordering| ordering == Ordering::Greater),
943        WorkflowBranchCondition::NumberGte(expected) => value
944            .and_then(|value| branch_condition_number_cmp(value, *expected))
945            .is_some_and(|ordering| ordering != Ordering::Less),
946        WorkflowBranchCondition::NumberLt(expected) => value
947            .and_then(|value| branch_condition_number_cmp(value, *expected))
948            .is_some_and(|ordering| ordering == Ordering::Less),
949        WorkflowBranchCondition::NumberLte(expected) => value
950            .and_then(|value| branch_condition_number_cmp(value, *expected))
951            .is_some_and(|ordering| ordering != Ordering::Greater),
952        WorkflowBranchCondition::StringContains(needle) => value
953            .map(branch_condition_text)
954            .is_some_and(|actual| actual.contains(needle)),
955    }
956}
957
958fn branch_condition_text(value: &Value) -> String {
959    match value {
960        Value::String(value) => value.clone(),
961        _ => value.to_string(),
962    }
963}
964
965fn branch_condition_number_cmp(value: &Value, expected: i64) -> Option<Ordering> {
966    let Value::Number(number) = value else {
967        return None;
968    };
969
970    if let Some(actual) = number.as_i64() {
971        return Some(actual.cmp(&expected));
972    }
973
974    if let Some(actual) = number.as_u64() {
975        return Some(if expected < 0 {
976            Ordering::Greater
977        } else {
978            actual.cmp(&(expected as u64))
979        });
980    }
981
982    let actual = number.as_f64()?;
983    actual.partial_cmp(&(expected as f64))
984}
985
986fn find_single_transition<'a>(
987    definition: &'a WorkflowDefinition,
988    state: &str,
989) -> Result<Option<&'a StateTransition>, WorkflowError> {
990    let mut matches = definition.transitions.iter().filter(|t| t.from == state);
991    let Some(first) = matches.next() else {
992        return Ok(None);
993    };
994    if matches.next().is_some() {
995        let count = definition.transitions_from(state).len();
996        return Err(WorkflowError::Other(format!(
997            "Ambiguous workflow definition '{}': state '{state}' has {count} outgoing transitions; use a Branch step or split states explicitly",
998            definition.name
999        )));
1000    }
1001    Ok(Some(first))
1002}
1003
1004fn validate_workflow_definition(definition: &WorkflowDefinition) -> Result<(), WorkflowError> {
1005    ensure_definition_text(&definition.name, "Workflow definition name")?;
1006    ensure_definition_text(&definition.initial_state, "Workflow initial state")?;
1007    if let Some(version) = &definition.version {
1008        ensure_definition_text(version, "Workflow definition version")?;
1009    }
1010
1011    let mut seen_states = std::collections::HashMap::<&str, usize>::new();
1012    for transition in &definition.transitions {
1013        ensure_definition_text(&transition.from, "Workflow transition source state")?;
1014        ensure_definition_text(&transition.to, "Workflow transition target state")?;
1015        *seen_states.entry(&transition.from).or_default() += 1;
1016        validate_workflow_steps(&transition.steps)?;
1017    }
1018
1019    if let Some((state, count)) = seen_states.into_iter().find(|(_, count)| *count > 1) {
1020        return Err(WorkflowError::Other(format!(
1021            "Ambiguous workflow definition '{}': state '{state}' has {count} outgoing transitions; use a Branch step or split states explicitly",
1022            definition.name
1023        )));
1024    }
1025
1026    Ok(())
1027}
1028
1029fn ensure_context_definition(
1030    ctx: &mut WorkflowContext,
1031    definition: &WorkflowDefinition,
1032) -> Result<(), WorkflowError> {
1033    if let Some(context_name) = &ctx.definition_name
1034        && context_name != &definition.name
1035    {
1036        return Err(WorkflowError::Other(format!(
1037            "Workflow context '{}' belongs to definition '{context_name}', not '{}'",
1038            ctx.workflow_id, definition.name
1039        )));
1040    }
1041
1042    if let Some(context_version) = &ctx.definition_version
1043        && Some(context_version) != definition.version.as_ref()
1044    {
1045        return Err(WorkflowError::Other(format!(
1046            "Workflow context '{}' uses definition version '{}', not '{}'",
1047            ctx.workflow_id,
1048            context_version,
1049            definition.version.as_deref().unwrap_or("<unversioned>")
1050        )));
1051    }
1052
1053    if ctx.definition_name.is_none() {
1054        ctx.definition_name = Some(definition.name.clone());
1055        ctx.updated_at = Utc::now();
1056    }
1057    if ctx.definition_version.is_none()
1058        && let Some(version) = &definition.version
1059    {
1060        ctx.definition_version = Some(version.clone());
1061        ctx.updated_at = Utc::now();
1062    }
1063
1064    Ok(())
1065}
1066
1067fn validate_workflow_steps(steps: &[WorkflowStep]) -> Result<(), WorkflowError> {
1068    for (idx, step) in steps.iter().enumerate() {
1069        ensure_step_position(step, idx, steps.len())?;
1070        match step {
1071            WorkflowStep::Query { cmd_json, store_as } => {
1072                normalize_query_wire_for_execution(cmd_json)?;
1073                ensure_optional_user_context_key(store_as.as_ref(), "Query store_as")?;
1074            }
1075            WorkflowStep::Wait {
1076                event, on_timeout, ..
1077            } => {
1078                ensure_wait_event_name(event)?;
1079                if steps_contain_wait(on_timeout) {
1080                    return Err(WorkflowError::Other(
1081                        "Wait steps inside on_timeout fallback are not supported".to_string(),
1082                    ));
1083                }
1084                validate_workflow_steps(on_timeout)?;
1085            }
1086            WorkflowStep::Branch {
1087                condition_key,
1088                branches,
1089                default,
1090            } => {
1091                ensure_context_lookup_key(condition_key, "Branch condition_key")?;
1092                ensure_unique_branch_values(condition_key, branches)?;
1093                for (_, branch_steps) in branches {
1094                    validate_workflow_steps(branch_steps)?;
1095                }
1096                validate_workflow_steps(default)?;
1097            }
1098            WorkflowStep::BranchWhen {
1099                condition_key,
1100                branches,
1101                default,
1102            } => {
1103                ensure_context_lookup_key(condition_key, "BranchWhen condition_key")?;
1104                ensure_unique_branch_conditions(condition_key, branches)?;
1105                for (_, branch_steps) in branches {
1106                    validate_workflow_steps(branch_steps)?;
1107                }
1108                validate_workflow_steps(default)?;
1109            }
1110            WorkflowStep::ForEach { list_key, steps } => {
1111                ensure_context_lookup_key(list_key, "ForEach list_key")?;
1112                validate_workflow_steps(steps)?;
1113            }
1114            WorkflowStep::Charge {
1115                amount_key,
1116                reference_key,
1117                description_key,
1118                payment_method_key,
1119                order_origin_key,
1120                store_as,
1121                ..
1122            } => {
1123                ensure_context_lookup_key(amount_key, "Charge amount_key")?;
1124                ensure_context_lookup_key(reference_key, "Charge reference_key")?;
1125                if let Some(description_key) = description_key {
1126                    ensure_context_lookup_key(description_key, "Charge description_key")?;
1127                }
1128                if let Some(payment_method_key) = payment_method_key {
1129                    ensure_context_lookup_key(payment_method_key, "Charge payment_method_key")?;
1130                }
1131                if let Some(order_origin_key) = order_origin_key {
1132                    ensure_context_lookup_key(order_origin_key, "Charge order_origin_key")?;
1133                }
1134                ensure_optional_user_context_key(store_as.as_ref(), "Charge store_as")?;
1135            }
1136            WorkflowStep::Notify {
1137                template,
1138                recipient_key,
1139                payload_key,
1140                ..
1141            } => {
1142                ensure_definition_text(template, "Notify template")?;
1143                ensure_context_lookup_key(recipient_key, "Notify recipient_key")?;
1144                if let Some(payload_key) = payload_key {
1145                    ensure_context_lookup_key(payload_key, "Notify payload_key")?;
1146                }
1147            }
1148            WorkflowStep::Transition { to } => {
1149                ensure_definition_text(to, "Transition target state")?;
1150            }
1151            WorkflowStep::Log { .. } => {}
1152        }
1153    }
1154
1155    Ok(())
1156}
1157
1158fn deadline_from_timeout(timeout: &std::time::Duration) -> Result<DateTime<Utc>, WorkflowError> {
1159    let timeout = chrono::Duration::from_std(*timeout).map_err(|_| {
1160        WorkflowError::Other("Wait timeout duration is too large to represent".to_string())
1161    })?;
1162    Utc::now()
1163        .checked_add_signed(timeout)
1164        .ok_or_else(|| WorkflowError::Other("Wait timeout deadline overflowed".to_string()))
1165}
1166
1167fn cursor_frame_for_index(kind: &StepListCursorKind, index: usize) -> WorkflowCursorFrame {
1168    match kind {
1169        StepListCursorKind::Steps => WorkflowCursorFrame::Steps { index },
1170        StepListCursorKind::Branch { selection } => WorkflowCursorFrame::Branch {
1171            selection: selection.clone(),
1172            index,
1173        },
1174        StepListCursorKind::ForEach { item_index, item } => WorkflowCursorFrame::ForEachItem {
1175            item_index: *item_index,
1176            index,
1177            item: item.clone(),
1178        },
1179    }
1180}
1181
1182fn cursor_frames_for_index(
1183    path_prefix: &[WorkflowCursorFrame],
1184    kind: &StepListCursorKind,
1185    index: usize,
1186) -> Vec<WorkflowCursorFrame> {
1187    let mut frames = Vec::with_capacity(path_prefix.len() + 1);
1188    frames.extend_from_slice(path_prefix);
1189    frames.push(cursor_frame_for_index(kind, index));
1190    frames
1191}
1192
1193async fn checkpoint_cursor<E: WorkflowExecutor>(
1194    executor: &E,
1195    ctx: &mut WorkflowContext,
1196    state: &str,
1197    frames: Vec<WorkflowCursorFrame>,
1198    wait: Option<WorkflowPendingWait>,
1199    timeout_fallback: bool,
1200) -> Result<(), WorkflowError> {
1201    if !timeout_fallback {
1202        clear_timeout_fallback(ctx);
1203    }
1204    let wait = if timeout_fallback && wait.is_none() {
1205        timeout_fallback_from_context(ctx)?
1206    } else {
1207        wait
1208    };
1209    if ctx.current_state == state {
1210        ctx.set_cursor(WorkflowCursor {
1211            state: state.to_string(),
1212            frames,
1213            wait,
1214        });
1215    } else {
1216        ctx.clear_cursor();
1217        if timeout_fallback {
1218            clear_timeout_fallback(ctx);
1219        }
1220    }
1221    executor.save_state(ctx).await
1222}
1223
1224async fn checkpoint_completed_step<E: WorkflowExecutor>(
1225    executor: &E,
1226    ctx: &mut WorkflowContext,
1227    state: &str,
1228    path_prefix: &[WorkflowCursorFrame],
1229    kind: &StepListCursorKind,
1230    next_index: usize,
1231    timeout_fallback: bool,
1232) -> Result<(), WorkflowError> {
1233    let frames = cursor_frames_for_index(path_prefix, kind, next_index);
1234    checkpoint_cursor(executor, ctx, state, frames, None, timeout_fallback).await
1235}
1236
1237fn restore_for_each_item(ctx: &mut WorkflowContext, previous_item: Option<Value>) {
1238    match previous_item {
1239        Some(item) => {
1240            ctx.data.insert(FOR_EACH_ITEM_KEY.to_string(), item);
1241        }
1242        None => {
1243            ctx.data.remove(FOR_EACH_ITEM_KEY);
1244        }
1245    }
1246    ctx.updated_at = Utc::now();
1247}
1248
1249fn side_effect_operation(
1250    ctx: &WorkflowContext,
1251    scope: &StepExecutionScope<'_>,
1252    step_index: usize,
1253    kind: WorkflowSideEffectKind,
1254) -> WorkflowSideEffect {
1255    let frames = cursor_frames_for_index(&scope.path_prefix, &scope.list_kind, step_index);
1256    let step_path = render_cursor_path(&frames);
1257    WorkflowSideEffect {
1258        workflow_id: ctx.workflow_id.clone(),
1259        state: scope.state.to_string(),
1260        step_path: step_path.clone(),
1261        kind,
1262        operation_id: side_effect_operation_id(
1263            &ctx.workflow_id,
1264            scope.state,
1265            ctx.transition_count(),
1266            kind,
1267            &step_path,
1268        ),
1269    }
1270}
1271
1272fn side_effect_operation_id(
1273    workflow_id: &str,
1274    state: &str,
1275    state_generation: usize,
1276    kind: WorkflowSideEffectKind,
1277    step_path: &str,
1278) -> String {
1279    serde_json::json!([
1280        "qail-workflow-side-effect",
1281        2,
1282        workflow_id,
1283        state,
1284        state_generation,
1285        kind.as_str(),
1286        step_path
1287    ])
1288    .to_string()
1289}
1290
1291fn render_cursor_path(frames: &[WorkflowCursorFrame]) -> String {
1292    frames
1293        .iter()
1294        .map(render_cursor_frame)
1295        .collect::<Vec<_>>()
1296        .join("/")
1297}
1298
1299fn render_cursor_frame(frame: &WorkflowCursorFrame) -> String {
1300    match frame {
1301        WorkflowCursorFrame::Steps { index } => format!("steps[{index}]"),
1302        WorkflowCursorFrame::Branch { selection, index } => {
1303            format!(
1304                "branch[{}].steps[{index}]",
1305                render_branch_selection(selection)
1306            )
1307        }
1308        WorkflowCursorFrame::ForEach { item_index, index } => {
1309            format!("for_each[{item_index}].steps[{index}]")
1310        }
1311        WorkflowCursorFrame::ForEachItem {
1312            item_index, index, ..
1313        } => format!("for_each[{item_index}].steps[{index}]"),
1314    }
1315}
1316
1317fn render_branch_selection(selection: &WorkflowBranchCursorSelection) -> String {
1318    match selection {
1319        WorkflowBranchCursorSelection::Branch(index) => index.to_string(),
1320        WorkflowBranchCursorSelection::Default => "default".to_string(),
1321    }
1322}
1323
1324fn log_value_to_string(value: &Value) -> String {
1325    match value {
1326        Value::String(value) => value.clone(),
1327        _ => value.to_string(),
1328    }
1329}
1330
1331fn resolve_log_message(ctx: &WorkflowContext, message: &str) -> String {
1332    let mut resolved = String::with_capacity(message.len());
1333    let mut rest = message;
1334
1335    while let Some(start) = rest.find('{') {
1336        resolved.push_str(&rest[..start]);
1337        let after_open = &rest[start + 1..];
1338        let Some(end) = after_open.find('}') else {
1339            resolved.push_str(&rest[start..]);
1340            return resolved;
1341        };
1342
1343        let key = &after_open[..end];
1344        if !key.is_empty()
1345            && let Some(value) = ctx.get(key)
1346        {
1347            resolved.push_str(&log_value_to_string(value));
1348        } else {
1349            resolved.push('{');
1350            resolved.push_str(key);
1351            resolved.push('}');
1352        }
1353        rest = &after_open[end + 1..];
1354    }
1355
1356    resolved.push_str(rest);
1357    resolved
1358}
1359
1360fn execute_steps<'a, E: WorkflowExecutor>(
1361    executor: &'a E,
1362    steps: &'a [WorkflowStep],
1363    ctx: &'a mut WorkflowContext,
1364    start_index: usize,
1365    cursor_frames: &'a [WorkflowCursorFrame],
1366    scope: StepExecutionScope<'a>,
1367) -> std::pin::Pin<
1368    Box<dyn std::future::Future<Output = Result<Option<StepPause>, WorkflowError>> + Send + 'a>,
1369> {
1370    Box::pin(async move {
1371        if start_index > steps.len() {
1372            return Err(invalid_cursor(format!(
1373                "step index {start_index} is past step count {}",
1374                steps.len()
1375            )));
1376        }
1377        if start_index == steps.len() && !cursor_frames.is_empty() {
1378            return Err(invalid_cursor(
1379                "cursor contains child frames after the step list ended",
1380            ));
1381        }
1382
1383        for (idx, step) in steps.iter().enumerate().skip(start_index) {
1384            ensure_step_position(step, idx, steps.len())?;
1385            let step_cursor = if idx == start_index {
1386                cursor_frames
1387            } else {
1388                &[]
1389            };
1390            match execute_step(executor, step, ctx, idx, step_cursor, &scope).await? {
1391                StepOutcome::Continue => {
1392                    if scope.checkpoint_steps {
1393                        checkpoint_completed_step(
1394                            executor,
1395                            ctx,
1396                            scope.state,
1397                            &scope.path_prefix,
1398                            &scope.list_kind,
1399                            idx + 1,
1400                            scope.checkpoint_timeout,
1401                        )
1402                        .await?;
1403                    }
1404                    if ctx.current_state != scope.state {
1405                        return Ok(None);
1406                    }
1407                }
1408                StepOutcome::Paused(pause) => return Ok(Some(pause)),
1409            }
1410        }
1411
1412        Ok(None)
1413    })
1414}
1415
1416fn selected_branch_steps<'a>(
1417    branches: &'a [(String, Vec<WorkflowStep>)],
1418    default: &'a [WorkflowStep],
1419    selection: &WorkflowBranchCursorSelection,
1420) -> Result<&'a [WorkflowStep], WorkflowError> {
1421    match selection {
1422        WorkflowBranchCursorSelection::Branch(idx) => branches
1423            .get(*idx)
1424            .map(|(_, steps)| steps.as_slice())
1425            .ok_or_else(|| invalid_cursor(format!("branch index {idx} no longer exists"))),
1426        WorkflowBranchCursorSelection::Default => Ok(default),
1427    }
1428}
1429
1430fn selected_branch_when_steps<'a>(
1431    branches: &'a [(WorkflowBranchCondition, Vec<WorkflowStep>)],
1432    default: &'a [WorkflowStep],
1433    selection: &WorkflowBranchCursorSelection,
1434) -> Result<&'a [WorkflowStep], WorkflowError> {
1435    match selection {
1436        WorkflowBranchCursorSelection::Branch(idx) => branches
1437            .get(*idx)
1438            .map(|(_, steps)| steps.as_slice())
1439            .ok_or_else(|| invalid_cursor(format!("branch index {idx} no longer exists"))),
1440        WorkflowBranchCursorSelection::Default => Ok(default),
1441    }
1442}
1443
1444fn validate_branch_resume_selection(
1445    branches: &[(String, Vec<WorkflowStep>)],
1446    selection: &WorkflowBranchCursorSelection,
1447    condition_key: &str,
1448    condition_value: &str,
1449) -> Result<(), WorkflowError> {
1450    match selection {
1451        WorkflowBranchCursorSelection::Branch(idx) => {
1452            let Some((branch_value, _)) = branches.get(*idx) else {
1453                return Err(invalid_cursor(format!(
1454                    "branch index {idx} no longer exists"
1455                )));
1456            };
1457            if branch_value != condition_value {
1458                return Err(invalid_cursor(format!(
1459                    "branch cursor selected index {idx} for value '{branch_value}', \
1460                     but current condition '{condition_key}' is '{condition_value}'"
1461                )));
1462            }
1463        }
1464        WorkflowBranchCursorSelection::Default => {
1465            if branches
1466                .iter()
1467                .any(|(branch_value, _)| branch_value == condition_value)
1468            {
1469                return Err(invalid_cursor(format!(
1470                    "default branch cursor no longer matches condition '{condition_key}' \
1471                     value '{condition_value}'"
1472                )));
1473            }
1474        }
1475    }
1476    Ok(())
1477}
1478
1479fn validate_branch_when_resume_selection(
1480    branches: &[(WorkflowBranchCondition, Vec<WorkflowStep>)],
1481    selection: &WorkflowBranchCursorSelection,
1482    condition_key: &str,
1483    condition_value: Option<&Value>,
1484) -> Result<(), WorkflowError> {
1485    match selection {
1486        WorkflowBranchCursorSelection::Branch(idx) => {
1487            let Some((condition, _)) = branches.get(*idx) else {
1488                return Err(invalid_cursor(format!(
1489                    "branch index {idx} no longer exists"
1490                )));
1491            };
1492            if !branch_condition_matches(condition, condition_value) {
1493                return Err(invalid_cursor(format!(
1494                    "branch cursor selected index {idx} for condition '{condition:?}', \
1495                     but current condition '{condition_key}' no longer matches"
1496                )));
1497            }
1498        }
1499        WorkflowBranchCursorSelection::Default => {
1500            if branches
1501                .iter()
1502                .any(|(condition, _)| branch_condition_matches(condition, condition_value))
1503            {
1504                return Err(invalid_cursor(format!(
1505                    "default branch cursor no longer matches condition '{condition_key}'"
1506                )));
1507            }
1508        }
1509    }
1510    Ok(())
1511}
1512
1513/// Execute a single workflow step.
1514fn execute_step<'a, E: WorkflowExecutor>(
1515    executor: &'a E,
1516    step: &'a WorkflowStep,
1517    ctx: &'a mut WorkflowContext,
1518    step_index: usize,
1519    cursor_frames: &'a [WorkflowCursorFrame],
1520    scope: &'a StepExecutionScope<'a>,
1521) -> std::pin::Pin<
1522    Box<dyn std::future::Future<Output = Result<StepOutcome, WorkflowError>> + Send + 'a>,
1523> {
1524    Box::pin(async move {
1525        match step {
1526            WorkflowStep::Query { cmd_json, store_as } => {
1527                ensure_no_child_cursor("Query", cursor_frames)?;
1528                ensure_optional_user_context_key(store_as.as_ref(), "Query store_as")?;
1529                let cmd_wire = normalize_query_wire_for_execution(cmd_json)?;
1530                let operation =
1531                    side_effect_operation(ctx, scope, step_index, WorkflowSideEffectKind::Query);
1532                let result = executor.execute_query_once(&operation, &cmd_wire).await?;
1533                if let Some(key) = store_as {
1534                    ctx.set(key, result);
1535                }
1536            }
1537
1538            WorkflowStep::Notify {
1539                channel,
1540                template,
1541                recipient_key,
1542                payload_key,
1543            } => {
1544                ensure_no_child_cursor("Notify", cursor_frames)?;
1545                let recipient = ctx
1546                    .get_str(recipient_key)
1547                    .ok_or_else(|| WorkflowError::MissingContextKey(recipient_key.clone()))?
1548                    .to_string();
1549
1550                let params = payload_key
1551                    .as_ref()
1552                    .map(|key| {
1553                        ctx.get(key)
1554                            .cloned()
1555                            .ok_or_else(|| WorkflowError::MissingContextKey(key.clone()))
1556                    })
1557                    .transpose()?
1558                    .unwrap_or(Value::Object(serde_json::Map::new()));
1559
1560                let operation =
1561                    side_effect_operation(ctx, scope, step_index, WorkflowSideEffectKind::Notify);
1562                executor
1563                    .send_notification_once(&operation, channel, &recipient, template, &params)
1564                    .await?;
1565            }
1566
1567            WorkflowStep::Wait {
1568                event,
1569                timeout,
1570                on_timeout,
1571            } => {
1572                ensure_no_child_cursor("Wait", cursor_frames)?;
1573                ensure_wait_event_name(event)?;
1574                // Wait steps are handled by the outer runner so state can be
1575                // persisted before returning to the caller.
1576                return Ok(StepOutcome::Paused(StepPause {
1577                    frames: cursor_frames_for_index(
1578                        &scope.path_prefix,
1579                        &scope.list_kind,
1580                        step_index + 1,
1581                    ),
1582                    wait: WorkflowPendingWait {
1583                        event: event.clone(),
1584                        deadline_at: deadline_from_timeout(timeout)?,
1585                        on_timeout: on_timeout.clone(),
1586                    },
1587                }));
1588            }
1589
1590            WorkflowStep::Branch {
1591                condition_key,
1592                branches,
1593                default,
1594            } => {
1595                ensure_unique_branch_values(condition_key, branches)?;
1596                let (selection, selected_steps, start_index, nested_cursor) = match cursor_frames
1597                    .first()
1598                {
1599                    Some(WorkflowCursorFrame::Branch { selection, index }) => {
1600                        let condition_value = ctx
1601                            .get_str(condition_key)
1602                            .ok_or_else(|| WorkflowError::MissingContextKey(condition_key.clone()))?
1603                            .to_string();
1604                        validate_branch_resume_selection(
1605                            branches,
1606                            selection,
1607                            condition_key,
1608                            &condition_value,
1609                        )?;
1610                        (
1611                            selection.clone(),
1612                            selected_branch_steps(branches, default, selection)?,
1613                            *index,
1614                            &cursor_frames[1..],
1615                        )
1616                    }
1617                    Some(_) => {
1618                        return Err(invalid_cursor(
1619                            "expected Branch frame for nested branch resume",
1620                        ));
1621                    }
1622                    None => {
1623                        let value = ctx
1624                            .get_str(condition_key)
1625                            .ok_or_else(|| WorkflowError::MissingContextKey(condition_key.clone()))?
1626                            .to_string();
1627                        match branches.iter().enumerate().find(|(_, (k, _))| k == &value) {
1628                            Some((idx, (_, steps))) => (
1629                                WorkflowBranchCursorSelection::Branch(idx),
1630                                steps.as_slice(),
1631                                0,
1632                                &[][..],
1633                            ),
1634                            None => (
1635                                WorkflowBranchCursorSelection::Default,
1636                                default.as_slice(),
1637                                0,
1638                                &[][..],
1639                            ),
1640                        }
1641                    }
1642                };
1643
1644                if let Some(pause) = execute_steps(
1645                    executor,
1646                    selected_steps,
1647                    ctx,
1648                    start_index,
1649                    nested_cursor,
1650                    scope.child(step_index, StepListCursorKind::Branch { selection }),
1651                )
1652                .await?
1653                {
1654                    return Ok(StepOutcome::Paused(pause));
1655                }
1656            }
1657
1658            WorkflowStep::BranchWhen {
1659                condition_key,
1660                branches,
1661                default,
1662            } => {
1663                ensure_unique_branch_conditions(condition_key, branches)?;
1664                let condition_value = ctx.get(condition_key).cloned();
1665                let (selection, selected_steps, start_index, nested_cursor) =
1666                    match cursor_frames.first() {
1667                        Some(WorkflowCursorFrame::Branch { selection, index }) => {
1668                            validate_branch_when_resume_selection(
1669                                branches,
1670                                selection,
1671                                condition_key,
1672                                condition_value.as_ref(),
1673                            )?;
1674                            (
1675                                selection.clone(),
1676                                selected_branch_when_steps(branches, default, selection)?,
1677                                *index,
1678                                &cursor_frames[1..],
1679                            )
1680                        }
1681                        Some(_) => {
1682                            return Err(invalid_cursor(
1683                                "expected Branch frame for nested branch resume",
1684                            ));
1685                        }
1686                        None => match branches.iter().enumerate().find(|(_, (condition, _))| {
1687                            branch_condition_matches(condition, condition_value.as_ref())
1688                        }) {
1689                            Some((idx, (_, steps))) => (
1690                                WorkflowBranchCursorSelection::Branch(idx),
1691                                steps.as_slice(),
1692                                0,
1693                                &[][..],
1694                            ),
1695                            None => (
1696                                WorkflowBranchCursorSelection::Default,
1697                                default.as_slice(),
1698                                0,
1699                                &[][..],
1700                            ),
1701                        },
1702                    };
1703
1704                if let Some(pause) = execute_steps(
1705                    executor,
1706                    selected_steps,
1707                    ctx,
1708                    start_index,
1709                    nested_cursor,
1710                    scope.child(step_index, StepListCursorKind::Branch { selection }),
1711                )
1712                .await?
1713                {
1714                    return Ok(StepOutcome::Paused(pause));
1715                }
1716            }
1717
1718            WorkflowStep::ForEach { list_key, steps } => {
1719                let list = ctx
1720                    .get(list_key)
1721                    .cloned()
1722                    .ok_or_else(|| WorkflowError::MissingContextKey(list_key.clone()))?;
1723
1724                let Value::Array(items) = list else {
1725                    return Err(WorkflowError::Other(format!(
1726                        "Expected JSON array for ForEach list '{}', got {}",
1727                        list_key,
1728                        json_type_name(&list)
1729                    )));
1730                };
1731
1732                let (start_item_index, start_step_index, nested_cursor) = match cursor_frames
1733                    .first()
1734                {
1735                    Some(WorkflowCursorFrame::ForEachItem {
1736                        item_index,
1737                        index,
1738                        item,
1739                    }) => {
1740                        if *item_index >= items.len() {
1741                            return Err(invalid_cursor(format!(
1742                                "for_each item index {item_index} is past item count {}",
1743                                items.len()
1744                            )));
1745                        }
1746                        if items.get(*item_index) != Some(item) {
1747                            return Err(invalid_cursor(format!(
1748                                "for_each cursor item at index {item_index} changed while the workflow was paused"
1749                            )));
1750                        }
1751                        (*item_index, *index, &cursor_frames[1..])
1752                    }
1753                    Some(WorkflowCursorFrame::ForEach { .. }) => {
1754                        return Err(invalid_cursor(
1755                            "legacy for_each cursor without item snapshot cannot be resumed safely",
1756                        ));
1757                    }
1758                    Some(_) => {
1759                        return Err(invalid_cursor(
1760                            "expected ForEach frame for nested loop resume",
1761                        ));
1762                    }
1763                    None => (0, 0, &[][..]),
1764                };
1765
1766                for (item_index, item) in items.into_iter().enumerate().skip(start_item_index) {
1767                    let item_snapshot = item.clone();
1768                    let previous_item = ctx.data.insert(FOR_EACH_ITEM_KEY.to_string(), item);
1769                    ctx.updated_at = Utc::now();
1770                    let item_step_start = if item_index == start_item_index {
1771                        start_step_index
1772                    } else {
1773                        0
1774                    };
1775                    let item_cursor = if item_index == start_item_index {
1776                        nested_cursor
1777                    } else {
1778                        &[]
1779                    };
1780
1781                    match execute_steps(
1782                        executor,
1783                        steps,
1784                        ctx,
1785                        item_step_start,
1786                        item_cursor,
1787                        scope.child(
1788                            step_index,
1789                            StepListCursorKind::ForEach {
1790                                item_index,
1791                                item: item_snapshot,
1792                            },
1793                        ),
1794                    )
1795                    .await
1796                    {
1797                        Ok(Some(pause)) => {
1798                            restore_for_each_item(ctx, previous_item);
1799                            return Ok(StepOutcome::Paused(pause));
1800                        }
1801                        Ok(None) => {
1802                            restore_for_each_item(ctx, previous_item);
1803                        }
1804                        Err(err) => {
1805                            restore_for_each_item(ctx, previous_item);
1806                            return Err(err);
1807                        }
1808                    }
1809                }
1810            }
1811
1812            WorkflowStep::Transition { to } => {
1813                ensure_no_child_cursor("Transition", cursor_frames)?;
1814                ctx.transition_to(to, None);
1815            }
1816
1817            WorkflowStep::Log { message } => {
1818                ensure_no_child_cursor("Log", cursor_frames)?;
1819                let resolved = resolve_log_message(ctx, message);
1820                // Engine logging — consumers can override via tracing subscriber
1821                eprintln!("[workflow:{}] {}", ctx.workflow_id, resolved);
1822            }
1823
1824            WorkflowStep::Charge {
1825                provider,
1826                amount_key,
1827                reference_key,
1828                description_key,
1829                payment_method_key,
1830                order_origin_key,
1831                store_as,
1832            } => {
1833                ensure_no_child_cursor("Charge", cursor_frames)?;
1834                ensure_optional_user_context_key(store_as.as_ref(), "Charge store_as")?;
1835                let amount = resolve_charge_amount(ctx, amount_key)?;
1836
1837                let reference_id = ctx
1838                    .get_str(reference_key)
1839                    .ok_or_else(|| WorkflowError::MissingContextKey(reference_key.clone()))?
1840                    .to_string();
1841
1842                let description = resolve_optional_charge_string(ctx, description_key.as_ref())?;
1843                let payment_method =
1844                    resolve_optional_charge_string(ctx, payment_method_key.as_ref())?;
1845                let order_origin = resolve_optional_order_origin(ctx, order_origin_key.as_ref())?;
1846
1847                let request = ChargeRequest {
1848                    amount,
1849                    currency: Currency::default(),
1850                    reference_id,
1851                    description,
1852                    payment_method,
1853                    order_origin,
1854                    idempotency_key: None,
1855                    return_url: None,
1856                    metadata: None,
1857                };
1858
1859                let operation =
1860                    side_effect_operation(ctx, scope, step_index, WorkflowSideEffectKind::Charge);
1861                let response = executor
1862                    .create_charge_once(&operation, provider, request.clone())
1863                    .await?;
1864
1865                if let Some(key) = store_as {
1866                    let response_json = serde_json::to_value(response.display_for(&request))
1867                        .map_err(|e| WorkflowError::Other(e.to_string()))?;
1868                    ctx.set(key, response_json);
1869                }
1870            }
1871        }
1872
1873        Ok(StepOutcome::Continue)
1874    })
1875}
1876
1877fn resolve_charge_amount(ctx: &WorkflowContext, amount_key: &str) -> Result<i64, WorkflowError> {
1878    let value = ctx.get(amount_key).ok_or_else(|| {
1879        WorkflowError::MissingContextKey(format!("{amount_key} (expected numeric amount)"))
1880    })?;
1881
1882    let amount = match value {
1883        Value::Number(number) => charge_amount_from_number(number, amount_key)?,
1884        _ => {
1885            return Err(WorkflowError::MissingContextKey(format!(
1886                "{amount_key} (expected numeric amount)"
1887            )));
1888        }
1889    };
1890
1891    if amount <= 0 {
1892        return Err(invalid_charge_amount(
1893            amount_key,
1894            "amount must be greater than zero",
1895        ));
1896    }
1897
1898    Ok(amount)
1899}
1900
1901fn resolve_optional_charge_string(
1902    ctx: &WorkflowContext,
1903    key: Option<&String>,
1904) -> Result<Option<String>, WorkflowError> {
1905    let Some(key) = key else {
1906        return Ok(None);
1907    };
1908    ctx.get_str(key)
1909        .map(|value| Some(value.to_string()))
1910        .ok_or_else(|| WorkflowError::MissingContextKey(format!("{key} (expected string)")))
1911}
1912
1913fn resolve_optional_order_origin(
1914    ctx: &WorkflowContext,
1915    key: Option<&String>,
1916) -> Result<Option<OrderOrigin>, WorkflowError> {
1917    let Some(key) = key else {
1918        return Ok(None);
1919    };
1920    let value = ctx.get_str(key).ok_or_else(|| {
1921        WorkflowError::MissingContextKey(format!("{key} (expected order origin)"))
1922    })?;
1923
1924    OrderOrigin::parse(value).map(Some).ok_or_else(|| {
1925        WorkflowError::Other(format!(
1926            "Charge order_origin_key '{key}' must resolve to one of: whatsapp, mcp, web, ios_app, android_app, api"
1927        ))
1928    })
1929}
1930
1931fn charge_amount_from_number(
1932    number: &serde_json::Number,
1933    amount_key: &str,
1934) -> Result<i64, WorkflowError> {
1935    if let Some(value) = number.as_i64() {
1936        return Ok(value);
1937    }
1938
1939    if let Some(value) = number.as_u64() {
1940        return i64::try_from(value).map_err(|_| {
1941            invalid_charge_amount(amount_key, "integer amount must fit in signed 64-bit range")
1942        });
1943    }
1944
1945    if let Some(value) = number.as_f64() {
1946        const MAX_SAFE_JSON_INTEGER: f64 = 9_007_199_254_740_991.0;
1947
1948        if !value.is_finite() {
1949            return Err(invalid_charge_amount(
1950                amount_key,
1951                "float amount must be finite",
1952            ));
1953        }
1954
1955        if value.fract() != 0.0 {
1956            return Err(invalid_charge_amount(
1957                amount_key,
1958                "float amount must be an integer in the smallest currency unit",
1959            ));
1960        }
1961
1962        if !(-MAX_SAFE_JSON_INTEGER..=MAX_SAFE_JSON_INTEGER).contains(&value) {
1963            return Err(invalid_charge_amount(
1964                amount_key,
1965                "float amount exceeds JSON safe integer range",
1966            ));
1967        }
1968
1969        return Ok(value as i64);
1970    }
1971
1972    Err(invalid_charge_amount(
1973        amount_key,
1974        "amount number is not representable",
1975    ))
1976}
1977
1978fn invalid_charge_amount(amount_key: &str, message: &str) -> WorkflowError {
1979    WorkflowError::Other(format!(
1980        "Invalid charge amount at '{amount_key}': {message}"
1981    ))
1982}
1983
1984fn normalize_query_wire_for_execution(cmd_json: &str) -> Result<String, WorkflowError> {
1985    if !is_current_workflow_query_wire(cmd_json) {
1986        return Err(WorkflowError::QueryFailed(
1987            "Legacy workflow query payload detected: cmd_json must use QAIL wire text \
1988             (QAIL-CMD/1 or QAIL-CMD/2). Migrate persisted workflow rows to wire text or purge/restart pending workflows."
1989                .to_string(),
1990        ));
1991    }
1992
1993    let cmd = qail_core::wire::decode_cmd_text(cmd_json).map_err(|e| {
1994        WorkflowError::QueryFailed(format!(
1995            "Invalid workflow query wire payload (expected QAIL-CMD/1 or QAIL-CMD/2): {}",
1996            e
1997        ))
1998    })?;
1999
2000    // Canonicalize payload before handing it to the executor.
2001    Ok(qail_core::wire::encode_cmd_text(&cmd))
2002}
2003
2004/// Run a workflow from its current state until it reaches a Wait or terminal state.
2005///
2006/// Returns the final state name after execution.
2007///
2008/// # Behavior
2009///
2010/// 1. Finds the transition matching `ctx.current_state`
2011/// 2. Executes each step in the transition
2012/// 3. If a `Wait` step is encountered, persists state and returns
2013/// 4. If a `Transition` step moves to a new state, looks for the next transition
2014/// 5. Continues until no more transitions or a terminal state is reached
2015///
2016/// # Example
2017///
2018/// ```rust,ignore
2019/// let mut ctx = WorkflowContext::new("booking-123", "operator_declined");
2020/// let final_state = run_workflow(&executor, &recovery_workflow, &mut ctx).await?;
2021/// // final_state might be "recovery_mode" (paused at Wait)
2022/// // or "cancelled" (if timeout was immediate)
2023/// ```
2024pub async fn run_workflow<E: WorkflowExecutor>(
2025    executor: &E,
2026    definition: &WorkflowDefinition,
2027    ctx: &mut WorkflowContext,
2028) -> Result<String, WorkflowError> {
2029    run_workflow_with_options(executor, definition, ctx, WorkflowRunOptions::default()).await
2030}
2031
2032/// Run a workflow with optional lease and idempotency controls.
2033pub async fn run_workflow_with_options<E: WorkflowExecutor>(
2034    executor: &E,
2035    definition: &WorkflowDefinition,
2036    ctx: &mut WorkflowContext,
2037    options: WorkflowRunOptions,
2038) -> Result<String, WorkflowError> {
2039    validate_workflow_definition(definition)?;
2040    validate_workflow_context_identity(ctx)?;
2041    let guard = match RuntimeGuard::enter(
2042        executor,
2043        &definition.name,
2044        &ctx.workflow_id,
2045        WorkflowOperationKind::Run,
2046        options,
2047    )
2048    .await?
2049    {
2050        RuntimeEntry::Active(guard) => guard,
2051        RuntimeEntry::Completed(state) => return Ok(state),
2052    };
2053
2054    let result = run_workflow_inner(executor, definition, ctx, RunMode::Normal).await;
2055    finish_runtime_operation(executor, guard, result).await
2056}
2057
2058async fn run_workflow_inner<'a, E: WorkflowExecutor>(
2059    executor: &E,
2060    definition: &WorkflowDefinition,
2061    ctx: &mut WorkflowContext,
2062    mode: RunMode<'a>,
2063) -> Result<String, WorkflowError> {
2064    validate_workflow_definition(definition)?;
2065    validate_workflow_context_identity(ctx)?;
2066    ensure_context_definition(ctx, definition)?;
2067    let run_start_transition_count = ctx.transition_count();
2068    let mut pending_cursor_frames = match ctx.cursor.clone() {
2069        Some(cursor) if cursor.state == ctx.current_state => {
2070            ensure_cursor_has_frames(&cursor)?;
2071            if timeout_fallback_from_context(ctx)?.is_some() {
2072                return Err(WorkflowError::Other(
2073                    "Workflow is executing a timeout fallback; resume with timeout_workflow"
2074                        .to_string(),
2075                ));
2076            }
2077            match (&cursor.wait, mode) {
2078                (Some(wait), RunMode::Normal) => {
2079                    return Err(WorkflowError::Other(format!(
2080                        "Workflow is paused awaiting event '{}'; resume with a matching event",
2081                        wait.event
2082                    )));
2083                }
2084                (Some(wait), RunMode::EventResume { event }) => {
2085                    if wait.event != event {
2086                        return Err(WorkflowError::Other(format!(
2087                            "Workflow is waiting for event '{}', received '{}'",
2088                            wait.event, event
2089                        )));
2090                    }
2091                    if Utc::now() > wait.deadline_at {
2092                        return Err(WorkflowError::Timeout {
2093                            event: wait.event.clone(),
2094                        });
2095                    }
2096                    ctx.clear_cursor();
2097                    Some(cursor.frames)
2098                }
2099                (None, RunMode::EventResume { .. }) => {
2100                    return Err(WorkflowError::Other(
2101                        "Workflow is not waiting for an external event".to_string(),
2102                    ));
2103                }
2104                (None, RunMode::Normal) => {
2105                    ctx.clear_cursor();
2106                    Some(cursor.frames)
2107                }
2108            }
2109        }
2110        Some(_) => match mode {
2111            RunMode::Normal => {
2112                ctx.clear_cursor();
2113                None
2114            }
2115            RunMode::EventResume { .. } => {
2116                return Err(invalid_cursor(
2117                    "resume cursor state does not match current workflow state",
2118                ));
2119            }
2120        },
2121        None => match mode {
2122            RunMode::Normal => None,
2123            RunMode::EventResume { .. } => {
2124                return Err(WorkflowError::Other(
2125                    "Workflow is not waiting for an external event".to_string(),
2126                ));
2127            }
2128        },
2129    };
2130
2131    loop {
2132        // Find transition from current state
2133        let transition = find_single_transition(definition, &ctx.current_state)?;
2134
2135        let transition = match transition {
2136            Some(t) => t,
2137            None => {
2138                // No transition = terminal state or waiting
2139                executor.save_state(ctx).await?;
2140                return Ok(ctx.current_state.clone());
2141            }
2142        };
2143
2144        // Safety: prevent infinite loops before executing the next transition.
2145        if ctx
2146            .transition_count()
2147            .saturating_sub(run_start_transition_count)
2148            >= 50
2149        {
2150            return Err(WorkflowError::Other(
2151                "Maximum transition count exceeded (50)".into(),
2152            ));
2153        }
2154
2155        let cursor_frames = pending_cursor_frames.take().unwrap_or_default();
2156        let (start_index, nested_cursor) = match cursor_frames.first() {
2157            Some(WorkflowCursorFrame::Steps { index }) => (*index, &cursor_frames[1..]),
2158            Some(_) => {
2159                return Err(invalid_cursor(
2160                    "top-level transition resume must start with a Steps frame",
2161                ));
2162            }
2163            None => (0, &[][..]),
2164        };
2165
2166        if let Some(pause) = execute_steps(
2167            executor,
2168            &transition.steps,
2169            ctx,
2170            start_index,
2171            nested_cursor,
2172            StepExecutionScope::new(&transition.from, StepListCursorKind::Steps, true),
2173        )
2174        .await?
2175        {
2176            clear_timeout_fallback(ctx);
2177            ctx.set_cursor(WorkflowCursor {
2178                state: transition.from.clone(),
2179                frames: pause.frames,
2180                wait: Some(pause.wait),
2181            });
2182            executor.save_state(ctx).await?;
2183            return Ok(ctx.current_state.clone());
2184        }
2185
2186        // After executing all steps, if a Transition step changed the state,
2187        // loop again to check for chained transitions.
2188        // If state didn't change (steps didn't include Transition), we're done.
2189        if ctx.current_state == transition.from {
2190            // No transition happened, move to `to` state
2191            ctx.transition_to(&transition.to, None);
2192        }
2193
2194        // Persist after each state change
2195        executor.save_state(ctx).await?;
2196    }
2197}
2198
2199/// Resume a workflow after a Wait event was received.
2200///
2201/// Call this when a webhook/callback arrives that matches the Wait event.
2202/// The workflow continues from where it paused.
2203pub async fn resume_workflow<E: WorkflowExecutor>(
2204    executor: &E,
2205    definition: &WorkflowDefinition,
2206    workflow_id: &str,
2207    event_data: Value,
2208) -> Result<String, WorkflowError> {
2209    resume_workflow_with_options(
2210        executor,
2211        definition,
2212        workflow_id,
2213        event_data,
2214        WorkflowRunOptions::default(),
2215    )
2216    .await
2217}
2218
2219/// Resume a workflow after a Wait event with optional lease/idempotency controls.
2220pub async fn resume_workflow_with_options<E: WorkflowExecutor>(
2221    executor: &E,
2222    definition: &WorkflowDefinition,
2223    workflow_id: &str,
2224    event_data: Value,
2225    options: WorkflowRunOptions,
2226) -> Result<String, WorkflowError> {
2227    let event_name = extract_resume_event_name(&event_data)?;
2228    resume_workflow_with_event_and_options(
2229        executor,
2230        definition,
2231        workflow_id,
2232        &event_name,
2233        event_data,
2234        options,
2235    )
2236    .await
2237}
2238
2239/// Resume a workflow after a named Wait event was received.
2240pub async fn resume_workflow_with_event<E: WorkflowExecutor>(
2241    executor: &E,
2242    definition: &WorkflowDefinition,
2243    workflow_id: &str,
2244    event_name: &str,
2245    event_data: Value,
2246) -> Result<String, WorkflowError> {
2247    resume_workflow_with_event_and_options(
2248        executor,
2249        definition,
2250        workflow_id,
2251        event_name,
2252        event_data,
2253        WorkflowRunOptions::default(),
2254    )
2255    .await
2256}
2257
2258/// Resume a workflow after a named Wait event with optional lease/idempotency controls.
2259pub async fn resume_workflow_with_event_and_options<E: WorkflowExecutor>(
2260    executor: &E,
2261    definition: &WorkflowDefinition,
2262    workflow_id: &str,
2263    event_name: &str,
2264    event_data: Value,
2265    options: WorkflowRunOptions,
2266) -> Result<String, WorkflowError> {
2267    validate_workflow_definition(definition)?;
2268    ensure_definition_text(workflow_id, "Workflow id")?;
2269    ensure_wait_event_name(event_name)?;
2270
2271    let guard = match RuntimeGuard::enter(
2272        executor,
2273        &definition.name,
2274        workflow_id,
2275        WorkflowOperationKind::Resume {
2276            event: event_name.to_string(),
2277        },
2278        options,
2279    )
2280    .await?
2281    {
2282        RuntimeEntry::Active(guard) => guard,
2283        RuntimeEntry::Completed(state) => return Ok(state),
2284    };
2285
2286    // Load persisted state
2287    let mut ctx = executor
2288        .load_state(workflow_id)
2289        .await?
2290        .ok_or_else(|| WorkflowError::Other(format!("Workflow not found: {}", workflow_id)))?;
2291
2292    // Store the event data in context
2293    ctx.set(RESUME_EVENT_KEY, event_data);
2294
2295    // Continue running from current state
2296    let result = run_workflow_inner(
2297        executor,
2298        definition,
2299        &mut ctx,
2300        RunMode::EventResume { event: event_name },
2301    )
2302    .await;
2303    finish_runtime_operation(executor, guard, result).await
2304}
2305
2306/// Execute the timeout fallback for a workflow currently paused at a Wait step.
2307pub async fn timeout_workflow<E: WorkflowExecutor>(
2308    executor: &E,
2309    definition: &WorkflowDefinition,
2310    workflow_id: &str,
2311) -> Result<String, WorkflowError> {
2312    timeout_workflow_with_options(
2313        executor,
2314        definition,
2315        workflow_id,
2316        WorkflowRunOptions::default(),
2317    )
2318    .await
2319}
2320
2321/// Execute a timeout fallback with optional lease/idempotency controls.
2322pub async fn timeout_workflow_with_options<E: WorkflowExecutor>(
2323    executor: &E,
2324    definition: &WorkflowDefinition,
2325    workflow_id: &str,
2326    options: WorkflowRunOptions,
2327) -> Result<String, WorkflowError> {
2328    timeout_workflow_with_options_at(executor, definition, workflow_id, options, Utc::now()).await
2329}
2330
2331async fn timeout_workflow_with_options_at<E: WorkflowExecutor>(
2332    executor: &E,
2333    definition: &WorkflowDefinition,
2334    workflow_id: &str,
2335    options: WorkflowRunOptions,
2336    now: DateTime<Utc>,
2337) -> Result<String, WorkflowError> {
2338    validate_workflow_definition(definition)?;
2339    ensure_definition_text(workflow_id, "Workflow id")?;
2340    let guard = match RuntimeGuard::enter(
2341        executor,
2342        &definition.name,
2343        workflow_id,
2344        WorkflowOperationKind::Timeout,
2345        options,
2346    )
2347    .await?
2348    {
2349        RuntimeEntry::Active(guard) => guard,
2350        RuntimeEntry::Completed(state) => return Ok(state),
2351    };
2352
2353    let result = timeout_workflow_inner(executor, definition, workflow_id, now).await;
2354    finish_runtime_operation(executor, guard, result).await
2355}
2356
2357/// Execute timeout fallbacks for workflows whose wait deadlines are due.
2358///
2359/// Apps provide due workflow ids by implementing
2360/// [`WorkflowExecutor::load_due_workflow_timeouts`]. Each workflow is executed
2361/// independently so one bad row does not stop the whole drain batch.
2362pub async fn timeout_due_workflows<E: WorkflowExecutor>(
2363    executor: &E,
2364    definition: &WorkflowDefinition,
2365    now: DateTime<Utc>,
2366    limit: usize,
2367    options: WorkflowRunOptions,
2368) -> Result<Vec<WorkflowTimeoutOutcome>, WorkflowError> {
2369    if limit == 0 {
2370        return Ok(Vec::new());
2371    }
2372    validate_workflow_definition(definition)?;
2373
2374    let workflow_ids = executor
2375        .load_due_workflow_timeouts(&definition.name, now, limit)
2376        .await?;
2377    let mut outcomes = Vec::with_capacity(workflow_ids.len().min(limit));
2378
2379    for workflow_id in workflow_ids.into_iter().take(limit) {
2380        let run_options = timeout_options_for_workflow(&options, definition, &workflow_id, now);
2381        let outcome = match timeout_workflow_with_options_at(
2382            executor,
2383            definition,
2384            &workflow_id,
2385            run_options,
2386            now,
2387        )
2388        .await
2389        {
2390            Ok(state) => WorkflowTimeoutOutcome {
2391                workflow_id,
2392                state: Some(state),
2393                error: None,
2394            },
2395            Err(err) => WorkflowTimeoutOutcome {
2396                workflow_id,
2397                state: None,
2398                error: Some(err.to_string()),
2399            },
2400        };
2401        outcomes.push(outcome);
2402    }
2403
2404    Ok(outcomes)
2405}
2406
2407fn timeout_options_for_workflow(
2408    options: &WorkflowRunOptions,
2409    definition: &WorkflowDefinition,
2410    workflow_id: &str,
2411    now: DateTime<Utc>,
2412) -> WorkflowRunOptions {
2413    let mut run_options = options.clone();
2414    run_options.idempotency_key = match &options.idempotency_key {
2415        Some(prefix) if prefix.trim().is_empty() => Some(prefix.clone()),
2416        Some(prefix) => Some(timeout_operation_idempotency_key(prefix, workflow_id, now)),
2417        None => Some(timeout_operation_idempotency_key(
2418            &definition.name,
2419            workflow_id,
2420            now,
2421        )),
2422    };
2423    run_options
2424}
2425
2426fn timeout_operation_idempotency_key(scope: &str, workflow_id: &str, now: DateTime<Utc>) -> String {
2427    serde_json::json!([
2428        "qail-workflow-timeout",
2429        2,
2430        scope,
2431        workflow_id,
2432        now.to_rfc3339_opts(SecondsFormat::Micros, true)
2433    ])
2434    .to_string()
2435}
2436
2437async fn timeout_workflow_inner<E: WorkflowExecutor>(
2438    executor: &E,
2439    definition: &WorkflowDefinition,
2440    workflow_id: &str,
2441    now: DateTime<Utc>,
2442) -> Result<String, WorkflowError> {
2443    let mut ctx = executor
2444        .load_state(workflow_id)
2445        .await?
2446        .ok_or_else(|| WorkflowError::Other(format!("Workflow not found: {}", workflow_id)))?;
2447    validate_workflow_context_identity(&ctx)?;
2448    ensure_context_definition(&mut ctx, definition)?;
2449
2450    let cursor = ctx
2451        .cursor
2452        .clone()
2453        .ok_or_else(|| WorkflowError::Other("Workflow is not paused at a Wait step".to_string()))?;
2454    ensure_cursor_has_frames(&cursor)?;
2455    if cursor.state != ctx.current_state {
2456        return Err(invalid_cursor(
2457            "timeout cursor state does not match current workflow state",
2458        ));
2459    }
2460    let timeout_fallback = timeout_fallback_from_context(&ctx)?;
2461    if let (Some(cursor_wait), Some(timeout)) = (&cursor.wait, &timeout_fallback)
2462        && cursor_wait != timeout
2463    {
2464        return Err(invalid_cursor(
2465            "timeout cursor wait metadata does not match timeout fallback metadata",
2466        ));
2467    }
2468
2469    let (wait, cursor_frames) = if let Some(timeout) = timeout_fallback {
2470        (timeout, cursor.frames.clone())
2471    } else {
2472        let wait = cursor.wait.clone().ok_or_else(|| {
2473            WorkflowError::Other("Workflow is not waiting for a timeout".to_string())
2474        })?;
2475        (wait, Vec::new())
2476    };
2477
2478    ensure_wait_event_name(&wait.event)?;
2479    if now < wait.deadline_at {
2480        return Err(WorkflowError::Other(format!(
2481            "Workflow wait for event '{}' has not timed out",
2482            wait.event
2483        )));
2484    }
2485    if wait.on_timeout.is_empty() {
2486        return Err(WorkflowError::Timeout { event: wait.event });
2487    }
2488    if steps_contain_wait(&wait.on_timeout) {
2489        return Err(WorkflowError::Other(
2490            "Wait steps inside on_timeout fallback are not supported".to_string(),
2491        ));
2492    }
2493    validate_workflow_steps(&wait.on_timeout)?;
2494
2495    set_timeout_fallback(&mut ctx, &wait)?;
2496    ctx.clear_cursor();
2497    ctx.set(
2498        RESUME_EVENT_KEY,
2499        serde_json::json!({
2500            "event": wait.event.clone(),
2501            "timeout": true,
2502        }),
2503    );
2504
2505    let (start_index, nested_cursor) = match cursor_frames.first() {
2506        Some(WorkflowCursorFrame::Steps { index }) => (*index, &cursor_frames[1..]),
2507        Some(_) => {
2508            return Err(invalid_cursor(
2509                "timeout fallback resume must start with a Steps frame",
2510            ));
2511        }
2512        None => (0, &[][..]),
2513    };
2514
2515    if execute_steps(
2516        executor,
2517        &wait.on_timeout,
2518        &mut ctx,
2519        start_index,
2520        nested_cursor,
2521        StepExecutionScope::new(&cursor.state, StepListCursorKind::Steps, true)
2522            .with_timeout_checkpoint(),
2523    )
2524    .await?
2525    .is_some()
2526    {
2527        return Err(WorkflowError::Other(
2528            "on_timeout fallback paused unexpectedly".to_string(),
2529        ));
2530    }
2531
2532    clear_timeout_fallback(&mut ctx);
2533    ctx.clear_cursor();
2534    if ctx.current_state == cursor.state {
2535        executor.save_state(&ctx).await?;
2536        return Ok(ctx.current_state.clone());
2537    }
2538
2539    run_workflow_inner(executor, definition, &mut ctx, RunMode::Normal).await
2540}
2541
2542fn extract_resume_event_name(event_data: &Value) -> Result<String, WorkflowError> {
2543    let event = ["event", "event_name", "type"]
2544        .iter()
2545        .find_map(|key| event_data.get(*key).and_then(Value::as_str))
2546        .ok_or_else(|| {
2547            WorkflowError::Other(
2548                "Resume event data must include a string 'event' field".to_string(),
2549            )
2550        })?;
2551    ensure_wait_event_name(event)?;
2552    Ok(event.to_string())
2553}
2554
2555fn steps_contain_wait(steps: &[WorkflowStep]) -> bool {
2556    steps.iter().any(|step| match step {
2557        WorkflowStep::Wait { .. } => true,
2558        WorkflowStep::Branch {
2559            branches, default, ..
2560        } => {
2561            branches
2562                .iter()
2563                .any(|(_, branch_steps)| steps_contain_wait(branch_steps))
2564                || steps_contain_wait(default)
2565        }
2566        WorkflowStep::BranchWhen {
2567            branches, default, ..
2568        } => {
2569            branches
2570                .iter()
2571                .any(|(_, branch_steps)| steps_contain_wait(branch_steps))
2572                || steps_contain_wait(default)
2573        }
2574        WorkflowStep::ForEach { steps, .. } => steps_contain_wait(steps),
2575        WorkflowStep::Query { .. }
2576        | WorkflowStep::Notify { .. }
2577        | WorkflowStep::Transition { .. }
2578        | WorkflowStep::Log { .. }
2579        | WorkflowStep::Charge { .. } => false,
2580    })
2581}
2582
2583#[cfg(test)]
2584mod tests {
2585    use super::*;
2586    use crate::context::StateChange;
2587    use crate::payment::{ChargeResponse, ChargeStatus, PaymentKind};
2588
2589    struct MockExecutor {
2590        queries: std::sync::Mutex<Vec<String>>,
2591        query_failures: std::sync::Mutex<std::collections::HashSet<String>>,
2592        notifications: std::sync::Mutex<Vec<(String, String)>>,
2593        charges: std::sync::Mutex<Vec<ChargeRequest>>,
2594        saved_state: std::sync::Mutex<Option<WorkflowContext>>,
2595        reject_leases: std::sync::Mutex<bool>,
2596        acquired_leases: std::sync::Mutex<Vec<WorkflowLease>>,
2597        released_leases: std::sync::Mutex<Vec<WorkflowLease>>,
2598        workflow_operations:
2599            std::sync::Mutex<std::collections::HashMap<String, WorkflowOperationStatus>>,
2600        workflow_operation_begin_failures: std::sync::Mutex<std::collections::HashSet<String>>,
2601        workflow_operation_complete_failures: std::sync::Mutex<std::collections::HashSet<String>>,
2602        workflow_operation_fail_failures: std::sync::Mutex<std::collections::HashSet<String>>,
2603        completed_workflow_operations: std::sync::Mutex<Vec<(String, String)>>,
2604        failed_workflow_operations: std::sync::Mutex<Vec<(String, String)>>,
2605        due_timeout_workflow_ids: std::sync::Mutex<Vec<String>>,
2606        skip_notify_side_effects: std::sync::Mutex<bool>,
2607        completed_side_effects: std::sync::Mutex<Vec<String>>,
2608        failed_side_effects: std::sync::Mutex<Vec<(String, String)>>,
2609    }
2610
2611    impl MockExecutor {
2612        fn new() -> Self {
2613            Self {
2614                queries: std::sync::Mutex::new(Vec::new()),
2615                query_failures: std::sync::Mutex::new(std::collections::HashSet::new()),
2616                notifications: std::sync::Mutex::new(Vec::new()),
2617                charges: std::sync::Mutex::new(Vec::new()),
2618                saved_state: std::sync::Mutex::new(None),
2619                reject_leases: std::sync::Mutex::new(false),
2620                acquired_leases: std::sync::Mutex::new(Vec::new()),
2621                released_leases: std::sync::Mutex::new(Vec::new()),
2622                workflow_operations: std::sync::Mutex::new(std::collections::HashMap::new()),
2623                workflow_operation_begin_failures: std::sync::Mutex::new(
2624                    std::collections::HashSet::new(),
2625                ),
2626                workflow_operation_complete_failures: std::sync::Mutex::new(
2627                    std::collections::HashSet::new(),
2628                ),
2629                workflow_operation_fail_failures: std::sync::Mutex::new(
2630                    std::collections::HashSet::new(),
2631                ),
2632                completed_workflow_operations: std::sync::Mutex::new(Vec::new()),
2633                failed_workflow_operations: std::sync::Mutex::new(Vec::new()),
2634                due_timeout_workflow_ids: std::sync::Mutex::new(Vec::new()),
2635                skip_notify_side_effects: std::sync::Mutex::new(false),
2636                completed_side_effects: std::sync::Mutex::new(Vec::new()),
2637                failed_side_effects: std::sync::Mutex::new(Vec::new()),
2638            }
2639        }
2640
2641        fn fail_query(&self, cmd_wire: impl Into<String>) {
2642            self.query_failures.lock().unwrap().insert(cmd_wire.into());
2643        }
2644
2645        fn reject_workflow_leases(&self) {
2646            *self.reject_leases.lock().unwrap() = true;
2647        }
2648
2649        fn set_workflow_operation_status(
2650            &self,
2651            idempotency_key: impl Into<String>,
2652            status: WorkflowOperationStatus,
2653        ) {
2654            self.workflow_operations
2655                .lock()
2656                .unwrap()
2657                .insert(idempotency_key.into(), status);
2658        }
2659
2660        fn fail_workflow_operation_begin(&self, idempotency_key: impl Into<String>) {
2661            self.workflow_operation_begin_failures
2662                .lock()
2663                .unwrap()
2664                .insert(idempotency_key.into());
2665        }
2666
2667        fn fail_workflow_operation_complete(&self, idempotency_key: impl Into<String>) {
2668            self.workflow_operation_complete_failures
2669                .lock()
2670                .unwrap()
2671                .insert(idempotency_key.into());
2672        }
2673
2674        fn fail_workflow_operation_fail(&self, idempotency_key: impl Into<String>) {
2675            self.workflow_operation_fail_failures
2676                .lock()
2677                .unwrap()
2678                .insert(idempotency_key.into());
2679        }
2680
2681        fn skip_notify_side_effects(&self) {
2682            *self.skip_notify_side_effects.lock().unwrap() = true;
2683        }
2684
2685        fn set_due_timeout_workflow_ids(&self, workflow_ids: Vec<&str>) {
2686            *self.due_timeout_workflow_ids.lock().unwrap() =
2687                workflow_ids.into_iter().map(ToString::to_string).collect();
2688        }
2689    }
2690
2691    fn charge_only_workflow(name: &str) -> WorkflowDefinition {
2692        WorkflowDefinition::new(name)
2693            .initial_state("created")
2694            .transition(
2695                "created",
2696                "awaiting_payment",
2697                vec![WorkflowStep::charge(
2698                    PaymentKind::Xendit,
2699                    "order.total",
2700                    "order.id",
2701                    Some("charge"),
2702                )],
2703            )
2704    }
2705
2706    #[async_trait]
2707    impl WorkflowExecutor for MockExecutor {
2708        async fn execute_query(&self, cmd_json: &str) -> Result<Value, WorkflowError> {
2709            self.queries.lock().unwrap().push(cmd_json.to_string());
2710            if self.query_failures.lock().unwrap().contains(cmd_json) {
2711                return Err(WorkflowError::QueryFailed(
2712                    "forced query failure".to_string(),
2713                ));
2714            }
2715            // Return mock results
2716            Ok(serde_json::json!([
2717                {"id": "op-1", "phone": "+628111"},
2718                {"id": "op-2", "phone": "+628222"},
2719            ]))
2720        }
2721
2722        async fn send_notification(
2723            &self,
2724            _channel: &ChannelKind,
2725            recipient: &str,
2726            template: &str,
2727            _params: &Value,
2728        ) -> Result<(), WorkflowError> {
2729            self.notifications
2730                .lock()
2731                .unwrap()
2732                .push((recipient.to_string(), template.to_string()));
2733            Ok(())
2734        }
2735
2736        async fn save_state(&self, ctx: &WorkflowContext) -> Result<(), WorkflowError> {
2737            *self.saved_state.lock().unwrap() = Some(ctx.clone());
2738            Ok(())
2739        }
2740
2741        async fn load_state(
2742            &self,
2743            workflow_id: &str,
2744        ) -> Result<Option<WorkflowContext>, WorkflowError> {
2745            Ok(self
2746                .saved_state
2747                .lock()
2748                .unwrap()
2749                .as_ref()
2750                .filter(|ctx| ctx.workflow_id == workflow_id)
2751                .cloned())
2752        }
2753
2754        async fn create_charge(
2755            &self,
2756            _provider: &PaymentKind,
2757            request: ChargeRequest,
2758        ) -> Result<ChargeResponse, WorkflowError> {
2759            self.charges.lock().unwrap().push(request.clone());
2760            Ok(ChargeResponse {
2761                charge_id: format!("mock-charge-{}", request.reference_id),
2762                status: ChargeStatus::Pending,
2763                redirect_url: None,
2764                qr_code: Some("00020101021226610014ID.CO.MOCK".into()),
2765                payment_code: None,
2766                expires_at: Some("2026-02-13T16:00:00Z".into()),
2767                raw: Some(serde_json::json!({"provider_secret": "hidden"})),
2768            })
2769        }
2770
2771        async fn acquire_workflow_lease(
2772            &self,
2773            lease: &WorkflowLease,
2774        ) -> Result<bool, WorkflowError> {
2775            if *self.reject_leases.lock().unwrap() {
2776                return Ok(false);
2777            }
2778            self.acquired_leases.lock().unwrap().push(lease.clone());
2779            Ok(true)
2780        }
2781
2782        async fn release_workflow_lease(&self, lease: &WorkflowLease) -> Result<(), WorkflowError> {
2783            self.released_leases.lock().unwrap().push(lease.clone());
2784            Ok(())
2785        }
2786
2787        async fn begin_workflow_operation(
2788            &self,
2789            operation: &WorkflowOperation,
2790        ) -> Result<WorkflowOperationStatus, WorkflowError> {
2791            if self
2792                .workflow_operation_begin_failures
2793                .lock()
2794                .unwrap()
2795                .contains(&operation.idempotency_key)
2796            {
2797                return Err(WorkflowError::Other(
2798                    "forced operation begin failure".to_string(),
2799                ));
2800            }
2801            Ok(self
2802                .workflow_operations
2803                .lock()
2804                .unwrap()
2805                .get(&operation.idempotency_key)
2806                .cloned()
2807                .unwrap_or(WorkflowOperationStatus::Started))
2808        }
2809
2810        async fn complete_workflow_operation(
2811            &self,
2812            operation: &WorkflowOperation,
2813            state: &str,
2814        ) -> Result<(), WorkflowError> {
2815            if self
2816                .workflow_operation_complete_failures
2817                .lock()
2818                .unwrap()
2819                .contains(&operation.idempotency_key)
2820            {
2821                return Err(WorkflowError::Other(
2822                    "forced operation complete failure".to_string(),
2823                ));
2824            }
2825            self.completed_workflow_operations
2826                .lock()
2827                .unwrap()
2828                .push((operation.idempotency_key.clone(), state.to_string()));
2829            Ok(())
2830        }
2831
2832        async fn fail_workflow_operation(
2833            &self,
2834            operation: &WorkflowOperation,
2835            error: &str,
2836        ) -> Result<(), WorkflowError> {
2837            if self
2838                .workflow_operation_fail_failures
2839                .lock()
2840                .unwrap()
2841                .contains(&operation.idempotency_key)
2842            {
2843                return Err(WorkflowError::Other(
2844                    "forced operation fail failure".to_string(),
2845                ));
2846            }
2847            self.failed_workflow_operations
2848                .lock()
2849                .unwrap()
2850                .push((operation.idempotency_key.clone(), error.to_string()));
2851            Ok(())
2852        }
2853
2854        async fn load_due_workflow_timeouts(
2855            &self,
2856            _workflow_name: &str,
2857            _now: DateTime<Utc>,
2858            limit: usize,
2859        ) -> Result<Vec<String>, WorkflowError> {
2860            Ok(self
2861                .due_timeout_workflow_ids
2862                .lock()
2863                .unwrap()
2864                .iter()
2865                .take(limit)
2866                .cloned()
2867                .collect())
2868        }
2869
2870        async fn begin_workflow_side_effect(
2871            &self,
2872            operation: &WorkflowSideEffect,
2873        ) -> Result<WorkflowSideEffectStatus, WorkflowError> {
2874            if operation.kind == WorkflowSideEffectKind::Notify
2875                && *self.skip_notify_side_effects.lock().unwrap()
2876            {
2877                return Ok(WorkflowSideEffectStatus::AlreadyCompleted { result: None });
2878            }
2879            Ok(WorkflowSideEffectStatus::Execute)
2880        }
2881
2882        async fn complete_workflow_side_effect(
2883            &self,
2884            operation: &WorkflowSideEffect,
2885            _result: Option<&Value>,
2886        ) -> Result<(), WorkflowError> {
2887            self.completed_side_effects
2888                .lock()
2889                .unwrap()
2890                .push(operation.operation_id.clone());
2891            Ok(())
2892        }
2893
2894        async fn fail_workflow_side_effect(
2895            &self,
2896            operation: &WorkflowSideEffect,
2897            error: &str,
2898        ) -> Result<(), WorkflowError> {
2899            self.failed_side_effects
2900                .lock()
2901                .unwrap()
2902                .push((operation.operation_id.clone(), error.to_string()));
2903            Ok(())
2904        }
2905    }
2906
2907    #[tokio::test]
2908    async fn test_simple_workflow() {
2909        let executor = MockExecutor::new();
2910
2911        let wf = WorkflowDefinition::new("test")
2912            .initial_state("start")
2913            .transition(
2914                "start",
2915                "done",
2916                vec![
2917                    WorkflowStep::log("Starting workflow"),
2918                    WorkflowStep::transition("done"),
2919                ],
2920            );
2921
2922        let mut ctx = WorkflowContext::new("wf-001", "start");
2923        let result = run_workflow(&executor, &wf, &mut ctx).await.unwrap();
2924        assert_eq!(result, "done");
2925        assert_eq!(ctx.transition_count(), 1);
2926    }
2927
2928    #[tokio::test]
2929    async fn versioned_workflow_stamps_context_metadata() {
2930        let executor = MockExecutor::new();
2931
2932        let wf = WorkflowDefinition::new("versioned_booking")
2933            .version("v1")
2934            .initial_state("start")
2935            .transition("start", "done", vec![WorkflowStep::transition("done")]);
2936
2937        let mut ctx = WorkflowContext::new("wf-versioned", "start");
2938        let result = run_workflow(&executor, &wf, &mut ctx).await.unwrap();
2939
2940        assert_eq!(result, "done");
2941        assert_eq!(ctx.definition_name.as_deref(), Some("versioned_booking"));
2942        assert_eq!(ctx.definition_version.as_deref(), Some("v1"));
2943    }
2944
2945    #[tokio::test]
2946    async fn versioned_resume_rejects_definition_version_drift() {
2947        let executor = MockExecutor::new();
2948
2949        let wf_v1 = WorkflowDefinition::new("versioned_resume")
2950            .version("v1")
2951            .initial_state("active")
2952            .transition(
2953                "active",
2954                "done",
2955                vec![
2956                    WorkflowStep::wait("operator.accepted", std::time::Duration::from_secs(3600)),
2957                    WorkflowStep::notify(ChannelKind::Email, "accepted", "customer.email"),
2958                    WorkflowStep::transition("done"),
2959                ],
2960            );
2961        let wf_v2 = WorkflowDefinition::new("versioned_resume")
2962            .version("v2")
2963            .initial_state("active")
2964            .transition(
2965                "active",
2966                "done",
2967                vec![
2968                    WorkflowStep::wait("operator.accepted", std::time::Duration::from_secs(3600)),
2969                    WorkflowStep::notify(ChannelKind::Email, "accepted", "customer.email"),
2970                    WorkflowStep::transition("done"),
2971                ],
2972            );
2973
2974        let mut ctx = WorkflowContext::new("wf-version-drift", "active");
2975        ctx.set(
2976            "customer",
2977            serde_json::json!({"email": "guest@example.com"}),
2978        );
2979        let result = run_workflow(&executor, &wf_v1, &mut ctx).await.unwrap();
2980        assert_eq!(result, "active");
2981
2982        let err = resume_workflow(
2983            &executor,
2984            &wf_v2,
2985            "wf-version-drift",
2986            serde_json::json!({"event": "operator.accepted"}),
2987        )
2988        .await
2989        .expect_err("version drift must fail before replay");
2990
2991        match err {
2992            WorkflowError::Other(msg) => {
2993                assert!(msg.contains("definition version 'v1'"), "got: {msg}");
2994                assert!(msg.contains("not 'v2'"), "got: {msg}");
2995            }
2996            other => panic!("expected version drift error, got {other:?}"),
2997        }
2998        assert!(executor.notifications.lock().unwrap().is_empty());
2999    }
3000
3001    #[tokio::test]
3002    async fn run_with_options_acquires_lease_and_completes_operation() {
3003        let executor = MockExecutor::new();
3004
3005        let wf = WorkflowDefinition::new("runtime_guarded_run")
3006            .initial_state("start")
3007            .transition("start", "done", vec![WorkflowStep::transition("done")]);
3008
3009        let mut ctx = WorkflowContext::new("wf-runtime-run", "start");
3010        let options = WorkflowRunOptions::default()
3011            .with_lease("worker-a", std::time::Duration::from_secs(30))
3012            .with_idempotency_key("run-1");
3013
3014        let result = run_workflow_with_options(&executor, &wf, &mut ctx, options)
3015            .await
3016            .unwrap();
3017
3018        assert_eq!(result, "done");
3019        let acquired_leases = executor.acquired_leases.lock().unwrap();
3020        let released_leases = executor.released_leases.lock().unwrap();
3021        assert_eq!(acquired_leases.len(), 1);
3022        assert_eq!(released_leases.len(), 1);
3023        assert_eq!(released_leases[0].owner, acquired_leases[0].owner);
3024        assert_ne!(
3025            acquired_leases[0].owner, "worker-a",
3026            "runtime lease owner must include a fencing token"
3027        );
3028        let lease_owner: serde_json::Value =
3029            serde_json::from_str(&acquired_leases[0].owner).unwrap();
3030        let lease_owner = lease_owner.as_array().unwrap();
3031        assert_eq!(lease_owner[0], "qail-workflow-lease");
3032        assert_eq!(lease_owner[2], "worker-a");
3033        assert_eq!(lease_owner[3], "runtime_guarded_run");
3034        assert_eq!(lease_owner[4], "wf-runtime-run");
3035        assert_eq!(lease_owner[6], "run-1");
3036        drop(acquired_leases);
3037        drop(released_leases);
3038        assert_eq!(
3039            executor
3040                .completed_workflow_operations
3041                .lock()
3042                .unwrap()
3043                .as_slice(),
3044            &[("run-1".to_string(), "done".to_string())]
3045        );
3046    }
3047
3048    #[test]
3049    fn runtime_lease_owner_changes_per_acquire_time() {
3050        let first_at = DateTime::parse_from_rfc3339("2026-06-17T12:00:00Z")
3051            .unwrap()
3052            .with_timezone(&Utc);
3053        let second_at = DateTime::parse_from_rfc3339("2026-06-17T12:00:01Z")
3054            .unwrap()
3055            .with_timezone(&Utc);
3056
3057        let first = runtime_lease_owner(
3058            "worker-a",
3059            "booking",
3060            "wf-1",
3061            &WorkflowOperationKind::Resume {
3062                event: "vendor.accepted".to_string(),
3063            },
3064            Some("event-1"),
3065            first_at,
3066        );
3067        let second = runtime_lease_owner(
3068            "worker-a",
3069            "booking",
3070            "wf-1",
3071            &WorkflowOperationKind::Resume {
3072                event: "vendor.accepted".to_string(),
3073            },
3074            Some("event-1"),
3075            second_at,
3076        );
3077
3078        assert_ne!(
3079            first, second,
3080            "stale release calls must not share the same lease owner as later acquisitions"
3081        );
3082    }
3083
3084    #[tokio::test]
3085    async fn lease_rejection_stops_workflow_before_side_effects() {
3086        let executor = MockExecutor::new();
3087        executor.reject_workflow_leases();
3088
3089        let wf = WorkflowDefinition::new("lease_guard")
3090            .initial_state("active")
3091            .transition(
3092                "active",
3093                "done",
3094                vec![
3095                    WorkflowStep::notify(ChannelKind::Email, "should_not_send", "customer.email"),
3096                    WorkflowStep::transition("done"),
3097                ],
3098            );
3099
3100        let mut ctx = WorkflowContext::new("wf-lease-guard", "active");
3101        ctx.set(
3102            "customer",
3103            serde_json::json!({"email": "guest@example.com"}),
3104        );
3105        let options = WorkflowRunOptions::default()
3106            .with_lease("worker-a", std::time::Duration::from_secs(30));
3107
3108        let err = run_workflow_with_options(&executor, &wf, &mut ctx, options)
3109            .await
3110            .expect_err("lease rejection must fail before executing steps");
3111
3112        match err {
3113            WorkflowError::Other(msg) => {
3114                assert!(msg.contains("already locked"), "got: {msg}");
3115            }
3116            other => panic!("expected lock error, got {other:?}"),
3117        }
3118        assert_eq!(ctx.current_state, "active");
3119        assert!(executor.notifications.lock().unwrap().is_empty());
3120    }
3121
3122    #[tokio::test]
3123    async fn resume_idempotency_completion_short_circuits_before_load_and_side_effects() {
3124        let executor = MockExecutor::new();
3125        executor.set_workflow_operation_status(
3126            "event-1",
3127            WorkflowOperationStatus::Completed {
3128                state: "done".to_string(),
3129            },
3130        );
3131
3132        let wf = WorkflowDefinition::new("resume_idempotency_guard")
3133            .initial_state("active")
3134            .transition(
3135                "active",
3136                "done",
3137                vec![
3138                    WorkflowStep::wait("payment.success", std::time::Duration::from_secs(3600)),
3139                    WorkflowStep::notify(ChannelKind::Email, "paid", "customer.email"),
3140                    WorkflowStep::transition("done"),
3141                ],
3142            );
3143
3144        let result = resume_workflow_with_options(
3145            &executor,
3146            &wf,
3147            "missing-workflow",
3148            serde_json::json!({"event": "payment.success"}),
3149            WorkflowRunOptions::default().with_idempotency_key("event-1"),
3150        )
3151        .await
3152        .unwrap();
3153
3154        assert_eq!(result, "done");
3155        assert!(executor.notifications.lock().unwrap().is_empty());
3156        assert!(
3157            executor
3158                .completed_workflow_operations
3159                .lock()
3160                .unwrap()
3161                .is_empty()
3162        );
3163    }
3164
3165    #[tokio::test]
3166    async fn completed_operation_short_circuits_before_rejected_lease() {
3167        let executor = MockExecutor::new();
3168        executor.reject_workflow_leases();
3169        executor.set_workflow_operation_status(
3170            "run-replay-1",
3171            WorkflowOperationStatus::Completed {
3172                state: "done".to_string(),
3173            },
3174        );
3175
3176        let wf = WorkflowDefinition::new("completed_before_lease")
3177            .initial_state("active")
3178            .transition(
3179                "active",
3180                "done",
3181                vec![
3182                    WorkflowStep::notify(ChannelKind::Email, "should_not_send", "customer.email"),
3183                    WorkflowStep::transition("done"),
3184                ],
3185            );
3186
3187        let mut ctx = WorkflowContext::new("wf-completed-before-lease", "active");
3188        ctx.set(
3189            "customer",
3190            serde_json::json!({"email": "guest@example.com"}),
3191        );
3192
3193        let result = run_workflow_with_options(
3194            &executor,
3195            &wf,
3196            &mut ctx,
3197            WorkflowRunOptions::default()
3198                .with_idempotency_key("run-replay-1")
3199                .with_lease("worker-a", std::time::Duration::from_secs(30)),
3200        )
3201        .await
3202        .unwrap();
3203
3204        assert_eq!(result, "done");
3205        assert!(executor.acquired_leases.lock().unwrap().is_empty());
3206        assert!(executor.released_leases.lock().unwrap().is_empty());
3207        assert!(executor.notifications.lock().unwrap().is_empty());
3208        assert!(
3209            executor
3210                .completed_workflow_operations
3211                .lock()
3212                .unwrap()
3213                .is_empty()
3214        );
3215    }
3216
3217    #[tokio::test]
3218    async fn started_operation_is_failed_when_lease_is_rejected() {
3219        let executor = MockExecutor::new();
3220        executor.reject_workflow_leases();
3221
3222        let wf = WorkflowDefinition::new("started_then_lease_rejected")
3223            .initial_state("active")
3224            .transition(
3225                "active",
3226                "done",
3227                vec![
3228                    WorkflowStep::notify(ChannelKind::Email, "should_not_send", "customer.email"),
3229                    WorkflowStep::transition("done"),
3230                ],
3231            );
3232
3233        let mut ctx = WorkflowContext::new("wf-started-lease-rejected", "active");
3234        ctx.set(
3235            "customer",
3236            serde_json::json!({"email": "guest@example.com"}),
3237        );
3238
3239        let err = run_workflow_with_options(
3240            &executor,
3241            &wf,
3242            &mut ctx,
3243            WorkflowRunOptions::default()
3244                .with_idempotency_key("run-lock-1")
3245                .with_lease("worker-a", std::time::Duration::from_secs(30)),
3246        )
3247        .await
3248        .expect_err("new idempotent operation must not execute without its lease");
3249
3250        match err {
3251            WorkflowError::Other(msg) => {
3252                assert!(msg.contains("already locked"), "got: {msg}");
3253            }
3254            other => panic!("expected lock error, got {other:?}"),
3255        }
3256        assert_eq!(ctx.current_state, "active");
3257        assert!(executor.acquired_leases.lock().unwrap().is_empty());
3258        assert!(executor.notifications.lock().unwrap().is_empty());
3259        assert_eq!(
3260            executor
3261                .failed_workflow_operations
3262                .lock()
3263                .unwrap()
3264                .as_slice(),
3265            &[(
3266                "run-lock-1".to_string(),
3267                "Workflow 'wf-started-lease-rejected' is already locked".to_string()
3268            )]
3269        );
3270    }
3271
3272    #[tokio::test]
3273    async fn empty_idempotency_key_does_not_release_unacquired_lease() {
3274        let executor = MockExecutor::new();
3275
3276        let wf = WorkflowDefinition::new("empty_idempotency_key")
3277            .initial_state("active")
3278            .transition("active", "done", vec![WorkflowStep::transition("done")]);
3279
3280        let mut ctx = WorkflowContext::new("wf-empty-idempotency", "active");
3281        let err = run_workflow_with_options(
3282            &executor,
3283            &wf,
3284            &mut ctx,
3285            WorkflowRunOptions::default()
3286                .with_idempotency_key(" ")
3287                .with_lease("worker-a", std::time::Duration::from_secs(30)),
3288        )
3289        .await
3290        .expect_err("empty idempotency key must fail before lease acquisition");
3291
3292        match err {
3293            WorkflowError::Other(msg) => {
3294                assert!(
3295                    msg.contains("idempotency key must not be empty"),
3296                    "got: {msg}"
3297                );
3298            }
3299            other => panic!("expected idempotency validation error, got {other:?}"),
3300        }
3301        assert!(executor.acquired_leases.lock().unwrap().is_empty());
3302        assert!(executor.released_leases.lock().unwrap().is_empty());
3303        assert_eq!(ctx.current_state, "active");
3304    }
3305
3306    #[tokio::test]
3307    async fn padded_runtime_guard_identity_fails_before_side_effects() {
3308        let executor = MockExecutor::new();
3309
3310        let wf = WorkflowDefinition::new("padded_runtime_guard")
3311            .initial_state("active")
3312            .transition(
3313                "active",
3314                "done",
3315                vec![
3316                    WorkflowStep::notify(ChannelKind::Email, "should_not_send", "customer.email"),
3317                    WorkflowStep::transition("done"),
3318                ],
3319            );
3320
3321        let mut padded_key_ctx = WorkflowContext::new("wf-padded-idempotency", "active");
3322        padded_key_ctx.set(
3323            "customer",
3324            serde_json::json!({"email": "guest@example.com"}),
3325        );
3326        let err = run_workflow_with_options(
3327            &executor,
3328            &wf,
3329            &mut padded_key_ctx,
3330            WorkflowRunOptions::default()
3331                .with_idempotency_key(" run-1 ")
3332                .with_lease("worker-a", std::time::Duration::from_secs(30)),
3333        )
3334        .await
3335        .expect_err("padded idempotency key must fail before side effects");
3336        match err {
3337            WorkflowError::Other(msg) => {
3338                assert!(
3339                    msg.contains("Workflow idempotency key must not have leading"),
3340                    "got: {msg}"
3341                );
3342            }
3343            other => panic!("expected idempotency key validation error, got {other:?}"),
3344        }
3345
3346        let mut padded_owner_ctx = WorkflowContext::new("wf-padded-owner", "active");
3347        padded_owner_ctx.set(
3348            "customer",
3349            serde_json::json!({"email": "guest@example.com"}),
3350        );
3351        let err = run_workflow_with_options(
3352            &executor,
3353            &wf,
3354            &mut padded_owner_ctx,
3355            WorkflowRunOptions::default()
3356                .with_idempotency_key("run-2")
3357                .with_lease(" worker-a ", std::time::Duration::from_secs(30)),
3358        )
3359        .await
3360        .expect_err("padded lease owner must fail before side effects");
3361        match err {
3362            WorkflowError::Other(msg) => {
3363                assert!(
3364                    msg.contains("Workflow lease owner must not have leading"),
3365                    "got: {msg}"
3366                );
3367            }
3368            other => panic!("expected lease owner validation error, got {other:?}"),
3369        }
3370
3371        assert!(executor.acquired_leases.lock().unwrap().is_empty());
3372        assert!(executor.released_leases.lock().unwrap().is_empty());
3373        assert!(executor.notifications.lock().unwrap().is_empty());
3374    }
3375
3376    #[tokio::test]
3377    async fn operation_begin_failure_does_not_release_unacquired_lease() {
3378        let executor = MockExecutor::new();
3379        executor.fail_workflow_operation_begin("begin-fails-1");
3380
3381        let wf = WorkflowDefinition::new("operation_begin_failure")
3382            .initial_state("active")
3383            .transition("active", "done", vec![WorkflowStep::transition("done")]);
3384
3385        let mut ctx = WorkflowContext::new("wf-begin-failure", "active");
3386        let err = run_workflow_with_options(
3387            &executor,
3388            &wf,
3389            &mut ctx,
3390            WorkflowRunOptions::default()
3391                .with_idempotency_key("begin-fails-1")
3392                .with_lease("worker-a", std::time::Duration::from_secs(30)),
3393        )
3394        .await
3395        .expect_err("operation begin failure must happen before lease acquisition");
3396
3397        match err {
3398            WorkflowError::Other(msg) => {
3399                assert!(msg.contains("forced operation begin failure"), "got: {msg}");
3400            }
3401            other => panic!("expected operation begin error, got {other:?}"),
3402        }
3403        assert!(executor.acquired_leases.lock().unwrap().is_empty());
3404        assert!(executor.released_leases.lock().unwrap().is_empty());
3405        assert_eq!(ctx.current_state, "active");
3406    }
3407
3408    #[tokio::test]
3409    async fn completed_operation_with_invalid_state_is_rejected_before_lease() {
3410        let executor = MockExecutor::new();
3411        executor.set_workflow_operation_status(
3412            "completed-invalid-state-1",
3413            WorkflowOperationStatus::Completed {
3414                state: " ".to_string(),
3415            },
3416        );
3417
3418        let wf = WorkflowDefinition::new("completed_invalid_state")
3419            .initial_state("active")
3420            .transition("active", "done", vec![WorkflowStep::transition("done")]);
3421
3422        let mut ctx = WorkflowContext::new("wf-completed-invalid-state", "active");
3423        let err = run_workflow_with_options(
3424            &executor,
3425            &wf,
3426            &mut ctx,
3427            WorkflowRunOptions::default()
3428                .with_idempotency_key("completed-invalid-state-1")
3429                .with_lease("worker-a", std::time::Duration::from_secs(30)),
3430        )
3431        .await
3432        .expect_err("invalid completed replay state must be rejected");
3433
3434        match err {
3435            WorkflowError::Other(msg) => {
3436                assert!(
3437                    msg.contains("Workflow completed operation state must not be empty"),
3438                    "got: {msg}"
3439                );
3440            }
3441            other => panic!("expected completed state validation error, got {other:?}"),
3442        }
3443        assert!(executor.acquired_leases.lock().unwrap().is_empty());
3444        assert!(executor.released_leases.lock().unwrap().is_empty());
3445        assert_eq!(ctx.current_state, "active");
3446    }
3447
3448    #[tokio::test]
3449    async fn invalid_definition_fails_before_runtime_guards() {
3450        let executor = MockExecutor::new();
3451
3452        let wf = WorkflowDefinition::new("invalid_definition")
3453            .version(" v2")
3454            .initial_state("active")
3455            .transition(
3456                "active",
3457                "done",
3458                vec![
3459                    WorkflowStep::notify(ChannelKind::Email, "should_not_send", "customer.email"),
3460                    WorkflowStep::transition("done"),
3461                ],
3462            );
3463
3464        let mut ctx = WorkflowContext::new("wf-invalid-definition", "active");
3465        ctx.set(
3466            "customer",
3467            serde_json::json!({"email": "guest@example.com"}),
3468        );
3469
3470        let err = run_workflow_with_options(
3471            &executor,
3472            &wf,
3473            &mut ctx,
3474            WorkflowRunOptions::default()
3475                .with_idempotency_key("invalid-run-1")
3476                .with_lease("worker-a", std::time::Duration::from_secs(30)),
3477        )
3478        .await
3479        .expect_err("invalid definition must fail before runtime guards");
3480
3481        match err {
3482            WorkflowError::Other(msg) => {
3483                assert!(
3484                    msg.contains("Workflow definition version must not have leading"),
3485                    "got: {msg}"
3486                );
3487            }
3488            other => panic!("expected definition validation error, got {other:?}"),
3489        }
3490        assert!(executor.acquired_leases.lock().unwrap().is_empty());
3491        assert!(executor.released_leases.lock().unwrap().is_empty());
3492        assert!(
3493            executor
3494                .completed_workflow_operations
3495                .lock()
3496                .unwrap()
3497                .is_empty()
3498        );
3499        assert!(
3500            executor
3501                .failed_workflow_operations
3502                .lock()
3503                .unwrap()
3504                .is_empty()
3505        );
3506        assert!(executor.notifications.lock().unwrap().is_empty());
3507    }
3508
3509    #[tokio::test]
3510    async fn empty_workflow_id_fails_before_runtime_guards() {
3511        let executor = MockExecutor::new();
3512
3513        let wf = WorkflowDefinition::new("empty_workflow_id")
3514            .initial_state("active")
3515            .transition("active", "done", vec![WorkflowStep::transition("done")]);
3516
3517        let mut ctx = WorkflowContext::new(" ", "active");
3518        let err = run_workflow_with_options(
3519            &executor,
3520            &wf,
3521            &mut ctx,
3522            WorkflowRunOptions::default()
3523                .with_idempotency_key("empty-id-1")
3524                .with_lease("worker-a", std::time::Duration::from_secs(30)),
3525        )
3526        .await
3527        .expect_err("empty workflow id must fail before runtime guards");
3528
3529        match err {
3530            WorkflowError::Other(msg) => {
3531                assert!(msg.contains("Workflow id must not be empty"), "got: {msg}");
3532            }
3533            other => panic!("expected workflow id validation error, got {other:?}"),
3534        }
3535        assert!(executor.acquired_leases.lock().unwrap().is_empty());
3536        assert!(executor.released_leases.lock().unwrap().is_empty());
3537        assert!(
3538            executor
3539                .completed_workflow_operations
3540                .lock()
3541                .unwrap()
3542                .is_empty()
3543        );
3544        assert!(
3545            executor
3546                .failed_workflow_operations
3547                .lock()
3548                .unwrap()
3549                .is_empty()
3550        );
3551    }
3552
3553    #[tokio::test]
3554    async fn empty_resume_workflow_id_fails_before_runtime_guards() {
3555        let executor = MockExecutor::new();
3556
3557        let wf = WorkflowDefinition::new("empty_resume_id")
3558            .initial_state("active")
3559            .transition(
3560                "active",
3561                "done",
3562                vec![
3563                    WorkflowStep::wait("vendor.ready", std::time::Duration::from_secs(3600)),
3564                    WorkflowStep::transition("done"),
3565                ],
3566            );
3567
3568        let err = resume_workflow_with_options(
3569            &executor,
3570            &wf,
3571            " ",
3572            serde_json::json!({"event": "vendor.ready"}),
3573            WorkflowRunOptions::default()
3574                .with_idempotency_key("empty-resume-id-1")
3575                .with_lease("worker-a", std::time::Duration::from_secs(30)),
3576        )
3577        .await
3578        .expect_err("empty workflow id must fail before runtime guards");
3579
3580        match err {
3581            WorkflowError::Other(msg) => {
3582                assert!(msg.contains("Workflow id must not be empty"), "got: {msg}");
3583            }
3584            other => panic!("expected workflow id validation error, got {other:?}"),
3585        }
3586        assert!(executor.acquired_leases.lock().unwrap().is_empty());
3587        assert!(executor.released_leases.lock().unwrap().is_empty());
3588        assert!(
3589            executor
3590                .completed_workflow_operations
3591                .lock()
3592                .unwrap()
3593                .is_empty()
3594        );
3595        assert!(
3596            executor
3597                .failed_workflow_operations
3598                .lock()
3599                .unwrap()
3600                .is_empty()
3601        );
3602    }
3603
3604    #[tokio::test]
3605    async fn operation_complete_failure_still_releases_lease() {
3606        let executor = MockExecutor::new();
3607        executor.fail_workflow_operation_complete("complete-fails-1");
3608
3609        let wf = WorkflowDefinition::new("complete_failure_release")
3610            .initial_state("active")
3611            .transition("active", "done", vec![WorkflowStep::transition("done")]);
3612
3613        let mut ctx = WorkflowContext::new("wf-complete-failure-release", "active");
3614        let err = run_workflow_with_options(
3615            &executor,
3616            &wf,
3617            &mut ctx,
3618            WorkflowRunOptions::default()
3619                .with_idempotency_key("complete-fails-1")
3620                .with_lease("worker-a", std::time::Duration::from_secs(30)),
3621        )
3622        .await
3623        .expect_err("operation completion failure should be returned");
3624
3625        match err {
3626            WorkflowError::Other(msg) => {
3627                assert!(
3628                    msg.contains("forced operation complete failure"),
3629                    "got: {msg}"
3630                );
3631            }
3632            other => panic!("expected completion failure, got {other:?}"),
3633        }
3634        assert_eq!(ctx.current_state, "done");
3635        assert_eq!(executor.acquired_leases.lock().unwrap().len(), 1);
3636        assert_eq!(executor.released_leases.lock().unwrap().len(), 1);
3637    }
3638
3639    #[tokio::test]
3640    async fn operation_fail_failure_still_releases_lease() {
3641        let executor = MockExecutor::new();
3642        executor.fail_workflow_operation_fail("fail-fails-1");
3643
3644        let wf = WorkflowDefinition::new("fail_failure_release")
3645            .initial_state("active")
3646            .transition(
3647                "active",
3648                "done",
3649                vec![
3650                    WorkflowStep::notify(ChannelKind::Email, "missing_recipient", "customer.email"),
3651                    WorkflowStep::transition("done"),
3652                ],
3653            );
3654
3655        let mut ctx = WorkflowContext::new("wf-fail-failure-release", "active");
3656        let err = run_workflow_with_options(
3657            &executor,
3658            &wf,
3659            &mut ctx,
3660            WorkflowRunOptions::default()
3661                .with_idempotency_key("fail-fails-1")
3662                .with_lease("worker-a", std::time::Duration::from_secs(30)),
3663        )
3664        .await
3665        .expect_err("original workflow error should be returned");
3666
3667        match err {
3668            WorkflowError::MissingContextKey(key) => {
3669                assert_eq!(key, "customer.email");
3670            }
3671            other => panic!("expected original workflow error, got {other:?}"),
3672        }
3673        assert_eq!(ctx.current_state, "active");
3674        assert_eq!(executor.acquired_leases.lock().unwrap().len(), 1);
3675        assert_eq!(executor.released_leases.lock().unwrap().len(), 1);
3676    }
3677
3678    #[tokio::test]
3679    async fn completed_notify_side_effect_skips_provider_call() {
3680        let executor = MockExecutor::new();
3681        executor.skip_notify_side_effects();
3682
3683        let wf = WorkflowDefinition::new("notify_side_effect_replay")
3684            .initial_state("active")
3685            .transition(
3686                "active",
3687                "done",
3688                vec![
3689                    WorkflowStep::notify(ChannelKind::Email, "booking_confirmed", "customer.email"),
3690                    WorkflowStep::transition("done"),
3691                ],
3692            );
3693
3694        let mut ctx = WorkflowContext::new("wf-notify-side-effect", "active");
3695        ctx.set(
3696            "customer",
3697            serde_json::json!({"email": "guest@example.com"}),
3698        );
3699
3700        let result = run_workflow(&executor, &wf, &mut ctx).await.unwrap();
3701
3702        assert_eq!(result, "done");
3703        assert!(
3704            executor.notifications.lock().unwrap().is_empty(),
3705            "completed notify side effect must not call the provider again"
3706        );
3707    }
3708
3709    #[tokio::test]
3710    async fn transition_limit_counts_only_current_run() {
3711        let executor = MockExecutor::new();
3712
3713        let wf = WorkflowDefinition::new("aged_workflow")
3714            .initial_state("active")
3715            .transition("active", "done", vec![WorkflowStep::transition("done")]);
3716
3717        let mut ctx = WorkflowContext::new("wf-aged-history", "active");
3718        for idx in 0..51 {
3719            ctx.history.push(StateChange {
3720                from: format!("past_{idx}"),
3721                to: format!("past_{}", idx + 1),
3722                at: Utc::now(),
3723                reason: None,
3724            });
3725        }
3726
3727        let result = run_workflow(&executor, &wf, &mut ctx)
3728            .await
3729            .expect("old workflow history must not trip per-run loop guard");
3730
3731        assert_eq!(result, "done");
3732        assert_eq!(ctx.transition_count(), 52);
3733    }
3734
3735    #[tokio::test]
3736    async fn transition_limit_blocks_next_transition_before_side_effects() {
3737        let executor = MockExecutor::new();
3738
3739        let mut wf = WorkflowDefinition::new("runaway_workflow").initial_state("s0");
3740        for idx in 0..50 {
3741            wf = wf.transition(
3742                format!("s{idx}"),
3743                format!("s{}", idx + 1),
3744                vec![WorkflowStep::transition(&format!("s{}", idx + 1))],
3745            );
3746        }
3747        wf = wf.transition(
3748            "s50",
3749            "s51",
3750            vec![
3751                WorkflowStep::notify(ChannelKind::Email, "runaway_side_effect", "ops.email"),
3752                WorkflowStep::transition("s51"),
3753            ],
3754        );
3755
3756        let mut ctx = WorkflowContext::new("wf-runaway", "s0");
3757        ctx.set("ops", serde_json::json!({"email": "ops@example.com"}));
3758
3759        let err = run_workflow(&executor, &wf, &mut ctx)
3760            .await
3761            .expect_err("the 51st transition must be blocked before side effects");
3762
3763        match err {
3764            WorkflowError::Other(msg) => {
3765                assert!(
3766                    msg.contains("Maximum transition count exceeded"),
3767                    "got: {msg}"
3768                );
3769            }
3770            other => panic!("expected transition limit error, got {other:?}"),
3771        }
3772        assert_eq!(ctx.current_state, "s50");
3773        assert_eq!(ctx.transition_count(), 50);
3774        assert!(
3775            executor.notifications.lock().unwrap().is_empty(),
3776            "blocked transition side effects must not execute"
3777        );
3778        let saved = executor
3779            .saved_state
3780            .lock()
3781            .unwrap()
3782            .clone()
3783            .expect("state after the last allowed transition should be saved");
3784        assert_eq!(saved.current_state, "s50");
3785    }
3786
3787    #[tokio::test]
3788    async fn test_workflow_with_branch() {
3789        let executor = MockExecutor::new();
3790
3791        let wf = WorkflowDefinition::new("branching")
3792            .initial_state("pending")
3793            .transition(
3794                "pending",
3795                "resolved",
3796                vec![
3797                    WorkflowStep::branch(
3798                        "reason",
3799                        vec![
3800                            ("full", vec![WorkflowStep::log("Fully booked")]),
3801                            ("maintenance", vec![WorkflowStep::log("Under maintenance")]),
3802                        ],
3803                        vec![WorkflowStep::log("Unknown reason")],
3804                    ),
3805                    WorkflowStep::transition("resolved"),
3806                ],
3807            );
3808
3809        let mut ctx = WorkflowContext::new("wf-002", "pending");
3810        ctx.set("reason", Value::String("maintenance".into()));
3811
3812        let result = run_workflow(&executor, &wf, &mut ctx).await.unwrap();
3813        assert_eq!(result, "resolved");
3814    }
3815
3816    #[tokio::test]
3817    async fn branch_when_routes_by_typed_numeric_condition() {
3818        let executor = MockExecutor::new();
3819
3820        let wf = WorkflowDefinition::new("typed_branching")
3821            .initial_state("pending")
3822            .transition(
3823                "pending",
3824                "resolved",
3825                vec![WorkflowStep::branch_when(
3826                    "payment.attempts",
3827                    vec![(
3828                        WorkflowBranchCondition::NumberGte(3),
3829                        vec![WorkflowStep::transition("manual_review")],
3830                    )],
3831                    vec![WorkflowStep::transition("retry")],
3832                )],
3833            );
3834
3835        let mut ctx = WorkflowContext::new("wf-typed-branch", "pending");
3836        ctx.set("payment", serde_json::json!({"attempts": 3}));
3837
3838        let result = run_workflow(&executor, &wf, &mut ctx).await.unwrap();
3839
3840        assert_eq!(result, "manual_review");
3841        assert_eq!(ctx.current_state, "manual_review");
3842    }
3843
3844    #[tokio::test]
3845    async fn branch_when_routes_by_decimal_numeric_condition() {
3846        let executor = MockExecutor::new();
3847
3848        let wf = WorkflowDefinition::new("decimal_branching")
3849            .initial_state("pending")
3850            .transition(
3851                "pending",
3852                "resolved",
3853                vec![WorkflowStep::branch_when(
3854                    "payment.risk_score",
3855                    vec![(
3856                        WorkflowBranchCondition::NumberGt(10),
3857                        vec![WorkflowStep::transition("manual_review")],
3858                    )],
3859                    vec![WorkflowStep::transition("retry")],
3860                )],
3861            );
3862
3863        let mut ctx = WorkflowContext::new("wf-decimal-branch", "pending");
3864        ctx.set("payment", serde_json::json!({"risk_score": 10.5}));
3865
3866        let result = run_workflow(&executor, &wf, &mut ctx).await.unwrap();
3867
3868        assert_eq!(result, "manual_review");
3869        assert_eq!(ctx.current_state, "manual_review");
3870    }
3871
3872    #[tokio::test]
3873    async fn branch_when_resume_rejects_condition_drift_before_side_effects() {
3874        let executor = MockExecutor::new();
3875
3876        let wf = WorkflowDefinition::new("branch_when_drift_source")
3877            .initial_state("active")
3878            .transition(
3879                "active",
3880                "done",
3881                vec![WorkflowStep::branch_when(
3882                    "payment.attempts",
3883                    vec![(
3884                        WorkflowBranchCondition::NumberGte(3),
3885                        vec![
3886                            WorkflowStep::wait(
3887                                "manual.approved",
3888                                std::time::Duration::from_secs(3600),
3889                            ),
3890                            WorkflowStep::notify(
3891                                ChannelKind::Email,
3892                                "manual_review_approved",
3893                                "customer.email",
3894                            ),
3895                        ],
3896                    )],
3897                    vec![],
3898                )],
3899            );
3900
3901        let mut ctx = WorkflowContext::new("wf-branch-when-drift", "active");
3902        ctx.set(
3903            "payment",
3904            serde_json::json!({
3905                "attempts": 3
3906            }),
3907        );
3908        ctx.set(
3909            "customer",
3910            serde_json::json!({"email": "guest@example.com"}),
3911        );
3912
3913        let result = run_workflow(&executor, &wf, &mut ctx).await.unwrap();
3914        assert_eq!(result, "active");
3915
3916        {
3917            let mut saved = executor.saved_state.lock().unwrap();
3918            let saved = saved.as_mut().expect("paused state should be saved");
3919            saved.set(
3920                "payment",
3921                serde_json::json!({
3922                    "attempts": 1
3923                }),
3924            );
3925        }
3926
3927        let err = resume_workflow(
3928            &executor,
3929            &wf,
3930            "wf-branch-when-drift",
3931            serde_json::json!({"event": "manual.approved"}),
3932        )
3933        .await
3934        .expect_err("branch predicate drift must reject resume");
3935
3936        match err {
3937            WorkflowError::Other(msg) => {
3938                assert!(msg.contains("Invalid workflow resume cursor"), "got: {msg}");
3939                assert!(msg.contains("no longer matches"), "got: {msg}");
3940            }
3941            other => panic!("expected invalid cursor error, got {other:?}"),
3942        }
3943        assert!(executor.notifications.lock().unwrap().is_empty());
3944    }
3945
3946    #[tokio::test]
3947    async fn branch_missing_context_key_errors_instead_of_taking_default() {
3948        let executor = MockExecutor::new();
3949
3950        let wf = WorkflowDefinition::new("branching_missing_key")
3951            .initial_state("pending")
3952            .transition(
3953                "pending",
3954                "resolved",
3955                vec![WorkflowStep::branch(
3956                    "payment.status",
3957                    vec![("paid", vec![WorkflowStep::transition("resolved")])],
3958                    vec![WorkflowStep::transition("fallback")],
3959                )],
3960            );
3961
3962        let mut ctx = WorkflowContext::new("wf-missing-branch-key", "pending");
3963        let err = run_workflow(&executor, &wf, &mut ctx)
3964            .await
3965            .expect_err("missing branch context must not route to default");
3966
3967        match err {
3968            WorkflowError::MissingContextKey(key) => assert_eq!(key, "payment.status"),
3969            other => panic!("expected MissingContextKey, got {other:?}"),
3970        }
3971        assert_eq!(ctx.current_state, "pending");
3972    }
3973
3974    #[tokio::test]
3975    async fn duplicate_transitions_from_same_state_fail_before_side_effects() {
3976        let executor = MockExecutor::new();
3977
3978        let wf = WorkflowDefinition::new("duplicate_transition_guard")
3979            .initial_state("active")
3980            .transition(
3981                "active",
3982                "vendor_a",
3983                vec![WorkflowStep::notify(
3984                    ChannelKind::WhatsApp,
3985                    "vendor_a",
3986                    "vendor.phone",
3987                )],
3988            )
3989            .transition(
3990                "active",
3991                "vendor_b",
3992                vec![WorkflowStep::notify(
3993                    ChannelKind::WhatsApp,
3994                    "vendor_b",
3995                    "vendor.phone",
3996                )],
3997            );
3998
3999        let mut ctx = WorkflowContext::new("wf-duplicate-transition", "active");
4000        ctx.set("vendor", serde_json::json!({"phone": "+628111"}));
4001
4002        let err = run_workflow(&executor, &wf, &mut ctx)
4003            .await
4004            .expect_err("duplicate transitions must fail closed");
4005
4006        match err {
4007            WorkflowError::Other(msg) => {
4008                assert!(msg.contains("Ambiguous workflow definition"), "got: {msg}");
4009                assert!(msg.contains("active"), "got: {msg}");
4010                assert!(msg.contains("2 outgoing transitions"), "got: {msg}");
4011            }
4012            other => panic!("expected ambiguous transition error, got {other:?}"),
4013        }
4014        assert!(executor.notifications.lock().unwrap().is_empty());
4015    }
4016
4017    #[tokio::test]
4018    async fn duplicate_future_transitions_fail_before_current_side_effects() {
4019        let executor = MockExecutor::new();
4020
4021        let wf = WorkflowDefinition::new("future_duplicate_transition_guard")
4022            .initial_state("active")
4023            .transition(
4024                "active",
4025                "routing",
4026                vec![
4027                    WorkflowStep::notify(ChannelKind::WhatsApp, "before_bad_state", "vendor.phone"),
4028                    WorkflowStep::transition("routing"),
4029                ],
4030            )
4031            .transition(
4032                "routing",
4033                "vendor_a",
4034                vec![WorkflowStep::notify(
4035                    ChannelKind::WhatsApp,
4036                    "vendor_a",
4037                    "vendor.phone",
4038                )],
4039            )
4040            .transition(
4041                "routing",
4042                "vendor_b",
4043                vec![WorkflowStep::notify(
4044                    ChannelKind::WhatsApp,
4045                    "vendor_b",
4046                    "vendor.phone",
4047                )],
4048            );
4049
4050        let mut ctx = WorkflowContext::new("wf-future-duplicate-transition", "active");
4051        ctx.set("vendor", serde_json::json!({"phone": "+628111"}));
4052
4053        let err = run_workflow(&executor, &wf, &mut ctx)
4054            .await
4055            .expect_err("future duplicate transitions must fail before current side effects");
4056
4057        match err {
4058            WorkflowError::Other(msg) => {
4059                assert!(msg.contains("Ambiguous workflow definition"), "got: {msg}");
4060                assert!(msg.contains("routing"), "got: {msg}");
4061            }
4062            other => panic!("expected ambiguous transition error, got {other:?}"),
4063        }
4064        assert_eq!(ctx.current_state, "active");
4065        assert!(
4066            executor.notifications.lock().unwrap().is_empty(),
4067            "definition validation must run before the initial notification"
4068        );
4069    }
4070
4071    #[tokio::test]
4072    async fn duplicate_branch_values_fail_before_side_effects() {
4073        let executor = MockExecutor::new();
4074
4075        let wf = WorkflowDefinition::new("duplicate_branch_guard")
4076            .initial_state("active")
4077            .transition(
4078                "active",
4079                "done",
4080                vec![WorkflowStep::branch(
4081                    "decision",
4082                    vec![
4083                        (
4084                            "accepted",
4085                            vec![WorkflowStep::notify(
4086                                ChannelKind::WhatsApp,
4087                                "accepted_a",
4088                                "vendor.phone",
4089                            )],
4090                        ),
4091                        (
4092                            "accepted",
4093                            vec![WorkflowStep::notify(
4094                                ChannelKind::WhatsApp,
4095                                "accepted_b",
4096                                "vendor.phone",
4097                            )],
4098                        ),
4099                    ],
4100                    vec![],
4101                )],
4102            );
4103
4104        let mut ctx = WorkflowContext::new("wf-duplicate-branch", "active");
4105        ctx.set("decision", Value::String("accepted".to_string()));
4106        ctx.set("vendor", serde_json::json!({"phone": "+628111"}));
4107
4108        let err = run_workflow(&executor, &wf, &mut ctx)
4109            .await
4110            .expect_err("duplicate branch values must fail closed");
4111
4112        match err {
4113            WorkflowError::Other(msg) => {
4114                assert!(msg.contains("Ambiguous workflow branch"), "got: {msg}");
4115                assert!(msg.contains("accepted"), "got: {msg}");
4116            }
4117            other => panic!("expected ambiguous branch error, got {other:?}"),
4118        }
4119        assert!(executor.notifications.lock().unwrap().is_empty());
4120    }
4121
4122    #[tokio::test]
4123    async fn branch_when_rejects_empty_string_contains_before_side_effects() {
4124        let executor = MockExecutor::new();
4125
4126        let wf = WorkflowDefinition::new("empty_string_contains_guard")
4127            .initial_state("active")
4128            .transition(
4129                "active",
4130                "done",
4131                vec![
4132                    WorkflowStep::notify(ChannelKind::Email, "before_bad_branch", "ops.email"),
4133                    WorkflowStep::branch_when(
4134                        "decision",
4135                        vec![(
4136                            WorkflowBranchCondition::StringContains(String::new()),
4137                            vec![WorkflowStep::transition("catch_all")],
4138                        )],
4139                        vec![WorkflowStep::transition("done")],
4140                    ),
4141                ],
4142            );
4143
4144        let mut ctx = WorkflowContext::new("wf-empty-string-contains", "active");
4145        ctx.set("ops", serde_json::json!({"email": "ops@example.com"}));
4146        ctx.set("decision", Value::String("accepted".to_string()));
4147
4148        let err = run_workflow(&executor, &wf, &mut ctx)
4149            .await
4150            .expect_err("empty StringContains predicate must fail before notification");
4151
4152        match err {
4153            WorkflowError::Other(msg) => {
4154                assert!(
4155                    msg.contains("StringContains condition must not be empty"),
4156                    "got: {msg}"
4157                );
4158            }
4159            other => panic!("expected branch condition validation error, got {other:?}"),
4160        }
4161        assert!(executor.notifications.lock().unwrap().is_empty());
4162    }
4163
4164    #[tokio::test]
4165    async fn branch_when_rejects_empty_one_of_before_side_effects() {
4166        let executor = MockExecutor::new();
4167
4168        let wf = WorkflowDefinition::new("empty_one_of_guard")
4169            .initial_state("active")
4170            .transition(
4171                "active",
4172                "done",
4173                vec![
4174                    WorkflowStep::notify(ChannelKind::Email, "before_bad_branch", "ops.email"),
4175                    WorkflowStep::branch_when(
4176                        "decision",
4177                        vec![(
4178                            WorkflowBranchCondition::OneOf(Vec::new()),
4179                            vec![WorkflowStep::transition("impossible")],
4180                        )],
4181                        vec![WorkflowStep::transition("done")],
4182                    ),
4183                ],
4184            );
4185
4186        let mut ctx = WorkflowContext::new("wf-empty-one-of", "active");
4187        ctx.set("ops", serde_json::json!({"email": "ops@example.com"}));
4188        ctx.set("decision", Value::String("accepted".to_string()));
4189
4190        let err = run_workflow(&executor, &wf, &mut ctx)
4191            .await
4192            .expect_err("empty OneOf predicate must fail before notification");
4193
4194        match err {
4195            WorkflowError::Other(msg) => {
4196                assert!(
4197                    msg.contains("OneOf condition must include at least one value"),
4198                    "got: {msg}"
4199                );
4200            }
4201            other => panic!("expected branch condition validation error, got {other:?}"),
4202        }
4203        assert!(executor.notifications.lock().unwrap().is_empty());
4204    }
4205
4206    #[tokio::test]
4207    async fn invalid_transition_target_fails_before_side_effects() {
4208        let executor = MockExecutor::new();
4209
4210        let wf = WorkflowDefinition::new("invalid_transition_target")
4211            .initial_state("active")
4212            .transition(
4213                "active",
4214                "done",
4215                vec![
4216                    WorkflowStep::notify(ChannelKind::Email, "before_bad_transition", "ops.email"),
4217                    WorkflowStep::transition(" "),
4218                ],
4219            );
4220
4221        let mut ctx = WorkflowContext::new("wf-invalid-transition-target", "active");
4222        ctx.set("ops", serde_json::json!({"email": "ops@example.com"}));
4223
4224        let err = run_workflow(&executor, &wf, &mut ctx)
4225            .await
4226            .expect_err("invalid transition target must fail during definition validation");
4227
4228        match err {
4229            WorkflowError::Other(msg) => {
4230                assert!(
4231                    msg.contains("Transition target state must not be empty"),
4232                    "got: {msg}"
4233                );
4234            }
4235            other => panic!("expected transition target validation error, got {other:?}"),
4236        }
4237        assert!(executor.notifications.lock().unwrap().is_empty());
4238    }
4239
4240    #[tokio::test]
4241    async fn invalid_notify_recipient_key_fails_before_side_effects() {
4242        let executor = MockExecutor::new();
4243
4244        let wf = WorkflowDefinition::new("invalid_notify_recipient")
4245            .initial_state("active")
4246            .transition(
4247                "active",
4248                "done",
4249                vec![
4250                    WorkflowStep::notify(ChannelKind::Email, "bad_notify", " "),
4251                    WorkflowStep::transition("done"),
4252                ],
4253            );
4254
4255        let mut ctx = WorkflowContext::new("wf-invalid-notify-recipient", "active");
4256        let err = run_workflow(&executor, &wf, &mut ctx)
4257            .await
4258            .expect_err("invalid notify recipient key must fail before send");
4259
4260        match err {
4261            WorkflowError::Other(msg) => {
4262                assert!(
4263                    msg.contains("Notify recipient_key must not be empty"),
4264                    "got: {msg}"
4265                );
4266            }
4267            other => panic!("expected notify recipient validation error, got {other:?}"),
4268        }
4269        assert!(executor.notifications.lock().unwrap().is_empty());
4270    }
4271
4272    #[tokio::test]
4273    async fn malformed_lookup_key_fails_before_side_effects() {
4274        let executor = MockExecutor::new();
4275
4276        let wf = WorkflowDefinition::new("malformed_lookup_key_guard")
4277            .initial_state("active")
4278            .transition(
4279                "active",
4280                "done",
4281                vec![
4282                    WorkflowStep::notify(ChannelKind::Email, "before_bad_lookup", "ops.email"),
4283                    WorkflowStep::notify(ChannelKind::Email, "bad_lookup", "customer."),
4284                    WorkflowStep::transition("done"),
4285                ],
4286            );
4287
4288        let mut ctx = WorkflowContext::new("wf-malformed-lookup-key", "active");
4289        ctx.set("ops", serde_json::json!({"email": "ops@example.com"}));
4290        ctx.set(
4291            "customer",
4292            serde_json::json!({"email": "guest@example.com"}),
4293        );
4294
4295        let err = run_workflow(&executor, &wf, &mut ctx)
4296            .await
4297            .expect_err("malformed lookup key must fail before first notification");
4298
4299        match err {
4300            WorkflowError::Other(msg) => {
4301                assert!(
4302                    msg.contains("empty dot-notation path segments"),
4303                    "got: {msg}"
4304                );
4305            }
4306            other => panic!("expected malformed lookup key error, got {other:?}"),
4307        }
4308        assert!(
4309            executor.notifications.lock().unwrap().is_empty(),
4310            "definition validation must fail before earlier notification side effects"
4311        );
4312    }
4313
4314    #[tokio::test]
4315    async fn invalid_charge_amount_key_fails_before_provider_call() {
4316        let executor = MockExecutor::new();
4317
4318        let wf = WorkflowDefinition::new("invalid_charge_amount_key")
4319            .initial_state("created")
4320            .transition(
4321                "created",
4322                "awaiting_payment",
4323                vec![WorkflowStep::charge(
4324                    PaymentKind::Xendit,
4325                    " ",
4326                    "order.id",
4327                    Some("charge"),
4328                )],
4329            );
4330
4331        let mut ctx = WorkflowContext::new("wf-invalid-charge-amount", "created");
4332        ctx.set(
4333            "order",
4334            serde_json::json!({"id": "booking-1", "total": 125_000}),
4335        );
4336        let err = run_workflow(&executor, &wf, &mut ctx)
4337            .await
4338            .expect_err("invalid charge amount key must fail before provider call");
4339
4340        match err {
4341            WorkflowError::Other(msg) => {
4342                assert!(
4343                    msg.contains("Charge amount_key must not be empty"),
4344                    "got: {msg}"
4345                );
4346            }
4347            other => panic!("expected charge amount key validation error, got {other:?}"),
4348        }
4349        assert!(executor.charges.lock().unwrap().is_empty());
4350    }
4351
4352    #[tokio::test]
4353    async fn malformed_charge_lookup_key_fails_before_provider_call() {
4354        let executor = MockExecutor::new();
4355
4356        let wf = WorkflowDefinition::new("malformed_charge_lookup_key")
4357            .initial_state("created")
4358            .transition(
4359                "created",
4360                "awaiting_payment",
4361                vec![WorkflowStep::charge(
4362                    PaymentKind::Xendit,
4363                    "order..total",
4364                    "order.id",
4365                    Some("charge"),
4366                )],
4367            );
4368
4369        let mut ctx = WorkflowContext::new("wf-malformed-charge-lookup", "created");
4370        ctx.set(
4371            "order",
4372            serde_json::json!({"id": "booking-1", "total": 125_000}),
4373        );
4374        let err = run_workflow(&executor, &wf, &mut ctx)
4375            .await
4376            .expect_err("malformed charge lookup key must fail before provider call");
4377
4378        match err {
4379            WorkflowError::Other(msg) => {
4380                assert!(
4381                    msg.contains("empty dot-notation path segments"),
4382                    "got: {msg}"
4383                );
4384            }
4385            other => panic!("expected malformed lookup key error, got {other:?}"),
4386        }
4387        assert!(executor.charges.lock().unwrap().is_empty());
4388    }
4389
4390    #[tokio::test]
4391    async fn charge_rejects_missing_optional_string_key_before_provider_call() {
4392        let executor = MockExecutor::new();
4393
4394        let wf = WorkflowDefinition::new("charge_missing_optional_string_key")
4395            .initial_state("created")
4396            .transition(
4397                "created",
4398                "awaiting_payment",
4399                vec![WorkflowStep::charge_with(
4400                    PaymentKind::Xendit,
4401                    "order.total",
4402                    "order.id",
4403                    Some("order.description"),
4404                    Some("order.payment_method"),
4405                    Some("charge"),
4406                )],
4407            );
4408
4409        let mut ctx = WorkflowContext::new("wf-charge-missing-optional-string", "created");
4410        ctx.set(
4411            "order",
4412            serde_json::json!({"id": "booking-1", "total": 125_000}),
4413        );
4414        let err = run_workflow(&executor, &wf, &mut ctx)
4415            .await
4416            .expect_err("missing configured charge string key must fail before provider call");
4417
4418        match err {
4419            WorkflowError::MissingContextKey(key) => {
4420                assert_eq!(key, "order.description (expected string)");
4421            }
4422            other => panic!("expected missing context key error, got {other:?}"),
4423        }
4424        assert!(executor.charges.lock().unwrap().is_empty());
4425    }
4426
4427    #[tokio::test]
4428    async fn charge_rejects_non_string_optional_key_before_provider_call() {
4429        let executor = MockExecutor::new();
4430
4431        let wf = WorkflowDefinition::new("charge_non_string_optional_key")
4432            .initial_state("created")
4433            .transition(
4434                "created",
4435                "awaiting_payment",
4436                vec![WorkflowStep::charge_with(
4437                    PaymentKind::Xendit,
4438                    "order.total",
4439                    "order.id",
4440                    None,
4441                    Some("order.payment_method"),
4442                    Some("charge"),
4443                )],
4444            );
4445
4446        let mut ctx = WorkflowContext::new("wf-charge-non-string-optional", "created");
4447        ctx.set(
4448            "order",
4449            serde_json::json!({
4450                "id": "booking-1",
4451                "total": 125_000,
4452                "payment_method": 100
4453            }),
4454        );
4455        let err = run_workflow(&executor, &wf, &mut ctx)
4456            .await
4457            .expect_err("non-string configured charge key must fail before provider call");
4458
4459        match err {
4460            WorkflowError::MissingContextKey(key) => {
4461                assert_eq!(key, "order.payment_method (expected string)");
4462            }
4463            other => panic!("expected missing context key error, got {other:?}"),
4464        }
4465        assert!(executor.charges.lock().unwrap().is_empty());
4466    }
4467
4468    #[tokio::test]
4469    async fn invalid_timeout_fallback_fails_before_pre_wait_side_effects() {
4470        let executor = MockExecutor::new();
4471
4472        let wf = WorkflowDefinition::new("invalid_timeout_fallback_guard")
4473            .initial_state("active")
4474            .transition(
4475                "active",
4476                "done",
4477                vec![
4478                    WorkflowStep::notify(ChannelKind::Email, "before_wait", "customer.email"),
4479                    WorkflowStep::wait_or(
4480                        "payment.success",
4481                        std::time::Duration::from_secs(3600),
4482                        vec![WorkflowStep::wait(
4483                            "manual_review",
4484                            std::time::Duration::from_secs(3600),
4485                        )],
4486                    ),
4487                    WorkflowStep::transition("done"),
4488                ],
4489            );
4490
4491        let mut ctx = WorkflowContext::new("wf-invalid-timeout-fallback", "active");
4492        ctx.set(
4493            "customer",
4494            serde_json::json!({"email": "guest@example.com"}),
4495        );
4496
4497        let err = run_workflow(&executor, &wf, &mut ctx)
4498            .await
4499            .expect_err("unsupported timeout fallback must fail before reaching the Wait");
4500
4501        match err {
4502            WorkflowError::Other(msg) => {
4503                assert!(
4504                    msg.contains("Wait steps inside on_timeout fallback are not supported"),
4505                    "got: {msg}"
4506                );
4507            }
4508            other => panic!("expected timeout fallback validation error, got {other:?}"),
4509        }
4510        assert!(ctx.cursor.is_none());
4511        assert!(
4512            executor.notifications.lock().unwrap().is_empty(),
4513            "definition validation must run before the pre-wait notification"
4514        );
4515    }
4516
4517    #[tokio::test]
4518    async fn nested_transition_stops_parent_block_before_follow_up_side_effects() {
4519        let executor = MockExecutor::new();
4520
4521        let wf = WorkflowDefinition::new("nested_transition_guard")
4522            .initial_state("active")
4523            .transition(
4524                "active",
4525                "fallback",
4526                vec![
4527                    WorkflowStep::branch(
4528                        "decision",
4529                        vec![("accepted", vec![WorkflowStep::transition("confirmed")])],
4530                        vec![],
4531                    ),
4532                    WorkflowStep::notify(
4533                        ChannelKind::WhatsApp,
4534                        "old_state_followup",
4535                        "vendor.phone",
4536                    ),
4537                ],
4538            );
4539
4540        let mut ctx = WorkflowContext::new("wf-nested-transition", "active");
4541        ctx.set("decision", Value::String("accepted".to_string()));
4542        ctx.set("vendor", serde_json::json!({"phone": "+628111"}));
4543
4544        let result = run_workflow(&executor, &wf, &mut ctx)
4545            .await
4546            .expect("nested transition should finish cleanly");
4547
4548        assert_eq!(result, "confirmed");
4549        assert_eq!(ctx.current_state, "confirmed");
4550        assert!(
4551            executor.notifications.lock().unwrap().is_empty(),
4552            "parent block must stop after nested transition changes state"
4553        );
4554    }
4555
4556    #[test]
4557    fn log_message_resolves_nested_context_placeholders() {
4558        let mut ctx = WorkflowContext::new("wf-log", "running");
4559        ctx.set(
4560            "item",
4561            serde_json::json!({
4562                "name": "Captain A",
4563                "phone": "+628111",
4564            }),
4565        );
4566        ctx.set("attempt", serde_json::json!(2));
4567
4568        assert_eq!(
4569            resolve_log_message(
4570                &ctx,
4571                "Processing {item.name} via {item.phone} attempt {attempt}"
4572            ),
4573            "Processing Captain A via +628111 attempt 2"
4574        );
4575        assert_eq!(
4576            resolve_log_message(&ctx, "Missing {item.email} stays"),
4577            "Missing {item.email} stays"
4578        );
4579    }
4580
4581    #[tokio::test]
4582    async fn test_workflow_for_each_notify() {
4583        let executor = MockExecutor::new();
4584
4585        let wf = WorkflowDefinition::new("broadcast")
4586            .initial_state("broadcasting")
4587            .transition(
4588                "broadcasting",
4589                "waiting",
4590                vec![WorkflowStep::for_each(
4591                    "operators",
4592                    vec![WorkflowStep::notify(
4593                        ChannelKind::WhatsApp,
4594                        "opportunity",
4595                        "item.phone",
4596                    )],
4597                )],
4598            );
4599
4600        let mut ctx = WorkflowContext::new("wf-003", "broadcasting");
4601        ctx.set(
4602            "operators",
4603            serde_json::json!([
4604                {"name": "Captain A", "phone": "+628111"},
4605                {"name": "Captain B", "phone": "+628222"},
4606                {"name": "Captain C", "phone": "+628333"},
4607            ]),
4608        );
4609
4610        let result = run_workflow(&executor, &wf, &mut ctx).await.unwrap();
4611        assert_eq!(result, "waiting");
4612
4613        // Verify all 3 captains were notified
4614        let notifs = executor.notifications.lock().unwrap();
4615        assert_eq!(notifs.len(), 3);
4616        assert_eq!(notifs[0].0, "+628111");
4617        assert_eq!(notifs[1].0, "+628222");
4618        assert_eq!(notifs[2].0, "+628333");
4619        drop(notifs);
4620
4621        let side_effects = executor.completed_side_effects.lock().unwrap();
4622        assert_eq!(side_effects.len(), 3);
4623        for operation_id in side_effects.iter() {
4624            assert!(
4625                !operation_id.contains("+628"),
4626                "side-effect operation id must not leak item payload: {operation_id}"
4627            );
4628            assert!(
4629                !operation_id.contains("Captain"),
4630                "side-effect operation id must not leak item payload: {operation_id}"
4631            );
4632        }
4633        assert_eq!(
4634            side_effects.as_slice(),
4635            &[
4636                side_effect_operation_id(
4637                    "wf-003",
4638                    "broadcasting",
4639                    0,
4640                    WorkflowSideEffectKind::Notify,
4641                    "steps[0]/for_each[0].steps[0]",
4642                ),
4643                side_effect_operation_id(
4644                    "wf-003",
4645                    "broadcasting",
4646                    0,
4647                    WorkflowSideEffectKind::Notify,
4648                    "steps[0]/for_each[1].steps[0]",
4649                ),
4650                side_effect_operation_id(
4651                    "wf-003",
4652                    "broadcasting",
4653                    0,
4654                    WorkflowSideEffectKind::Notify,
4655                    "steps[0]/for_each[2].steps[0]",
4656                ),
4657            ]
4658        );
4659    }
4660
4661    #[test]
4662    fn side_effect_operation_id_does_not_collide_on_delimiters() {
4663        let first = side_effect_operation_id(
4664            "tenant:wf",
4665            "broadcasting",
4666            7,
4667            WorkflowSideEffectKind::Notify,
4668            "steps[0]",
4669        );
4670        let second = side_effect_operation_id(
4671            "tenant",
4672            "wf:broadcasting",
4673            7,
4674            WorkflowSideEffectKind::Notify,
4675            "steps[0]",
4676        );
4677
4678        assert_ne!(
4679            first, second,
4680            "side-effect operation ids must not collide when fields contain delimiters"
4681        );
4682    }
4683
4684    #[tokio::test]
4685    async fn side_effect_operation_id_changes_when_state_is_reentered() {
4686        let executor = MockExecutor::new();
4687
4688        let wf = WorkflowDefinition::new("reentered_state_side_effect")
4689            .initial_state("active")
4690            .transition(
4691                "active",
4692                "idle",
4693                vec![
4694                    WorkflowStep::notify(ChannelKind::Email, "active_notice", "customer.email"),
4695                    WorkflowStep::transition("idle"),
4696                ],
4697            );
4698
4699        let mut ctx = WorkflowContext::new("wf-reentered-state", "active");
4700        ctx.set(
4701            "customer",
4702            serde_json::json!({"email": "guest@example.com"}),
4703        );
4704
4705        let first = run_workflow(&executor, &wf, &mut ctx).await.unwrap();
4706        assert_eq!(first, "idle");
4707        ctx.transition_to("active", Some("manual requeue".to_string()));
4708        let second = run_workflow(&executor, &wf, &mut ctx).await.unwrap();
4709        assert_eq!(second, "idle");
4710
4711        let notifications = executor.notifications.lock().unwrap();
4712        assert_eq!(
4713            notifications.len(),
4714            2,
4715            "same state re-entry must execute the side effect again"
4716        );
4717        drop(notifications);
4718
4719        let side_effects = executor.completed_side_effects.lock().unwrap();
4720        assert_eq!(
4721            side_effects.as_slice(),
4722            &[
4723                side_effect_operation_id(
4724                    "wf-reentered-state",
4725                    "active",
4726                    0,
4727                    WorkflowSideEffectKind::Notify,
4728                    "steps[0]",
4729                ),
4730                side_effect_operation_id(
4731                    "wf-reentered-state",
4732                    "active",
4733                    2,
4734                    WorkflowSideEffectKind::Notify,
4735                    "steps[0]",
4736                ),
4737            ]
4738        );
4739    }
4740
4741    #[tokio::test]
4742    async fn notify_with_missing_payload_key_fails_before_send() {
4743        let executor = MockExecutor::new();
4744
4745        let wf = WorkflowDefinition::new("notify_payload_guard")
4746            .initial_state("active")
4747            .transition(
4748                "active",
4749                "done",
4750                vec![WorkflowStep::notify_with_payload(
4751                    ChannelKind::Email,
4752                    "booking_confirmed",
4753                    "customer.email",
4754                    "booking.template_payload",
4755                )],
4756            );
4757
4758        let mut ctx = WorkflowContext::new("wf-notify-payload-guard", "active");
4759        ctx.set(
4760            "customer",
4761            serde_json::json!({"email": "guest@example.com"}),
4762        );
4763
4764        let err = run_workflow(&executor, &wf, &mut ctx)
4765            .await
4766            .expect_err("missing explicit notification payload must fail closed");
4767
4768        match err {
4769            WorkflowError::MissingContextKey(key) => assert_eq!(key, "booking.template_payload"),
4770            other => panic!("expected MissingContextKey, got {other:?}"),
4771        }
4772        assert!(
4773            executor.notifications.lock().unwrap().is_empty(),
4774            "notification provider must not receive an empty payload for a missing explicit payload key"
4775        );
4776    }
4777
4778    #[tokio::test]
4779    async fn test_workflow_pauses_at_wait() {
4780        let executor = MockExecutor::new();
4781
4782        let wf = WorkflowDefinition::new("wait_test")
4783            .initial_state("active")
4784            .transition(
4785                "active",
4786                "resolved",
4787                vec![
4788                    WorkflowStep::log("Before wait"),
4789                    WorkflowStep::wait("user_response", std::time::Duration::from_secs(3600)),
4790                    // This should NOT execute (paused at Wait)
4791                    WorkflowStep::log("After wait"),
4792                ],
4793            );
4794
4795        let mut ctx = WorkflowContext::new("wf-004", "active");
4796        let result = run_workflow(&executor, &wf, &mut ctx).await.unwrap();
4797
4798        // Should pause at "active" (Wait encountered before Transition)
4799        assert_eq!(result, "active");
4800    }
4801
4802    #[tokio::test]
4803    async fn test_workflow_pauses_at_nested_branch_wait() {
4804        let executor = MockExecutor::new();
4805
4806        let wf = WorkflowDefinition::new("nested_branch_wait")
4807            .initial_state("active")
4808            .transition(
4809                "active",
4810                "resolved",
4811                vec![
4812                    WorkflowStep::branch(
4813                        "customer.tier",
4814                        vec![(
4815                            "vip",
4816                            vec![WorkflowStep::wait(
4817                                "vip_payment",
4818                                std::time::Duration::from_secs(7200),
4819                            )],
4820                        )],
4821                        vec![WorkflowStep::log("Standard customer")],
4822                    ),
4823                    WorkflowStep::notify(ChannelKind::Email, "booking_confirmed", "customer.email"),
4824                    WorkflowStep::transition("resolved"),
4825                ],
4826            );
4827
4828        let mut ctx = WorkflowContext::new("wf-nested-branch-wait", "active");
4829        ctx.set(
4830            "customer",
4831            serde_json::json!({
4832                "tier": "vip",
4833                "email": "guest@example.com"
4834            }),
4835        );
4836
4837        let result = run_workflow(&executor, &wf, &mut ctx).await.unwrap();
4838        assert_eq!(result, "active");
4839        assert_eq!(ctx.current_state, "active");
4840
4841        let notifs = executor.notifications.lock().unwrap();
4842        assert!(
4843            notifs.is_empty(),
4844            "steps after a nested branch Wait must not execute"
4845        );
4846    }
4847
4848    #[tokio::test]
4849    async fn branch_resume_rejects_reordered_branch_definition() {
4850        let executor = MockExecutor::new();
4851
4852        let wf = WorkflowDefinition::new("branch_reorder_source")
4853            .initial_state("active")
4854            .transition(
4855                "active",
4856                "done",
4857                vec![WorkflowStep::branch(
4858                    "customer.tier",
4859                    vec![
4860                        (
4861                            "vip",
4862                            vec![
4863                                WorkflowStep::wait(
4864                                    "approved",
4865                                    std::time::Duration::from_secs(3600),
4866                                ),
4867                                WorkflowStep::notify(
4868                                    ChannelKind::Email,
4869                                    "vip_approved",
4870                                    "customer.email",
4871                                ),
4872                            ],
4873                        ),
4874                        (
4875                            "standard",
4876                            vec![
4877                                WorkflowStep::wait(
4878                                    "approved",
4879                                    std::time::Duration::from_secs(3600),
4880                                ),
4881                                WorkflowStep::notify(
4882                                    ChannelKind::Email,
4883                                    "standard_approved",
4884                                    "customer.email",
4885                                ),
4886                            ],
4887                        ),
4888                    ],
4889                    vec![],
4890                )],
4891            );
4892
4893        let mut ctx = WorkflowContext::new("wf-branch-reorder", "active");
4894        ctx.set(
4895            "customer",
4896            serde_json::json!({
4897                "tier": "vip",
4898                "email": "guest@example.com"
4899            }),
4900        );
4901
4902        let result = run_workflow(&executor, &wf, &mut ctx).await.unwrap();
4903        assert_eq!(result, "active");
4904
4905        let drifted_wf = WorkflowDefinition::new("branch_reorder_source")
4906            .initial_state("active")
4907            .transition(
4908                "active",
4909                "done",
4910                vec![WorkflowStep::branch(
4911                    "customer.tier",
4912                    vec![
4913                        (
4914                            "standard",
4915                            vec![
4916                                WorkflowStep::wait(
4917                                    "approved",
4918                                    std::time::Duration::from_secs(3600),
4919                                ),
4920                                WorkflowStep::notify(
4921                                    ChannelKind::Email,
4922                                    "standard_approved",
4923                                    "customer.email",
4924                                ),
4925                            ],
4926                        ),
4927                        (
4928                            "vip",
4929                            vec![
4930                                WorkflowStep::wait(
4931                                    "approved",
4932                                    std::time::Duration::from_secs(3600),
4933                                ),
4934                                WorkflowStep::notify(
4935                                    ChannelKind::Email,
4936                                    "vip_approved",
4937                                    "customer.email",
4938                                ),
4939                            ],
4940                        ),
4941                    ],
4942                    vec![],
4943                )],
4944            );
4945
4946        let err = resume_workflow(
4947            &executor,
4948            &drifted_wf,
4949            "wf-branch-reorder",
4950            serde_json::json!({"event": "approved"}),
4951        )
4952        .await
4953        .expect_err("branch definition reorder must not silently resume wrong arm");
4954
4955        match err {
4956            WorkflowError::Other(msg) => {
4957                assert!(msg.contains("Invalid workflow resume cursor"));
4958                assert!(msg.contains("standard"));
4959                assert!(msg.contains("vip"));
4960            }
4961            other => panic!("expected invalid cursor error, got {other:?}"),
4962        }
4963        assert!(
4964            executor.notifications.lock().unwrap().is_empty(),
4965            "drifted resume must fail before sending the wrong branch notification"
4966        );
4967    }
4968
4969    #[tokio::test]
4970    async fn branch_resume_rejects_default_branch_drift() {
4971        let executor = MockExecutor::new();
4972
4973        let wf = WorkflowDefinition::new("branch_default_source")
4974            .initial_state("active")
4975            .transition(
4976                "active",
4977                "done",
4978                vec![WorkflowStep::branch(
4979                    "customer.tier",
4980                    vec![(
4981                        "vip",
4982                        vec![WorkflowStep::notify(
4983                            ChannelKind::Email,
4984                            "vip",
4985                            "customer.email",
4986                        )],
4987                    )],
4988                    vec![
4989                        WorkflowStep::wait("approved", std::time::Duration::from_secs(3600)),
4990                        WorkflowStep::notify(ChannelKind::Email, "default", "customer.email"),
4991                    ],
4992                )],
4993            );
4994
4995        let mut ctx = WorkflowContext::new("wf-branch-default-drift", "active");
4996        ctx.set(
4997            "customer",
4998            serde_json::json!({
4999                "tier": "gold",
5000                "email": "guest@example.com"
5001            }),
5002        );
5003
5004        let result = run_workflow(&executor, &wf, &mut ctx).await.unwrap();
5005        assert_eq!(result, "active");
5006
5007        let drifted_wf = WorkflowDefinition::new("branch_default_source")
5008            .initial_state("active")
5009            .transition(
5010                "active",
5011                "done",
5012                vec![WorkflowStep::branch(
5013                    "customer.tier",
5014                    vec![
5015                        (
5016                            "vip",
5017                            vec![WorkflowStep::notify(
5018                                ChannelKind::Email,
5019                                "vip",
5020                                "customer.email",
5021                            )],
5022                        ),
5023                        (
5024                            "gold",
5025                            vec![WorkflowStep::notify(
5026                                ChannelKind::Email,
5027                                "gold",
5028                                "customer.email",
5029                            )],
5030                        ),
5031                    ],
5032                    vec![
5033                        WorkflowStep::wait("approved", std::time::Duration::from_secs(3600)),
5034                        WorkflowStep::notify(ChannelKind::Email, "default", "customer.email"),
5035                    ],
5036                )],
5037            );
5038
5039        let err = resume_workflow(
5040            &executor,
5041            &drifted_wf,
5042            "wf-branch-default-drift",
5043            serde_json::json!({"event": "approved"}),
5044        )
5045        .await
5046        .expect_err("default branch cursor must reject new matching branch arms");
5047
5048        match err {
5049            WorkflowError::Other(msg) => {
5050                assert!(msg.contains("default branch cursor"));
5051                assert!(msg.contains("gold"));
5052            }
5053            other => panic!("expected invalid cursor error, got {other:?}"),
5054        }
5055        assert!(executor.notifications.lock().unwrap().is_empty());
5056    }
5057
5058    #[tokio::test]
5059    async fn test_workflow_pauses_at_nested_for_each_wait() {
5060        let executor = MockExecutor::new();
5061
5062        let wf = WorkflowDefinition::new("nested_for_each_wait")
5063            .initial_state("active")
5064            .transition(
5065                "active",
5066                "resolved",
5067                vec![
5068                    WorkflowStep::for_each(
5069                        "operators",
5070                        vec![
5071                            WorkflowStep::wait(
5072                                "operator_accept",
5073                                std::time::Duration::from_secs(3600),
5074                            ),
5075                            WorkflowStep::notify(ChannelKind::WhatsApp, "after_wait", "item.phone"),
5076                        ],
5077                    ),
5078                    WorkflowStep::transition("resolved"),
5079                ],
5080            );
5081
5082        let mut ctx = WorkflowContext::new("wf-nested-for-each-wait", "active");
5083        ctx.set(
5084            "operators",
5085            serde_json::json!([
5086                {"name": "Captain A", "phone": "+628111"},
5087                {"name": "Captain B", "phone": "+628222"}
5088            ]),
5089        );
5090
5091        let result = run_workflow(&executor, &wf, &mut ctx).await.unwrap();
5092        assert_eq!(result, "active");
5093        assert_eq!(ctx.current_state, "active");
5094
5095        let notifs = executor.notifications.lock().unwrap();
5096        assert!(
5097            notifs.is_empty(),
5098            "steps after a nested for_each Wait must not execute"
5099        );
5100    }
5101
5102    #[tokio::test]
5103    async fn test_workflow_resumes_for_each_after_wait_without_replaying_items() {
5104        let executor = MockExecutor::new();
5105
5106        let wf = WorkflowDefinition::new("for_each_resume")
5107            .initial_state("active")
5108            .transition(
5109                "active",
5110                "resolved",
5111                vec![
5112                    WorkflowStep::for_each(
5113                        "operators",
5114                        vec![
5115                            WorkflowStep::notify(
5116                                ChannelKind::WhatsApp,
5117                                "before_wait",
5118                                "item.phone",
5119                            ),
5120                            WorkflowStep::wait(
5121                                "operator_accept",
5122                                std::time::Duration::from_secs(3600),
5123                            ),
5124                            WorkflowStep::notify(ChannelKind::WhatsApp, "after_wait", "item.phone"),
5125                        ],
5126                    ),
5127                    WorkflowStep::transition("resolved"),
5128                ],
5129            );
5130
5131        let mut ctx = WorkflowContext::new("wf-for-each-resume", "active");
5132        ctx.set(
5133            "operators",
5134            serde_json::json!([
5135                {"name": "Captain A", "phone": "+628111"},
5136                {"name": "Captain B", "phone": "+628222"}
5137            ]),
5138        );
5139
5140        let result = run_workflow(&executor, &wf, &mut ctx).await.unwrap();
5141        assert_eq!(result, "active");
5142        assert!(
5143            ctx.cursor.is_some(),
5144            "first pause should persist a resume cursor"
5145        );
5146
5147        {
5148            let notifs = executor.notifications.lock().unwrap();
5149            assert_eq!(
5150                notifs.as_slice(),
5151                &[("+628111".to_string(), "before_wait".to_string())]
5152            );
5153        }
5154
5155        let result = resume_workflow(
5156            &executor,
5157            &wf,
5158            "wf-for-each-resume",
5159            serde_json::json!({"event": "operator_accept", "accepted": true}),
5160        )
5161        .await
5162        .unwrap();
5163        assert_eq!(result, "active");
5164
5165        {
5166            let notifs = executor.notifications.lock().unwrap();
5167            assert_eq!(
5168                notifs.as_slice(),
5169                &[
5170                    ("+628111".to_string(), "before_wait".to_string()),
5171                    ("+628111".to_string(), "after_wait".to_string()),
5172                    ("+628222".to_string(), "before_wait".to_string()),
5173                ],
5174                "resume must continue within the active item and must not replay item 0"
5175            );
5176        }
5177
5178        let result = resume_workflow(
5179            &executor,
5180            &wf,
5181            "wf-for-each-resume",
5182            serde_json::json!({"event": "operator_accept", "accepted": true}),
5183        )
5184        .await
5185        .unwrap();
5186        assert_eq!(result, "resolved");
5187
5188        let notifs = executor.notifications.lock().unwrap();
5189        assert_eq!(
5190            notifs.as_slice(),
5191            &[
5192                ("+628111".to_string(), "before_wait".to_string()),
5193                ("+628111".to_string(), "after_wait".to_string()),
5194                ("+628222".to_string(), "before_wait".to_string()),
5195                ("+628222".to_string(), "after_wait".to_string()),
5196            ],
5197            "each side effect should run exactly once per operator"
5198        );
5199
5200        let saved = executor.saved_state.lock().unwrap();
5201        let saved = saved.as_ref().expect("final state should be saved");
5202        assert_eq!(saved.current_state, "resolved");
5203        assert!(
5204            saved.cursor.is_none(),
5205            "completed workflow should not retain a resume cursor"
5206        );
5207    }
5208
5209    #[tokio::test]
5210    async fn for_each_resume_rejects_reordered_items_before_notification() {
5211        let executor = MockExecutor::new();
5212
5213        let wf = WorkflowDefinition::new("for_each_reorder_guard")
5214            .initial_state("active")
5215            .transition(
5216                "active",
5217                "resolved",
5218                vec![
5219                    WorkflowStep::for_each(
5220                        "operators",
5221                        vec![
5222                            WorkflowStep::wait(
5223                                "operator_accept",
5224                                std::time::Duration::from_secs(3600),
5225                            ),
5226                            WorkflowStep::notify(ChannelKind::WhatsApp, "accepted", "item.phone"),
5227                        ],
5228                    ),
5229                    WorkflowStep::transition("resolved"),
5230                ],
5231            );
5232
5233        let mut ctx = WorkflowContext::new("wf-for-each-reorder", "active");
5234        ctx.set(
5235            "operators",
5236            serde_json::json!([
5237                {"name": "Captain A", "phone": "+628111"},
5238                {"name": "Captain B", "phone": "+628222"}
5239            ]),
5240        );
5241
5242        let result = run_workflow(&executor, &wf, &mut ctx).await.unwrap();
5243        assert_eq!(result, "active");
5244
5245        {
5246            let mut saved = executor.saved_state.lock().unwrap();
5247            let saved = saved.as_mut().expect("paused state should be saved");
5248            saved.set(
5249                "operators",
5250                serde_json::json!([
5251                    {"name": "Captain B", "phone": "+628222"},
5252                    {"name": "Captain A", "phone": "+628111"}
5253                ]),
5254            );
5255        }
5256
5257        let err = resume_workflow(
5258            &executor,
5259            &wf,
5260            "wf-for-each-reorder",
5261            serde_json::json!({"event": "operator_accept"}),
5262        )
5263        .await
5264        .expect_err("reordered operator list must not resume on the wrong item");
5265
5266        match err {
5267            WorkflowError::Other(msg) => {
5268                assert!(msg.contains("Invalid workflow resume cursor"), "got: {msg}");
5269                assert!(msg.contains("for_each cursor item"), "got: {msg}");
5270                assert!(msg.contains("changed"), "got: {msg}");
5271            }
5272            other => panic!("expected invalid cursor error, got {other:?}"),
5273        }
5274        assert!(
5275            executor.notifications.lock().unwrap().is_empty(),
5276            "wrong operator must not receive the post-wait notification"
5277        );
5278    }
5279
5280    #[tokio::test]
5281    async fn legacy_for_each_cursor_is_rejected_before_notification() {
5282        let executor = MockExecutor::new();
5283
5284        let wf = WorkflowDefinition::new("legacy_for_each_cursor_guard")
5285            .initial_state("active")
5286            .transition(
5287                "active",
5288                "resolved",
5289                vec![
5290                    WorkflowStep::for_each(
5291                        "operators",
5292                        vec![
5293                            WorkflowStep::wait(
5294                                "operator_accept",
5295                                std::time::Duration::from_secs(3600),
5296                            ),
5297                            WorkflowStep::notify(ChannelKind::WhatsApp, "accepted", "item.phone"),
5298                        ],
5299                    ),
5300                    WorkflowStep::transition("resolved"),
5301                ],
5302            );
5303
5304        let mut ctx = WorkflowContext::new("wf-legacy-for-each-cursor", "active");
5305        ctx.set(
5306            "operators",
5307            serde_json::json!([
5308                {"name": "Captain A", "phone": "+628111"},
5309                {"name": "Captain B", "phone": "+628222"}
5310            ]),
5311        );
5312        ctx.set_cursor(WorkflowCursor {
5313            state: "active".to_string(),
5314            frames: vec![
5315                WorkflowCursorFrame::Steps { index: 0 },
5316                WorkflowCursorFrame::ForEach {
5317                    item_index: 1,
5318                    index: 1,
5319                },
5320            ],
5321            wait: Some(WorkflowPendingWait {
5322                event: "operator_accept".to_string(),
5323                deadline_at: Utc::now() + chrono::Duration::hours(1),
5324                on_timeout: vec![],
5325            }),
5326        });
5327        executor.save_state(&ctx).await.unwrap();
5328
5329        let err = resume_workflow(
5330            &executor,
5331            &wf,
5332            "wf-legacy-for-each-cursor",
5333            serde_json::json!({"event": "operator_accept"}),
5334        )
5335        .await
5336        .expect_err("legacy for_each cursors must fail closed");
5337
5338        match err {
5339            WorkflowError::Other(msg) => {
5340                assert!(msg.contains("Invalid workflow resume cursor"), "got: {msg}");
5341                assert!(msg.contains("legacy for_each cursor"), "got: {msg}");
5342            }
5343            other => panic!("expected invalid cursor error, got {other:?}"),
5344        }
5345        assert!(
5346            executor.notifications.lock().unwrap().is_empty(),
5347            "legacy cursor must not resume onto an unverified loop item"
5348        );
5349    }
5350
5351    #[tokio::test]
5352    async fn test_nested_for_each_restores_outer_item_binding() {
5353        let executor = MockExecutor::new();
5354
5355        let wf = WorkflowDefinition::new("nested_for_each_items")
5356            .initial_state("active")
5357            .transition(
5358                "active",
5359                "done",
5360                vec![
5361                    WorkflowStep::for_each(
5362                        "operators",
5363                        vec![
5364                            WorkflowStep::for_each(
5365                                "tasks",
5366                                vec![WorkflowStep::notify(
5367                                    ChannelKind::WhatsApp,
5368                                    "task",
5369                                    "item.phone",
5370                                )],
5371                            ),
5372                            WorkflowStep::notify(ChannelKind::WhatsApp, "summary", "item.phone"),
5373                        ],
5374                    ),
5375                    WorkflowStep::transition("done"),
5376                ],
5377            );
5378
5379        let mut ctx = WorkflowContext::new("wf-nested-items", "active");
5380        ctx.set(
5381            "operators",
5382            serde_json::json!([
5383                {"name": "Captain A", "phone": "+628111"},
5384                {"name": "Captain B", "phone": "+628222"}
5385            ]),
5386        );
5387        ctx.set(
5388            "tasks",
5389            serde_json::json!([
5390                {"name": "Task 1", "phone": "+629999"}
5391            ]),
5392        );
5393
5394        let result = run_workflow(&executor, &wf, &mut ctx).await.unwrap();
5395        assert_eq!(result, "done");
5396
5397        let notifs = executor.notifications.lock().unwrap();
5398        assert_eq!(
5399            notifs.as_slice(),
5400            &[
5401                ("+629999".to_string(), "task".to_string()),
5402                ("+628111".to_string(), "summary".to_string()),
5403                ("+629999".to_string(), "task".to_string()),
5404                ("+628222".to_string(), "summary".to_string()),
5405            ],
5406            "inner ForEach must not delete the outer item binding"
5407        );
5408    }
5409
5410    #[tokio::test]
5411    async fn test_nested_for_each_wait_restores_item_binding_across_resume() {
5412        let executor = MockExecutor::new();
5413
5414        let wf = WorkflowDefinition::new("nested_for_each_wait_items")
5415            .initial_state("active")
5416            .transition(
5417                "active",
5418                "done",
5419                vec![
5420                    WorkflowStep::for_each(
5421                        "operators",
5422                        vec![
5423                            WorkflowStep::for_each(
5424                                "tasks",
5425                                vec![
5426                                    WorkflowStep::wait(
5427                                        "task_done",
5428                                        std::time::Duration::from_secs(3600),
5429                                    ),
5430                                    WorkflowStep::notify(
5431                                        ChannelKind::WhatsApp,
5432                                        "task",
5433                                        "item.phone",
5434                                    ),
5435                                ],
5436                            ),
5437                            WorkflowStep::notify(ChannelKind::WhatsApp, "summary", "item.phone"),
5438                        ],
5439                    ),
5440                    WorkflowStep::transition("done"),
5441                ],
5442            );
5443
5444        let mut ctx = WorkflowContext::new("wf-nested-wait-items", "active");
5445        ctx.set(
5446            "operators",
5447            serde_json::json!([
5448                {"name": "Captain A", "phone": "+628111"},
5449                {"name": "Captain B", "phone": "+628222"}
5450            ]),
5451        );
5452        ctx.set(
5453            "tasks",
5454            serde_json::json!([
5455                {"name": "Task 1", "phone": "+629999"}
5456            ]),
5457        );
5458
5459        let result = run_workflow(&executor, &wf, &mut ctx).await.unwrap();
5460        assert_eq!(result, "active");
5461        assert!(
5462            ctx.get("item").is_none(),
5463            "paused workflow state must not persist an active loop item"
5464        );
5465
5466        let result = resume_workflow(
5467            &executor,
5468            &wf,
5469            "wf-nested-wait-items",
5470            serde_json::json!({"event": "task_done"}),
5471        )
5472        .await
5473        .unwrap();
5474        assert_eq!(result, "active");
5475        assert!(
5476            executor
5477                .saved_state
5478                .lock()
5479                .unwrap()
5480                .as_ref()
5481                .and_then(|ctx| ctx.get("item"))
5482                .is_none(),
5483            "paused resume state must keep item reconstruction cursor-driven"
5484        );
5485
5486        {
5487            let notifs = executor.notifications.lock().unwrap();
5488            assert_eq!(
5489                notifs.as_slice(),
5490                &[
5491                    ("+629999".to_string(), "task".to_string()),
5492                    ("+628111".to_string(), "summary".to_string()),
5493                ],
5494                "inner wait resume must restore the outer item before summary"
5495            );
5496        }
5497
5498        let result = resume_workflow(
5499            &executor,
5500            &wf,
5501            "wf-nested-wait-items",
5502            serde_json::json!({"event": "task_done"}),
5503        )
5504        .await
5505        .unwrap();
5506        assert_eq!(result, "done");
5507
5508        let notifs = executor.notifications.lock().unwrap();
5509        assert_eq!(
5510            notifs.as_slice(),
5511            &[
5512                ("+629999".to_string(), "task".to_string()),
5513                ("+628111".to_string(), "summary".to_string()),
5514                ("+629999".to_string(), "task".to_string()),
5515                ("+628222".to_string(), "summary".to_string()),
5516            ],
5517            "nested for_each wait must not leak the inner item into the outer loop"
5518        );
5519        assert!(
5520            executor
5521                .saved_state
5522                .lock()
5523                .unwrap()
5524                .as_ref()
5525                .and_then(|ctx| ctx.get("item"))
5526                .is_none(),
5527            "completed workflow state must not retain a stale loop item"
5528        );
5529    }
5530
5531    #[tokio::test]
5532    async fn test_resume_rejects_unexpected_wait_event() {
5533        let executor = MockExecutor::new();
5534
5535        let wf = WorkflowDefinition::new("wait_event_guard")
5536            .initial_state("active")
5537            .transition(
5538                "active",
5539                "done",
5540                vec![
5541                    WorkflowStep::wait("payment.success", std::time::Duration::from_secs(3600)),
5542                    WorkflowStep::notify(ChannelKind::Email, "paid", "customer.email"),
5543                    WorkflowStep::transition("done"),
5544                ],
5545            );
5546
5547        let mut ctx = WorkflowContext::new("wf-event-guard", "active");
5548        ctx.set(
5549            "customer",
5550            serde_json::json!({"email": "guest@example.com"}),
5551        );
5552        let result = run_workflow(&executor, &wf, &mut ctx).await.unwrap();
5553        assert_eq!(result, "active");
5554
5555        let wait = ctx
5556            .cursor
5557            .as_ref()
5558            .and_then(|cursor| cursor.wait.as_ref())
5559            .expect("wait metadata should be persisted");
5560        assert_eq!(wait.event, "payment.success");
5561
5562        let err = resume_workflow(
5563            &executor,
5564            &wf,
5565            "wf-event-guard",
5566            serde_json::json!({"event": "operator.declined"}),
5567        )
5568        .await
5569        .expect_err("wrong event must not resume the workflow");
5570
5571        match err {
5572            WorkflowError::Other(msg) => {
5573                assert!(msg.contains("payment.success"));
5574                assert!(msg.contains("operator.declined"));
5575            }
5576            other => panic!("expected event mismatch error, got: {other}"),
5577        }
5578        assert!(executor.notifications.lock().unwrap().is_empty());
5579    }
5580
5581    #[tokio::test]
5582    async fn resume_event_rejects_padded_event_name_before_replay() {
5583        let executor = MockExecutor::new();
5584
5585        let wf = WorkflowDefinition::new("wait_event_padding_guard")
5586            .initial_state("active")
5587            .transition(
5588                "active",
5589                "done",
5590                vec![
5591                    WorkflowStep::wait("payment.success", std::time::Duration::from_secs(3600)),
5592                    WorkflowStep::notify(ChannelKind::Email, "paid", "customer.email"),
5593                    WorkflowStep::transition("done"),
5594                ],
5595            );
5596
5597        let mut ctx = WorkflowContext::new("wf-event-padding-guard", "active");
5598        ctx.set(
5599            "customer",
5600            serde_json::json!({"email": "guest@example.com"}),
5601        );
5602        let result = run_workflow(&executor, &wf, &mut ctx).await.unwrap();
5603        assert_eq!(result, "active");
5604
5605        let err = resume_workflow(
5606            &executor,
5607            &wf,
5608            "wf-event-padding-guard",
5609            serde_json::json!({"event": " payment.success "}),
5610        )
5611        .await
5612        .expect_err("padded event name must not resume the workflow");
5613
5614        match err {
5615            WorkflowError::Other(msg) => {
5616                assert!(msg.contains("leading or trailing whitespace"), "got: {msg}");
5617            }
5618            other => panic!("expected padded event error, got {other:?}"),
5619        }
5620        assert!(
5621            executor.notifications.lock().unwrap().is_empty(),
5622            "padded resume event must fail before notification replay"
5623        );
5624    }
5625
5626    #[tokio::test]
5627    async fn resume_event_without_wait_cursor_does_not_run_transition() {
5628        let executor = MockExecutor::new();
5629
5630        let wf = WorkflowDefinition::new("vendor_callback_guard")
5631            .initial_state("awaiting_vendor")
5632            .transition(
5633                "awaiting_vendor",
5634                "notified",
5635                vec![
5636                    WorkflowStep::notify(ChannelKind::WhatsApp, "vendor_ready", "vendor.phone"),
5637                    WorkflowStep::transition("notified"),
5638                ],
5639            );
5640
5641        let mut ctx = WorkflowContext::new("wf-no-wait-cursor", "awaiting_vendor");
5642        ctx.set("vendor", serde_json::json!({"phone": "+628111"}));
5643        executor.save_state(&ctx).await.unwrap();
5644
5645        let err = resume_workflow(
5646            &executor,
5647            &wf,
5648            "wf-no-wait-cursor",
5649            serde_json::json!({"event": "vendor.ready"}),
5650        )
5651        .await
5652        .expect_err("event resume must not run a workflow with no wait cursor");
5653
5654        match err {
5655            WorkflowError::Other(msg) => {
5656                assert!(
5657                    msg.contains("not waiting for an external event"),
5658                    "got: {msg}"
5659                );
5660            }
5661            other => panic!("expected not-waiting error, got {other:?}"),
5662        }
5663        assert!(
5664            executor.notifications.lock().unwrap().is_empty(),
5665            "webhook resume without a wait cursor must not send vendor notifications"
5666        );
5667    }
5668
5669    #[tokio::test]
5670    async fn resume_event_rejects_stale_cursor_state_before_transition() {
5671        let executor = MockExecutor::new();
5672
5673        let wf = WorkflowDefinition::new("stale_cursor_guard")
5674            .initial_state("awaiting_vendor")
5675            .transition(
5676                "awaiting_vendor",
5677                "notified",
5678                vec![
5679                    WorkflowStep::notify(ChannelKind::WhatsApp, "vendor_ready", "vendor.phone"),
5680                    WorkflowStep::transition("notified"),
5681                ],
5682            );
5683
5684        let mut ctx = WorkflowContext::new("wf-stale-cursor", "awaiting_vendor");
5685        ctx.set("vendor", serde_json::json!({"phone": "+628111"}));
5686        ctx.set_cursor(WorkflowCursor {
5687            state: "old_state".to_string(),
5688            frames: vec![WorkflowCursorFrame::Steps { index: 1 }],
5689            wait: Some(WorkflowPendingWait {
5690                event: "vendor.ready".to_string(),
5691                deadline_at: Utc::now() + chrono::Duration::hours(1),
5692                on_timeout: vec![],
5693            }),
5694        });
5695        executor.save_state(&ctx).await.unwrap();
5696
5697        let err = resume_workflow(
5698            &executor,
5699            &wf,
5700            "wf-stale-cursor",
5701            serde_json::json!({"event": "vendor.ready"}),
5702        )
5703        .await
5704        .expect_err("stale cursor state must not resume the current transition");
5705
5706        match err {
5707            WorkflowError::Other(msg) => {
5708                assert!(msg.contains("Invalid workflow resume cursor"), "got: {msg}");
5709                assert!(msg.contains("state does not match"), "got: {msg}");
5710            }
5711            other => panic!("expected invalid cursor error, got {other:?}"),
5712        }
5713        assert!(
5714            executor.notifications.lock().unwrap().is_empty(),
5715            "stale cursor mismatch must fail before side effects"
5716        );
5717    }
5718
5719    #[tokio::test]
5720    async fn normal_resume_rejects_empty_cursor_frames_before_replay() {
5721        let executor = MockExecutor::new();
5722
5723        let wf = WorkflowDefinition::new("empty_cursor_guard")
5724            .initial_state("active")
5725            .transition(
5726                "active",
5727                "done",
5728                vec![
5729                    WorkflowStep::notify(ChannelKind::WhatsApp, "vendor_ready", "vendor.phone"),
5730                    WorkflowStep::transition("done"),
5731                ],
5732            );
5733
5734        let mut ctx = WorkflowContext::new("wf-empty-cursor", "active");
5735        ctx.set("vendor", serde_json::json!({"phone": "+628111"}));
5736        ctx.set_cursor(WorkflowCursor {
5737            state: "active".to_string(),
5738            frames: vec![],
5739            wait: None,
5740        });
5741
5742        let err = run_workflow(&executor, &wf, &mut ctx)
5743            .await
5744            .expect_err("empty cursor frames must not replay the transition from step zero");
5745
5746        match err {
5747            WorkflowError::Other(msg) => {
5748                assert!(msg.contains("Invalid workflow resume cursor"), "got: {msg}");
5749                assert!(msg.contains("no frames"), "got: {msg}");
5750            }
5751            other => panic!("expected invalid cursor error, got {other:?}"),
5752        }
5753        assert!(
5754            executor.notifications.lock().unwrap().is_empty(),
5755            "empty checkpoint cursor must fail before replaying notifications"
5756        );
5757    }
5758
5759    #[tokio::test]
5760    async fn event_resume_rejects_empty_wait_cursor_frames_before_replay() {
5761        let executor = MockExecutor::new();
5762
5763        let wf = WorkflowDefinition::new("empty_wait_cursor_guard")
5764            .initial_state("active")
5765            .transition(
5766                "active",
5767                "done",
5768                vec![
5769                    WorkflowStep::wait("vendor.ready", std::time::Duration::from_secs(3600)),
5770                    WorkflowStep::notify(ChannelKind::WhatsApp, "vendor_ready", "vendor.phone"),
5771                    WorkflowStep::transition("done"),
5772                ],
5773            );
5774
5775        let mut ctx = WorkflowContext::new("wf-empty-wait-cursor", "active");
5776        ctx.set("vendor", serde_json::json!({"phone": "+628111"}));
5777        ctx.set_cursor(WorkflowCursor {
5778            state: "active".to_string(),
5779            frames: vec![],
5780            wait: Some(WorkflowPendingWait {
5781                event: "vendor.ready".to_string(),
5782                deadline_at: Utc::now() + chrono::Duration::hours(1),
5783                on_timeout: vec![],
5784            }),
5785        });
5786        executor.save_state(&ctx).await.unwrap();
5787
5788        let err = resume_workflow(
5789            &executor,
5790            &wf,
5791            "wf-empty-wait-cursor",
5792            serde_json::json!({"event": "vendor.ready"}),
5793        )
5794        .await
5795        .expect_err("empty wait cursor frames must not resume from step zero");
5796
5797        match err {
5798            WorkflowError::Other(msg) => {
5799                assert!(msg.contains("Invalid workflow resume cursor"), "got: {msg}");
5800                assert!(msg.contains("no frames"), "got: {msg}");
5801            }
5802            other => panic!("expected invalid cursor error, got {other:?}"),
5803        }
5804        assert!(
5805            executor.notifications.lock().unwrap().is_empty(),
5806            "empty wait cursor must fail before replaying notifications"
5807        );
5808    }
5809
5810    #[tokio::test]
5811    async fn wait_with_empty_event_name_is_rejected_before_pause() {
5812        let executor = MockExecutor::new();
5813
5814        let wf = WorkflowDefinition::new("empty_wait_event")
5815            .initial_state("active")
5816            .transition(
5817                "active",
5818                "done",
5819                vec![
5820                    WorkflowStep::wait("", std::time::Duration::from_secs(3600)),
5821                    WorkflowStep::notify(ChannelKind::Email, "after_wait", "customer.email"),
5822                ],
5823            );
5824
5825        let mut ctx = WorkflowContext::new("wf-empty-wait", "active");
5826        ctx.set(
5827            "customer",
5828            serde_json::json!({"email": "guest@example.com"}),
5829        );
5830
5831        let err = run_workflow(&executor, &wf, &mut ctx)
5832            .await
5833            .expect_err("empty wait event must fail before persisting a pause");
5834
5835        match err {
5836            WorkflowError::Other(msg) => {
5837                assert!(
5838                    msg.contains("Wait event name must not be empty"),
5839                    "got: {msg}"
5840                );
5841            }
5842            other => panic!("expected empty event error, got {other:?}"),
5843        }
5844        assert!(ctx.cursor.is_none());
5845        assert!(executor.notifications.lock().unwrap().is_empty());
5846    }
5847
5848    #[tokio::test]
5849    async fn wait_with_padded_event_name_is_rejected_before_pause() {
5850        let executor = MockExecutor::new();
5851
5852        let wf = WorkflowDefinition::new("padded_wait_event")
5853            .initial_state("active")
5854            .transition(
5855                "active",
5856                "done",
5857                vec![
5858                    WorkflowStep::wait(" payment.success ", std::time::Duration::from_secs(3600)),
5859                    WorkflowStep::notify(ChannelKind::Email, "after_wait", "customer.email"),
5860                ],
5861            );
5862
5863        let mut ctx = WorkflowContext::new("wf-padded-wait", "active");
5864        ctx.set(
5865            "customer",
5866            serde_json::json!({"email": "guest@example.com"}),
5867        );
5868
5869        let err = run_workflow(&executor, &wf, &mut ctx)
5870            .await
5871            .expect_err("padded wait event must fail before persisting a pause");
5872
5873        match err {
5874            WorkflowError::Other(msg) => {
5875                assert!(msg.contains("leading or trailing whitespace"), "got: {msg}");
5876            }
5877            other => panic!("expected padded event error, got {other:?}"),
5878        }
5879        assert!(ctx.cursor.is_none());
5880        assert!(executor.notifications.lock().unwrap().is_empty());
5881    }
5882
5883    #[tokio::test]
5884    async fn transition_must_be_final_to_avoid_lost_follow_up_side_effects() {
5885        let executor = MockExecutor::new();
5886
5887        let wf = WorkflowDefinition::new("transition_order_guard")
5888            .initial_state("active")
5889            .transition(
5890                "active",
5891                "done",
5892                vec![
5893                    WorkflowStep::transition("done"),
5894                    WorkflowStep::notify(ChannelKind::WhatsApp, "after_transition", "vendor.phone"),
5895                ],
5896            );
5897
5898        let mut ctx = WorkflowContext::new("wf-transition-order", "active");
5899        ctx.set("vendor", serde_json::json!({"phone": "+628111"}));
5900
5901        let err = run_workflow(&executor, &wf, &mut ctx)
5902            .await
5903            .expect_err("transition before side effects must fail closed");
5904
5905        match err {
5906            WorkflowError::Other(msg) => {
5907                assert!(
5908                    msg.contains("Transition steps must be the final step"),
5909                    "got: {msg}"
5910                );
5911            }
5912            other => panic!("expected transition order error, got {other:?}"),
5913        }
5914        assert_eq!(ctx.current_state, "active");
5915        assert!(executor.notifications.lock().unwrap().is_empty());
5916    }
5917
5918    #[tokio::test]
5919    async fn test_timeout_workflow_executes_on_timeout_fallback() {
5920        let executor = MockExecutor::new();
5921
5922        let wf = WorkflowDefinition::new("wait_timeout")
5923            .initial_state("active")
5924            .transition(
5925                "active",
5926                "done",
5927                vec![
5928                    WorkflowStep::wait_or(
5929                        "payment.success",
5930                        std::time::Duration::from_secs(0),
5931                        vec![
5932                            WorkflowStep::notify(
5933                                ChannelKind::Email,
5934                                "payment_timeout",
5935                                "customer.email",
5936                            ),
5937                            WorkflowStep::transition("timed_out"),
5938                        ],
5939                    ),
5940                    WorkflowStep::notify(ChannelKind::Email, "paid", "customer.email"),
5941                    WorkflowStep::transition("done"),
5942                ],
5943            );
5944
5945        let mut ctx = WorkflowContext::new("wf-timeout", "active");
5946        ctx.set(
5947            "customer",
5948            serde_json::json!({"email": "guest@example.com"}),
5949        );
5950        let result = run_workflow(&executor, &wf, &mut ctx).await.unwrap();
5951        assert_eq!(result, "active");
5952
5953        let result = timeout_workflow(&executor, &wf, "wf-timeout")
5954            .await
5955            .unwrap();
5956        assert_eq!(result, "timed_out");
5957
5958        let notifs = executor.notifications.lock().unwrap();
5959        assert_eq!(
5960            notifs.as_slice(),
5961            &[(
5962                "guest@example.com".to_string(),
5963                "payment_timeout".to_string()
5964            )]
5965        );
5966    }
5967
5968    #[tokio::test]
5969    async fn persisted_timeout_fallback_is_validated_before_side_effects() {
5970        let executor = MockExecutor::new();
5971
5972        let wf = WorkflowDefinition::new("persisted_timeout_fallback_guard")
5973            .initial_state("active")
5974            .transition(
5975                "active",
5976                "done",
5977                vec![WorkflowStep::wait_or(
5978                    "payment.success",
5979                    std::time::Duration::from_secs(0),
5980                    vec![WorkflowStep::transition("timed_out")],
5981                )],
5982            );
5983
5984        let mut ctx = WorkflowContext::new("wf-persisted-timeout-fallback", "active");
5985        ctx.set(
5986            "customer",
5987            serde_json::json!({"email": "guest@example.com"}),
5988        );
5989        let result = run_workflow(&executor, &wf, &mut ctx).await.unwrap();
5990        assert_eq!(result, "active");
5991
5992        {
5993            let mut saved = executor.saved_state.lock().unwrap();
5994            let saved = saved
5995                .as_mut()
5996                .expect("workflow should be saved while waiting for timeout");
5997            let cursor = saved
5998                .cursor
5999                .as_mut()
6000                .expect("saved workflow should have a wait cursor");
6001            let wait = cursor
6002                .wait
6003                .as_mut()
6004                .expect("saved workflow should be waiting for timeout");
6005            wait.on_timeout = vec![
6006                WorkflowStep::notify(ChannelKind::Email, "bad_timeout", "customer.email"),
6007                WorkflowStep::transition(" "),
6008            ];
6009        }
6010
6011        let err = timeout_workflow(&executor, &wf, "wf-persisted-timeout-fallback")
6012            .await
6013            .expect_err("invalid persisted timeout fallback must fail before side effects");
6014
6015        match err {
6016            WorkflowError::Other(msg) => {
6017                assert!(
6018                    msg.contains("Transition target state must not be empty"),
6019                    "got: {msg}"
6020                );
6021            }
6022            other => panic!("expected timeout fallback validation error, got {other:?}"),
6023        }
6024        assert!(
6025            executor.notifications.lock().unwrap().is_empty(),
6026            "persisted timeout fallback validation must run before notification"
6027        );
6028    }
6029
6030    #[tokio::test]
6031    async fn persisted_timeout_fallback_before_deadline_does_not_execute() {
6032        let executor = MockExecutor::new();
6033
6034        let wf = WorkflowDefinition::new("future_persisted_timeout_fallback")
6035            .initial_state("active")
6036            .transition(
6037                "active",
6038                "done",
6039                vec![WorkflowStep::wait_or(
6040                    "payment.success",
6041                    std::time::Duration::from_secs(3600),
6042                    vec![
6043                        WorkflowStep::notify(
6044                            ChannelKind::Email,
6045                            "payment_timeout",
6046                            "customer.email",
6047                        ),
6048                        WorkflowStep::transition("timed_out"),
6049                    ],
6050                )],
6051            );
6052
6053        let mut ctx = WorkflowContext::new("wf-future-persisted-timeout", "active");
6054        ctx.set(
6055            "customer",
6056            serde_json::json!({"email": "guest@example.com"}),
6057        );
6058        let wait = WorkflowPendingWait {
6059            event: "payment.success".to_string(),
6060            deadline_at: Utc::now() + chrono::Duration::hours(1),
6061            on_timeout: vec![
6062                WorkflowStep::notify(ChannelKind::Email, "payment_timeout", "customer.email"),
6063                WorkflowStep::transition("timed_out"),
6064            ],
6065        };
6066        ctx.set_cursor(WorkflowCursor {
6067            state: "active".to_string(),
6068            frames: vec![WorkflowCursorFrame::Steps { index: 1 }],
6069            wait: Some(wait.clone()),
6070        });
6071        set_timeout_fallback(&mut ctx, &wait).unwrap();
6072        executor.save_state(&ctx).await.unwrap();
6073
6074        let err = timeout_workflow(&executor, &wf, "wf-future-persisted-timeout")
6075            .await
6076            .expect_err("persisted timeout fallback must still respect the wait deadline");
6077
6078        match err {
6079            WorkflowError::Other(msg) => {
6080                assert!(msg.contains("has not timed out"), "got: {msg}");
6081            }
6082            other => panic!("expected not-timed-out error, got {other:?}"),
6083        }
6084        assert!(
6085            executor.notifications.lock().unwrap().is_empty(),
6086            "future timeout fallback metadata must fail before notification"
6087        );
6088    }
6089
6090    #[tokio::test]
6091    async fn persisted_timeout_wait_event_is_validated_before_side_effects() {
6092        let executor = MockExecutor::new();
6093
6094        let wf = WorkflowDefinition::new("persisted_timeout_event_guard")
6095            .initial_state("active")
6096            .transition(
6097                "active",
6098                "done",
6099                vec![WorkflowStep::wait_or(
6100                    "payment.success",
6101                    std::time::Duration::from_secs(0),
6102                    vec![
6103                        WorkflowStep::notify(ChannelKind::Email, "bad_timeout", "customer.email"),
6104                        WorkflowStep::transition("timed_out"),
6105                    ],
6106                )],
6107            );
6108
6109        let mut ctx = WorkflowContext::new("wf-persisted-timeout-event", "active");
6110        ctx.set(
6111            "customer",
6112            serde_json::json!({"email": "guest@example.com"}),
6113        );
6114        let result = run_workflow(&executor, &wf, &mut ctx).await.unwrap();
6115        assert_eq!(result, "active");
6116
6117        {
6118            let mut saved = executor.saved_state.lock().unwrap();
6119            let saved = saved
6120                .as_mut()
6121                .expect("workflow should be saved while waiting for timeout");
6122            let cursor = saved
6123                .cursor
6124                .as_mut()
6125                .expect("saved workflow should have a wait cursor");
6126            let wait = cursor
6127                .wait
6128                .as_mut()
6129                .expect("saved workflow should be waiting for timeout");
6130            wait.event = " payment.success ".to_string();
6131        }
6132
6133        let err = timeout_workflow(&executor, &wf, "wf-persisted-timeout-event")
6134            .await
6135            .expect_err("invalid persisted wait event must fail before side effects");
6136
6137        match err {
6138            WorkflowError::Other(msg) => {
6139                assert!(
6140                    msg.contains("Wait event name must not have leading or trailing whitespace"),
6141                    "got: {msg}"
6142                );
6143            }
6144            other => panic!("expected wait event validation error, got {other:?}"),
6145        }
6146        assert!(
6147            executor.notifications.lock().unwrap().is_empty(),
6148            "persisted timeout wait validation must run before notification"
6149        );
6150    }
6151
6152    #[tokio::test]
6153    async fn timeout_due_workflows_drains_due_waits_with_generated_idempotency() {
6154        let executor = MockExecutor::new();
6155
6156        let wf = WorkflowDefinition::new("wait_timeout_drain")
6157            .initial_state("active")
6158            .transition(
6159                "active",
6160                "done",
6161                vec![WorkflowStep::wait_or(
6162                    "payment.success",
6163                    std::time::Duration::from_secs(0),
6164                    vec![
6165                        WorkflowStep::notify(
6166                            ChannelKind::Email,
6167                            "payment_timeout",
6168                            "customer.email",
6169                        ),
6170                        WorkflowStep::transition("timed_out"),
6171                    ],
6172                )],
6173            );
6174
6175        let mut ctx = WorkflowContext::new("wf-timeout-drain", "active");
6176        ctx.set(
6177            "customer",
6178            serde_json::json!({"email": "guest@example.com"}),
6179        );
6180        let result = run_workflow(&executor, &wf, &mut ctx).await.unwrap();
6181        assert_eq!(result, "active");
6182
6183        executor.set_due_timeout_workflow_ids(vec!["wf-timeout-drain"]);
6184        let now = Utc::now();
6185        let outcomes =
6186            timeout_due_workflows(&executor, &wf, now, 10, WorkflowRunOptions::default())
6187                .await
6188                .unwrap();
6189
6190        assert_eq!(
6191            outcomes,
6192            vec![WorkflowTimeoutOutcome {
6193                workflow_id: "wf-timeout-drain".to_string(),
6194                state: Some("timed_out".to_string()),
6195                error: None,
6196            }]
6197        );
6198        assert_eq!(
6199            executor
6200                .completed_workflow_operations
6201                .lock()
6202                .unwrap()
6203                .as_slice(),
6204            &[(
6205                timeout_operation_idempotency_key("wait_timeout_drain", "wf-timeout-drain", now),
6206                "timed_out".to_string(),
6207            )]
6208        );
6209    }
6210
6211    #[tokio::test]
6212    async fn timeout_due_workflows_uses_batch_now_for_timeout_execution() {
6213        let executor = MockExecutor::new();
6214
6215        let wf = WorkflowDefinition::new("future_wait_timeout_drain")
6216            .initial_state("active")
6217            .transition(
6218                "active",
6219                "done",
6220                vec![WorkflowStep::wait_or(
6221                    "payment.success",
6222                    std::time::Duration::from_secs(3600),
6223                    vec![WorkflowStep::transition("timed_out")],
6224                )],
6225            );
6226
6227        let mut ctx = WorkflowContext::new("wf-future-timeout-drain", "active");
6228        let result = run_workflow(&executor, &wf, &mut ctx).await.unwrap();
6229        assert_eq!(result, "active");
6230
6231        executor.set_due_timeout_workflow_ids(vec!["wf-future-timeout-drain"]);
6232        let outcomes = timeout_due_workflows(
6233            &executor,
6234            &wf,
6235            Utc::now() + chrono::Duration::hours(2),
6236            10,
6237            WorkflowRunOptions::default(),
6238        )
6239        .await
6240        .unwrap();
6241
6242        assert_eq!(
6243            outcomes,
6244            vec![WorkflowTimeoutOutcome {
6245                workflow_id: "wf-future-timeout-drain".to_string(),
6246                state: Some("timed_out".to_string()),
6247                error: None,
6248            }]
6249        );
6250    }
6251
6252    #[test]
6253    fn timeout_batch_idempotency_key_is_scoped_per_workflow() {
6254        let wf = WorkflowDefinition::new("wait_timeout_drain")
6255            .initial_state("active")
6256            .transition("active", "done", vec![]);
6257        let options = WorkflowRunOptions::default().with_idempotency_key("batch-20260617");
6258        let now = DateTime::parse_from_rfc3339("2026-06-17T12:00:00Z")
6259            .unwrap()
6260            .with_timezone(&Utc);
6261
6262        let first = timeout_options_for_workflow(&options, &wf, "wf-a", now);
6263        let second = timeout_options_for_workflow(&options, &wf, "wf-b", now);
6264        let first_key = first.idempotency_key.clone();
6265        let second_key = second.idempotency_key.clone();
6266
6267        assert_eq!(
6268            first_key,
6269            Some(timeout_operation_idempotency_key(
6270                "batch-20260617",
6271                "wf-a",
6272                now
6273            ))
6274        );
6275        assert_eq!(
6276            second_key,
6277            Some(timeout_operation_idempotency_key(
6278                "batch-20260617",
6279                "wf-b",
6280                now
6281            ))
6282        );
6283        assert_ne!(first.idempotency_key, second.idempotency_key);
6284    }
6285
6286    #[test]
6287    fn timeout_batch_idempotency_key_does_not_collide_on_delimiters() {
6288        let now = DateTime::parse_from_rfc3339("2026-06-17T12:00:00Z")
6289            .unwrap()
6290            .with_timezone(&Utc);
6291        let first = timeout_operation_idempotency_key("tenant:batch", "wf-a", now);
6292        let second = timeout_operation_idempotency_key("tenant", "batch:wf-a", now);
6293
6294        assert_ne!(
6295            first, second,
6296            "timeout operation idempotency keys must not collide when fields contain delimiters"
6297        );
6298    }
6299
6300    #[test]
6301    fn timeout_batch_idempotency_key_changes_between_scheduler_runs() {
6302        let first_run = DateTime::parse_from_rfc3339("2026-06-17T12:00:00Z")
6303            .unwrap()
6304            .with_timezone(&Utc);
6305        let second_run = DateTime::parse_from_rfc3339("2026-06-17T12:01:00Z")
6306            .unwrap()
6307            .with_timezone(&Utc);
6308
6309        assert_ne!(
6310            timeout_operation_idempotency_key("batch", "wf-a", first_run),
6311            timeout_operation_idempotency_key("batch", "wf-a", second_run),
6312            "later timeout attempts for the same workflow must not collide with an old completed timeout operation"
6313        );
6314    }
6315
6316    #[tokio::test]
6317    async fn test_timeout_fallback_without_transition_does_not_run_success_path() {
6318        let executor = MockExecutor::new();
6319
6320        let wf = WorkflowDefinition::new("wait_timeout_no_transition")
6321            .initial_state("active")
6322            .transition(
6323                "active",
6324                "confirmed",
6325                vec![
6326                    WorkflowStep::wait_or(
6327                        "payment.success",
6328                        std::time::Duration::from_secs(0),
6329                        vec![WorkflowStep::notify(
6330                            ChannelKind::Email,
6331                            "payment_timeout",
6332                            "customer.email",
6333                        )],
6334                    ),
6335                    WorkflowStep::notify(ChannelKind::Email, "booking_confirmed", "customer.email"),
6336                    WorkflowStep::transition("confirmed"),
6337                ],
6338            );
6339
6340        let mut ctx = WorkflowContext::new("wf-timeout-no-transition", "active");
6341        ctx.set(
6342            "customer",
6343            serde_json::json!({"email": "guest@example.com"}),
6344        );
6345        let result = run_workflow(&executor, &wf, &mut ctx).await.unwrap();
6346        assert_eq!(result, "active");
6347
6348        let result = timeout_workflow(&executor, &wf, "wf-timeout-no-transition")
6349            .await
6350            .unwrap();
6351        assert_eq!(result, "active");
6352
6353        let notifs = executor.notifications.lock().unwrap();
6354        assert_eq!(
6355            notifs.as_slice(),
6356            &[(
6357                "guest@example.com".to_string(),
6358                "payment_timeout".to_string()
6359            )],
6360            "timeout fallback must not fall through into the post-wait success path"
6361        );
6362    }
6363
6364    #[tokio::test]
6365    async fn test_step_checkpoint_prevents_side_effect_replay_after_failure() {
6366        let executor = MockExecutor::new();
6367        let failing_query =
6368            qail_core::wire::encode_cmd_text(&qail_core::Qail::get("bookings").limit(1));
6369        executor.fail_query(failing_query.clone());
6370
6371        let wf = WorkflowDefinition::new("checkpoint_failure")
6372            .initial_state("active")
6373            .transition(
6374                "active",
6375                "done",
6376                vec![
6377                    WorkflowStep::notify(ChannelKind::Email, "charged", "customer.email"),
6378                    WorkflowStep::Query {
6379                        cmd_json: failing_query,
6380                        store_as: None,
6381                    },
6382                    WorkflowStep::transition("done"),
6383                ],
6384            );
6385
6386        let mut ctx = WorkflowContext::new("wf-checkpoint", "active");
6387        ctx.set(
6388            "customer",
6389            serde_json::json!({"email": "guest@example.com"}),
6390        );
6391        let err = run_workflow(&executor, &wf, &mut ctx)
6392            .await
6393            .expect_err("legacy query should fail after notification checkpoint");
6394        assert!(matches!(err, WorkflowError::QueryFailed(_)));
6395        assert_eq!(executor.notifications.lock().unwrap().len(), 1);
6396
6397        let mut restored = executor
6398            .saved_state
6399            .lock()
6400            .unwrap()
6401            .clone()
6402            .expect("checkpoint should be saved after notification");
6403        let cursor = restored.cursor.as_ref().expect("checkpoint cursor");
6404        assert_eq!(cursor.frames, vec![WorkflowCursorFrame::Steps { index: 1 }]);
6405        assert!(cursor.wait.is_none());
6406        assert!(restored.get(TIMEOUT_FALLBACK_KEY).is_none());
6407
6408        let err = run_workflow(&executor, &wf, &mut restored)
6409            .await
6410            .expect_err("query should still fail on retry");
6411        assert!(matches!(err, WorkflowError::QueryFailed(_)));
6412        assert_eq!(
6413            executor.notifications.lock().unwrap().len(),
6414            1,
6415            "retry must resume after the completed notification step"
6416        );
6417    }
6418
6419    #[tokio::test]
6420    async fn failed_query_step_marks_side_effect_failed() {
6421        let executor = MockExecutor::new();
6422        let failing_query =
6423            qail_core::wire::encode_cmd_text(&qail_core::Qail::get("bookings").limit(1));
6424        executor.fail_query(failing_query.clone());
6425
6426        let wf = WorkflowDefinition::new("failed_side_effect")
6427            .initial_state("active")
6428            .transition(
6429                "active",
6430                "done",
6431                vec![
6432                    WorkflowStep::Query {
6433                        cmd_json: failing_query,
6434                        store_as: None,
6435                    },
6436                    WorkflowStep::transition("done"),
6437                ],
6438            );
6439
6440        let mut ctx = WorkflowContext::new("wf-side-effect-fail", "active");
6441        let err = run_workflow(&executor, &wf, &mut ctx)
6442            .await
6443            .expect_err("query should fail");
6444        assert!(matches!(err, WorkflowError::QueryFailed(_)));
6445
6446        let expected_operation_id = side_effect_operation_id(
6447            "wf-side-effect-fail",
6448            "active",
6449            0,
6450            WorkflowSideEffectKind::Query,
6451            "steps[0]",
6452        );
6453        let failed_side_effects = executor.failed_side_effects.lock().unwrap();
6454        assert_eq!(failed_side_effects.len(), 1);
6455        assert_eq!(failed_side_effects[0].0, expected_operation_id);
6456        assert!(
6457            failed_side_effects[0].1.contains("forced query failure"),
6458            "failure hook must preserve the app/provider error"
6459        );
6460    }
6461
6462    #[tokio::test]
6463    async fn test_timeout_checkpoint_prevents_side_effect_replay_after_failure() {
6464        let executor = MockExecutor::new();
6465        let failing_query =
6466            qail_core::wire::encode_cmd_text(&qail_core::Qail::get("bookings").limit(1));
6467        executor.fail_query(failing_query.clone());
6468
6469        let wf = WorkflowDefinition::new("timeout_checkpoint_failure")
6470            .initial_state("active")
6471            .transition(
6472                "active",
6473                "done",
6474                vec![WorkflowStep::wait_or(
6475                    "payment.success",
6476                    std::time::Duration::from_secs(0),
6477                    vec![
6478                        WorkflowStep::notify(
6479                            ChannelKind::Email,
6480                            "payment_timeout",
6481                            "customer.email",
6482                        ),
6483                        WorkflowStep::Query {
6484                            cmd_json: failing_query,
6485                            store_as: None,
6486                        },
6487                        WorkflowStep::transition("timed_out"),
6488                    ],
6489                )],
6490            );
6491
6492        let mut ctx = WorkflowContext::new("wf-timeout-checkpoint", "active");
6493        ctx.set(
6494            "customer",
6495            serde_json::json!({"email": "guest@example.com"}),
6496        );
6497        let result = run_workflow(&executor, &wf, &mut ctx).await.unwrap();
6498        assert_eq!(result, "active");
6499
6500        let err = timeout_workflow(&executor, &wf, "wf-timeout-checkpoint")
6501            .await
6502            .expect_err("legacy query should fail after timeout notification checkpoint");
6503        assert!(matches!(err, WorkflowError::QueryFailed(_)));
6504        assert_eq!(executor.notifications.lock().unwrap().len(), 1);
6505
6506        let saved = executor
6507            .saved_state
6508            .lock()
6509            .unwrap()
6510            .clone()
6511            .expect("timeout checkpoint should be saved after notification");
6512        let cursor = saved.cursor.as_ref().expect("timeout checkpoint cursor");
6513        assert_eq!(cursor.frames, vec![WorkflowCursorFrame::Steps { index: 1 }]);
6514        let wait = cursor
6515            .wait
6516            .as_ref()
6517            .expect("timeout fallback checkpoint must remain scheduler-visible");
6518        assert_eq!(wait.event, "payment.success");
6519        assert_eq!(wait.on_timeout.len(), 3);
6520        assert!(
6521            saved.get(TIMEOUT_FALLBACK_KEY).is_some(),
6522            "timeout fallback cursor must retain internal on_timeout metadata"
6523        );
6524
6525        let err = timeout_workflow(&executor, &wf, "wf-timeout-checkpoint")
6526            .await
6527            .expect_err("timeout fallback query should still fail on retry");
6528        assert!(matches!(err, WorkflowError::QueryFailed(_)));
6529        assert_eq!(
6530            executor.notifications.lock().unwrap().len(),
6531            1,
6532            "timeout retry must resume after the completed notification step"
6533        );
6534    }
6535
6536    #[tokio::test]
6537    async fn test_workflow_errors_on_for_each_non_array() {
6538        let executor = MockExecutor::new();
6539
6540        let wf = WorkflowDefinition::new("bad_for_each")
6541            .initial_state("active")
6542            .transition(
6543                "active",
6544                "resolved",
6545                vec![WorkflowStep::for_each(
6546                    "operators",
6547                    vec![WorkflowStep::notify(
6548                        ChannelKind::WhatsApp,
6549                        "opportunity",
6550                        "item.phone",
6551                    )],
6552                )],
6553            );
6554
6555        let mut ctx = WorkflowContext::new("wf-bad-for-each", "active");
6556        ctx.set("operators", serde_json::json!({"phone": "+628111"}));
6557
6558        let err = run_workflow(&executor, &wf, &mut ctx)
6559            .await
6560            .expect_err("ForEach should reject non-array context values");
6561
6562        match err {
6563            WorkflowError::Other(msg) => {
6564                assert!(msg.contains("Expected JSON array for ForEach list 'operators'"));
6565                assert!(msg.contains("object"));
6566            }
6567            other => panic!("expected Other error, got: {other}"),
6568        }
6569        assert!(
6570            executor.notifications.lock().unwrap().is_empty(),
6571            "ForEach type errors must fail before running nested steps"
6572        );
6573    }
6574
6575    #[tokio::test]
6576    async fn test_workflow_with_charge() {
6577        let executor = MockExecutor::new();
6578
6579        let wf = WorkflowDefinition::new("booking_payment")
6580            .initial_state("created")
6581            .transition(
6582                "created",
6583                "awaiting_payment",
6584                vec![
6585                    WorkflowStep::charge_with_origin(
6586                        PaymentKind::Xendit,
6587                        "order.total",
6588                        "order.id",
6589                        None,
6590                        Some("order.payment_method"),
6591                        Some("order.origin"),
6592                        Some("charge"),
6593                    ),
6594                    WorkflowStep::wait("payment.success", std::time::Duration::from_secs(3600)),
6595                ],
6596            )
6597            .transition(
6598                "awaiting_payment",
6599                "confirmed",
6600                vec![
6601                    WorkflowStep::notify(ChannelKind::Email, "booking_confirmed", "customer.email"),
6602                    WorkflowStep::transition("confirmed"),
6603                ],
6604            );
6605
6606        let mut ctx = WorkflowContext::new("wf-payment-001", "created");
6607        ctx.set(
6608            "order",
6609            serde_json::json!({
6610                "id": "booking-789",
6611                "total": 150000,
6612                "payment_method": "QRIS",
6613                "origin": "mcp"
6614            }),
6615        );
6616        ctx.set(
6617            "customer",
6618            serde_json::json!({
6619                "email": "guest@example.com"
6620            }),
6621        );
6622
6623        let result = run_workflow(&executor, &wf, &mut ctx).await.unwrap();
6624
6625        // Should pause at Wait step (awaiting_payment webhook)
6626        assert_eq!(result, "created");
6627
6628        // Verify charge was stored in context
6629        let charge = ctx.get("charge").expect("charge should be in context");
6630        assert_eq!(
6631            charge.get("charge_id").and_then(|v| v.as_str()),
6632            Some("mock-charge-booking-789")
6633        );
6634        assert_eq!(
6635            charge.get("status").and_then(|v| v.as_str()),
6636            Some("Pending")
6637        );
6638        assert!(charge.get("qr_code").is_some());
6639        assert_eq!(
6640            charge.get("payment_method").and_then(|v| v.as_str()),
6641            Some("QRIS")
6642        );
6643        assert_eq!(
6644            charge.get("order_origin").and_then(|v| v.as_str()),
6645            Some("mcp")
6646        );
6647        assert!(
6648            charge.get("raw").is_none(),
6649            "stored charge context must be safe for chat display"
6650        );
6651
6652        let charges = executor.charges.lock().unwrap();
6653        assert_eq!(charges.len(), 1);
6654        assert_eq!(charges[0].amount, 150000);
6655        assert_eq!(charges[0].order_origin, Some(OrderOrigin::Mcp));
6656        assert!(
6657            charges[0]
6658                .idempotency_key
6659                .as_deref()
6660                .is_some_and(|key| key.contains("qail-workflow-side-effect")),
6661            "provider charge request must carry a stable workflow idempotency key"
6662        );
6663    }
6664
6665    #[tokio::test]
6666    async fn charge_rejects_invalid_origin_before_provider_call() {
6667        let executor = MockExecutor::new();
6668
6669        let wf = WorkflowDefinition::new("charge_invalid_origin")
6670            .initial_state("created")
6671            .transition(
6672                "created",
6673                "awaiting_payment",
6674                vec![WorkflowStep::charge_with_origin(
6675                    PaymentKind::Xendit,
6676                    "order.total",
6677                    "order.id",
6678                    None,
6679                    None,
6680                    Some("order.origin"),
6681                    Some("charge"),
6682                )],
6683            );
6684
6685        let mut ctx = WorkflowContext::new("wf-charge-invalid-origin", "created");
6686        ctx.set(
6687            "order",
6688            serde_json::json!({
6689                "id": "booking-invalid-origin",
6690                "total": 150000,
6691                "origin": "telegram"
6692            }),
6693        );
6694
6695        let err = run_workflow(&executor, &wf, &mut ctx)
6696            .await
6697            .expect_err("invalid order origin must fail before provider call");
6698
6699        match err {
6700            WorkflowError::Other(msg) => {
6701                assert!(
6702                    msg.contains("Charge order_origin_key 'order.origin'"),
6703                    "got: {msg}"
6704                );
6705            }
6706            other => panic!("expected invalid order origin error, got {other:?}"),
6707        }
6708        assert!(executor.charges.lock().unwrap().is_empty());
6709    }
6710
6711    #[tokio::test]
6712    async fn charge_rejects_reserved_store_key_before_provider_call() {
6713        let executor = MockExecutor::new();
6714
6715        let wf = WorkflowDefinition::new("charge_reserved_key")
6716            .initial_state("created")
6717            .transition(
6718                "created",
6719                "awaiting_payment",
6720                vec![WorkflowStep::charge(
6721                    PaymentKind::Xendit,
6722                    "order.total",
6723                    "order.id",
6724                    Some("item"),
6725                )],
6726            );
6727
6728        let mut ctx = WorkflowContext::new("wf-charge-reserved-key", "created");
6729        ctx.set(
6730            "order",
6731            serde_json::json!({
6732                "id": "booking-reserved",
6733                "total": 150000
6734            }),
6735        );
6736
6737        let err = run_workflow(&executor, &wf, &mut ctx)
6738            .await
6739            .expect_err("reserved charge output key must fail before provider call");
6740
6741        match err {
6742            WorkflowError::Other(msg) => {
6743                assert!(msg.contains("reserved context key 'item'"), "got: {msg}");
6744            }
6745            other => panic!("expected reserved context key error, got {other:?}"),
6746        }
6747        assert!(
6748            executor.charges.lock().unwrap().is_empty(),
6749            "charge provider must not be called after reserved key validation fails"
6750        );
6751    }
6752
6753    #[tokio::test]
6754    async fn charge_rejects_dotted_store_key_before_provider_call() {
6755        let executor = MockExecutor::new();
6756
6757        let wf = WorkflowDefinition::new("charge_dotted_key")
6758            .initial_state("created")
6759            .transition(
6760                "created",
6761                "awaiting_payment",
6762                vec![WorkflowStep::charge(
6763                    PaymentKind::Xendit,
6764                    "order.total",
6765                    "order.id",
6766                    Some("payment.charge"),
6767                )],
6768            );
6769
6770        let mut ctx = WorkflowContext::new("wf-charge-dotted-key", "created");
6771        ctx.set(
6772            "order",
6773            serde_json::json!({
6774                "id": "booking-dotted",
6775                "total": 150000
6776            }),
6777        );
6778
6779        let err = run_workflow(&executor, &wf, &mut ctx)
6780            .await
6781            .expect_err("dotted charge output key must fail before provider call");
6782
6783        match err {
6784            WorkflowError::Other(msg) => {
6785                assert!(msg.contains("top-level context key"), "got: {msg}");
6786            }
6787            other => panic!("expected dotted context key error, got {other:?}"),
6788        }
6789        assert!(
6790            executor.charges.lock().unwrap().is_empty(),
6791            "charge provider must not be called after dotted key validation fails"
6792        );
6793    }
6794
6795    #[tokio::test]
6796    async fn charge_accepts_safe_integer_float_amount() {
6797        let executor = MockExecutor::new();
6798        let wf = charge_only_workflow("booking_payment_float");
6799
6800        let mut ctx = WorkflowContext::new("wf-payment-float", "created");
6801        ctx.set(
6802            "order",
6803            serde_json::json!({
6804                "id": "booking-float",
6805                "total": 150000.0
6806            }),
6807        );
6808
6809        let result = run_workflow(&executor, &wf, &mut ctx).await.unwrap();
6810
6811        assert_eq!(result, "awaiting_payment");
6812        let charges = executor.charges.lock().unwrap();
6813        assert_eq!(charges.len(), 1);
6814        assert_eq!(charges[0].amount, 150000);
6815    }
6816
6817    #[tokio::test]
6818    async fn charge_rejects_fractional_amount_before_provider_call() {
6819        let executor = MockExecutor::new();
6820        let wf = charge_only_workflow("booking_payment_fractional");
6821
6822        let mut ctx = WorkflowContext::new("wf-payment-fractional", "created");
6823        ctx.set(
6824            "order",
6825            serde_json::json!({
6826                "id": "booking-fractional",
6827                "total": 150000.75
6828            }),
6829        );
6830
6831        let err = run_workflow(&executor, &wf, &mut ctx)
6832            .await
6833            .expect_err("fractional charge amount must fail before provider call");
6834
6835        match err {
6836            WorkflowError::Other(msg) => {
6837                assert!(msg.contains("Invalid charge amount"));
6838                assert!(msg.contains("smallest currency unit"));
6839            }
6840            other => panic!("expected invalid amount error, got {other:?}"),
6841        }
6842        assert!(executor.charges.lock().unwrap().is_empty());
6843    }
6844
6845    #[tokio::test]
6846    async fn charge_rejects_non_positive_amount_before_provider_call() {
6847        let executor = MockExecutor::new();
6848        let wf = charge_only_workflow("booking_payment_negative");
6849
6850        let mut ctx = WorkflowContext::new("wf-payment-negative", "created");
6851        ctx.set(
6852            "order",
6853            serde_json::json!({
6854                "id": "booking-negative",
6855                "total": -1
6856            }),
6857        );
6858
6859        let err = run_workflow(&executor, &wf, &mut ctx)
6860            .await
6861            .expect_err("non-positive charge amount must fail before provider call");
6862
6863        match err {
6864            WorkflowError::Other(msg) => {
6865                assert!(msg.contains("Invalid charge amount"));
6866                assert!(msg.contains("greater than zero"));
6867            }
6868            other => panic!("expected invalid amount error, got {other:?}"),
6869        }
6870        assert!(executor.charges.lock().unwrap().is_empty());
6871    }
6872
6873    #[tokio::test]
6874    async fn charge_rejects_oversized_unsigned_amount_before_provider_call() {
6875        let executor = MockExecutor::new();
6876        let wf = charge_only_workflow("booking_payment_oversized");
6877
6878        let mut ctx = WorkflowContext::new("wf-payment-oversized", "created");
6879        ctx.set(
6880            "order",
6881            serde_json::json!({
6882                "id": "booking-oversized",
6883                "total": u64::MAX
6884            }),
6885        );
6886
6887        let err = run_workflow(&executor, &wf, &mut ctx)
6888            .await
6889            .expect_err("oversized charge amount must fail before provider call");
6890
6891        match err {
6892            WorkflowError::Other(msg) => {
6893                assert!(msg.contains("Invalid charge amount"));
6894                assert!(msg.contains("signed 64-bit"));
6895            }
6896            other => panic!("expected invalid amount error, got {other:?}"),
6897        }
6898        assert!(executor.charges.lock().unwrap().is_empty());
6899    }
6900
6901    #[tokio::test]
6902    async fn test_query_step_rejects_legacy_non_wire_payload() {
6903        let executor = MockExecutor::new();
6904
6905        let wf = WorkflowDefinition::new("legacy_query")
6906            .initial_state("start")
6907            .transition(
6908                "start",
6909                "done",
6910                vec![
6911                    WorkflowStep::Query {
6912                        cmd_json: "get users limit 1".to_string(),
6913                        store_as: Some("rows".to_string()),
6914                    },
6915                    WorkflowStep::transition("done"),
6916                ],
6917            );
6918
6919        let mut ctx = WorkflowContext::new("wf-legacy-query-001", "start");
6920        let err = run_workflow(&executor, &wf, &mut ctx)
6921            .await
6922            .expect_err("legacy non-wire query payload must fail");
6923
6924        match err {
6925            WorkflowError::QueryFailed(msg) => {
6926                assert!(
6927                    msg.contains("QAIL-CMD/1"),
6928                    "error should mention required wire magic"
6929                );
6930                assert!(
6931                    msg.contains("purge/restart pending workflows"),
6932                    "error should include cutover guidance"
6933                );
6934            }
6935            other => panic!("expected QueryFailed, got: {other}"),
6936        }
6937
6938        assert!(
6939            executor.queries.lock().unwrap().is_empty(),
6940            "legacy payload must fail before executor query is invoked"
6941        );
6942    }
6943
6944    #[tokio::test]
6945    async fn future_legacy_query_payload_fails_before_current_side_effects() {
6946        let executor = MockExecutor::new();
6947
6948        let wf = WorkflowDefinition::new("future_legacy_query")
6949            .initial_state("active")
6950            .transition(
6951                "active",
6952                "query_state",
6953                vec![
6954                    WorkflowStep::notify(ChannelKind::Email, "before_bad_query", "customer.email"),
6955                    WorkflowStep::transition("query_state"),
6956                ],
6957            )
6958            .transition(
6959                "query_state",
6960                "done",
6961                vec![
6962                    WorkflowStep::Query {
6963                        cmd_json: "get users limit 1".to_string(),
6964                        store_as: Some("rows".to_string()),
6965                    },
6966                    WorkflowStep::transition("done"),
6967                ],
6968            );
6969
6970        let mut ctx = WorkflowContext::new("wf-future-legacy-query", "active");
6971        ctx.set(
6972            "customer",
6973            serde_json::json!({"email": "guest@example.com"}),
6974        );
6975
6976        let err = run_workflow(&executor, &wf, &mut ctx)
6977            .await
6978            .expect_err("future legacy query payload must fail before current side effects");
6979
6980        match err {
6981            WorkflowError::QueryFailed(msg) => {
6982                assert!(
6983                    msg.contains("QAIL-CMD/1"),
6984                    "error should mention required wire magic"
6985                );
6986            }
6987            other => panic!("expected QueryFailed, got: {other}"),
6988        }
6989        assert_eq!(ctx.current_state, "active");
6990        assert!(
6991            executor.notifications.lock().unwrap().is_empty(),
6992            "definition validation must run before the current-state notification"
6993        );
6994        assert!(
6995            executor.queries.lock().unwrap().is_empty(),
6996            "legacy payload must fail before executor query is invoked"
6997        );
6998    }
6999
7000    #[tokio::test]
7001    async fn test_query_step_accepts_wire_payload_and_executes() {
7002        let executor = MockExecutor::new();
7003        let cmd = qail_core::Qail::get("users").columns(["id"]).limit(1);
7004        let wire = qail_core::wire::encode_cmd_text(&cmd);
7005
7006        let wf = WorkflowDefinition::new("wire_query")
7007            .initial_state("start")
7008            .transition(
7009                "start",
7010                "done",
7011                vec![
7012                    WorkflowStep::Query {
7013                        cmd_json: wire.clone(),
7014                        store_as: Some("rows".to_string()),
7015                    },
7016                    WorkflowStep::transition("done"),
7017                ],
7018            );
7019
7020        let mut ctx = WorkflowContext::new("wf-wire-query-001", "start");
7021        let result = run_workflow(&executor, &wf, &mut ctx)
7022            .await
7023            .expect("wire payload should execute");
7024        assert_eq!(result, "done");
7025
7026        let queries = executor.queries.lock().unwrap();
7027        assert_eq!(queries.len(), 1);
7028        assert_eq!(
7029            queries[0], wire,
7030            "executor should receive canonical wire payload"
7031        );
7032    }
7033
7034    #[tokio::test]
7035    async fn query_step_executes_the_exact_command_it_persisted() {
7036        let executor = MockExecutor::new();
7037        // Canonical text of these is `add rows` / `get orders ...` without the
7038        // values, DISTINCT or FOR UPDATE.
7039        let insert = qail_core::Qail::add("rows")
7040            .set_value("id", 1)
7041            .set_value("status", "a");
7042        let mut locked = qail_core::Qail::get("orders")
7043            .columns(["id"])
7044            .where_eq("id", 1)
7045            .for_update();
7046        locked.distinct = true;
7047
7048        let steps = vec![
7049            WorkflowStep::query(&insert, None),
7050            WorkflowStep::query(&locked, Some("rows")),
7051            WorkflowStep::transition("done"),
7052        ];
7053        let persisted: Vec<WorkflowStep> =
7054            serde_json::from_str(&serde_json::to_string(&steps).unwrap()).unwrap();
7055        let wf = WorkflowDefinition::new("exact_query")
7056            .initial_state("start")
7057            .transition("start", "done", persisted);
7058
7059        let mut ctx = WorkflowContext::new("wf-exact-query-001", "start");
7060        let result = run_workflow(&executor, &wf, &mut ctx)
7061            .await
7062            .expect("exact payloads should execute");
7063        assert_eq!(result, "done");
7064
7065        let queries = executor.queries.lock().unwrap();
7066        let decoded: Vec<_> = queries
7067            .iter()
7068            .map(|wire| qail_core::wire::decode_cmd_text(wire).unwrap())
7069            .collect();
7070        assert_eq!(decoded, vec![insert, locked]);
7071    }
7072
7073    #[test]
7074    fn query_scope_wire_normalization_retains_pending_scope() {
7075        let mut cmd = qail_core::Qail::add("scope_transport").set_value("id", 1);
7076        cmd.conflict_update_scope.push(qail_core::ast::builders::eq(
7077            "scope_transport.tenant_id",
7078            "a",
7079        ));
7080        let wire = qail_core::wire::encode_cmd_text(&cmd);
7081        assert!(wire.starts_with("QAIL-CMD/2\n"));
7082        assert!(is_current_workflow_query_wire(&wire));
7083        let normalized = normalize_query_wire_for_execution(&wire).unwrap();
7084        assert_eq!(qail_core::wire::decode_cmd_text(&normalized).unwrap(), cmd);
7085        assert!(normalize_query_wire_for_execution("QAIL-CMD/2\n2\n{}").is_err());
7086        assert!(normalize_query_wire_for_execution("QAIL-CMD/3\n0\n").is_err());
7087    }
7088
7089    #[tokio::test]
7090    async fn query_rejects_reserved_store_key_before_execution() {
7091        let executor = MockExecutor::new();
7092        let cmd = qail_core::Qail::get("users").columns(["id"]).limit(1);
7093        let wire = qail_core::wire::encode_cmd_text(&cmd);
7094
7095        let wf = WorkflowDefinition::new("query_reserved_key")
7096            .initial_state("start")
7097            .transition(
7098                "start",
7099                "done",
7100                vec![
7101                    WorkflowStep::Query {
7102                        cmd_json: wire,
7103                        store_as: Some("__qail_timeout_fallback".to_string()),
7104                    },
7105                    WorkflowStep::transition("done"),
7106                ],
7107            );
7108
7109        let mut ctx = WorkflowContext::new("wf-query-reserved-key", "start");
7110        let err = run_workflow(&executor, &wf, &mut ctx)
7111            .await
7112            .expect_err("reserved query output key must fail before query execution");
7113
7114        match err {
7115            WorkflowError::Other(msg) => {
7116                assert!(
7117                    msg.contains("reserved context key '__qail_timeout_fallback'"),
7118                    "got: {msg}"
7119                );
7120            }
7121            other => panic!("expected reserved context key error, got {other:?}"),
7122        }
7123        assert!(
7124            executor.queries.lock().unwrap().is_empty(),
7125            "query executor must not be called after reserved key validation fails"
7126        );
7127    }
7128
7129    #[tokio::test]
7130    async fn query_rejects_dotted_store_key_before_execution() {
7131        let executor = MockExecutor::new();
7132        let cmd = qail_core::Qail::get("users").columns(["id"]).limit(1);
7133        let wire = qail_core::wire::encode_cmd_text(&cmd);
7134
7135        let wf = WorkflowDefinition::new("query_dotted_key")
7136            .initial_state("start")
7137            .transition(
7138                "start",
7139                "done",
7140                vec![
7141                    WorkflowStep::Query {
7142                        cmd_json: wire,
7143                        store_as: Some("query.rows".to_string()),
7144                    },
7145                    WorkflowStep::transition("done"),
7146                ],
7147            );
7148
7149        let mut ctx = WorkflowContext::new("wf-query-dotted-key", "start");
7150        let err = run_workflow(&executor, &wf, &mut ctx)
7151            .await
7152            .expect_err("dotted query output key must fail before query execution");
7153
7154        match err {
7155            WorkflowError::Other(msg) => {
7156                assert!(msg.contains("top-level context key"), "got: {msg}");
7157            }
7158            other => panic!("expected dotted context key error, got {other:?}"),
7159        }
7160        assert!(
7161            executor.queries.lock().unwrap().is_empty(),
7162            "query executor must not be called after dotted key validation fails"
7163        );
7164    }
7165
7166    #[test]
7167    fn test_collect_legacy_query_payload_issues_reports_nested_paths() {
7168        let wire_cmd = qail_core::wire::encode_cmd_text(&qail_core::Qail::get("users").limit(1));
7169        let wf = WorkflowDefinition::new("legacy_audit")
7170            .initial_state("start")
7171            .transition(
7172                "start",
7173                "done",
7174                vec![
7175                    WorkflowStep::Query {
7176                        cmd_json: "get users limit 1".to_string(),
7177                        store_as: Some("rows".to_string()),
7178                    },
7179                    WorkflowStep::Wait {
7180                        event: "timeout".to_string(),
7181                        timeout: std::time::Duration::from_secs(10),
7182                        on_timeout: vec![WorkflowStep::Query {
7183                            cmd_json: "{\"legacy\":true}".to_string(),
7184                            store_as: None,
7185                        }],
7186                    },
7187                    WorkflowStep::branch(
7188                        "kind",
7189                        vec![(
7190                            "wire_ok",
7191                            vec![WorkflowStep::Query {
7192                                cmd_json: wire_cmd,
7193                                store_as: None,
7194                            }],
7195                        )],
7196                        vec![WorkflowStep::ForEach {
7197                            list_key: "items".to_string(),
7198                            steps: vec![WorkflowStep::Query {
7199                                cmd_json: "select * from legacy".to_string(),
7200                                store_as: None,
7201                            }],
7202                        }],
7203                    ),
7204                ],
7205            );
7206
7207        let issues = collect_legacy_query_payload_issues(&wf);
7208        assert_eq!(
7209            issues.len(),
7210            3,
7211            "expected all legacy payloads to be detected"
7212        );
7213
7214        let paths: Vec<String> = issues.iter().map(|i| i.step_path.clone()).collect();
7215        assert!(
7216            paths.iter().any(|p| p == "steps[0]"),
7217            "top-level legacy query should be reported"
7218        );
7219        assert!(
7220            paths.iter().any(|p| p == "steps[1].on_timeout.steps[0]"),
7221            "nested wait/on_timeout legacy query should be reported"
7222        );
7223        assert!(
7224            paths
7225                .iter()
7226                .any(|p| p == "steps[2].default.steps[0].for_each.steps[0]"),
7227            "nested branch/default/foreach legacy query should be reported"
7228        );
7229    }
7230}