Skip to main content

runifold_workflow/
execution.rs

1use std::{collections::BTreeMap, future::Future, pin::Pin};
2
3use runifold_core::{
4    ChildEvent, DomainEvent, EventId, Instant, LifecycleEvent, RetrySafety, RunContext, RunError,
5    RunErrorKind, RunEventKind, Usage,
6};
7use serde_json::Value;
8
9use crate::checkpoint::WorkflowCheckpointCursor;
10use crate::parallel::execute_parallel;
11use crate::race::execute_race;
12use crate::remediation::{execute_repairable_node, prepare_remediation_resume};
13use crate::workflow::WorkflowNodeKind;
14use crate::{
15    ParallelBranchCheckpoint, StepId, Workflow, WorkflowCheckpoint, WorkflowCheckpointPhase,
16    WorkflowCheckpointState, WorkflowError, WorkflowInterruptDecision, WorkflowInterruptOutcome,
17    WorkflowInterruptRequest, WorkflowOutcome, WorkflowResumePolicy, WorkflowWait,
18    WorkflowWaitOutcome, WorkflowWake,
19};
20
21/// A boxed, sendable workflow execution future.
22#[cfg(not(target_arch = "wasm32"))]
23pub type WorkflowFuture<'a, T> = Pin<Box<dyn Future<Output = T> + Send + 'a>>;
24
25/// A boxed workflow execution future on single-threaded WASM.
26#[cfg(target_arch = "wasm32")]
27pub type WorkflowFuture<'a, T> = Pin<Box<dyn Future<Output = T> + 'a>>;
28
29pub(crate) enum WorkflowExecution {
30    Completed(WorkflowOutcome),
31    Suspended(WorkflowWait),
32}
33
34impl WorkflowExecution {
35    fn require_completed(self) -> Result<WorkflowOutcome, WorkflowError> {
36        match self {
37            Self::Completed(outcome) => Ok(outcome),
38            Self::Suspended(_) => Err(WorkflowError::DurableWaitRequiresWorker),
39        }
40    }
41}
42
43impl Workflow {
44    /// Executes this workflow from the first node.
45    pub fn run<'a>(
46        &'a self,
47        input: impl Into<Value> + Send + 'a,
48        run: &'a RunContext,
49    ) -> WorkflowFuture<'a, Result<WorkflowOutcome, WorkflowError>> {
50        let state = self.initial_state(input.into(), run.budget().usage());
51        Box::pin(async move {
52            self.execute_state(state, run, None)
53                .await?
54                .require_completed()
55        })
56    }
57
58    /// Executes with write-ahead workflow checkpoint persistence.
59    pub fn run_checkpointed<'a>(
60        &'a self,
61        input: impl Into<Value> + Send + 'a,
62        run: &'a RunContext,
63        checkpoint: &'a WorkflowCheckpoint,
64    ) -> WorkflowFuture<'a, Result<WorkflowOutcome, WorkflowError>> {
65        Box::pin(async move {
66            self.validate_authority(run)?;
67            let state = self.initial_state(input.into(), run.budget().usage());
68            let mut cursor = WorkflowCheckpointCursor::create(checkpoint, run, &state).await?;
69            self.execute_state(state, run, Some(&mut cursor))
70                .await?
71                .require_completed()
72        })
73    }
74
75    pub(crate) fn run_checkpointed_controlled<'a>(
76        &'a self,
77        input: Value,
78        run: &'a RunContext,
79        checkpoint: &'a WorkflowCheckpoint,
80    ) -> WorkflowFuture<'a, Result<WorkflowExecution, WorkflowError>> {
81        Box::pin(async move {
82            self.validate_authority(run)?;
83            let state = self.initial_state(input, run.budget().usage());
84            let mut cursor = WorkflowCheckpointCursor::create(checkpoint, run, &state).await?;
85            self.execute_state(state, run, Some(&mut cursor)).await
86        })
87    }
88
89    /// Resumes a persisted workflow execution.
90    pub fn resume<'a>(
91        &'a self,
92        checkpoint: &'a WorkflowCheckpoint,
93        run: &'a RunContext,
94        policy: WorkflowResumePolicy,
95    ) -> WorkflowFuture<'a, Result<WorkflowOutcome, WorkflowError>> {
96        Box::pin(async move {
97            self.resume_controlled(checkpoint, run, policy, None)
98                .await?
99                .require_completed()
100        })
101    }
102
103    pub(crate) fn resume_controlled<'a>(
104        &'a self,
105        checkpoint: &'a WorkflowCheckpoint,
106        run: &'a RunContext,
107        policy: WorkflowResumePolicy,
108        wake: Option<WorkflowWake>,
109    ) -> WorkflowFuture<'a, Result<WorkflowExecution, WorkflowError>> {
110        Box::pin(async move {
111            let (envelope, mut state) = checkpoint.load_async().await?;
112            self.validate_checkpoint_identity(&state)?;
113            if let Some(outcome) = state.outcome() {
114                validate_exact_usage(state.usage, run.budget().usage())?;
115                return Ok(WorkflowExecution::Completed(outcome));
116            }
117            let mut waiting_wake = None;
118            match &state.phase {
119                WorkflowCheckpointPhase::StepInFlight { step } => {
120                    if policy == WorkflowResumePolicy::RejectAmbiguous {
121                        return Err(WorkflowError::AmbiguousCheckpoint { step: step.clone() });
122                    }
123                    validate_usage_floor(state.usage, run.budget().usage())?;
124                    state.usage = run.budget().usage();
125                    state.phase = WorkflowCheckpointPhase::Ready;
126                }
127                WorkflowCheckpointPhase::Remediating { .. } => {
128                    prepare_remediation_resume(&mut state, run, policy)?;
129                }
130                WorkflowCheckpointPhase::Waiting { wait, .. } => {
131                    validate_exact_usage(state.usage, run.budget().usage())?;
132                    let wake = wake.ok_or(WorkflowError::DurableWaitRequiresWorker)?;
133                    if !wake.matches(wait) {
134                        return Err(WorkflowError::WakeMismatch);
135                    }
136                    waiting_wake = Some((wait.clone(), wake));
137                }
138                WorkflowCheckpointPhase::ParallelInFlight { step, branches } => {
139                    let all_completed = branches
140                        .values()
141                        .all(|branch| matches!(branch, ParallelBranchCheckpoint::Completed { .. }));
142                    if !all_completed && policy == WorkflowResumePolicy::RejectAmbiguous {
143                        return Err(WorkflowError::AmbiguousCheckpoint { step: step.clone() });
144                    }
145                    if all_completed {
146                        validate_exact_usage(state.usage, run.budget().usage())?;
147                    } else {
148                        validate_usage_floor(state.usage, run.budget().usage())?;
149                        state.usage = run.budget().usage();
150                    }
151                }
152                WorkflowCheckpointPhase::RaceInFlight { step, branches } => {
153                    let has_winner = branches
154                        .values()
155                        .any(|branch| matches!(branch, ParallelBranchCheckpoint::Completed { .. }));
156                    let all_failed = branches
157                        .values()
158                        .all(|branch| matches!(branch, ParallelBranchCheckpoint::Failed { .. }));
159                    if !has_winner && !all_failed && policy == WorkflowResumePolicy::RejectAmbiguous
160                    {
161                        return Err(WorkflowError::AmbiguousCheckpoint { step: step.clone() });
162                    }
163                    if has_winner || all_failed {
164                        validate_exact_usage(state.usage, run.budget().usage())?;
165                    } else {
166                        validate_usage_floor(state.usage, run.budget().usage())?;
167                        state.usage = run.budget().usage();
168                    }
169                }
170                WorkflowCheckpointPhase::Ready => {
171                    validate_exact_usage(state.usage, run.budget().usage())?;
172                }
173                WorkflowCheckpointPhase::Completed { .. } => {
174                    unreachable!("completed workflow checkpoints return before phase recovery")
175                }
176            }
177            let mut cursor = WorkflowCheckpointCursor::loaded(checkpoint, envelope);
178            if let Some((wait, wake)) = waiting_wake {
179                let node = &self.nodes[state.next_index];
180                let output = wake_output(&wait, wake, &state.value)?;
181                commit_node(&mut state, &node.id, output, run, &mut Some(&mut cursor)).await?;
182            }
183            self.execute_state(state, run, Some(&mut cursor)).await
184        })
185    }
186
187    fn initial_state(&self, input: Value, usage: Usage) -> WorkflowCheckpointState {
188        WorkflowCheckpointState {
189            workflow: self.name.clone(),
190            workflow_version: self.version,
191            layout: self.step_ids().cloned().collect(),
192            next_index: 0,
193            value: input,
194            outputs: BTreeMap::new(),
195            usage,
196            phase: WorkflowCheckpointPhase::Ready,
197        }
198    }
199
200    async fn execute_state(
201        &self,
202        mut state: WorkflowCheckpointState,
203        run: &RunContext,
204        mut checkpoint: Option<&mut WorkflowCheckpointCursor>,
205    ) -> Result<WorkflowExecution, WorkflowError> {
206        self.validate_authority(run)?;
207        let started = run
208            .record(
209                RunEventKind::Lifecycle(LifecycleEvent::Started),
210                run.caused_by(),
211            )?
212            .map(|event| event.meta.event_id);
213        let result = self
214            .run_loop(&mut state, run, started, &mut checkpoint)
215            .await;
216        run.record(terminal_event(&self.name, &result), started)?;
217        result
218    }
219
220    async fn run_loop(
221        &self,
222        state: &mut WorkflowCheckpointState,
223        run: &RunContext,
224        caused_by: Option<EventId>,
225        checkpoint: &mut Option<&mut WorkflowCheckpointCursor>,
226    ) -> Result<WorkflowExecution, WorkflowError> {
227        while state.next_index < self.nodes.len() {
228            check_lifecycle(run)?;
229            let node = &self.nodes[state.next_index];
230            let output = match &node.kind {
231                WorkflowNodeKind::Parallel(branches) => {
232                    self.execute_parallel_node(node, branches, state, run, caused_by, checkpoint)
233                        .await?
234                }
235                WorkflowNodeKind::Race(branches) => {
236                    self.execute_race_node(node, branches, state, run, caused_by, checkpoint)
237                        .await?
238                }
239                WorkflowNodeKind::Repairable(repairable) => {
240                    execute_repairable_node(
241                        &self.name, node, repairable, state, run, caused_by, checkpoint,
242                    )
243                    .await?
244                }
245                WorkflowNodeKind::Timer(wait)
246                | WorkflowNodeKind::Signal(wait)
247                | WorkflowNodeKind::SignalOrTimeout(wait) => {
248                    state.phase = WorkflowCheckpointPhase::Waiting {
249                        step: node.id.clone(),
250                        wait: wait.clone(),
251                    };
252                    state.usage = run.budget().usage();
253                    save_checkpoint(checkpoint, state).await?;
254                    record_domain(
255                        run,
256                        "workflow.suspended",
257                        serde_json::json!({
258                            "workflow": self.name,
259                            "step": node.id,
260                            "wait": wait,
261                        }),
262                        caused_by,
263                    )?;
264                    return Ok(WorkflowExecution::Suspended(wait.clone()));
265                }
266                WorkflowNodeKind::Interrupt(prompt) => {
267                    let wait = WorkflowWait::Interrupt {
268                        request: WorkflowInterruptRequest::new(
269                            prompt.clone(),
270                            state.value.clone(),
271                        )?,
272                    };
273                    state.phase = WorkflowCheckpointPhase::Waiting {
274                        step: node.id.clone(),
275                        wait: wait.clone(),
276                    };
277                    state.usage = run.budget().usage();
278                    save_checkpoint(checkpoint, state).await?;
279                    record_domain(
280                        run,
281                        "workflow.interrupted",
282                        serde_json::json!({
283                            "workflow": self.name,
284                            "step": node.id,
285                            "wait": wait,
286                        }),
287                        caused_by,
288                    )?;
289                    return Ok(WorkflowExecution::Suspended(wait));
290                }
291                _ => {
292                    self.execute_serial_node(node, state, run, caused_by, checkpoint)
293                        .await?
294                }
295            };
296            commit_node(state, &node.id, output, run, checkpoint).await?;
297        }
298
299        let outcome = WorkflowOutcome {
300            output: state.value.clone(),
301            steps: state.outputs.clone(),
302            usage: run.budget().usage(),
303        };
304        state.usage = outcome.usage;
305        state.phase = WorkflowCheckpointPhase::Completed {
306            outcome: outcome.clone(),
307        };
308        save_checkpoint(checkpoint, state).await?;
309        Ok(WorkflowExecution::Completed(outcome))
310    }
311
312    async fn execute_serial_node(
313        &self,
314        node: &crate::workflow::WorkflowNode,
315        state: &mut WorkflowCheckpointState,
316        run: &RunContext,
317        caused_by: Option<EventId>,
318        checkpoint: &mut Option<&mut WorkflowCheckpointCursor>,
319    ) -> Result<Value, WorkflowError> {
320        state.phase = WorkflowCheckpointPhase::StepInFlight {
321            step: node.id.clone(),
322        };
323        state.usage = run.budget().usage();
324        save_checkpoint(checkpoint, state).await?;
325
326        let step_started = record_domain(
327            run,
328            "step.started",
329            serde_json::json!({
330                "workflow": self.name,
331                "step": node.id,
332                "index": state.next_index,
333            }),
334            caused_by,
335        )?;
336        let mut child = run.child(node.capabilities.clone()).map_err(|error| {
337            WorkflowError::AuthorityEscalation {
338                step: node.id.clone(),
339                capability: error.capability,
340            }
341        })?;
342        if let Some(event_id) = step_started {
343            child = child.with_cause(event_id);
344        }
345        run.record(
346            RunEventKind::Child(ChildEvent::Started {
347                child_run_id: child.run_id(),
348            }),
349            step_started,
350        )?;
351
352        let execution = node.execute(state.value.clone(), &child).await;
353        let (output, branch) = match execution {
354            Ok(result) => result,
355            Err(source) => {
356                run.record(
357                    RunEventKind::Child(ChildEvent::Failed {
358                        child_run_id: child.run_id(),
359                    }),
360                    step_started,
361                )?;
362                record_domain(
363                    run,
364                    "step.failed",
365                    serde_json::json!({
366                        "workflow": self.name,
367                        "step": node.id,
368                    }),
369                    step_started,
370                )?;
371                return Err(WorkflowError::Step {
372                    step: node.id.clone(),
373                    source: Box::new(source),
374                });
375            }
376        };
377
378        run.record(
379            RunEventKind::Child(ChildEvent::Completed {
380                child_run_id: child.run_id(),
381            }),
382            step_started,
383        )?;
384        record_domain(
385            run,
386            "step.completed",
387            serde_json::json!({
388                "workflow": self.name,
389                "step": node.id,
390                "branch": branch,
391            }),
392            step_started,
393        )?;
394
395        Ok(output)
396    }
397
398    async fn execute_parallel_node(
399        &self,
400        node: &crate::workflow::WorkflowNode,
401        branches: &[crate::ParallelBranch],
402        state: &mut WorkflowCheckpointState,
403        run: &RunContext,
404        caused_by: Option<EventId>,
405        checkpoint: &mut Option<&mut WorkflowCheckpointCursor>,
406    ) -> Result<Value, WorkflowError> {
407        let step_started = record_domain(
408            run,
409            "step.started",
410            serde_json::json!({
411                "workflow": self.name,
412                "step": node.id,
413                "index": state.next_index,
414                "kind": "parallel",
415            }),
416            caused_by,
417        )?;
418        let result = execute_parallel(
419            &self.name,
420            node,
421            branches,
422            state,
423            run,
424            step_started,
425            checkpoint,
426        )
427        .await;
428        let output = match result {
429            Ok(output) => output,
430            Err(error) => {
431                record_domain(
432                    run,
433                    "step.failed",
434                    serde_json::json!({
435                        "workflow": self.name,
436                        "step": node.id,
437                    }),
438                    step_started,
439                )?;
440                return Err(error);
441            }
442        };
443        record_domain(
444            run,
445            "step.completed",
446            serde_json::json!({
447                "workflow": self.name,
448                "step": node.id,
449                "kind": "parallel",
450            }),
451            step_started,
452        )?;
453        Ok(output)
454    }
455
456    async fn execute_race_node(
457        &self,
458        node: &crate::workflow::WorkflowNode,
459        branches: &[crate::ParallelBranch],
460        state: &mut WorkflowCheckpointState,
461        run: &RunContext,
462        caused_by: Option<EventId>,
463        checkpoint: &mut Option<&mut WorkflowCheckpointCursor>,
464    ) -> Result<Value, WorkflowError> {
465        let step_started = record_domain(
466            run,
467            "step.started",
468            serde_json::json!({
469                "workflow": self.name,
470                "step": node.id,
471                "index": state.next_index,
472                "kind": "race",
473            }),
474            caused_by,
475        )?;
476        let result = execute_race(
477            &self.name,
478            node,
479            branches,
480            state,
481            run,
482            step_started,
483            checkpoint,
484        )
485        .await;
486        match result {
487            Ok(output) => {
488                record_domain(
489                    run,
490                    "step.completed",
491                    serde_json::json!({
492                        "workflow": self.name,
493                        "step": node.id,
494                        "kind": "race",
495                    }),
496                    step_started,
497                )?;
498                Ok(output)
499            }
500            Err(error) => {
501                record_domain(
502                    run,
503                    "step.failed",
504                    serde_json::json!({
505                        "workflow": self.name,
506                        "step": node.id,
507                        "kind": "race",
508                    }),
509                    step_started,
510                )?;
511                Err(error)
512            }
513        }
514    }
515
516    fn validate_authority(&self, run: &RunContext) -> Result<(), WorkflowError> {
517        for node in self.nodes.iter() {
518            match &node.kind {
519                WorkflowNodeKind::Parallel(branches) | WorkflowNodeKind::Race(branches) => {
520                    for branch in branches.iter() {
521                        if let Some(missing) =
522                            branch.capabilities.first_missing_from(run.capabilities())
523                        {
524                            return Err(WorkflowError::AuthorityEscalation {
525                                step: node.id.clone(),
526                                capability: missing.name.clone(),
527                            });
528                        }
529                    }
530                }
531                WorkflowNodeKind::Repairable(repairable) => {
532                    if let Some(missing) = node
533                        .capabilities
534                        .first_missing_from(run.capabilities())
535                        .or_else(|| {
536                            repairable
537                                .reviewer_capabilities
538                                .first_missing_from(run.capabilities())
539                        })
540                    {
541                        return Err(WorkflowError::AuthorityEscalation {
542                            step: node.id.clone(),
543                            capability: missing.name.clone(),
544                        });
545                    }
546                }
547                _ => {
548                    if let Some(missing) = node.capabilities.first_missing_from(run.capabilities())
549                    {
550                        return Err(WorkflowError::AuthorityEscalation {
551                            step: node.id.clone(),
552                            capability: missing.name.clone(),
553                        });
554                    }
555                }
556            }
557        }
558        Ok(())
559    }
560
561    fn validate_checkpoint_identity(
562        &self,
563        state: &WorkflowCheckpointState,
564    ) -> Result<(), WorkflowError> {
565        let layout_matches = self.step_ids().eq(state.layout.iter());
566        let completed_layout = &state.layout[..state.next_index.min(state.layout.len())];
567        let outputs_match = state.outputs.len() == state.next_index
568            && completed_layout
569                .iter()
570                .all(|step| state.outputs.contains_key(step));
571        let phase_matches = match &state.phase {
572            WorkflowCheckpointPhase::Ready => state.next_index <= self.nodes.len(),
573            WorkflowCheckpointPhase::StepInFlight { step } => self
574                .nodes
575                .get(state.next_index)
576                .is_some_and(|node| node.id == *step),
577            WorkflowCheckpointPhase::Remediating {
578                step,
579                attempt,
580                original_input,
581                checkpoint,
582            } => {
583                let valid_attempt = *attempt > 0;
584                let input_matches = *original_input == state.value;
585                let node_matches = self.nodes.get(state.next_index).is_some_and(|node| {
586                    node.id == *step && matches!(node.kind, WorkflowNodeKind::Repairable(_))
587                });
588                let checkpoint_matches = match checkpoint {
589                    crate::WorkflowRemediationCheckpoint::GenerationReady { input }
590                    | crate::WorkflowRemediationCheckpoint::GenerationInFlight { input } => {
591                        *attempt > 1 || *input == *original_input
592                    }
593                    crate::WorkflowRemediationCheckpoint::ReviewReady { .. }
594                    | crate::WorkflowRemediationCheckpoint::ReviewInFlight { .. }
595                    | crate::WorkflowRemediationCheckpoint::Approved { .. }
596                    | crate::WorkflowRemediationCheckpoint::Rejected { .. }
597                    | crate::WorkflowRemediationCheckpoint::Exhausted { .. } => true,
598                };
599                valid_attempt && input_matches && node_matches && checkpoint_matches
600            }
601            WorkflowCheckpointPhase::Waiting { step, wait } => {
602                self.nodes.get(state.next_index).is_some_and(|node| {
603                    if node.id != *step {
604                        return false;
605                    }
606                    matches!(
607                        &node.kind,
608                        WorkflowNodeKind::Timer(expected)
609                            | WorkflowNodeKind::Signal(expected)
610                            | WorkflowNodeKind::SignalOrTimeout(expected) if expected == wait
611                    ) || matches!(
612                        (&node.kind, wait),
613                        (
614                            WorkflowNodeKind::Interrupt(prompt),
615                            WorkflowWait::Interrupt { request }
616                        ) if request.prompt == *prompt && request.proposal == state.value
617                    )
618                })
619            }
620            WorkflowCheckpointPhase::ParallelInFlight { step, branches } => {
621                self.nodes.get(state.next_index).is_some_and(|node| {
622                    node.id == *step && parallel_layout_matches(&node.kind, branches)
623                })
624            }
625            WorkflowCheckpointPhase::RaceInFlight { step, branches } => self
626                .nodes
627                .get(state.next_index)
628                .is_some_and(|node| node.id == *step && race_layout_matches(&node.kind, branches)),
629            WorkflowCheckpointPhase::Completed { outcome } => {
630                state.next_index == self.nodes.len()
631                    && outcome.output == state.value
632                    && outcome.steps == state.outputs
633                    && outcome.usage == state.usage
634            }
635        };
636        if state.workflow != self.name
637            || state.workflow_version != self.version
638            || !layout_matches
639            || state.next_index > self.nodes.len()
640            || !outputs_match
641            || !phase_matches
642        {
643            return Err(WorkflowError::CheckpointIdentityMismatch);
644        }
645        Ok(())
646    }
647}
648
649async fn commit_node(
650    state: &mut WorkflowCheckpointState,
651    step: &StepId,
652    output: Value,
653    run: &RunContext,
654    checkpoint: &mut Option<&mut WorkflowCheckpointCursor>,
655) -> Result<(), WorkflowError> {
656    state.outputs.insert(step.clone(), output.clone());
657    state.value = output;
658    state.next_index += 1;
659    state.usage = run.budget().usage();
660    state.phase = WorkflowCheckpointPhase::Ready;
661    save_checkpoint(checkpoint, state).await
662}
663
664fn wake_output(
665    wait: &WorkflowWait,
666    wake: WorkflowWake,
667    current: &Value,
668) -> Result<Value, WorkflowError> {
669    match (wait, wake) {
670        (WorkflowWait::Timer { .. }, WorkflowWake::Timer) => Ok(current.clone()),
671        (WorkflowWait::Signal { .. }, WorkflowWake::Signal { payload, .. }) => Ok(payload),
672        (
673            WorkflowWait::SignalOrTimeout { .. },
674            WorkflowWake::Signal {
675                signal_id,
676                name,
677                payload,
678            },
679        ) => Ok(serde_json::to_value(WorkflowWaitOutcome::Signal {
680            signal_id,
681            name,
682            payload,
683        })?),
684        (WorkflowWait::SignalOrTimeout { .. }, WorkflowWake::Timeout) => {
685            Ok(serde_json::to_value(WorkflowWaitOutcome::TimedOut)?)
686        }
687        (WorkflowWait::Interrupt { request }, WorkflowWake::Signal { name, payload, .. })
688            if name == request.signal_name() =>
689        {
690            let decision: WorkflowInterruptDecision = serde_json::from_value(payload)?;
691            decision.validate()?;
692            let outcome = match decision {
693                WorkflowInterruptDecision::Approve => WorkflowInterruptOutcome::Approved {
694                    value: request.proposal.clone(),
695                },
696                WorkflowInterruptDecision::Edit { value } => {
697                    WorkflowInterruptOutcome::Edited { value }
698                }
699                WorkflowInterruptDecision::Reject { reason } => {
700                    WorkflowInterruptOutcome::Rejected { reason }
701                }
702            };
703            Ok(serde_json::to_value(outcome)?)
704        }
705        _ => Err(WorkflowError::WakeMismatch),
706    }
707}
708
709fn parallel_layout_matches(
710    kind: &WorkflowNodeKind,
711    checkpoint_branches: &BTreeMap<StepId, ParallelBranchCheckpoint>,
712) -> bool {
713    let WorkflowNodeKind::Parallel(branches) = kind else {
714        return false;
715    };
716    branches.len() == checkpoint_branches.len()
717        && branches.iter().all(|branch| {
718            checkpoint_branches
719                .keys()
720                .any(|checkpoint| checkpoint.as_str() == branch.id)
721        })
722}
723
724fn race_layout_matches(
725    kind: &WorkflowNodeKind,
726    checkpoint_branches: &BTreeMap<StepId, ParallelBranchCheckpoint>,
727) -> bool {
728    let WorkflowNodeKind::Race(branches) = kind else {
729        return false;
730    };
731    let completed = checkpoint_branches
732        .values()
733        .filter(|branch| matches!(branch, ParallelBranchCheckpoint::Completed { .. }))
734        .count();
735    let winner_is_terminal = completed == 0
736        || checkpoint_branches.values().all(|branch| {
737            matches!(
738                branch,
739                ParallelBranchCheckpoint::Completed { .. }
740                    | ParallelBranchCheckpoint::Failed { .. }
741                    | ParallelBranchCheckpoint::Cancelled
742            )
743        });
744    completed <= 1
745        && winner_is_terminal
746        && branches.len() == checkpoint_branches.len()
747        && branches.iter().all(|branch| {
748            checkpoint_branches
749                .keys()
750                .any(|checkpoint| checkpoint.as_str() == branch.id)
751        })
752}
753
754pub(crate) async fn save_checkpoint(
755    checkpoint: &mut Option<&mut WorkflowCheckpointCursor>,
756    state: &WorkflowCheckpointState,
757) -> Result<(), WorkflowError> {
758    if let Some(checkpoint) = checkpoint.as_deref_mut() {
759        checkpoint.save(state).await?;
760    }
761    Ok(())
762}
763
764pub(crate) fn check_lifecycle(run: &RunContext) -> Result<(), WorkflowError> {
765    if run.cancellation().is_cancelled() {
766        return Err(WorkflowError::Cancelled);
767    }
768    if run
769        .deadline()
770        .is_some_and(|deadline| deadline <= Instant::now())
771    {
772        return Err(WorkflowError::DeadlineExceeded);
773    }
774    Ok(())
775}
776
777pub(crate) fn record_domain(
778    run: &RunContext,
779    name: &str,
780    payload: Value,
781    caused_by: Option<EventId>,
782) -> Result<Option<EventId>, WorkflowError> {
783    Ok(run
784        .record(
785            RunEventKind::Domain(DomainEvent {
786                namespace: "runifold.workflow".into(),
787                name: name.into(),
788                payload,
789            }),
790            caused_by,
791        )?
792        .map(|event| event.meta.event_id))
793}
794
795fn terminal_event(
796    workflow: &str,
797    result: &Result<WorkflowExecution, WorkflowError>,
798) -> RunEventKind {
799    match result {
800        Ok(WorkflowExecution::Completed(outcome)) => {
801            RunEventKind::Lifecycle(LifecycleEvent::Completed {
802                output: serde_json::json!({
803                    "workflow": workflow,
804                    "steps": outcome.steps.len(),
805                    "usage": outcome.usage,
806                }),
807            })
808        }
809        Ok(WorkflowExecution::Suspended(wait)) => {
810            RunEventKind::Lifecycle(LifecycleEvent::Completed {
811                output: serde_json::json!({
812                    "workflow": workflow,
813                    "state": "suspended",
814                    "wait": wait,
815                }),
816            })
817        }
818        Err(WorkflowError::Cancelled) => RunEventKind::Lifecycle(LifecycleEvent::Cancelled),
819        Err(error) => RunEventKind::Lifecycle(LifecycleEvent::Failed {
820            error: workflow_run_error(error),
821        }),
822    }
823}
824
825fn workflow_run_error(error: &WorkflowError) -> RunError {
826    let (kind, retry_safety) = match error {
827        WorkflowError::AuthorityEscalation { .. } => {
828            (RunErrorKind::CapabilityDenied, RetrySafety::Safe)
829        }
830        WorkflowError::Cancelled => (RunErrorKind::Cancelled, RetrySafety::Safe),
831        WorkflowError::DeadlineExceeded => (RunErrorKind::DeadlineExceeded, RetrySafety::Unknown),
832        WorkflowError::DurableWaitRequiresWorker | WorkflowError::WakeMismatch => {
833            (RunErrorKind::InvalidInput, RetrySafety::Safe)
834        }
835        WorkflowError::Budget(_) => (RunErrorKind::BudgetExceeded, RetrySafety::Safe),
836        WorkflowError::Build(_)
837        | WorkflowError::Wait(_)
838        | WorkflowError::CheckpointIdentityMismatch
839        | WorkflowError::CheckpointUsageMismatch => (RunErrorKind::InvalidInput, RetrySafety::Safe),
840        WorkflowError::AmbiguousCheckpoint { .. }
841        | WorkflowError::Serialization(_)
842        | WorkflowError::Step { .. }
843        | WorkflowError::Review { .. }
844        | WorkflowError::RemediationRejected { .. }
845        | WorkflowError::RemediationExhausted { .. }
846        | WorkflowError::ParallelBranch { .. }
847        | WorkflowError::RaceAllFailed { .. }
848        | WorkflowError::ChildRun(_)
849        | WorkflowError::Journal(_)
850        | WorkflowError::Checkpoint(_) => (RunErrorKind::Invocation, RetrySafety::Unknown),
851    };
852    RunError {
853        kind,
854        message: error.to_string(),
855        retry_safety,
856        metadata: BTreeMap::new(),
857    }
858}
859
860pub(crate) fn validate_exact_usage(expected: Usage, actual: Usage) -> Result<(), WorkflowError> {
861    if expected != actual {
862        return Err(WorkflowError::CheckpointUsageMismatch);
863    }
864    Ok(())
865}
866
867pub(crate) fn validate_usage_floor(floor: Usage, actual: Usage) -> Result<(), WorkflowError> {
868    let covers = actual.tokens >= floor.tokens
869        && actual.cost_microusd >= floor.cost_microusd
870        && actual.duration_micros >= floor.duration_micros
871        && actual.turns >= floor.turns
872        && actual.tool_calls >= floor.tool_calls
873        && actual.delegations >= floor.delegations;
874    if !covers {
875        return Err(WorkflowError::CheckpointUsageMismatch);
876    }
877    Ok(())
878}