1use std::future::Future;
4use std::ops::ControlFlow;
5use std::sync::Arc;
6
7use backon::{BlockingRetryable, Retryable};
8use bytes::Bytes;
9use sayiir_core::codec::EnvelopeCodec;
10use sayiir_core::error::{BoxError, WorkflowError};
11use sayiir_core::snapshot::ExecutionPosition;
12use sayiir_core::workflow::WorkflowContinuation;
13use sayiir_persistence::SignalStore;
14
15use crate::error::RuntimeError;
16
17use super::control_flow::{
18 ParkReason, StepOutcome, StepResult, compute_signal_timeout, compute_wake_at,
19 save_park_checkpoint,
20};
21use super::fork::{
22 JoinResolution, collect_cached_branches, execute_fork_branches_sequential, resolve_join,
23 settle_fork_outcome,
24};
25use super::helpers::{
26 TaskStepParams, check_guards, execute_task_step, policy_to_backoff, resolve_branch,
27};
28use super::loop_runner::{
29 LoopConfig, LoopExit, LoopNext, NoHooks, resolve_loop_iteration, run_loop_async,
30};
31
32#[allow(clippy::too_many_lines)]
47pub fn execute_continuation_sync<F, E>(
48 continuation: &WorkflowContinuation,
49 input: Bytes,
50 execute_task: &F,
51 envelope_codec: &E,
52) -> Result<Bytes, RuntimeError>
53where
54 F: Fn(&str, Bytes) -> Result<Bytes, BoxError>,
55 E: EnvelopeCodec,
56{
57 let mut current = continuation;
58 let mut current_input = input;
59
60 loop {
61 match current {
62 WorkflowContinuation::Task {
63 id,
64 retry_policy,
65 next,
66 ..
67 } => {
68 let output = (|| execute_task(id, current_input.clone()))
69 .retry(policy_to_backoff(retry_policy.as_ref()))
70 .sleep(std::thread::sleep)
71 .notify(|e, dur: std::time::Duration| {
72 tracing::info!(
73 task_id = %id,
74 delay_ms = dur.as_millis(),
75 error = %e,
76 "Retrying task (sync)"
77 );
78 })
79 .call()
80 .map_err(RuntimeError::from)?;
81
82 match next {
83 Some(next_cont) => {
84 current = next_cont;
85 current_input = output;
86 }
87 None => return Ok(output),
88 }
89 }
90 WorkflowContinuation::Fork { branches, join, .. } => {
91 let mut branch_results = Vec::with_capacity(branches.len());
93
94 for branch in branches {
95 let branch_id = branch.id().to_string();
96 let output = execute_continuation_sync(
97 branch,
98 current_input.clone(),
99 execute_task,
100 envelope_codec,
101 )?;
102 branch_results.push((branch_id, output));
103 }
104
105 match resolve_join(join.as_deref(), &branch_results, envelope_codec)? {
106 JoinResolution::Continue { next, input } => {
107 current = next;
108 current_input = input;
109 }
110 JoinResolution::Done(output) => return Ok(output),
111 }
112 }
113 WorkflowContinuation::Delay { duration, next, .. } => {
114 std::thread::sleep(*duration);
115 match next {
116 Some(next_cont) => {
117 current = next_cont;
118 }
119 None => return Ok(current_input),
120 }
121 }
122 WorkflowContinuation::AwaitSignal { id, .. } => {
123 return Err(WorkflowError::ResumeError(format!(
125 "AwaitSignal '{id}' not supported in sync executor"
126 ))
127 .into());
128 }
129 WorkflowContinuation::Branch {
130 id,
131 branches,
132 default,
133 next,
134 ..
135 } => {
136 let key_bytes =
137 execute_task(&sayiir_core::workflow::key_fn_id(id), current_input.clone())?;
138 let (key, chosen) =
139 resolve_branch(id, &key_bytes, branches, default.as_deref(), envelope_codec)?;
140 let branch_output = execute_continuation_sync(
141 chosen,
142 current_input.clone(),
143 execute_task,
144 envelope_codec,
145 )?;
146 let envelope_bytes = envelope_codec
147 .encode_branch_envelope(&key, &branch_output)
148 .map_err(RuntimeError::from)?;
149
150 match next {
151 Some(next_cont) => {
152 current = next_cont;
153 current_input = envelope_bytes;
154 }
155 None => return Ok(envelope_bytes),
156 }
157 }
158 WorkflowContinuation::Loop {
159 id,
160 body,
161 max_iterations,
162 on_max,
163 next,
164 } => {
165 let cfg = LoopConfig {
166 id,
167 body,
168 max_iterations: *max_iterations,
169 on_max: *on_max,
170 start_iteration: 0,
171 };
172 let mut loop_input = current_input.clone();
173 let mut exited = false;
174 for iteration in 0..cfg.max_iterations {
175 let output = execute_continuation_sync(
176 body,
177 loop_input.clone(),
178 execute_task,
179 envelope_codec,
180 )?;
181 match resolve_loop_iteration(&output, iteration, &cfg)? {
182 ControlFlow::Break(LoopExit(inner)) => match next {
183 Some(next_cont) => {
184 current = next_cont;
185 current_input = inner;
186 exited = true;
187 break;
188 }
189 None => return Ok(inner),
190 },
191 ControlFlow::Continue(LoopNext(inner)) => {
192 loop_input = inner;
193 }
194 }
195 }
196 if !exited {
197 return Err(WorkflowError::MaxIterationsExceeded {
202 loop_id: sayiir_core::TaskId::from(id),
203 max_iterations: *max_iterations,
204 }
205 .into());
206 }
207 }
208 WorkflowContinuation::ChildWorkflow { child, next, .. } => {
209 let output = execute_continuation_sync(
210 child,
211 current_input.clone(),
212 execute_task,
213 envelope_codec,
214 )?;
215 match next {
216 Some(next_cont) => {
217 current = next_cont;
218 current_input = output;
219 }
220 None => return Ok(output),
221 }
222 }
223 }
224 }
225}
226
227pub async fn execute_continuation_async<E: EnvelopeCodec + Clone + 'static>(
241 continuation: &WorkflowContinuation,
242 input: Bytes,
243 envelope_codec: &E,
244) -> Result<Bytes, RuntimeError> {
245 execute_async_inner(continuation, input, true, envelope_codec).await
246}
247
248async fn run_task_with_retry(
253 id: &str,
254 input: Bytes,
255 func: &dyn sayiir_core::task::CoreTask<
256 Input = Bytes,
257 Output = Bytes,
258 Future = sayiir_core::task::BytesFuture,
259 >,
260 timeout: Option<&std::time::Duration>,
261 retry_policy: Option<&sayiir_core::task::RetryPolicy>,
262) -> Result<Bytes, RuntimeError> {
263 (|| async {
264 let task_input = input.clone();
265 if let Some(d) = timeout {
266 match tokio::time::timeout(*d, func.run(task_input)).await {
267 Ok(result) => result.map_err(RuntimeError::from),
268 Err(_) => Err(WorkflowError::TaskTimedOut {
269 task_id: sayiir_core::TaskId::from(id),
270 timeout: *d,
271 }
272 .into()),
273 }
274 } else {
275 func.run(task_input).await.map_err(RuntimeError::from)
276 }
277 })
278 .retry(policy_to_backoff(retry_policy))
279 .notify(|e, dur: std::time::Duration| {
280 tracing::info!(
281 task_id = %id,
282 delay_ms = dur.as_millis(),
283 error = %e,
284 "Retrying task"
285 );
286 })
287 .await
288}
289
290#[allow(clippy::too_many_lines)]
298fn execute_async_inner<'a, E: EnvelopeCodec + Clone + 'static>(
299 continuation: &'a WorkflowContinuation,
300 input: Bytes,
301 parallel_branches: bool,
302 envelope_codec: &'a E,
303) -> std::pin::Pin<Box<dyn Future<Output = Result<Bytes, RuntimeError>> + Send + 'a>> {
304 Box::pin(async move {
305 let mut current = continuation;
306 let mut current_input = input;
307
308 loop {
309 match current {
310 WorkflowContinuation::Task {
311 id,
312 func: Some(func),
313 timeout,
314 retry_policy,
315 next,
316 ..
317 } => {
318 let output = run_task_with_retry(
319 id,
320 current_input.clone(),
321 func.as_ref(),
322 timeout.as_ref(),
323 retry_policy.as_ref(),
324 )
325 .await?;
326
327 match next {
328 Some(next_cont) => {
329 current = next_cont;
330 current_input = output;
331 }
332 None => return Ok(output),
333 }
334 }
335 WorkflowContinuation::Task { func: None, id, .. } => {
336 return Err(WorkflowError::TaskNotImplemented(id.clone()).into());
337 }
338 WorkflowContinuation::Delay { duration, next, .. } => {
339 tokio::time::sleep(*duration).await;
340 match next {
341 Some(next_cont) => {
342 current = next_cont;
343 }
344 None => return Ok(current_input),
345 }
346 }
347 WorkflowContinuation::AwaitSignal { id, .. } => {
348 return Err(WorkflowError::ResumeError(format!(
350 "AwaitSignal '{id}' not supported in non-durable async executor"
351 ))
352 .into());
353 }
354 WorkflowContinuation::Branch {
355 id,
356 key_fn: Some(key_fn),
357 branches,
358 default,
359 next,
360 } => {
361 let key_bytes = key_fn
362 .run(current_input.clone())
363 .await
364 .map_err(RuntimeError::from)?;
365 let (key, chosen) = resolve_branch(
366 id,
367 &key_bytes,
368 branches,
369 default.as_deref(),
370 envelope_codec,
371 )?;
372 let branch_output =
373 execute_async_inner(chosen, current_input.clone(), false, envelope_codec)
374 .await?;
375 let envelope_bytes = envelope_codec
376 .encode_branch_envelope(&key, &branch_output)
377 .map_err(RuntimeError::from)?;
378
379 match next {
380 Some(next_cont) => {
381 current = next_cont;
382 current_input = envelope_bytes;
383 }
384 None => return Ok(envelope_bytes),
385 }
386 }
387 WorkflowContinuation::Branch {
388 key_fn: None, id, ..
389 } => {
390 return Err(WorkflowError::TaskNotImplemented(
391 sayiir_core::workflow::key_fn_id(id),
392 )
393 .into());
394 }
395 WorkflowContinuation::Loop {
396 id,
397 body,
398 max_iterations,
399 on_max,
400 next,
401 } => {
402 let cfg = LoopConfig {
403 id,
404 body,
405 max_iterations: *max_iterations,
406 on_max: *on_max,
407 start_iteration: 0,
408 };
409 let output = run_loop_async(
410 &cfg,
411 current_input.clone(),
412 |input| execute_async_inner(body, input, false, envelope_codec),
413 &mut NoHooks,
414 )
415 .await?;
416 match next {
417 Some(next_cont) => {
418 current = next_cont;
419 current_input = output;
420 }
421 None => return Ok(output),
422 }
423 }
424 WorkflowContinuation::ChildWorkflow { child, next, .. } => {
425 let output =
426 execute_async_inner(child, current_input.clone(), false, envelope_codec)
427 .await?;
428 match next {
429 Some(next_cont) => {
430 current = next_cont;
431 current_input = output;
432 }
433 None => return Ok(output),
434 }
435 }
436 WorkflowContinuation::Fork { branches, join, .. } => {
437 let branch_results = if parallel_branches && branches.len() > 1 {
438 let mut set = tokio::task::JoinSet::new();
440 for branch in branches {
441 let branch_id = branch.id().to_string();
442 let branch = Arc::clone(branch);
443 let branch_input = current_input.clone();
444 let branch_codec = envelope_codec.clone();
445 set.spawn(async move {
446 execute_async_inner(&branch, branch_input, false, &branch_codec)
447 .await
448 .map(|output| (branch_id, output))
449 });
450 }
451
452 let mut results = Vec::with_capacity(set.len());
453 while let Some(res) = set.join_next().await {
454 results.push(res??);
455 }
456 results
457 } else {
458 let mut results = Vec::with_capacity(branches.len());
460 for branch in branches {
461 let branch_id = branch.id().to_string();
462 let output = execute_async_inner(
463 branch,
464 current_input.clone(),
465 false,
466 envelope_codec,
467 )
468 .await?;
469 results.push((branch_id, output));
470 }
471 results
472 };
473
474 match resolve_join(join.as_deref(), &branch_results, envelope_codec)? {
475 JoinResolution::Continue { next, input } => {
476 current = next;
477 current_input = input;
478 }
479 JoinResolution::Done(output) => return Ok(output),
480 }
481 }
482 }
483 }
484 })
485}
486
487#[allow(clippy::too_many_lines)]
509#[tracing::instrument(
510 name = "workflow",
511 skip_all,
512 fields(instance_id = %snapshot.instance_id),
513)]
514pub async fn execute_continuation_with_checkpointing<F, Fut, B, E>(
515 continuation: &WorkflowContinuation,
516 input: Bytes,
517 snapshot: &mut sayiir_core::snapshot::WorkflowSnapshot,
518 backend: &B,
519 execute_task: &F,
520 envelope_codec: &E,
521) -> Result<Bytes, RuntimeError>
522where
523 B: SignalStore,
524 F: Fn(&str, Bytes) -> Fut + Send + Sync,
525 Fut: Future<Output = Result<Bytes, BoxError>> + Send,
526 E: EnvelopeCodec,
527{
528 tracing::debug!("executing workflow with checkpointing");
529 let mut current = continuation;
530 let mut current_input = input;
531
532 loop {
533 let step: StepResult = match current {
534 WorkflowContinuation::Task {
535 id,
536 timeout,
537 retry_policy,
538 next,
539 ..
540 } => {
541 let task_params = TaskStepParams {
542 id,
543 timeout: timeout.as_ref(),
544 retry_policy: retry_policy.as_ref(),
545 next: next.as_deref(),
546 };
547 let output = execute_task_step(
548 &task_params,
549 current_input.clone(),
550 snapshot,
551 backend,
552 |i| execute_task(id, i),
553 )
554 .await?;
555 Ok(ControlFlow::Continue(output))
556 }
557 WorkflowContinuation::Delay { id, duration, next } => {
558 check_guards(
559 backend,
560 &snapshot.instance_id,
561 Some(sayiir_core::TaskId::from(id)),
562 )
563 .await?;
564
565 if snapshot
566 .get_task_result(&sayiir_core::TaskId::from(id))
567 .is_some()
568 {
569 Ok(ControlFlow::Continue(current_input.clone()))
570 } else {
571 let wake_at = compute_wake_at(duration)?;
572 Ok(ControlFlow::Break(StepOutcome::Park(ParkReason::Delay {
573 delay_id: sayiir_core::TaskId::from(id),
574 wake_at,
575 next_task: next.as_deref().map(WorkflowContinuation::first_task_hint),
576 passthrough: current_input.clone(),
577 })))
578 }
579 }
580 WorkflowContinuation::AwaitSignal {
581 id,
582 signal_name,
583 timeout,
584 next,
585 } => {
586 check_guards(
587 backend,
588 &snapshot.instance_id,
589 Some(sayiir_core::TaskId::from(id)),
590 )
591 .await?;
592
593 if snapshot
594 .get_task_result(&sayiir_core::TaskId::from(id))
595 .is_some()
596 {
597 let payload = snapshot
598 .get_task_result_bytes(&sayiir_core::TaskId::from(id))
599 .unwrap_or(current_input.clone());
600 Ok(ControlFlow::Continue(payload))
601 } else {
602 match backend
604 .consume_event(&snapshot.instance_id, signal_name)
605 .await
606 {
607 Ok(Some(payload)) => {
608 snapshot.mark_task_completed(sayiir_core::TaskId::from(id), payload);
609 if let Some(next_cont) = next.as_deref() {
610 let hint = next_cont.first_task_hint();
611 snapshot.update_position(ExecutionPosition::AtTask {
612 task_id: hint.id,
613 });
614 snapshot.set_task_hint(&hint);
615 }
616 backend.save_snapshot(snapshot).await?;
617 let output = snapshot
618 .get_task_result_bytes(&sayiir_core::TaskId::from(id))
619 .unwrap_or(current_input.clone());
620 Ok(ControlFlow::Continue(output))
621 }
622 Ok(None) => Ok(ControlFlow::Break(StepOutcome::Park(
623 ParkReason::AwaitingSignal {
624 signal_id: sayiir_core::TaskId::from(id),
625 signal_name: signal_name.clone(),
626 timeout: compute_signal_timeout(timeout.as_ref()),
627 next_task: next
628 .as_deref()
629 .map(WorkflowContinuation::first_task_hint),
630 },
631 ))),
632 Err(e) => Err(RuntimeError::from(e)),
633 }
634 }
635 }
636 WorkflowContinuation::Fork {
637 id: fork_id,
638 branches,
639 join,
640 } => {
641 check_guards(backend, &snapshot.instance_id, None).await?;
642
643 let branch_results =
644 if let Some(cached) = collect_cached_branches(branches, snapshot) {
645 cached
646 } else {
647 let outcome = execute_fork_branches_sequential(
648 branches,
649 ¤t_input,
650 snapshot,
651 backend,
652 execute_task,
653 envelope_codec,
654 )
655 .await?;
656 settle_fork_outcome(fork_id, outcome, join.as_deref(), snapshot, backend)
657 .await?
658 };
659
660 match resolve_join(join.as_deref(), &branch_results, envelope_codec)? {
661 JoinResolution::Continue { input, .. } => Ok(ControlFlow::Continue(input)),
662 JoinResolution::Done(output) => {
663 Ok(ControlFlow::Break(StepOutcome::Done(output)))
664 }
665 }
666 }
667 WorkflowContinuation::Branch {
668 id,
669 branches,
670 default,
671 ..
672 } => {
673 check_guards(
674 backend,
675 &snapshot.instance_id,
676 Some(sayiir_core::TaskId::from(id)),
677 )
678 .await?;
679
680 if let Some(result) = snapshot.get_task_result(&sayiir_core::TaskId::from(id)) {
681 Ok(ControlFlow::Continue(result.output.clone()))
682 } else {
683 let key_bytes =
684 execute_task(&sayiir_core::workflow::key_fn_id(id), current_input.clone())
685 .await
686 .map_err(RuntimeError::from)?;
687 let (key, chosen) = resolve_branch(
688 id,
689 &key_bytes,
690 branches,
691 default.as_deref(),
692 envelope_codec,
693 )?;
694 let branch_output = super::fork::execute_branch_with_checkpointing(
695 chosen,
696 current_input.clone(),
697 &snapshot.instance_id,
698 backend,
699 execute_task,
700 envelope_codec,
701 )
702 .await?;
703
704 let envelope_bytes = envelope_codec
705 .encode_branch_envelope(&key, &branch_output)
706 .map_err(RuntimeError::from)?;
707
708 snapshot
709 .mark_task_completed(sayiir_core::TaskId::from(id), envelope_bytes.clone());
710 backend.save_snapshot(snapshot).await?;
711
712 Ok(ControlFlow::Continue(envelope_bytes))
713 }
714 }
715 WorkflowContinuation::Loop {
716 id,
717 body,
718 max_iterations,
719 on_max,
720 ..
721 } => {
722 check_guards(
723 backend,
724 &snapshot.instance_id,
725 Some(sayiir_core::TaskId::from(id)),
726 )
727 .await?;
728
729 if let Some(result) = snapshot.get_task_result(&sayiir_core::TaskId::from(id)) {
732 Ok(ControlFlow::Continue(result.output.clone()))
733 } else {
734 let cfg = LoopConfig {
735 id,
736 body,
737 max_iterations: *max_iterations,
738 on_max: *on_max,
739 start_iteration: snapshot.loop_iteration(&sayiir_core::TaskId::from(id)),
740 };
741 let mut loop_input = current_input.clone();
742 let mut final_output = None;
743
744 for iteration in cfg.start_iteration..cfg.max_iterations {
745 let output = Box::pin(execute_continuation_with_checkpointing(
746 body,
747 loop_input.clone(),
748 snapshot,
749 backend,
750 execute_task,
751 envelope_codec,
752 ))
753 .await?;
754
755 let body_ser = body.to_serializable();
756 for tid in &body_ser.task_ids() {
757 snapshot.remove_task_result(&sayiir_core::TaskId::from(*tid));
758 }
759
760 match resolve_loop_iteration(&output, iteration, &cfg)? {
761 ControlFlow::Break(LoopExit(inner)) => {
762 snapshot.clear_loop_iteration(&sayiir_core::TaskId::from(id));
763 snapshot.mark_task_completed(
764 sayiir_core::TaskId::from(id),
765 inner.clone(),
766 );
767 backend.save_snapshot(snapshot).await?;
768 final_output = Some(inner);
769 break;
770 }
771 ControlFlow::Continue(LoopNext(inner)) => {
772 snapshot.set_loop_iteration(
773 sayiir_core::TaskId::from(id),
774 iteration + 1,
775 );
776 snapshot.update_position(ExecutionPosition::InLoop {
777 loop_id: sayiir_core::TaskId::from(id),
778 iteration: iteration + 1,
779 next_task_id: Some(sayiir_core::TaskId::from(
780 body.first_task_id(),
781 )),
782 });
783 backend.save_snapshot(snapshot).await?;
784 loop_input = inner;
785 }
786 }
787 }
788
789 match final_output {
790 Some(output) => Ok(ControlFlow::Continue(output)),
791 None => Err(RuntimeError::from(WorkflowError::MaxIterationsExceeded {
792 loop_id: sayiir_core::TaskId::from(id),
793 max_iterations: *max_iterations,
794 })),
795 }
796 }
797 }
798 WorkflowContinuation::ChildWorkflow { id, child, .. } => {
799 check_guards(
800 backend,
801 &snapshot.instance_id,
802 Some(sayiir_core::TaskId::from(id)),
803 )
804 .await?;
805
806 if let Some(result) = snapshot.get_task_result(&sayiir_core::TaskId::from(id)) {
807 Ok(ControlFlow::Continue(result.output.clone()))
808 } else {
809 let output = Box::pin(execute_continuation_with_checkpointing(
810 child,
811 current_input.clone(),
812 snapshot,
813 backend,
814 execute_task,
815 envelope_codec,
816 ))
817 .await?;
818
819 snapshot.mark_task_completed(sayiir_core::TaskId::from(id), output.clone());
820 backend.save_snapshot(snapshot).await?;
821
822 Ok(ControlFlow::Continue(output))
823 }
824 }
825 };
826
827 match step? {
828 ControlFlow::Continue(output) => match current.get_next() {
829 Some(next) => {
830 current = next;
831 current_input = output;
832 }
833 None => return Ok(output),
834 },
835 ControlFlow::Break(StepOutcome::Done(output)) => return Ok(output),
836 ControlFlow::Break(StepOutcome::Park(reason)) => {
837 return Err(save_park_checkpoint(reason, snapshot, backend).await);
838 }
839 }
840 }
841}