1use std::future::{Future, poll_fn};
2use std::panic::{AssertUnwindSafe, catch_unwind};
3use std::pin::Pin;
4use std::sync::Arc;
5use std::task::{Context, Poll};
6use std::time::Duration;
7
8use agents_kit::AgentBackend;
9use futures_util::{FutureExt, Stream};
10use serde_json::{Map, Value};
11use storage_kit::StateStoreBackend;
12use tokio::sync::{OwnedSemaphorePermit, mpsc, oneshot};
13
14use crate::AppError;
15use crate::app::{AppRuntime, DriverCommand, FinalResult, RunTaskKind};
16use crate::storage::StorageFailure;
17use crate::workflow::action::ActionInputs;
18use crate::workflow::definition::{JsonBoundary, Step};
19use crate::workflow::execution::{
20 BspDecision, ExecutionEngine, StagedActivationRecord, StagedOutcomeRecord,
21};
22use crate::workflow::run::{
23 ActionPosition, CheckpointPhaseRecord, ChildAdmissionReceipt, ControlRecord,
24 ControlRecordState, ControlTransition, FailureCategory, FrontierRecord, LoadedRun,
25 MAX_DRIVER_TRANSITIONS, RECONCILIATION_INITIAL_DELAY, RECONCILIATION_MAX_DELAY, RunAdmission,
26 RunRecord, RuntimeOperationFailure, ScheduleCause, WorkflowRunRuntime,
27};
28use crate::workflow::{
29 ActionBackend, ActionContext, ActionEvent, ActionMode, ActionOutcome, ActionRequest,
30 ChildBatch, ChildCompletion, ChildExecutor, ConditionBackend, ExecutionEvent, ExecutionOptions,
31 InlineExecutor, JsonState, LifecycleEvent, LifecycleHooks, NoActions, NoConditions, NoHooks,
32 RunSnapshot, RunStatus, StateTyping, WorkflowFailure, WorkflowRun, WorkflowState,
33};
34
35#[must_use = "execution owns lazy advancement; finish or abort it explicitly"]
90pub struct Execution<
91 'a,
92 CS: StateStoreBackend,
93 A: AgentBackend,
94 X: ActionBackend = NoActions,
95 C: ConditionBackend = NoConditions,
96 T: StateTyping = JsonState,
97 H: LifecycleHooks = NoHooks,
98 E: ChildExecutor = InlineExecutor,
99> {
100 runtime: WorkflowRunRuntime<CS, A, X, C, T, H, E>,
101 app: &'a AppRuntime<CS, A, X, C, T, H, E>,
102 lane: ExecutionLane,
103 admission: Option<OwnedSemaphorePermit>,
104 children: Vec<ChildExecution<'a, ChildFuture<Self>, E>>,
105 wake: Option<Pin<Box<tokio::sync::futures::OwnedNotified>>>,
106 timer: Option<Pin<Box<tokio::time::Sleep>>>,
107 generation: Option<u64>,
108 parent_deadline: Option<tokio::time::Instant>,
109 state: SessionState,
110 sources: Vec<CallerSource<X>>,
111 next_source: usize,
112 native_partials: Option<(mpsc::Sender<ExecutionEvent>, mpsc::Receiver<ExecutionEvent>)>,
113 prefetched: Option<(usize, Result<Option<ActionEvent>, FailureCategory>)>,
114 transitions: usize,
115 observed: Option<RunRecord>,
116}
117
118#[must_use = "execution owns lazy advancement; finish or abort it explicitly"]
164pub struct SyncExecution<
165 'a,
166 CS: StateStoreBackend,
167 A: AgentBackend,
168 X: ActionBackend = NoActions,
169 C: ConditionBackend = NoConditions,
170 T: StateTyping = JsonState,
171 H: LifecycleHooks = NoHooks,
172 E: ChildExecutor = InlineExecutor,
173> {
174 execution: Execution<'a, CS, A, X, C, T, H, E>,
175 runtime: tokio::runtime::Handle,
176}
177
178impl<
179 'a,
180 CS: StateStoreBackend,
181 A: AgentBackend,
182 X: ActionBackend,
183 C: ConditionBackend,
184 T: StateTyping,
185 H: LifecycleHooks,
186 E: ChildExecutor,
187> SyncExecution<'a, CS, A, X, C, T, H, E>
188{
189 pub fn open(
255 run: &'a mut WorkflowRun<CS, A, X, C, T, H, E>,
256 options: ExecutionOptions,
257 ) -> Result<Self, AppError> {
258 Self::ensure_blocking()?;
259 let runtime = run.runtime.app.execution_handle()?;
260 let execution = runtime.block_on(Execution::open(run, options, ActionMode::Synchronous))?;
261 Ok(Self { execution, runtime })
262 }
263
264 fn ensure_blocking() -> Result<(), AppError> {
265 if tokio::runtime::Handle::try_current().is_ok() {
266 return Err(AppError::InvalidInput(
267 "synchronous execution requires a thread outside an entered Tokio runtime".into(),
268 ));
269 }
270 Ok(())
271 }
272
273 pub fn try_next(&mut self) -> Result<Option<ExecutionEvent>, AppError> {
323 Self::ensure_blocking()?;
324 self.runtime.block_on(self.execution.next())
325 }
326
327 pub fn step(&mut self) -> Result<Option<ExecutionEvent>, AppError> {
377 Self::ensure_blocking()?;
378 self.runtime.block_on(self.execution.step())
379 }
380
381 pub fn finish(self) -> Result<RunSnapshot, AppError> {
432 Self::ensure_blocking()?;
433 self.runtime.block_on(self.execution.finish())
434 }
435
436 pub fn abort(self) -> Result<RunSnapshot, AppError> {
488 Self::ensure_blocking()?;
489 self.runtime.block_on(self.execution.abort())
490 }
491}
492
493struct CallerSource<X: ActionBackend> {
494 activation: FrontierRecord,
495 context: Arc<ActionContext>,
496 deadline: tokio::time::Instant,
497 state: CallerSourceState<X>,
498}
499
500enum CallerSourceState<X: ActionBackend> {
501 Polling(Pin<Box<X::Stream>>),
502 Failed(FailureCategory),
503 Ended,
504 Eof,
505}
506
507impl<X: ActionBackend> CallerSource<X> {
508 fn stop(&mut self) -> Result<(), FailureCategory> {
509 let previous = std::mem::replace(&mut self.state, CallerSourceState::Ended);
510 match previous {
511 CallerSourceState::Polling(source) => catch_unwind(AssertUnwindSafe(|| drop(source)))
512 .map_err(|_| FailureCategory::Execution),
513 previous => {
514 self.state = previous;
515 Ok(())
516 }
517 }
518 }
519
520 fn poll(
521 &mut self,
522 context: &mut Context<'_>,
523 ) -> Poll<Result<Option<ActionEvent>, FailureCategory>> {
524 let event = match &mut self.state {
525 CallerSourceState::Polling(source) => {
526 match catch_unwind(AssertUnwindSafe(|| source.as_mut().poll_next(context))) {
527 Ok(Poll::Pending) => return Poll::Pending,
528 Ok(Poll::Ready(Some(Ok(event)))) => Ok(Some(event)),
529 Ok(Poll::Ready(Some(Err(_)))) => Err(FailureCategory::Action),
530 Ok(Poll::Ready(None)) => Ok(None),
531 Err(_) => Err(FailureCategory::Execution),
532 }
533 }
534 CallerSourceState::Failed(category) => {
535 let category = *category;
536 self.state = CallerSourceState::Ended;
537 return Poll::Ready(Err(category));
538 }
539 CallerSourceState::Ended | CallerSourceState::Eof => return Poll::Pending,
540 };
541 Poll::Ready(event)
542 }
543}
544
545#[derive(Clone, Copy, PartialEq)]
546enum ExecutionLane {
547 Caller,
548 Worker,
549}
550
551struct ChildExecution<'a, F, E>
552where
553 F: Future<Output = Result<ChildCompletion, AppError>> + 'a,
554 E: ChildExecutor,
555{
556 sequence: u64,
557 parent: Option<Arc<RunRecord>>,
558 context: Arc<ActionContext>,
559 lineage: String,
560 deadline: tokio::time::Instant,
561 state: ChildExecutionState<'a, F, E>,
562 inputs: Vec<Map<String, Value>>,
563}
564
565enum ChildExecutionState<'a, F, E>
566where
567 F: Future<Output = Result<ChildCompletion, AppError>> + 'a,
568 E: ChildExecutor,
569{
570 Awaiting,
571 Prepared(Vec<RunRecord>),
572 Local(Pin<Box<E::Execution<'a, F>>>),
573 Pipeline(ChildBatch<F>),
574 Worker(oneshot::Receiver<Result<SessionReceipt, RuntimeOperationFailure>>),
575 Receipt(Box<Result<SessionReceipt, RuntimeOperationFailure>>),
576 Complete(Result<Vec<ChildCompletion>, FailureCategory>),
577 Closed,
578}
579
580impl<
581 'a,
582 CS: StateStoreBackend,
583 A: AgentBackend,
584 X: ActionBackend,
585 C: ConditionBackend,
586 T: StateTyping,
587 H: LifecycleHooks,
588 E: ChildExecutor,
589> ChildExecution<'a, ChildFuture<Execution<'a, CS, A, X, C, T, H, E>>, E>
590{
591 fn start_local(
592 &mut self,
593 runtime: &WorkflowRunRuntime<CS, A, X, C, T, H, E>,
594 app: &'a AppRuntime<CS, A, X, C, T, H, E>,
595 lane: ExecutionLane,
596 children: Vec<RunRecord>,
597 ) -> Result<(), AppError> {
598 if tokio::time::Instant::now() >= self.deadline {
599 runtime.retain_source_failure(&self.context, &self.lineage, FailureCategory::Timeout);
600 self.state = ChildExecutionState::Complete(Err(FailureCategory::Timeout));
601 return Ok(());
602 }
603 let parent = self.parent.as_ref().ok_or_else(|| {
604 AppError::Integration("child group parent evidence is unavailable".into())
605 })?;
606 let definition = app.definition_for(parent)?;
607 let control = parent
608 .checkpoint
609 .control(self.sequence)
610 .ok_or_else(|| AppError::Conflict("child control is unavailable".into()))?;
611 let step = definition
612 .node(&control.node)
613 .ok_or_else(|| AppError::Conflict("child definition changed".into()))?
614 .step();
615 let (parallelism, _) = step
616 .child_plan()
617 .ok_or_else(|| AppError::Conflict("child definition changed".into()))?;
618 let slots = Arc::new(tokio::sync::Semaphore::new(parallelism));
619 let notification = runtime.notifier()?;
620 let mut tasks = Vec::with_capacity(children.len());
621 for record in children {
622 record.validate_child(parent, &definition)?;
623 let child = WorkflowRunRuntime::attach_record(Arc::clone(&runtime.app), &record)?;
624 if !record.checkpoint.is_terminal() {
625 child.cell.bind_child(runtime.key.clone())?;
626 child
627 .cell
628 .retain_child_observation(Arc::clone(&self.context))?;
629 }
630 tasks.push(ChildFuture::new(
631 Execution::owned(child, app, lane),
632 Arc::clone(parent),
633 self.sequence,
634 self.deadline,
635 Arc::clone(&slots),
636 Arc::clone(¬ification),
637 ));
638 }
639 let batch = ChildBatch::new(tasks, self.context.mode)?;
640 self.state = if matches!(step, Step::Pipeline { .. }) {
641 ChildExecutionState::Pipeline(batch)
642 } else {
643 match catch_unwind(AssertUnwindSafe(|| app.child_executor().execute(batch))) {
644 Ok(future) => ChildExecutionState::Local(Box::pin(future)),
645 Err(_) => {
646 runtime.retain_source_failure(
647 &self.context,
648 &self.lineage,
649 FailureCategory::Execution,
650 );
651 ChildExecutionState::Complete(Err(FailureCategory::Execution))
652 }
653 }
654 };
655 Ok(())
656 }
657
658 fn stop(&mut self, runtime: &WorkflowRunRuntime<CS, A, X, C, T, H, E>) {
659 let previous = std::mem::replace(&mut self.state, ChildExecutionState::Closed);
660 if let ChildExecutionState::Complete(Err(category)) = &previous {
661 runtime.retain_source_failure(&self.context, &self.lineage, *category);
662 }
663 if catch_unwind(AssertUnwindSafe(|| drop(previous))).is_err() {
664 runtime.retain_source_failure(&self.context, &self.lineage, FailureCategory::Execution);
667 }
668 }
669
670 fn poll_ready(
671 &mut self,
672 cx: &mut Context<'_>,
673 runtime: &WorkflowRunRuntime<CS, A, X, C, T, H, E>,
674 ) -> Poll<()> {
675 if matches!(
676 self.state,
677 ChildExecutionState::Complete(_)
678 | ChildExecutionState::Receipt(_)
679 | ChildExecutionState::Closed
680 ) {
681 return Poll::Ready(());
682 }
683 if matches!(
684 self.state,
685 ChildExecutionState::Awaiting | ChildExecutionState::Prepared(_)
686 ) {
687 return Poll::Pending;
688 }
689 if matches!(self.state, ChildExecutionState::Worker(_)) {
690 return self.poll_receipt(cx);
691 }
692 let result = if tokio::time::Instant::now() >= self.deadline {
693 Err(FailureCategory::Timeout)
694 } else {
695 match catch_unwind(AssertUnwindSafe(|| match &mut self.state {
696 ChildExecutionState::Local(future) => future.as_mut().poll(cx),
697 ChildExecutionState::Pipeline(batch) => Pin::new(batch).poll(cx),
698 _ => unreachable!(),
699 })) {
700 Ok(Poll::Pending) => return Poll::Pending,
701 Ok(Poll::Ready(result)) => result.map_err(|_| FailureCategory::Child),
702 Err(_) => Err(FailureCategory::Execution),
703 }
704 };
705 self.complete(runtime, result);
706 Poll::Ready(())
707 }
708
709 fn poll_receipt(&mut self, cx: &mut Context<'_>) -> Poll<()> {
710 let ChildExecutionState::Worker(receipt) = &mut self.state else {
711 unreachable!()
712 };
713 match Pin::new(receipt).poll(cx) {
714 Poll::Pending => Poll::Pending,
715 Poll::Ready(result) => {
716 self.state = ChildExecutionState::Receipt(Box::new(
717 result
718 .map_err(|_| {
719 RuntimeOperationFailure::from(AppError::Closed(
720 "child executor receipt closed".into(),
721 ))
722 })
723 .and_then(|result| result),
724 ));
725 Poll::Ready(())
726 }
727 }
728 }
729
730 fn complete(
731 &mut self,
732 runtime: &WorkflowRunRuntime<CS, A, X, C, T, H, E>,
733 result: Result<Vec<ChildCompletion>, FailureCategory>,
734 ) {
735 if let Err(category) = result.as_ref() {
736 runtime.retain_source_failure(&self.context, &self.lineage, *category);
737 }
738 let previous = std::mem::replace(&mut self.state, ChildExecutionState::Complete(result));
739 if catch_unwind(AssertUnwindSafe(|| drop(previous))).is_err()
740 && matches!(&self.state, ChildExecutionState::Complete(Ok(_)))
741 {
742 runtime.retain_source_failure(&self.context, &self.lineage, FailureCategory::Execution);
743 self.state = ChildExecutionState::Complete(Err(FailureCategory::Execution));
744 }
745 }
746}
747
748struct ChildFuture<EX> {
749 execution: EX,
750 parent: Arc<RunRecord>,
751 sequence: u64,
752 slots: Arc<tokio::sync::Semaphore>,
753 permit: Option<tokio::sync::OwnedSemaphorePermit>,
754 notification: Arc<tokio::sync::Notify>,
755 wake: Pin<Box<tokio::sync::futures::OwnedNotified>>,
756 state: ChildFutureState,
757}
758
759enum ChildFutureState {
760 Admission,
761 Driving,
762 Reconcile(Option<oneshot::Receiver<Result<LoadedRun, AppError>>>),
763 Retry(tokio::time::Instant),
764 Finishing,
765 Waiting,
766 Finished,
767}
768
769impl<
770 'a,
771 CS: StateStoreBackend,
772 A: AgentBackend,
773 X: ActionBackend,
774 C: ConditionBackend,
775 T: StateTyping,
776 H: LifecycleHooks,
777 E: ChildExecutor,
778> ChildFuture<Execution<'a, CS, A, X, C, T, H, E>>
779{
780 fn new(
781 mut execution: Execution<'a, CS, A, X, C, T, H, E>,
782 parent: Arc<RunRecord>,
783 sequence: u64,
784 deadline: tokio::time::Instant,
785 slots: Arc<tokio::sync::Semaphore>,
786 notification: Arc<tokio::sync::Notify>,
787 ) -> Self {
788 let wake = Box::pin(Arc::clone(¬ification).notified_owned());
789 execution.parent_deadline = Some(deadline);
790 Self {
791 execution,
792 parent,
793 sequence,
794 slots,
795 permit: None,
796 notification,
797 wake,
798 state: ChildFutureState::Admission,
799 }
800 }
801
802 fn poll_reconciliation(&mut self, cx: &mut Context<'_>) -> Poll<Result<(), AppError>> {
803 let ChildFutureState::Reconcile(receipt) = &mut self.state else {
804 unreachable!()
805 };
806 if receipt.is_none() {
807 match &mut self.execution.state {
811 SessionState::Publishing { receipt, .. } => {
812 if Pin::new(receipt).poll(cx).is_pending() {
813 return Poll::Pending;
814 }
815 }
816 SessionState::ObservingItem { receipt, .. } => {
817 if Pin::new(receipt).poll(cx).is_pending() {
818 return Poll::Pending;
819 }
820 }
821 _ => {}
822 }
823 self.execution.state = SessionState::Finished;
824 *receipt = Some(
825 self.execution
826 .runtime
827 .reconcile_child_panic(self.execution.parent_deadline),
828 );
829 }
830 let loaded = match Pin::new(receipt.as_mut().expect("reconciliation was admitted")).poll(cx)
831 {
832 Poll::Pending => return Poll::Pending,
833 Poll::Ready(Ok(result)) => result?,
834 Poll::Ready(Err(_)) => {
835 self.execution
836 .runtime
837 .app
838 .tasks_failed(FinalResult::Integration(
839 "child panic reconciliation receipt closed".into(),
840 ));
841 return Poll::Ready(Err(AppError::Integration(
842 "child panic reconciliation receipt closed".into(),
843 )));
844 }
845 };
846 self.execution.state = SessionState::Loaded(loaded);
847 self.state = ChildFutureState::Driving;
848 Poll::Ready(Ok(()))
849 }
850
851 fn poll_completion(
852 &mut self,
853 cx: &mut Context<'_>,
854 ) -> Poll<Result<Option<ChildCompletion>, AppError>> {
855 match self.execution.poll_publication(cx, false) {
856 Poll::Pending => Poll::Pending,
857 Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
858 Poll::Ready(Ok(Some(_))) => Poll::Ready(Ok(None)),
859 Poll::Ready(Ok(None)) => {
860 let SessionState::Loaded(loaded) =
861 std::mem::replace(&mut self.execution.state, SessionState::Finished)
862 else {
863 return Poll::Ready(Err(AppError::Integration(
864 "child terminal observation is unavailable".into(),
865 )));
866 };
867 if !loaded.record.checkpoint.is_terminal() {
868 self.execution.state = SessionState::Loaded(loaded);
869 self.state = ChildFutureState::Waiting;
870 return Poll::Ready(Ok(None));
871 }
872 let definition = self.execution.runtime.app.definition_for(&self.parent)?;
873 let result = if matches!(
874 loaded.record.checkpoint.phase,
875 CheckpointPhaseRecord::Succeeded
876 ) {
877 ChildCompletion::admitted(loaded.record, &self.parent, &definition)
878 } else {
879 Err(AppError::Integration(
880 "child workflow did not succeed".into(),
881 ))
882 };
883 Poll::Ready(result.map(Some))
884 }
885 }
886 }
887}
888
889impl<
890 CS: StateStoreBackend,
891 A: AgentBackend,
892 X: ActionBackend,
893 C: ConditionBackend,
894 T: StateTyping,
895 H: LifecycleHooks,
896 E: ChildExecutor,
897> Future for ChildFuture<Execution<'_, CS, A, X, C, T, H, E>>
898{
899 type Output = Result<ChildCompletion, AppError>;
900
901 fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
902 let child = self.get_mut();
903 let observed = (|| -> Poll<Result<ChildCompletion, AppError>> {
904 loop {
905 match child.state {
906 ChildFutureState::Admission => {
907 child.wake.as_mut().enable();
908 match Arc::clone(&child.slots).try_acquire_owned() {
909 Ok(permit) => {
910 child.permit = Some(permit);
911 child.state = ChildFutureState::Driving;
912 }
913 Err(tokio::sync::TryAcquireError::NoPermits) => {
914 if child.wake.as_mut().poll(cx).is_pending() {
915 return Poll::Pending;
916 }
917 child.wake =
918 Box::pin(Arc::clone(&child.notification).notified_owned());
919 }
920 Err(tokio::sync::TryAcquireError::Closed) => {
921 return Poll::Ready(Err(AppError::Closed(
922 "child execution capacity is closed".into(),
923 )));
924 }
925 }
926 }
927 ChildFutureState::Reconcile(_) => match child.poll_reconciliation(cx) {
928 Poll::Pending => return Poll::Pending,
929 Poll::Ready(result) => result?,
930 },
931 ChildFutureState::Retry(deadline) => {
932 if !child.execution.poll_timer(cx, Some(deadline)) {
933 return Poll::Pending;
934 }
935 child.execution.state = SessionState::Ready;
936 child.state = ChildFutureState::Driving;
937 }
938 ChildFutureState::Driving => {
939 let observed = catch_unwind(AssertUnwindSafe(|| {
940 child.execution.poll_advance(cx, true)
941 }));
942 match observed {
943 Err(_) => {
944 child.state = ChildFutureState::Reconcile(None);
947 }
948 Ok(Poll::Pending) => return Poll::Pending,
949 Ok(Poll::Ready(Err(error))) => {
950 if let Some(delay) = child.execution.runtime.cell.complete_drive() {
951 child.state = ChildFutureState::Retry(
952 tokio::time::Instant::now() + delay,
953 );
954 } else {
955 return Poll::Ready(Err(error));
956 }
957 }
958 Ok(Poll::Ready(Ok(Some(_)))) => continue,
959 Ok(Poll::Ready(Ok(None))) => {
960 child.execution.publish(
961 SessionPublication::FinishChild {
962 parent: Arc::clone(&child.parent),
963 sequence: child.sequence,
964 },
965 false,
966 )?;
967 child.state = ChildFutureState::Finishing;
968 }
969 }
970 }
971 ChildFutureState::Finishing => match child.poll_completion(cx) {
972 Poll::Pending => return Poll::Pending,
973 Poll::Ready(Err(error)) => return Poll::Ready(Err(error)),
974 Poll::Ready(Ok(Some(completion))) => return Poll::Ready(Ok(completion)),
975 Poll::Ready(Ok(None)) => continue,
976 },
977 ChildFutureState::Waiting => {
978 if !child.execution.poll_notification(cx)? {
981 return Poll::Pending;
982 }
983 child.execution.state = SessionState::Ready;
984 child.state = ChildFutureState::Driving;
985 }
986 ChildFutureState::Finished => {
987 return Poll::Ready(Err(AppError::Conflict(
988 "child future already completed".into(),
989 )));
990 }
991 }
992 }
993 })();
994 if observed.is_ready() {
995 child.release();
996 child.state = ChildFutureState::Finished;
997 }
998 observed
999 }
1000}
1001
1002impl<EX> ChildFuture<EX> {
1003 fn release(&mut self) {
1004 if self.permit.take().is_some() {
1005 self.notification.notify_waiters();
1006 }
1007 }
1008}
1009
1010impl<EX> Drop for ChildFuture<EX> {
1011 fn drop(&mut self) {
1012 self.release();
1013 }
1014}
1015
1016enum SessionState {
1017 Ready,
1018 Loaded(LoadedRun),
1019 ActivationReady {
1020 loaded: LoadedRun,
1021 activation: FrontierRecord,
1022 context: Arc<ActionContext>,
1023 remaining: std::vec::IntoIter<FrontierRecord>,
1024 staged: Vec<StagedActivationRecord>,
1025 deadline: tokio::time::Instant,
1026 deadline_millis: u64,
1027 },
1028 Sources {
1029 loaded: LoadedRun,
1030 },
1031 ObservingItem {
1032 loaded: LoadedRun,
1033 index: usize,
1034 receipt: oneshot::Receiver<Result<Option<Map<String, Value>>, AppError>>,
1035 },
1036 Publishing {
1037 receipt: oneshot::Receiver<Result<SessionReceipt, RuntimeOperationFailure>>,
1038 purpose: PublicationPurpose,
1039 },
1040 Eof,
1041 Finished,
1042}
1043
1044enum PublicationPurpose {
1045 Load,
1046 Reload,
1047 ObserveChildren,
1048 AdmitChildren,
1049 VerifyChildren,
1050 Start {
1051 activation: FrontierRecord,
1052 context: Arc<ActionContext>,
1053 remaining: std::vec::IntoIter<FrontierRecord>,
1054 deadline: tokio::time::Instant,
1055 deadline_millis: u64,
1056 },
1057 Continue {
1058 remaining: std::vec::IntoIter<FrontierRecord>,
1059 ordinary: bool,
1060 },
1061 Descendants {
1062 context: Arc<ActionContext>,
1063 lineage: String,
1064 inputs: Vec<Map<String, Value>>,
1065 remaining: std::vec::IntoIter<FrontierRecord>,
1066 ordinary: bool,
1067 },
1068 Commit {
1069 ordinary: bool,
1070 },
1071 Advance,
1072 EndSource,
1073}
1074
1075pub(crate) enum SessionReceipt {
1076 Checkpoint {
1077 loaded: LoadedRun,
1078 staged: Vec<StagedActivationRecord>,
1079 },
1080 Children {
1081 loaded: LoadedRun,
1082 sequence: u64,
1083 children: Vec<RunRecord>,
1084 verified: bool,
1085 },
1086}
1087
1088impl SessionReceipt {
1089 fn record(&self) -> &RunRecord {
1090 match self {
1091 Self::Checkpoint { loaded, .. } | Self::Children { loaded, .. } => &loaded.record,
1092 }
1093 }
1094
1095 fn with_loaded(self, loaded: LoadedRun) -> Self {
1096 match self {
1097 Self::Checkpoint { staged, .. } => Self::Checkpoint { loaded, staged },
1098 Self::Children {
1099 sequence,
1100 children,
1101 verified,
1102 ..
1103 } => Self::Children {
1104 loaded,
1105 sequence,
1106 children,
1107 verified,
1108 },
1109 }
1110 }
1111}
1112
1113enum SessionPublication {
1114 ObserveChildren {
1115 loaded: LoadedRun,
1116 sequence: u64,
1117 },
1118 VerifyChildren {
1119 parent: Arc<RunRecord>,
1120 sequence: u64,
1121 completions: Vec<ChildCompletion>,
1122 },
1123 ChildOutcome {
1124 control: ControlRecord,
1125 transition: ControlTransition,
1126 },
1127 FinishChild {
1128 parent: Arc<RunRecord>,
1129 sequence: u64,
1130 },
1131 SettleStopping {
1132 loaded: LoadedRun,
1133 },
1134 Load {
1135 terminal: Option<u64>,
1136 },
1137 Reload {
1138 receipt: SessionReceipt,
1139 deadline: Option<tokio::time::Instant>,
1140 },
1141 BeginActivation {
1142 loaded: LoadedRun,
1143 activation: FrontierRecord,
1144 context: Arc<ActionContext>,
1145 remaining: std::vec::IntoIter<FrontierRecord>,
1146 staged: Vec<StagedActivationRecord>,
1147 deadline: tokio::time::Instant,
1148 deadline_millis: u64,
1149 },
1150 Decision {
1151 loaded: LoadedRun,
1152 decision: BspDecision,
1153 activation_deadline: Option<u64>,
1154 context: Option<Arc<ActionContext>>,
1155 staged: Vec<StagedActivationRecord>,
1156 remaining: std::vec::IntoIter<FrontierRecord>,
1157 },
1158 Transition {
1159 loaded: LoadedRun,
1160 transition: ControlTransition,
1161 },
1162 Retry {
1163 loaded: LoadedRun,
1164 },
1165 EndSource {
1166 loaded: LoadedRun,
1167 },
1168 Release,
1169}
1170
1171impl<
1172 'a,
1173 CS: StateStoreBackend,
1174 A: AgentBackend,
1175 X: ActionBackend,
1176 C: ConditionBackend,
1177 T: StateTyping,
1178 H: LifecycleHooks,
1179 E: ChildExecutor,
1180> Execution<'a, CS, A, X, C, T, H, E>
1181{
1182 pub(crate) async fn open(
1183 run: &'a mut WorkflowRun<CS, A, X, C, T, H, E>,
1184 mut options: ExecutionOptions,
1185 mode: ActionMode,
1186 ) -> Result<Self, AppError> {
1187 let admission = run.runtime.app.reserve_caller()?;
1188 if !options.state.values().is_empty() {
1189 return Err(AppError::InvalidInput(
1190 "existing execution cannot replace initial state".into(),
1191 ));
1192 }
1193 JsonBoundary::validate_inputs(&options.inputs)?;
1194 run.runtime.app.ensure_open().await?;
1195 let loaded = run.runtime.load().await?;
1196 let definition = run.runtime.app.definition_for(&loaded.record)?;
1197 let graph = run.runtime.app.graph(&definition);
1198 options.resolve_selectors(&graph)?;
1199 let pending_controls = !loaded.record.checkpoint.controls.is_empty();
1200 let discard_staged = loaded.record.checkpoint.rollback_stage_keys(&definition);
1201 let generation = run.runtime.cell.claim_execution(mode)?;
1202 run.runtime.cell.configure_execution(generation, options)?;
1203 let mut execution = Self {
1204 runtime: run.runtime.clone(),
1205 app: &run.runtime.app,
1206 lane: ExecutionLane::Caller,
1207 admission: Some(admission),
1208 children: Vec::new(),
1209 wake: None,
1210 timer: None,
1211 generation: Some(generation),
1212 parent_deadline: None,
1213 state: SessionState::Ready,
1214 sources: Vec::new(),
1215 next_source: 0,
1216 native_partials: None,
1217 prefetched: None,
1218 transitions: 0,
1219 observed: Some(loaded.record.clone()),
1220 };
1221 if loaded.record.checkpoint.can_retry_failed() {
1222 execution.publish(SessionPublication::Retry { loaded }, false)?;
1223 poll_fn(|cx| execution.poll_publication(cx, false)).await?;
1224 } else if !loaded.record.checkpoint.is_terminal()
1225 && (loaded.record.checkpoint.admission != RunAdmission::Prepared
1226 || !discard_staged.is_empty())
1227 {
1228 execution.publish_transition(loaded, ControlTransition::Pause { discard_staged })?;
1229 poll_fn(|cx| execution.poll_publication(cx, false)).await?;
1230 }
1231 drop(execution.admission.take());
1233 execution.runtime.cell.admit_lifecycle(generation)?;
1234 if pending_controls {
1235 execution.runtime.schedule(ScheduleCause::Reconcile);
1236 }
1237 Ok(execution)
1238 }
1239
1240 pub(super) fn background(run: &'a mut WorkflowRun<CS, A, X, C, T, H, E>) -> Self {
1241 Self::owned(run.runtime.clone(), &run.runtime.app, ExecutionLane::Worker)
1242 }
1243
1244 fn owned(
1245 runtime: WorkflowRunRuntime<CS, A, X, C, T, H, E>,
1246 app: &'a AppRuntime<CS, A, X, C, T, H, E>,
1247 lane: ExecutionLane,
1248 ) -> Self {
1249 Self {
1250 runtime,
1251 app,
1252 lane,
1253 admission: None,
1254 children: Vec::new(),
1255 wake: None,
1256 timer: None,
1257 generation: None,
1258 parent_deadline: None,
1259 state: SessionState::Ready,
1260 sources: Vec::new(),
1261 next_source: 0,
1262 native_partials: None,
1263 prefetched: None,
1264 transitions: 0,
1265 observed: None,
1266 }
1267 }
1268
1269 pub async fn next(&mut self) -> Result<Option<ExecutionEvent>, AppError> {
1323 let event = poll_fn(|cx| self.poll_advance(cx, false)).await;
1324 if (matches!(&event, Ok(Some(ExecutionEvent::Committed { .. })))
1325 || matches!(&event, Ok(None)) && matches!(&self.state, SessionState::Eof)
1326 || event.is_err() && self.runtime.cell.lifecycle_generation().is_none())
1327 && let Some(generation) = self.generation
1328 {
1329 self.runtime.cell.returned_boundary(generation);
1330 }
1331 event
1332 }
1333
1334 fn poll_advance(
1335 &mut self,
1336 cx: &mut Context<'_>,
1337 ordinary: bool,
1338 ) -> Poll<Result<Option<ExecutionEvent>, AppError>> {
1339 loop {
1340 if !matches!(&self.state, SessionState::Sources { .. })
1344 || self
1345 .children
1346 .iter()
1347 .any(|group| matches!(group.state, ChildExecutionState::Prepared(_)))
1348 {
1349 self.poll_child_groups(cx)?;
1350 }
1351 match &mut self.state {
1352 SessionState::ObservingItem { .. } => match self.poll_item(cx) {
1353 Poll::Pending => return Poll::Pending,
1354 Poll::Ready(Err(error)) => return Poll::Ready(Err(error)),
1355 Poll::Ready(Ok(Some(event))) if !ordinary => {
1356 return Poll::Ready(Ok(Some(event)));
1357 }
1358 Poll::Ready(Ok(_)) => continue,
1359 },
1360 SessionState::Finished => return Poll::Ready(Ok(None)),
1361 SessionState::Eof => self.state = SessionState::Ready,
1362 SessionState::ActivationReady { .. } => self.activate_ready(ordinary)?,
1363 SessionState::Publishing { .. } => {
1364 match self.poll_publication(cx, false) {
1365 Poll::Pending => return Poll::Pending,
1366 Poll::Ready(Err(error)) => return Poll::Ready(Err(error)),
1367 Poll::Ready(Ok(Some(event))) => {
1368 if ordinary && matches!(event, ExecutionEvent::Partial { .. }) {
1369 continue;
1370 }
1371 return Poll::Ready(Ok(Some(event)));
1372 }
1373 Poll::Ready(Ok(None)) => {}
1374 }
1375 if matches!(&self.state, SessionState::Eof) {
1376 return Poll::Ready(Ok(None));
1377 }
1378 }
1379 SessionState::Ready => {
1380 self.ensure_owned()?;
1381 self.arm_notification()?;
1382 let terminal = self
1383 .observed
1384 .as_ref()
1385 .filter(|record| {
1386 self.generation.is_none() && record.checkpoint.is_terminal()
1387 })
1388 .map(|record| record.checkpoint.sequence);
1389 self.publish(SessionPublication::Load { terminal }, ordinary)?;
1390 }
1391 SessionState::Loaded(_) => {
1392 if self.advance_loaded(ordinary)? {
1393 return Poll::Ready(Ok(None));
1394 }
1395 }
1396 SessionState::Sources { .. } => {
1397 self.start_child_groups(cx)?;
1398 if !matches!(self.state, SessionState::Sources { .. }) {
1399 continue;
1400 }
1401 self.poll_child_groups(cx)?;
1402 if self.publish_child_completion(ordinary, false)? {
1403 continue;
1404 }
1405 if self.prefetched.is_none()
1406 && !self.sources.is_empty()
1407 && self.sources.iter().all(|source| {
1408 matches!(
1409 source.state,
1410 CallerSourceState::Ended | CallerSourceState::Eof
1411 )
1412 })
1413 {
1414 let SessionState::Sources { loaded, .. } = &self.state else {
1415 unreachable!()
1416 };
1417 if !self.retains_external_continuation(&loaded.record) {
1418 self.finish_wave(ordinary)?;
1419 continue;
1420 }
1421 }
1422 if self.prefetched.is_none() {
1423 match self.poll_sources_observation(cx) {
1424 Poll::Pending => return Poll::Pending,
1425 Poll::Ready(Err(error)) => return Poll::Ready(Err(error)),
1426 Poll::Ready(Ok(Some(event))) if !ordinary => {
1427 return Poll::Ready(Ok(Some(event)));
1428 }
1429 Poll::Ready(Ok(Some(_))) => continue,
1430 Poll::Ready(Ok(None)) => {}
1431 }
1432 if !matches!(&self.state, SessionState::Sources { .. }) {
1433 continue;
1434 }
1435 }
1436 if let Some((index, event)) = self.prefetched.take() {
1437 self.consume_source_event(index, event, ordinary)?;
1438 } else {
1439 return Poll::Ready(Ok(None));
1440 }
1441 }
1442 }
1443 }
1444 }
1445
1446 fn advance_loaded(&mut self, ordinary: bool) -> Result<bool, AppError> {
1447 let SessionState::Loaded(loaded) =
1448 std::mem::replace(&mut self.state, SessionState::Finished)
1449 else {
1450 unreachable!()
1451 };
1452 self.ensure_owned()?;
1453 self.runtime.supervise_controls(&loaded.record);
1454 if loaded.record.checkpoint.is_stopping() {
1455 self.stop_children();
1456 self.stop_sources();
1457 self.publish(SessionPublication::SettleStopping { loaded }, ordinary)?;
1458 return Ok(false);
1459 }
1460 if loaded.record.checkpoint.is_terminal() {
1461 self.stop_children();
1462 self.stop_sources();
1463 self.observed = Some(loaded.record);
1464 return Ok(true);
1465 }
1466 self.observed = Some(loaded.record.clone());
1467 if self.children.iter().any(|group| {
1468 matches!(
1469 group.state,
1470 ChildExecutionState::Worker(_) | ChildExecutionState::Receipt(_)
1471 ) && group
1472 .parent
1473 .as_ref()
1474 .is_some_and(|parent| !loaded.record.owns_child_activation(parent, group.sequence))
1475 }) {
1476 self.state = SessionState::Sources { loaded };
1480 return Ok(false);
1481 }
1482 if self.generation.is_none()
1483 && self.parent_deadline.is_none()
1484 && loaded.record.checkpoint.is_paused()
1485 {
1486 self.state = SessionState::Sources { loaded };
1489 return Ok(true);
1490 }
1491 if self.runtime.cell.execution_boundary()? == Some(false) && self.halt_after(&loaded)? {
1492 return Ok(false);
1493 }
1494 if self.transitions >= MAX_DRIVER_TRANSITIONS {
1495 self.publish_decision(
1496 loaded,
1497 BspDecision::Failed {
1498 category: FailureCategory::Limit,
1499 message: "workflow transition bound exhausted".into(),
1500 observation: None,
1501 },
1502 ordinary,
1503 )?;
1504 return Ok(false);
1505 }
1506 let mut decision = match self.runtime.decide_checkpoint(&loaded) {
1507 Ok(decision) => decision,
1508 Err(failure) => {
1509 if self.generation.is_none() && self.runtime.schedule_operation_failure(&failure) {
1510 self.observed = None;
1511 }
1512 return Err(failure.into_error());
1513 }
1514 };
1515 if let BspDecision::Activate(selected) = &mut decision {
1516 selected.retain(|activation| {
1517 !self
1518 .sources
1519 .iter()
1520 .any(|source| source.activation == *activation)
1521 });
1522 if selected.is_empty() {
1523 decision = BspDecision::AwaitControls;
1524 }
1525 }
1526 let boundary = self.runtime.cell.execution_boundary()?;
1527 if boundary.is_some()
1528 && let Some(node) =
1529 decision.pending_node(&self.runtime.cell.execution_options()?.halt_before)
1530 {
1531 let node = node.to_owned();
1532 self.publish_transition(loaded, ControlTransition::Halt { node, result: None })?;
1533 return Ok(false);
1534 }
1535 if boundary == Some(true)
1536 && !matches!(&decision, BspDecision::Routed { .. })
1537 && self.halt_after(&loaded)?
1538 {
1539 return Ok(false);
1540 }
1541 if !matches!(&decision, BspDecision::Routed { .. }) {
1542 self.runtime.cell.clear_execution_boundary()?;
1543 }
1544 match decision {
1545 BspDecision::AwaitControls => {
1546 let pending = loaded
1547 .record
1548 .checkpoint
1549 .controls
1550 .values()
1551 .find(|control| {
1552 matches!(control.state, ControlRecordState::Child { .. })
1553 && !self.children.iter().any(|group| {
1554 group.sequence == control.sequence
1555 && !matches!(group.state, ChildExecutionState::Awaiting)
1556 })
1557 })
1558 .map(|control| control.sequence);
1559 if let Some(sequence) = pending {
1560 self.publish(
1561 SessionPublication::ObserveChildren { loaded, sequence },
1562 ordinary,
1563 )?;
1564 } else {
1565 self.state = SessionState::Sources { loaded };
1566 }
1567 }
1568 BspDecision::Activate(selected) => {
1569 self.transitions += 1;
1570 let mut remaining = selected.into_iter();
1571 let activation = remaining.next().ok_or_else(|| {
1572 AppError::Integration("native selected prefix is empty".into())
1573 })?;
1574 self.begin_activation(loaded, activation, remaining, Vec::new(), ordinary)?;
1575 }
1576 decision => {
1577 self.transitions += 1;
1578 self.publish_decision(loaded, decision, ordinary)?;
1579 }
1580 }
1581 Ok(false)
1582 }
1583
1584 pub async fn step(&mut self) -> Result<Option<ExecutionEvent>, AppError> {
1637 loop {
1638 match poll_fn(|cx| self.poll_advance(cx, true)).await {
1639 Ok(Some(ExecutionEvent::Partial { .. })) => {}
1640 event => {
1641 if (matches!(&event, Ok(Some(ExecutionEvent::Committed { .. })))
1642 || matches!(&event, Ok(None)) && matches!(&self.state, SessionState::Eof)
1643 || event.is_err() && self.runtime.cell.lifecycle_generation().is_none())
1644 && let Some(generation) = self.generation
1645 {
1646 self.runtime.cell.returned_boundary(generation);
1647 }
1648 return event;
1649 }
1650 }
1651 }
1652 }
1653
1654 pub async fn finish(mut self) -> Result<RunSnapshot, AppError> {
1707 if !matches!(&self.state, SessionState::Eof) {
1708 while poll_fn(|cx| self.poll_advance(cx, true)).await?.is_some() {}
1709 }
1710 self.close().await
1711 }
1712
1713 pub async fn abort(mut self) -> Result<RunSnapshot, AppError> {
1768 self.close().await
1769 }
1770
1771 async fn close(&mut self) -> Result<RunSnapshot, AppError> {
1772 self.stop_sources();
1773 self.children.retain(|group| {
1774 !matches!(
1775 group.state,
1776 ChildExecutionState::Awaiting | ChildExecutionState::Prepared(_)
1777 )
1778 });
1779 poll_fn(|cx| -> Poll<Result<(), AppError>> {
1780 loop {
1781 self.poll_child_groups(cx)?;
1782 let deadline = self.source_deadline(None);
1783 if self.poll_timer(cx, deadline) {
1784 self.poll_child_groups(cx)?;
1785 }
1786 if matches!(self.state, SessionState::Publishing { .. }) {
1787 match self.poll_publication(cx, true) {
1788 Poll::Pending => return Poll::Pending,
1789 Poll::Ready(result) => {
1790 result?;
1791 continue;
1792 }
1793 }
1794 }
1795 if let SessionState::ObservingItem { receipt, .. } = &mut self.state {
1796 let observed = match Pin::new(receipt).poll(cx) {
1797 Poll::Pending => return Poll::Pending,
1798 Poll::Ready(result) => result.map_err(|_| {
1799 AppError::Closed("stream observation receipt closed".into())
1800 })?,
1801 };
1802 self.finish_item(observed)?;
1803 continue;
1804 }
1805 if self.children.is_empty() {
1806 return Poll::Ready(Ok(()));
1807 }
1808 if let SessionState::Loaded(_) = &self.state {
1809 let SessionState::Loaded(loaded) =
1810 std::mem::replace(&mut self.state, SessionState::Finished)
1811 else {
1812 unreachable!()
1813 };
1814 self.state = SessionState::Sources { loaded };
1815 }
1816 if matches!(self.state, SessionState::Sources { .. }) {
1817 if self.publish_child_completion(false, true)? {
1818 continue;
1819 }
1820 if self.poll_notification(cx)? {
1823 let SessionState::Sources { loaded } =
1824 std::mem::replace(&mut self.state, SessionState::Finished)
1825 else {
1826 unreachable!()
1827 };
1828 self.publish(
1829 SessionPublication::Reload {
1830 receipt: SessionReceipt::Checkpoint {
1831 loaded,
1832 staged: Vec::new(),
1833 },
1834 deadline,
1835 },
1836 false,
1837 )?;
1838 continue;
1839 }
1840 }
1841 return Poll::Pending;
1842 }
1843 })
1844 .await?;
1845 if !self.sources.is_empty() {
1846 if !matches!(&self.state, SessionState::Sources { .. }) {
1847 let deadline = LifecycleEvent::deadline(self.parent_deadline);
1848 let loaded = if matches!(&self.state, SessionState::Loaded(_)) {
1849 self.runtime
1853 .observe_generation(deadline)
1854 .await
1855 .map_err(|failure| {
1856 self.runtime.retain_execution_read_failure(&failure);
1857 failure.into_error()
1858 })?;
1859 let SessionState::Loaded(loaded) =
1860 std::mem::replace(&mut self.state, SessionState::Finished)
1861 else {
1862 unreachable!()
1863 };
1864 loaded
1865 } else {
1866 self.runtime
1867 .load_execution_observation(None, Some(deadline))
1868 .await
1869 .map_err(|failure| {
1870 self.runtime.retain_execution_read_failure(&failure);
1871 failure.into_error()
1872 })?
1873 .ok_or_else(|| AppError::NotFound("workflow run does not exist".into()))?
1874 };
1875 self.state = SessionState::Sources { loaded };
1876 }
1877 let buffered_failure = self
1878 .prefetched
1879 .take()
1880 .and_then(|(index, event)| event.err().map(|category| (index, category)));
1881 for index in 0..self.sources.len() {
1882 let source = &mut self.sources[index];
1883 let failure = buffered_failure
1884 .filter(|(buffered, _)| *buffered == index)
1885 .map(|(_, category)| category)
1886 .or(match source.state {
1887 CallerSourceState::Failed(category) => Some(category),
1888 _ => None,
1889 });
1890 source.state = CallerSourceState::Ended;
1891 if let Some(category) = failure {
1892 self.stage_source_failure(index, category, false)?;
1893 poll_fn(|cx| self.poll_publication(cx, true)).await?;
1894 self.runtime
1895 .observe_generation(LifecycleEvent::deadline(self.parent_deadline))
1896 .await
1897 .map_err(|failure| {
1898 self.runtime.retain_execution_read_failure(&failure);
1899 failure.into_error()
1900 })?;
1901 let SessionState::Loaded(loaded) =
1902 std::mem::replace(&mut self.state, SessionState::Finished)
1903 else {
1904 unreachable!("closing publication retains its original checkpoint receipt")
1905 };
1906 self.state = SessionState::Sources { loaded };
1907 }
1908 }
1909 let SessionState::Sources { loaded, .. } = &self.state else {
1910 unreachable!()
1911 };
1912 if loaded
1913 .record
1914 .checkpoint
1915 .staged
1916 .iter()
1917 .any(|activation| matches!(activation.outcome, StagedOutcomeRecord::Failed { .. }))
1918 {
1919 self.finish_wave(false)?;
1920 poll_fn(|cx| self.poll_publication(cx, true)).await?;
1921 }
1922 }
1923 if let SessionState::Sources { loaded, .. } =
1924 std::mem::replace(&mut self.state, SessionState::Finished)
1925 {
1926 self.observed = Some(loaded.record);
1927 }
1928 self.sources.clear();
1929 let foreground = self.generation.is_some();
1930 if foreground {
1931 self.publish(SessionPublication::Release, false)?;
1932 poll_fn(|cx| self.poll_publication(cx, true)).await?;
1933 self.state = SessionState::Finished;
1934 self.generation = None;
1935 }
1936 let snapshot = if foreground {
1937 self.runtime.snapshot().await?
1938 } else {
1939 self.runtime.app.ensure_snapshot_read()?;
1940 let terminal = self
1941 .observed
1942 .as_ref()
1943 .filter(|record| record.checkpoint.is_terminal())
1944 .map(|record| record.checkpoint.sequence);
1945 let loaded = self
1946 .runtime
1947 .load_execution_observation(terminal, self.parent_deadline)
1948 .await
1949 .map_err(|failure| {
1950 self.runtime.retain_execution_read_failure(&failure);
1951 failure.into_error()
1952 })?
1953 .ok_or_else(|| AppError::NotFound("workflow run does not exist".into()))?;
1954 let mut deadline = LifecycleEvent::deadline(self.parent_deadline);
1955 if loaded.record.checkpoint.is_terminal()
1956 && let Some(generation) = self.runtime.cell.lifecycle_generation()
1957 {
1958 deadline = self
1959 .runtime
1960 .cell
1961 .begin_action_cleanup(generation)?
1962 .min(deadline);
1963 }
1964 if !self.retains_execution_lifecycle(&loaded.record)
1965 && (loaded.record.checkpoint.controls.is_empty()
1966 || loaded.record.checkpoint.is_public_outcome())
1967 {
1968 self.runtime
1969 .finish_execution_observation(
1970 &loaded.record,
1971 None,
1972 loaded.record.checkpoint.is_terminal().then_some(deadline),
1973 false,
1974 )
1975 .await?;
1976 }
1977 if loaded.record.checkpoint.is_terminal() {
1978 self.runtime
1981 .observe_generation(deadline)
1982 .await
1983 .map_err(|failure| {
1984 self.runtime.retain_execution_read_failure(&failure);
1985 failure.into_error()
1986 })?;
1987 let snapshot = self.runtime.project(&loaded.record)?;
1988 self.runtime.app.ensure_snapshot_read()?;
1989 snapshot
1990 } else {
1991 self.runtime.snapshot().await?
1992 }
1993 };
1994 if !foreground
1995 && matches!(
1996 snapshot.status,
1997 RunStatus::Succeeded
1998 | RunStatus::Cancelled
1999 | RunStatus::Failed(_)
2000 | RunStatus::Halted { .. }
2001 )
2002 {
2003 self.runtime.cell.clear_execution_options(None);
2004 }
2005 if foreground && matches!(snapshot.status, crate::workflow::RunStatus::Running) {
2006 self.runtime.wait().await
2007 } else {
2008 Ok(snapshot)
2009 }
2010 }
2011
2012 fn stop_sources(&mut self) {
2013 for source in &mut self.sources {
2014 let failure = match source.state {
2015 CallerSourceState::Failed(category) => Some(category),
2016 _ => None,
2017 };
2018 let destroyed = source.stop().err();
2019 if let Some(category) = failure.or(destroyed) {
2020 self.runtime.retain_source_failure(
2021 &source.context,
2022 &source.activation.lineage,
2023 category,
2024 );
2025 source.state = CallerSourceState::Failed(category);
2026 }
2027 }
2028 }
2029
2030 fn stop_children(&mut self) {
2031 for group in &mut self.children {
2032 group.stop(&self.runtime);
2033 }
2034 self.children.clear();
2035 }
2036
2037 fn plan_child_group(
2038 &mut self,
2039 loaded: LoadedRun,
2040 sequence: u64,
2041 children: Vec<RunRecord>,
2042 verified: bool,
2043 ) -> Result<(), AppError> {
2044 if loaded.record.checkpoint.is_stopping() || loaded.record.checkpoint.is_terminal() {
2045 self.state = SessionState::Loaded(loaded);
2046 return Ok(());
2047 }
2048 if let Some(group) = self
2049 .children
2050 .iter_mut()
2051 .find(|group| group.sequence == sequence)
2052 && let Some(ControlRecord {
2053 state: ControlRecordState::Child { admitted, .. },
2054 ..
2055 }) = loaded.record.checkpoint.control(sequence)
2056 {
2057 for values in group.inputs.iter_mut().take(admitted.len()) {
2060 values.clear();
2061 }
2062 }
2063 let decision = catch_unwind(AssertUnwindSafe(|| {
2064 self.runtime
2065 .plan_children(&loaded, sequence, children, verified)
2066 }))
2067 .map_err(|_| AppError::Integration("child planning callback panicked".into()))??;
2068 match decision {
2069 BspDecision::ExecuteChildren { sequence, children } => {
2070 let control = loaded
2071 .record
2072 .checkpoint
2073 .control(sequence)
2074 .ok_or_else(|| AppError::Conflict("child control is unavailable".into()))?;
2075 let context = self.runtime.child_context(&loaded.record, sequence)?;
2076 let definition = self.runtime.app.definition_for(&loaded.record)?;
2077 let step = definition
2078 .node(&control.node)
2079 .ok_or_else(|| AppError::Integration("child definition is unavailable".into()))?
2080 .step();
2081 step.validate_mode(context.mode)?;
2082 let deadline = tokio::time::Instant::now()
2083 .checked_add(WorkflowRunRuntime::<CS, A, X, C, T, H, E>::remaining(
2084 control.deadline_millis,
2085 )?)
2086 .ok_or_else(|| {
2087 AppError::Timeout("child deadline is outside the clock range".into())
2088 })?;
2089 if let Some(group) = self
2090 .children
2091 .iter_mut()
2092 .find(|group| group.sequence == sequence)
2093 {
2094 if !matches!(group.state, ChildExecutionState::Awaiting) {
2095 return Err(AppError::Conflict(
2096 "child group already has a polling owner".into(),
2097 ));
2098 }
2099 group.state = ChildExecutionState::Prepared(children);
2100 } else {
2101 self.children.push(ChildExecution {
2102 sequence,
2103 parent: None,
2104 context,
2105 lineage: control.lineage.clone(),
2106 deadline,
2107 state: ChildExecutionState::Prepared(children),
2108 inputs: Vec::new(),
2109 });
2110 }
2111 self.state = SessionState::Loaded(loaded);
2112 Ok(())
2113 }
2114 BspDecision::AwaitControls => {
2115 self.state = SessionState::Loaded(loaded);
2116 Ok(())
2117 }
2118 decision => {
2119 let transition = match decision {
2120 BspDecision::Stage(activations) => ControlTransition::Stage { activations },
2121 BspDecision::Failed {
2122 category,
2123 message,
2124 observation,
2125 } => ControlTransition::Failed {
2126 category,
2127 message,
2128 observation,
2129 },
2130 decision => return self.publish_decision(loaded, decision, false),
2131 };
2132 let control = loaded
2133 .record
2134 .checkpoint
2135 .control(sequence)
2136 .cloned()
2137 .ok_or_else(|| {
2138 AppError::Conflict("completed child control is unavailable".into())
2139 })?;
2140 self.publish(
2141 SessionPublication::ChildOutcome {
2142 control,
2143 transition,
2144 },
2145 false,
2146 )
2147 }
2148 }
2149 }
2150
2151 fn start_child_groups(&mut self, cx: &mut Context<'_>) -> Result<(), AppError> {
2152 if !self
2153 .children
2154 .iter()
2155 .any(|group| matches!(group.state, ChildExecutionState::Prepared(_)))
2156 {
2157 return Ok(());
2158 }
2159 if self.poll_notification(cx)? {
2160 let deadline = self.source_deadline(None);
2161 let SessionState::Sources { loaded } =
2162 std::mem::replace(&mut self.state, SessionState::Finished)
2163 else {
2164 unreachable!()
2165 };
2166 self.publish(
2167 SessionPublication::Reload {
2168 receipt: SessionReceipt::Checkpoint {
2169 loaded,
2170 staged: Vec::new(),
2171 },
2172 deadline,
2173 },
2174 false,
2175 )?;
2176 return Ok(());
2177 }
2178 let SessionState::Sources { loaded, .. } = &self.state else {
2179 return Ok(());
2180 };
2181 let parent = Arc::new(loaded.record.clone());
2182 for index in 0..self.children.len() {
2183 if !matches!(self.children[index].state, ChildExecutionState::Prepared(_)) {
2184 continue;
2185 }
2186 let group = &mut self.children[index];
2187 let ChildExecutionState::Prepared(children) =
2188 std::mem::replace(&mut group.state, ChildExecutionState::Closed)
2189 else {
2190 unreachable!()
2191 };
2192 group.parent = Some(Arc::clone(&parent));
2193 if group.context.mode == ActionMode::Asynchronous && self.lane == ExecutionLane::Caller
2194 {
2195 let (sender, receipt) = oneshot::channel();
2196 if !self.runtime.app.spawn_driver(
2197 self.runtime.key.clone(),
2198 Duration::ZERO,
2199 DriverCommand::Child {
2200 sequence: group.sequence,
2201 generation: self.generation,
2202 parent: Arc::clone(&parent),
2203 children,
2204 receipt: sender,
2205 },
2206 ) {
2207 return Err(AppError::Busy(
2208 "child group already has an accepted owner".into(),
2209 ));
2210 }
2211 group.state = ChildExecutionState::Worker(receipt);
2212 } else {
2213 group.start_local(&self.runtime, self.app, self.lane, children)?;
2214 }
2215 }
2216 Ok(())
2217 }
2218
2219 fn poll_child_groups(&mut self, cx: &mut Context<'_>) -> Result<(), AppError> {
2220 let stopped = match &self.state {
2221 SessionState::Loaded(loaded)
2222 | SessionState::Sources { loaded }
2223 | SessionState::ObservingItem { loaded, .. } => {
2224 loaded.record.checkpoint.is_stopping() || loaded.record.checkpoint.is_terminal()
2225 }
2226 _ => false,
2227 };
2228 if stopped {
2229 self.stop_children();
2230 return Ok(());
2231 }
2232 if let SessionState::Loaded(loaded)
2233 | SessionState::Sources { loaded }
2234 | SessionState::ObservingItem { loaded, .. } = &self.state
2235 {
2236 self.children.retain(|group| {
2237 !matches!(group.state, ChildExecutionState::Awaiting)
2238 || loaded.record.checkpoint.control(group.sequence).is_some()
2239 });
2240 if self.children.iter().any(|group| {
2241 !matches!(
2242 group.state,
2243 ChildExecutionState::Awaiting
2244 | ChildExecutionState::Worker(_)
2245 | ChildExecutionState::Receipt(_)
2246 ) && group.parent.as_ref().is_some_and(|parent| {
2247 !loaded.record.owns_child_activation(parent, group.sequence)
2248 })
2249 }) {
2250 return Err(AppError::Conflict(
2251 "child parent reservation changed".into(),
2252 ));
2253 }
2254 }
2255 let observing_parent = matches!(
2256 self.state,
2257 SessionState::Publishing {
2258 purpose: PublicationPurpose::Reload,
2259 ..
2260 }
2261 );
2262 for group in &mut self.children {
2263 if observing_parent
2264 && tokio::time::Instant::now() >= group.deadline
2265 && matches!(
2266 group.state,
2267 ChildExecutionState::Local(_) | ChildExecutionState::Pipeline(_)
2268 )
2269 {
2270 continue;
2273 }
2274 let _ = group.poll_ready(cx, &self.runtime);
2275 }
2276 Ok(())
2277 }
2278
2279 fn publish_child_completion(
2280 &mut self,
2281 ordinary: bool,
2282 closing: bool,
2283 ) -> Result<bool, AppError> {
2284 let Some(index) = self.children.iter().position(|group| {
2285 matches!(
2286 group.state,
2287 ChildExecutionState::Complete(_) | ChildExecutionState::Receipt(_)
2288 )
2289 }) else {
2290 return Ok(false);
2291 };
2292 let group = &mut self.children[index];
2293 let sequence = group.sequence;
2294 let parent = group.parent.take();
2295 let context = Arc::clone(&group.context);
2296 let lineage = group.lineage.clone();
2297 let completed = std::mem::replace(&mut group.state, ChildExecutionState::Awaiting);
2298 if closing {
2299 self.children.remove(index);
2300 }
2301 if let ChildExecutionState::Receipt(result) = completed {
2302 match (*result).map_err(RuntimeOperationFailure::into_error)? {
2303 SessionReceipt::Children {
2304 loaded,
2305 sequence,
2306 children,
2307 verified: true,
2308 } => {
2309 if closing {
2310 self.state = SessionState::Loaded(loaded);
2311 } else {
2312 self.plan_child_group(loaded, sequence, children, true)?;
2313 }
2314 }
2315 SessionReceipt::Checkpoint { loaded, .. } => {
2316 self.state = SessionState::Loaded(loaded)
2317 }
2318 SessionReceipt::Children {
2319 verified: false, ..
2320 } => {
2321 return Err(AppError::Integration(
2322 "child worker returned an unverified execution result".into(),
2323 ));
2324 }
2325 }
2326 return Ok(true);
2327 }
2328 let ChildExecutionState::Complete(result) = completed else {
2329 unreachable!()
2330 };
2331 let SessionState::Sources { loaded } =
2332 std::mem::replace(&mut self.state, SessionState::Finished)
2333 else {
2334 unreachable!()
2335 };
2336 match result {
2337 Ok(completions) => self.publish(
2338 SessionPublication::VerifyChildren {
2339 parent: parent.ok_or_else(|| {
2340 AppError::Integration("child group parent evidence is unavailable".into())
2341 })?,
2342 sequence,
2343 completions,
2344 },
2345 ordinary,
2346 )?,
2347 Err(category) => {
2348 let activation = loaded
2349 .record
2350 .checkpoint
2351 .frontier
2352 .iter()
2353 .find(|activation| {
2354 activation.node == context.node && activation.lineage == lineage
2355 })
2356 .cloned()
2357 .ok_or_else(|| AppError::Conflict("child activation changed".into()))?;
2358 self.publish_failure(loaded, &activation, &context, category)?;
2359 }
2360 }
2361 Ok(true)
2362 }
2363
2364 fn retains_external_continuation(&self, record: &RunRecord) -> bool {
2365 !self.children.is_empty()
2366 || self.runtime.app.has_child_work(&self.runtime.key)
2367 || record.checkpoint.controls.values().any(|control| {
2368 matches!(&control.state, ControlRecordState::Attempt(_))
2369 && self
2370 .runtime
2371 .app
2372 .has_control_work(&self.runtime.key, control.sequence)
2373 })
2374 }
2375
2376 fn retains_execution_lifecycle(&self, record: &RunRecord) -> bool {
2377 self.retains_execution_work(record)
2378 || record
2379 .checkpoint
2380 .controls
2381 .values()
2382 .any(|control| matches!(control.state, ControlRecordState::Inputs { .. }))
2383 }
2384
2385 fn retains_execution_work(&self, record: &RunRecord) -> bool {
2386 self.sources.iter().any(|source| {
2387 matches!(
2388 source.state,
2389 CallerSourceState::Polling(_) | CallerSourceState::Failed(_)
2390 )
2391 }) || self.prefetched.is_some()
2392 || self.retains_external_continuation(record)
2393 || record.checkpoint.controls.values().any(|control| {
2394 matches!(control.state, ControlRecordState::Sleep)
2395 && self
2396 .runtime
2397 .app
2398 .has_control_work(&self.runtime.key, control.sequence)
2399 })
2400 }
2401
2402 fn ensure_owned(&self) -> Result<(), AppError> {
2403 self.runtime.app.ensure_runtime_owned()?;
2404 if self
2405 .generation
2406 .is_some_and(|generation| !self.runtime.cell.execution_is_live(generation))
2407 {
2408 return Err(AppError::Closed(
2409 "run execution generation is closed".into(),
2410 ));
2411 }
2412 Ok(())
2413 }
2414
2415 fn source_deadline(&self, excluded: Option<usize>) -> Option<tokio::time::Instant> {
2416 self.sources
2417 .iter()
2418 .enumerate()
2419 .filter_map(|(index, source)| {
2420 (Some(index) != excluded && matches!(source.state, CallerSourceState::Polling(_)))
2421 .then_some(source.deadline)
2422 })
2423 .chain(self.children.iter().filter_map(|group| {
2424 (!matches!(
2425 group.state,
2426 ChildExecutionState::Complete(_) | ChildExecutionState::Receipt(_)
2427 ))
2428 .then_some(group.deadline)
2429 }))
2430 .min()
2431 }
2432
2433 fn expire_sources(&mut self, excluded: Option<usize>) {
2434 let now = tokio::time::Instant::now();
2435 for (index, source) in self.sources.iter_mut().enumerate() {
2436 if Some(index) == excluded
2437 || now < source.deadline
2438 || !matches!(source.state, CallerSourceState::Polling(_))
2439 {
2440 continue;
2441 }
2442 self.runtime.retain_source_failure(
2443 &source.context,
2444 &source.activation.lineage,
2445 FailureCategory::Timeout,
2446 );
2447 let _ = source.stop();
2448 if self
2449 .prefetched
2450 .as_ref()
2451 .is_some_and(|(buffered, _)| *buffered == index)
2452 {
2453 self.prefetched = Some((index, Err(FailureCategory::Timeout)));
2454 } else {
2455 source.state = CallerSourceState::Failed(FailureCategory::Timeout);
2456 }
2457 }
2458 }
2459
2460 fn poll_sources(&mut self, context: &mut Context<'_>, excluded: Option<usize>) {
2461 self.expire_sources(excluded);
2462 if self.prefetched.is_some() {
2463 return;
2464 }
2465 let count = self.sources.len();
2466 for offset in 0..count {
2467 let index = (self.next_source + offset) % count;
2468 if Some(index) == excluded {
2469 continue;
2470 }
2471 if let Poll::Ready(mut event) = self.sources[index].poll(context) {
2472 let source = &mut self.sources[index];
2473 if let Err(category) = &event {
2474 self.runtime.retain_source_failure(
2477 &source.context,
2478 &source.activation.lineage,
2479 *category,
2480 );
2481 }
2482 if matches!(
2483 &event,
2484 Ok(Some(ActionEvent::Completed { .. })) | Ok(None) | Err(_)
2485 ) {
2486 let destroyed = source.stop();
2487 if event.is_ok()
2488 && let Err(category) = destroyed
2489 {
2490 self.runtime.retain_source_failure(
2491 &source.context,
2492 &source.activation.lineage,
2493 category,
2494 );
2495 event = Err(category);
2496 }
2497 if matches!(&event, Ok(None)) {
2498 source.state = CallerSourceState::Eof;
2499 }
2500 }
2501 self.next_source = (index + 1) % count;
2502 self.prefetched = Some((index, event));
2503 return;
2504 }
2505 }
2506 }
2507
2508 fn arm_notification(&mut self) -> Result<(), AppError> {
2509 if self.wake.is_none() {
2510 self.wake = Some(Box::pin(self.runtime.notifier()?.notified_owned()));
2511 }
2512 let wake = self.wake.as_mut().unwrap();
2513 wake.as_mut().enable();
2514 Ok(())
2517 }
2518
2519 fn poll_notification(&mut self, cx: &mut Context<'_>) -> Result<bool, AppError> {
2520 self.arm_notification()?;
2521 let ready = self.wake.as_mut().unwrap().as_mut().poll(cx).is_ready();
2522 if ready {
2523 self.wake = None;
2524 }
2525 Ok(ready)
2526 }
2527
2528 fn poll_timer(&mut self, cx: &mut Context<'_>, deadline: Option<tokio::time::Instant>) -> bool {
2529 let Some(deadline) = deadline else {
2530 self.timer = None;
2531 return false;
2532 };
2533 let timer = self
2534 .timer
2535 .get_or_insert_with(|| Box::pin(tokio::time::sleep_until(deadline)));
2536 if timer.deadline() != deadline {
2537 timer.as_mut().reset(deadline);
2538 }
2539 timer.as_mut().poll(cx).is_ready()
2540 }
2541
2542 fn poll_native_partial(&mut self, cx: &mut Context<'_>) -> Option<ExecutionEvent> {
2543 let Some((_, receiver)) = &mut self.native_partials else {
2544 return None;
2545 };
2546 match receiver.poll_recv(cx) {
2547 Poll::Ready(Some(event)) => Some(event),
2548 Poll::Ready(None) => {
2549 self.native_partials = None;
2550 None
2551 }
2552 Poll::Pending => None,
2553 }
2554 }
2555
2556 fn poll_sources_observation(
2557 &mut self,
2558 cx: &mut Context<'_>,
2559 ) -> Poll<Result<Option<ExecutionEvent>, AppError>> {
2560 self.ensure_owned()?;
2561 self.expire_sources(None);
2562 let deadline = self.source_deadline(None);
2563 if self.poll_timer(cx, deadline) {
2564 self.expire_sources(None);
2565 }
2566 let pending_failure = self
2567 .sources
2568 .iter()
2569 .any(|source| matches!(source.state, CallerSourceState::Failed(_)));
2570 if !pending_failure && self.poll_notification(cx)? {
2571 let SessionState::Sources { loaded } =
2572 std::mem::replace(&mut self.state, SessionState::Finished)
2573 else {
2574 unreachable!()
2575 };
2576 self.publish(
2577 SessionPublication::Reload {
2578 receipt: SessionReceipt::Checkpoint {
2579 loaded,
2580 staged: Vec::new(),
2581 },
2582 deadline,
2583 },
2584 false,
2585 )?;
2586 return Poll::Ready(Ok(None));
2587 }
2588 self.poll_sources(cx, None);
2589 if let Some(event) = self.poll_native_partial(cx) {
2590 return Poll::Ready(Ok(Some(event)));
2591 }
2592 if self.prefetched.is_some() {
2593 return Poll::Ready(Ok(None));
2594 }
2595 let SessionState::Sources { loaded, .. } = &self.state else {
2596 unreachable!()
2597 };
2598 let known_eof = self
2599 .sources
2600 .iter()
2601 .any(|source| matches!(source.state, CallerSourceState::Eof));
2602 if (self.generation.is_some() || !self.retains_execution_work(&loaded.record))
2603 && !known_eof
2604 && !self.children.iter().any(|group| {
2605 matches!(
2606 group.state,
2607 ChildExecutionState::Complete(_) | ChildExecutionState::Receipt(_)
2608 )
2609 })
2610 && loaded.record.checkpoint.is_actionable()
2611 {
2612 return Poll::Ready(Ok(None));
2613 }
2614 Poll::Pending
2615 }
2616
2617 fn poll_item(
2618 &mut self,
2619 cx: &mut Context<'_>,
2620 ) -> Poll<Result<Option<ExecutionEvent>, AppError>> {
2621 let SessionState::ObservingItem { index, .. } = &self.state else {
2622 unreachable!()
2623 };
2624 let index = *index;
2625 self.expire_sources(Some(index));
2626 let deadline = self.source_deadline(Some(index));
2627 if self.poll_timer(cx, deadline) {
2628 self.expire_sources(Some(index));
2629 }
2630 let SessionState::ObservingItem { receipt, .. } = &mut self.state else {
2631 unreachable!()
2632 };
2633 if let Poll::Ready(result) = Pin::new(receipt).poll(cx) {
2634 let result = result
2635 .map_err(|_| AppError::Closed("stream observation receipt closed".into()))
2636 .and_then(|result| result);
2637 return Poll::Ready(self.finish_item(result).map(|result| {
2638 result.map(|result| ExecutionEvent::Partial {
2639 node: self.sources[index].activation.node.clone(),
2640 result,
2641 })
2642 }));
2643 }
2644 self.poll_sources(cx, Some(index));
2645 if let Some(event) = self.poll_native_partial(cx) {
2646 return Poll::Ready(Ok(Some(event)));
2647 }
2648 Poll::Pending
2649 }
2650
2651 fn finish_item(
2652 &mut self,
2653 result: Result<Option<Map<String, Value>>, AppError>,
2654 ) -> Result<Option<Map<String, Value>>, AppError> {
2655 let SessionState::ObservingItem {
2656 loaded, receipt, ..
2657 } = std::mem::replace(&mut self.state, SessionState::Finished)
2658 else {
2659 unreachable!()
2660 };
2661 drop(receipt);
2662 if result.is_err() {
2663 self.observed = Some(loaded.record);
2664 for source in &mut self.sources {
2665 let _ = source.stop();
2666 }
2667 } else {
2668 self.state = SessionState::Sources { loaded };
2669 }
2670 result
2671 }
2672
2673 fn consume_source_event(
2674 &mut self,
2675 index: usize,
2676 event: Result<Option<ActionEvent>, FailureCategory>,
2677 ordinary: bool,
2678 ) -> Result<(), AppError> {
2679 match event {
2680 Ok(Some(ActionEvent::Partial { result })) => {
2681 if JsonBoundary::validate_size(&result).is_err() {
2682 let _ = self.sources[index].stop();
2683 return self.stage_source_failure(index, FailureCategory::State, ordinary);
2684 }
2685 self.publish_item(index, ActionEvent::Partial { result })
2686 }
2687 Ok(Some(
2688 event @ (ActionEvent::SpanStarted { .. }
2689 | ActionEvent::SpanAttribute { .. }
2690 | ActionEvent::SpanFinished { .. }),
2691 )) => self.publish_item(index, event),
2692 Ok(Some(ActionEvent::Completed { result, mut writes })) => {
2693 let SessionState::Sources { loaded, .. } = &self.state else {
2694 unreachable!()
2695 };
2696 let source = &self.sources[index];
2697 let staged = catch_unwind(AssertUnwindSafe(|| {
2698 let definition = self.runtime.app.definition_for(&loaded.record)?;
2699 if let Some(Step::Action {
2700 action,
2701 revision,
2702 project_result: true,
2703 }) = definition
2704 .node(&source.activation.node)
2705 .map(|node| node.step())
2706 {
2707 writes.insert(
2708 0,
2709 self.runtime
2710 .app
2711 .action(action, revision)?
2712 .declaration()
2713 .result_projection(&result),
2714 );
2715 }
2716 StagedActivationRecord::action(
2717 &source.activation,
2718 &loaded.record.checkpoint.state,
2719 &writes,
2720 result,
2721 self.runtime.app.typing(),
2722 )
2723 }));
2724 match staged {
2725 Ok(Ok(staged)) => self.stage_source(index, staged, ordinary),
2726 Ok(Err(_)) => {
2727 self.stage_source_failure(index, FailureCategory::State, ordinary)
2728 }
2729 Err(_) => {
2730 self.stage_source_failure(index, FailureCategory::Execution, ordinary)
2731 }
2732 }
2733 }
2734 Ok(None) => Ok(()),
2735 Err(category) => self.stage_source_failure(index, category, ordinary),
2736 }
2737 }
2738
2739 fn stage_source_failure(
2740 &mut self,
2741 index: usize,
2742 category: FailureCategory,
2743 ordinary: bool,
2744 ) -> Result<(), AppError> {
2745 let source = &self.sources[index];
2746 self.runtime
2747 .retain_source_failure(&source.context, &source.activation.lineage, category);
2748 let staged = StagedActivationRecord::failed(
2749 source.activation.node.clone(),
2750 source.activation.lineage.clone(),
2751 category,
2752 "caller action execution failed".into(),
2753 );
2754 self.stage_source(index, staged, ordinary)
2755 }
2756
2757 fn stage_source(
2758 &mut self,
2759 index: usize,
2760 staged: StagedActivationRecord,
2761 ordinary: bool,
2762 ) -> Result<(), AppError> {
2763 let SessionState::Sources { loaded } =
2764 std::mem::replace(&mut self.state, SessionState::Finished)
2765 else {
2766 unreachable!()
2767 };
2768 self.observed = Some(loaded.record.clone());
2769 self.publish(
2770 SessionPublication::Decision {
2771 loaded,
2772 decision: BspDecision::Stage(vec![staged]),
2773 activation_deadline: None,
2774 context: Some(Arc::clone(&self.sources[index].context)),
2775 staged: Vec::new(),
2776 remaining: Vec::new().into_iter(),
2777 },
2778 ordinary,
2779 )
2780 }
2781
2782 fn finish_wave(&mut self, ordinary: bool) -> Result<(), AppError> {
2783 let SessionState::Sources { loaded } =
2784 std::mem::replace(&mut self.state, SessionState::Finished)
2785 else {
2786 unreachable!()
2787 };
2788 let definition = self.runtime.app.definition_for(&loaded.record)?;
2789 let failed = loaded
2790 .record
2791 .checkpoint
2792 .selected_staged(&definition)
2793 .filter_map(|activation| {
2794 if let StagedOutcomeRecord::Failed { category, message } = &activation.outcome {
2795 Some((activation, *category, message))
2796 } else {
2797 None
2798 }
2799 })
2800 .min_by(|left, right| {
2801 (&left.0.node, &left.0.lineage).cmp(&(&right.0.node, &right.0.lineage))
2802 });
2803 if let Some((activation, category, message)) = failed {
2804 let message = message.clone();
2805 let observation = match self.sources.iter().find(|source| {
2806 source.activation.node == activation.node
2807 && source.activation.lineage == activation.lineage
2808 }) {
2809 Some(source) => ActionPosition {
2810 node: source.context.node.clone(),
2811 ordinal: source.context.action_ordinal,
2812 },
2813 None => loaded.record.checkpoint.proposed_position(
2814 &definition,
2815 &activation.node,
2816 &activation.lineage,
2817 &[],
2818 )?,
2819 };
2820 self.sources.clear();
2821 self.publish_decision(
2822 loaded,
2823 BspDecision::Failed {
2824 category,
2825 message,
2826 observation: Some(observation),
2827 },
2828 ordinary,
2829 )
2830 } else if self
2831 .sources
2832 .iter()
2833 .any(|source| matches!(source.state, CallerSourceState::Eof))
2834 {
2835 self.sources.clear();
2836 self.publish(SessionPublication::EndSource { loaded }, ordinary)
2837 } else {
2838 self.sources.clear();
2839 self.observed = Some(loaded.record);
2840 self.state = SessionState::Ready;
2841 Ok(())
2842 }
2843 }
2844
2845 fn begin_activation(
2846 &mut self,
2847 loaded: LoadedRun,
2848 activation: FrontierRecord,
2849 remaining: std::vec::IntoIter<FrontierRecord>,
2850 staged: Vec<StagedActivationRecord>,
2851 ordinary: bool,
2852 ) -> Result<(), AppError> {
2853 self.ensure_owned()?;
2854 let definition = self.runtime.app.definition_for(&loaded.record)?;
2855 if let Some(node) = definition.node(&activation.node) {
2856 node.step()
2857 .validate_mode(self.runtime.cell.execution_mode()?)?;
2858 }
2859 let (started, deadline_millis) = WorkflowRunRuntime::<CS, A, X, C, T, H, E>::node_deadline(
2860 &definition,
2861 &activation.node,
2862 )?;
2863 let deadline = tokio::time::Instant::now()
2864 .checked_add(Duration::from_millis(deadline_millis - started))
2865 .ok_or_else(|| {
2866 AppError::Timeout("activation deadline is outside the clock range".into())
2867 })?;
2868 let context = match self.runtime.cell.action_context(
2869 loaded.record.checkpoint.base_sequence,
2870 &activation.node,
2871 &activation.lineage,
2872 ) {
2873 Some(context) => context,
2874 None => Arc::new(loaded.record.action_context(
2875 &definition,
2876 &activation,
2877 &staged,
2878 self.runtime.cell.execution_mode()?,
2879 )?),
2880 };
2881 if loaded.record.checkpoint.loses_join_first(
2882 &definition,
2883 &activation.node,
2884 &activation.lineage,
2885 ) {
2886 self.state = SessionState::ActivationReady {
2887 loaded,
2888 activation,
2889 context,
2890 remaining,
2891 staged,
2892 deadline,
2893 deadline_millis,
2894 };
2895 return Ok(());
2896 }
2897 if let Some(Step::Action {
2898 action, revision, ..
2899 }) = definition.node(&activation.node).map(|node| node.step())
2900 {
2901 let binding = self.runtime.app.action(action, revision)?;
2902 let validated = catch_unwind(AssertUnwindSafe(|| {
2903 binding.declaration().validate_reads(&WorkflowState {
2904 values: loaded.record.checkpoint.state.clone(),
2905 })?;
2906 binding.declaration().resolve_inputs(
2907 &self.runtime.cell.execution_options()?.inputs,
2908 self.runtime.cell.execution_mode()?,
2909 )
2910 }));
2911 match validated {
2912 Ok(Err(AppError::InvalidInput(message))) if self.generation.is_none() => {
2913 return self.publish_decision(
2914 loaded,
2915 BspDecision::Failed {
2916 category: FailureCategory::Execution,
2917 message,
2918 observation: None,
2919 },
2920 ordinary,
2921 );
2922 }
2923 Ok(Ok(ActionInputs::Required(required))) => {
2924 return self.publish(
2925 SessionPublication::Decision {
2926 loaded,
2927 decision: BspDecision::Inputs {
2928 node: activation.node,
2929 lineage: activation.lineage,
2930 required,
2931 },
2932 activation_deadline: Some(deadline_millis),
2933 context: None,
2934 staged,
2935 remaining,
2936 },
2937 ordinary,
2938 );
2939 }
2940 Ok(result) => {
2941 result?;
2942 }
2943 Err(_) => {
2944 return self.publish_failure(
2945 loaded,
2946 &activation,
2947 &context,
2948 FailureCategory::Execution,
2949 );
2950 }
2951 }
2952 }
2953 self.publish(
2954 SessionPublication::BeginActivation {
2955 loaded,
2956 activation,
2957 context,
2958 remaining,
2959 staged,
2960 deadline,
2961 deadline_millis,
2962 },
2963 ordinary,
2964 )
2965 }
2966
2967 fn activate_ready(&mut self, ordinary: bool) -> Result<(), AppError> {
2968 self.ensure_owned()?;
2969 let SessionState::ActivationReady {
2970 loaded,
2971 activation,
2972 context,
2973 mut remaining,
2974 mut staged,
2975 deadline,
2976 deadline_millis,
2977 } = std::mem::replace(&mut self.state, SessionState::Finished)
2978 else {
2979 unreachable!()
2980 };
2981 let definition = self.runtime.app.definition_for(&loaded.record)?;
2982 let skipped = loaded.record.checkpoint.loses_join_first(
2983 &definition,
2984 &activation.node,
2985 &activation.lineage,
2986 );
2987 if !skipped
2988 && matches!(
2989 definition.node(&activation.node).map(|node| node.step()),
2990 Some(Step::Action { .. })
2991 )
2992 {
2993 self.construct_source(&loaded, activation, context, deadline)?;
2994 return match remaining.next() {
2995 Some(activation) => {
2996 self.begin_activation(loaded, activation, remaining, staged, ordinary)
2997 }
2998 None if staged.is_empty() => {
2999 self.observed = None;
3000 self.state = SessionState::Sources { loaded };
3001 Ok(())
3002 }
3003 None => self.publish_decision(loaded, BspDecision::Stage(staged), ordinary),
3004 };
3005 }
3006 if !skipped && tokio::time::Instant::now() >= deadline {
3007 return self.publish_failure(loaded, &activation, &context, FailureCategory::Timeout);
3008 }
3009 let engine = ExecutionEngine::new(
3010 &definition,
3011 self.runtime.app.typing(),
3012 self.runtime.app.conditions(),
3013 );
3014 let decision = match catch_unwind(AssertUnwindSafe(|| {
3015 let decision = engine.activate(
3016 &loaded.record.checkpoint,
3017 &activation,
3018 loaded
3019 .record
3020 .parent
3021 .as_ref()
3022 .map_or(0, |parent| parent.depth),
3023 |workflow, revision| {
3024 let revision = match revision {
3025 Some(revision) => revision,
3026 None => loaded.record.child_revision(&workflow)?.to_owned(),
3027 };
3028 self.runtime.app.definition_revision(&workflow, &revision)?;
3029 Ok((workflow, revision))
3030 },
3031 )?;
3032 if let BspDecision::Descendants { inputs, .. } = &decision {
3033 let step = definition
3034 .node(&activation.node)
3035 .ok_or_else(|| {
3036 AppError::Integration("child declaration is unavailable".into())
3037 })?
3038 .step();
3039 for task in inputs {
3040 let child = self
3041 .runtime
3042 .app
3043 .definition_revision(&task.workflow, &task.revision)?;
3044 step.validate_child_binding(&definition, &child, &task.input)?;
3045 }
3046 }
3047 Ok(decision)
3048 })) {
3049 Ok(Ok(decision)) => decision,
3050 Ok(Err(error)) => BspDecision::Failed {
3051 category: FailureCategory::Evaluation,
3052 message: match error {
3053 AppError::InvalidInput(message) => message,
3054 _ => "workflow evaluation failed".into(),
3055 },
3056 observation: Some(loaded.record.checkpoint.proposed_position(
3057 &definition,
3058 &activation.node,
3059 &activation.lineage,
3060 &staged,
3061 )?),
3062 },
3063 Err(_) => BspDecision::Failed {
3064 category: FailureCategory::Execution,
3065 message: "workflow callback panicked".into(),
3066 observation: Some(loaded.record.checkpoint.proposed_position(
3067 &definition,
3068 &activation.node,
3069 &activation.lineage,
3070 &staged,
3071 )?),
3072 },
3073 };
3074 match decision {
3075 BspDecision::Stage(mut activations) => {
3076 staged.append(&mut activations);
3077 match remaining.next() {
3078 Some(activation) => {
3079 self.begin_activation(loaded, activation, remaining, staged, ordinary)
3080 }
3081 None => self.publish_decision(loaded, BspDecision::Stage(staged), ordinary),
3082 }
3083 }
3084 decision => self.publish(
3085 SessionPublication::Decision {
3086 loaded,
3087 decision,
3088 activation_deadline: Some(deadline_millis),
3089 context: Some(context),
3090 staged,
3091 remaining,
3092 },
3093 ordinary,
3094 ),
3095 }
3096 }
3097
3098 fn construct_source(
3099 &mut self,
3100 loaded: &LoadedRun,
3101 activation: FrontierRecord,
3102 context: Arc<ActionContext>,
3103 deadline: tokio::time::Instant,
3104 ) -> Result<(), AppError> {
3105 if tokio::time::Instant::now() >= deadline {
3106 self.runtime.retain_source_failure(
3107 &context,
3108 &activation.lineage,
3109 FailureCategory::Timeout,
3110 );
3111 self.sources.push(CallerSource {
3112 activation,
3113 context,
3114 deadline,
3115 state: CallerSourceState::Failed(FailureCategory::Timeout),
3116 });
3117 return Ok(());
3118 }
3119 let definition = self.runtime.app.definition_for(&loaded.record)?;
3120 let Some(Step::Action {
3121 action, revision, ..
3122 }) = definition.node(&activation.node).map(|node| node.step())
3123 else {
3124 return Err(AppError::Integration(
3125 "action decision has no exact declaration".into(),
3126 ));
3127 };
3128 let binding = self.runtime.app.action(action, revision)?;
3129 let state = WorkflowState {
3130 values: loaded.record.checkpoint.state.clone(),
3131 };
3132 let constructed = catch_unwind(AssertUnwindSafe(|| {
3133 binding.declaration().validate_reads(&state)?;
3134 let inputs = binding
3135 .declaration()
3136 .resolve_inputs(
3137 &self.runtime.cell.execution_options()?.inputs,
3138 self.runtime.cell.execution_mode()?,
3139 )?
3140 .into_values()?;
3141 Ok::<_, AppError>(binding.start(ActionRequest {
3142 state,
3143 inputs,
3144 context: context.as_ref().clone(),
3145 }))
3146 }));
3147 let state = match constructed {
3148 Ok(Ok(source)) => CallerSourceState::Polling(Box::pin(source)),
3149 Ok(Err(error)) => return Err(error),
3150 Err(_) => CallerSourceState::Failed(FailureCategory::Execution),
3151 };
3152 if let CallerSourceState::Failed(category) = &state {
3153 self.runtime
3154 .retain_source_failure(&context, &activation.lineage, *category);
3155 }
3156 self.sources.push(CallerSource {
3157 activation,
3158 context,
3159 deadline,
3160 state,
3161 });
3162 self.expire_sources(None);
3163 Ok(())
3164 }
3165
3166 fn halt_after(&mut self, loaded: &LoadedRun) -> Result<bool, AppError> {
3167 let outcomes = &loaded.record.checkpoint.outcomes;
3168 let Some(last) = outcomes.last() else {
3169 return Ok(false);
3170 };
3171 let options = self.runtime.cell.execution_options()?;
3172 let selected = outcomes
3173 .iter()
3174 .find(|outcome| options.halt_after.iter().any(|node| node == &outcome.node));
3175 let outcome = match selected {
3176 Some(outcome) => outcome,
3177 None if self.runtime.cell.halt_after(&last.node)? => last,
3178 None => return Ok(false),
3179 };
3180 self.publish_transition(
3181 LoadedRun {
3182 record: loaded.record.clone(),
3183 revision: loaded.revision,
3184 },
3185 ControlTransition::Halt {
3186 node: outcome.node.clone(),
3187 result: Some(outcome.result.clone()),
3188 },
3189 )?;
3190 Ok(true)
3191 }
3192
3193 fn publish_item(&mut self, index: usize, event: ActionEvent) -> Result<(), AppError> {
3194 let SessionState::Sources { loaded } = &self.state else {
3195 return Err(AppError::Integration(
3196 "stream item has no source owner".into(),
3197 ));
3198 };
3199 let source = &self.sources[index];
3200 let context = Arc::clone(&source.context);
3201 let deadline = source.deadline;
3202 let generation = self.generation;
3203 if let Some(generation) = generation {
3204 self.runtime.cell.begin_publication(generation)?;
3205 }
3206 let runtime = self.runtime.clone();
3207 let loaded = LoadedRun {
3208 record: loaded.record.clone(),
3209 revision: loaded.revision,
3210 };
3211 let activation = source.activation.clone();
3212 let (sender, receipt) = oneshot::channel();
3213 let kind = generation.map_or(RunTaskKind::Operation, |generation| {
3214 RunTaskKind::Foreground { generation }
3215 });
3216 if !self
3217 .runtime
3218 .app
3219 .spawn_run(self.runtime.key.clone(), kind, None, async move {
3220 let mut result = match runtime
3223 .observe_source_event(&context, event, deadline)
3224 .await
3225 {
3226 Ok(result) => Ok(result),
3227 Err(failure) => Err(runtime
3228 .fail_hook(&loaded, Some((&activation, &context)), failure, Vec::new())
3229 .await),
3230 };
3231 if let Some(generation) = generation
3232 && runtime.cell.end_publication(generation)
3233 && let Err(error) = runtime.finish_execution(generation, None).await
3234 && result.is_ok()
3235 {
3236 result = Err(error);
3237 }
3238 let _ = sender.send(result);
3239 })
3240 {
3241 if let Some(generation) = generation {
3242 self.runtime.cell.end_publication(generation);
3243 }
3244 return Err(AppError::Closed(
3245 "stream observation admission closed".into(),
3246 ));
3247 }
3248 let SessionState::Sources { loaded } =
3249 std::mem::replace(&mut self.state, SessionState::Finished)
3250 else {
3251 unreachable!()
3252 };
3253 self.state = SessionState::ObservingItem {
3254 loaded,
3255 index,
3256 receipt,
3257 };
3258 Ok(())
3259 }
3260
3261 fn publish_failure(
3262 &mut self,
3263 loaded: LoadedRun,
3264 activation: &FrontierRecord,
3265 context: &ActionContext,
3266 category: FailureCategory,
3267 ) -> Result<(), AppError> {
3268 let definition = self.runtime.app.definition_for(&loaded.record)?;
3269 let position = ActionPosition {
3270 node: context.node.clone(),
3271 ordinal: context.action_ordinal,
3272 };
3273 let (position, category, message) = match loaded.record.checkpoint.preceding_failure(
3274 &definition,
3275 &position,
3276 &activation.lineage,
3277 )? {
3278 Some((position, category, message)) => (position, category, message.to_owned()),
3279 None => (position, category, "caller action execution failed".into()),
3280 };
3281 self.publish_decision(
3282 loaded,
3283 BspDecision::Failed {
3284 category,
3285 message,
3286 observation: Some(position),
3287 },
3288 false,
3289 )
3290 }
3291
3292 fn publish_decision(
3293 &mut self,
3294 loaded: LoadedRun,
3295 decision: BspDecision,
3296 ordinary: bool,
3297 ) -> Result<(), AppError> {
3298 self.publish(
3299 SessionPublication::Decision {
3300 loaded,
3301 decision,
3302 activation_deadline: None,
3303 context: None,
3304 staged: Vec::new(),
3305 remaining: Vec::new().into_iter(),
3306 },
3307 ordinary,
3308 )
3309 }
3310
3311 fn publish_transition(
3312 &mut self,
3313 loaded: LoadedRun,
3314 transition: ControlTransition,
3315 ) -> Result<(), AppError> {
3316 self.publish(SessionPublication::Transition { loaded, transition }, false)
3317 }
3318
3319 fn publish(
3320 &mut self,
3321 mut publication: SessionPublication,
3322 ordinary: bool,
3323 ) -> Result<(), AppError> {
3324 let mut staged_prefix = Vec::new();
3325 let mut admitted_inputs = Vec::new();
3326 if let SessionPublication::Decision {
3327 loaded,
3328 decision: BspDecision::AdmitChildren { sequence, children },
3329 ..
3330 } = &publication
3331 && let Some(group) = self
3332 .children
3333 .iter()
3334 .find(|group| group.sequence == *sequence)
3335 {
3336 let original = loaded
3337 .record
3338 .checkpoint
3339 .control(*sequence)
3340 .and_then(|control| match &control.state {
3341 ControlRecordState::Child { admitted, .. } => Some(admitted.len()),
3342 _ => None,
3343 })
3344 .ok_or_else(|| {
3345 AppError::Integration("child admission evidence is unavailable".into())
3346 })?;
3347 if !group.inputs.is_empty() {
3348 for index in original..children.len() {
3349 let values = group.inputs.get(index).ok_or_else(|| {
3350 AppError::Integration("child runtime input membership changed".into())
3351 })?;
3352 admitted_inputs.push(values.clone());
3355 }
3356 }
3357 }
3358 let purpose = match &mut publication {
3359 SessionPublication::Decision {
3360 decision:
3361 BspDecision::Descendants {
3362 lineage,
3363 runtime_inputs,
3364 ..
3365 },
3366 context,
3367 remaining,
3368 ..
3369 } => PublicationPurpose::Descendants {
3370 context: Arc::clone(context.as_ref().ok_or_else(|| {
3371 AppError::Integration(
3372 "child runtime inputs have no original action context".into(),
3373 )
3374 })?),
3375 lineage: lineage.clone(),
3376 inputs: std::mem::take(runtime_inputs),
3377 remaining: std::mem::take(remaining),
3378 ordinary,
3379 },
3380 SessionPublication::Load { .. } => PublicationPurpose::Load,
3381 SessionPublication::Reload { .. } => PublicationPurpose::Reload,
3382 SessionPublication::ObserveChildren { .. } => PublicationPurpose::ObserveChildren,
3383 SessionPublication::VerifyChildren { .. } => PublicationPurpose::VerifyChildren,
3384 SessionPublication::FinishChild { .. } | SessionPublication::SettleStopping { .. } => {
3385 PublicationPurpose::Load
3386 }
3387 SessionPublication::Decision {
3388 decision: BspDecision::AdmitChildren { .. },
3389 ..
3390 } => PublicationPurpose::AdmitChildren,
3391 SessionPublication::BeginActivation {
3392 activation,
3393 context,
3394 remaining,
3395 staged,
3396 deadline,
3397 deadline_millis,
3398 ..
3399 } => {
3400 staged_prefix = std::mem::take(staged);
3403 PublicationPurpose::Start {
3404 activation: activation.clone(),
3405 context: Arc::clone(context),
3406 remaining: std::mem::take(remaining),
3407 deadline: *deadline,
3408 deadline_millis: *deadline_millis,
3409 }
3410 }
3411 SessionPublication::Decision { remaining, .. } if !remaining.as_slice().is_empty() => {
3412 PublicationPurpose::Continue {
3413 remaining: std::mem::take(remaining),
3414 ordinary,
3415 }
3416 }
3417 SessionPublication::Decision {
3418 decision: BspDecision::Barrier { .. },
3419 ..
3420 } => PublicationPurpose::Commit { ordinary },
3421 SessionPublication::EndSource { .. } => PublicationPurpose::EndSource,
3422 _ => PublicationPurpose::Advance,
3423 };
3424 if let SessionPublication::Decision {
3425 decision: BspDecision::Agent(request),
3426 ..
3427 } = &mut publication
3428 && request.is_streaming()
3429 {
3430 let (sender, _) = self.native_partials.get_or_insert_with(|| mpsc::channel(1));
3431 request.observe_partials(sender.clone());
3432 }
3433 let generation = self.generation;
3434 if let Some(generation) = generation {
3435 self.runtime.cell.begin_publication(generation)?;
3436 if matches!(&publication, SessionPublication::Release) {
3437 self.runtime.cell.revoke_execution(generation);
3438 }
3439 }
3440 let child_observation = match &publication {
3441 SessionPublication::Reload { receipt, .. } => receipt
3442 .record()
3443 .checkpoint
3444 .controls
3445 .values()
3446 .filter(|control| {
3447 matches!(control.state, ControlRecordState::Child { .. })
3448 && !self.children.iter().any(|group| {
3449 group.sequence == control.sequence
3450 && matches!(
3451 group.state,
3452 ChildExecutionState::Worker(_)
3453 | ChildExecutionState::Receipt(_)
3454 )
3455 })
3456 })
3457 .min_by_key(|control| control.deadline_millis)
3458 .map(|control| control.sequence),
3459 _ => None,
3460 };
3461 let options = self.runtime.cell.execution_options()?;
3462 let parent_deadline = self.parent_deadline;
3463 let reconcile_read = generation.is_none()
3464 && self.runtime.cell.child_parent().is_some()
3465 && matches!(
3466 &publication,
3467 SessionPublication::Load { .. }
3468 | SessionPublication::Reload { .. }
3469 | SessionPublication::ObserveChildren { .. }
3470 | SessionPublication::VerifyChildren { .. }
3471 );
3472 let execution_read = matches!(
3473 &publication,
3474 SessionPublication::Load { .. }
3475 | SessionPublication::Reload { .. }
3476 | SessionPublication::ObserveChildren { .. }
3477 | SessionPublication::FinishChild { .. }
3478 | SessionPublication::BeginActivation { .. }
3479 );
3480 let runtime = self.runtime.clone();
3481 let (sender, receipt) = oneshot::channel();
3482 let kind = generation.map_or(RunTaskKind::Operation, |generation| {
3483 RunTaskKind::Foreground { generation }
3484 });
3485 if !self
3486 .runtime
3487 .app
3488 .spawn_run(self.runtime.key.clone(), kind, self.admission.take(), async move {
3489 let inputs = &options.inputs;
3490 let observed = AssertUnwindSafe(async {
3491 match publication {
3492 SessionPublication::ObserveChildren { loaded, sequence } => async {
3493 let (loaded, children) = runtime.observe_children(loaded, sequence).await?;
3494 Ok(SessionReceipt::Children { loaded, sequence, children, verified: false })
3495 }.await,
3496 SessionPublication::VerifyChildren { parent, sequence, completions } => {
3497 match runtime.verify_child_completions(&parent, sequence, completions).await {
3498 Ok((loaded, children)) => {
3499 let control = parent.checkpoint.control(sequence).ok_or_else(|| {
3500 AppError::Conflict("verified child control is unavailable".into())
3501 })?;
3502 let complete = matches!(&control.state,
3503 ControlRecordState::Child { inputs, .. } if children.len() == inputs.len());
3504 if loaded.record.checkpoint.is_stopping() || loaded.record.checkpoint.is_terminal() || !complete {
3505 runtime.schedule_child_retention(control);
3506 Ok(SessionReceipt::Children { loaded, sequence, children, verified: true })
3507 } else {
3508 let transition = runtime.plan_child_completion(&loaded, control, children)?;
3512 runtime.publish_child_outcome(control, transition).await
3513 }
3514 },
3515 Err(RuntimeOperationFailure::Application(error)) => runtime
3516 .finish_child_admission(sequence, ChildAdmissionReceipt::Rejected(error))
3517 .await
3518 .map(|(loaded, children)| SessionReceipt::Children {
3519 loaded, sequence, children, verified: false,
3520 }),
3521 Err(failure) => Err(failure),
3522 }
3523 },
3524 SessionPublication::Decision { loaded, decision: BspDecision::AdmitChildren { sequence, children }, .. } => {
3525 let receipt = runtime.register_children(loaded, sequence, children, admitted_inputs).await;
3526 runtime.finish_child_admission(sequence, receipt).await.map(|(loaded, children)| {
3527 SessionReceipt::Children { loaded, sequence, children, verified: false }
3528 })
3529 },
3530 SessionPublication::ChildOutcome { control, transition } => {
3531 runtime.publish_child_outcome(&control, transition).await
3532 },
3533 SessionPublication::FinishChild { parent, sequence } => async {
3534 let loaded = runtime.load_execution_observation(None, parent_deadline).await?
3535 .ok_or_else(|| AppError::NotFound("admitted child is unavailable".into()))?;
3536 let definition = runtime.app.definition_for(&parent)?;
3537 loaded.record.validate_child(&parent, &definition)?;
3538 if loaded.record.parent.as_ref().is_none_or(|relation| relation.control_sequence != sequence) {
3539 return Err(AppError::Conflict("child completion group changed".into()).into());
3540 }
3541 if loaded.record.checkpoint.is_stopping()
3542 || loaded.record.checkpoint.is_terminal()
3543 || !loaded.record.checkpoint.controls.values().any(|control| {
3544 matches!(control.state, ControlRecordState::Inputs { .. })
3545 })
3546 {
3547 runtime.finish_execution_observation(&loaded.record, None, None, false).await?;
3548 runtime.cell.lifecycle_observation(loaded.record.checkpoint.sequence, false)?;
3549 }
3550 Ok(SessionReceipt::Checkpoint { loaded, staged: Vec::new() })
3551 }.await,
3552 SessionPublication::SettleStopping { loaded } => async {
3553 if let Some(loaded) = Box::pin(runtime.settle_stopping(loaded)).await {
3554 let observed = runtime.finish_execution_observation(&loaded.record, None, None, false).await;
3555 if runtime.cell.child_parent().is_none() {
3556 runtime.maintain_retention(None).await;
3557 }
3558 observed?;
3559 }
3560 let loaded = runtime.load_execution_observation(None, parent_deadline).await
3564 .inspect_err(|failure| runtime.retain_execution_read_failure(failure))?
3565 .ok_or_else(|| AppError::NotFound("workflow run does not exist".into()))?;
3566 Ok(SessionReceipt::Checkpoint { loaded, staged: Vec::new() })
3567 }.await,
3568 SessionPublication::Load { terminal } => async {
3569 let loaded = runtime.load_execution_observation(terminal, parent_deadline).await?
3570 .ok_or_else(|| AppError::NotFound("workflow run does not exist".into()))?;
3571 if !loaded.record.checkpoint.is_stopping()
3572 && !loaded.record.checkpoint.is_terminal()
3573 && loaded.record.checkpoint.controls.values().any(|control| {
3574 matches!(control.state, ControlRecordState::Inputs { .. })
3575 })
3576 && let Err(failure) = runtime.begin_execution_observation().await
3577 {
3578 return Err(runtime.fail_hook(&loaded, None, failure, Vec::new()).await.into());
3579 }
3580 Ok(SessionReceipt::Checkpoint { loaded, staged: Vec::new() })
3581 }.await,
3582 SessionPublication::Reload { receipt, deadline } => {
3583 let observed = tokio::select! {
3584 biased;
3585 _ = async { match deadline {
3586 Some(deadline) => tokio::time::sleep_until(deadline).await,
3587 None => std::future::pending().await,
3588 }}, if child_observation.is_none() && parent_deadline.is_none() => Ok(None),
3589 observed = async {
3590 let observed = match child_observation {
3595 Some(sequence) => runtime.observe_child_parent(receipt.record(), sequence, false, None).await?,
3596 None => runtime.load_execution_observation(None, deadline.into_iter().chain(parent_deadline).min()).await?
3597 .ok_or_else(|| AppError::NotFound("workflow run does not exist".into()))?,
3598 };
3599 Ok::<_, RuntimeOperationFailure>(Some(observed))
3600 } => observed,
3601 };
3602 observed.map(|observed| match observed {
3603 Some(loaded) => receipt.with_loaded(loaded),
3604 None => receipt,
3605 })
3606 },
3607 SessionPublication::BeginActivation { loaded, activation, context, deadline, .. } => async {
3608 if let Err(failure) = runtime.observe_activation_start(&loaded, &activation, &context, inputs, deadline).await {
3609 return Err(runtime.fail_hook(&loaded, Some((&activation, &context)), failure, std::mem::take(&mut staged_prefix)).await.into());
3610 }
3611 let current = runtime.load_execution_observation(
3612 None, parent_deadline.into_iter().chain(Some(deadline)).min(),
3613 ).await?.ok_or_else(|| AppError::NotFound("workflow run does not exist".into()))?;
3614 if !(current.record.checkpoint.is_terminal()
3615 || current.record.checkpoint.is_stopping()
3616 || current.record.same_run(&loaded.record)
3617 && current.record.checkpoint.base_sequence == loaded.record.checkpoint.base_sequence
3618 && current.record.checkpoint.frontier.iter().any(|candidate| candidate == &activation)
3619 && current.record.checkpoint.control_for(&activation.node, &activation.lineage).is_none()
3620 && !current.record.checkpoint.staged.iter().any(|staged| staged.node == activation.node && staged.lineage == activation.lineage))
3621 {
3622 return Err(AppError::Conflict("action pre-hook authority changed".into()).into());
3623 }
3624 Ok(SessionReceipt::Checkpoint { loaded: current, staged: std::mem::take(&mut staged_prefix) })
3625 }.await,
3626 publication => async {
3627 match publication {
3628 SessionPublication::Load { .. } | SessionPublication::Reload { .. }
3629 | SessionPublication::ObserveChildren { .. } | SessionPublication::VerifyChildren { .. }
3630 | SessionPublication::ChildOutcome { .. } | SessionPublication::FinishChild { .. } | SessionPublication::SettleStopping { .. }
3631 | SessionPublication::BeginActivation { .. } => unreachable!(),
3632 SessionPublication::Decision { loaded, decision, activation_deadline, context, staged, .. } => {
3633 if matches!(decision, BspDecision::Inputs { .. })
3634 && let Err(failure) = runtime.begin_execution_observation().await
3635 {
3636 return Err(runtime.fail_hook(&loaded, None, failure, staged).await);
3637 }
3638 runtime.finish_staged_spans(&loaded, &decision, context.as_deref()).await?;
3639 runtime.apply_decision(loaded, decision, activation_deadline, staged).await
3640 }
3641 SessionPublication::Transition { loaded, transition } => {
3642 let loaded = runtime.commit_for_recovery(loaded, transition).await
3643 .map_err(RuntimeOperationFailure::into_error)?;
3644 runtime.observe_generation(LifecycleEvent::deadline(parent_deadline)).await
3648 .map_err(|failure| {
3649 runtime.retain_execution_read_failure(&failure);
3650 failure.into_error()
3651 })?;
3652 Ok(loaded)
3653 }
3654 SessionPublication::Retry { loaded } => {
3655 let loaded = runtime.readmit_failed(loaded).await?;
3656 runtime.observe_generation(LifecycleEvent::deadline(parent_deadline)).await
3657 .map_err(|failure| {
3658 runtime.retain_execution_read_failure(&failure);
3659 failure.into_error()
3660 })?;
3661 Ok(loaded)
3662 }
3663 SessionPublication::EndSource { loaded } => {
3664 let loaded = runtime.end_source(loaded, generation).await?;
3665 runtime.observe_generation(LifecycleEvent::deadline(parent_deadline)).await
3666 .map_err(|failure| {
3667 runtime.retain_execution_read_failure(&failure);
3668 failure.into_error()
3669 })?;
3670 Ok(loaded)
3671 }
3672 SessionPublication::Release => {
3673 if let Some(generation) = generation { runtime.finish_execution(generation, None).await?; }
3674 runtime.load_execution_observation(None, parent_deadline).await
3675 .map_err(|failure| {
3676 runtime.retain_execution_read_failure(&failure);
3677 failure.into_error()
3678 })?
3679 .ok_or_else(|| AppError::NotFound("workflow run does not exist".into()))
3680 }
3681 }
3682 }.await.map(|loaded| SessionReceipt::Checkpoint { loaded, staged: std::mem::take(&mut staged_prefix) })
3683 .map_err(RuntimeOperationFailure::from),
3684 }
3685 }).catch_unwind().await;
3686 let mut result: Result<SessionReceipt, RuntimeOperationFailure> = match observed {
3687 Ok(result) => result,
3688 Err(payload) if reconcile_read => {
3689 drop(payload);
3690 match runtime.reconcile_child_panic(parent_deadline).await {
3691 Ok(result) => result.map(|loaded| SessionReceipt::Checkpoint {
3692 loaded, staged: Vec::new(),
3693 }).map_err(RuntimeOperationFailure::from),
3694 Err(_) => {
3695 runtime.app.tasks_failed(FinalResult::Integration(
3696 "child read panic reconciliation receipt closed".into(),
3697 ));
3698 Err(AppError::Integration("child read panic reconciliation receipt closed".into()).into())
3699 }
3700 }
3701 }
3702 Err(payload) => std::panic::resume_unwind(payload),
3703 };
3704 if execution_read
3705 && let Err(RuntimeOperationFailure::Storage(failure)) = &result
3706 {
3707 runtime.retain_execution_read_failure(failure);
3708 }
3709 if let Some(generation) = generation
3710 && runtime.cell.end_publication(generation)
3711 {
3712 let loaded = match &result {
3713 Ok(SessionReceipt::Checkpoint { loaded, .. })
3714 if (loaded.record.checkpoint.is_paused() || loaded.record.checkpoint.is_stopping())
3715 && loaded.record.checkpoint.controls.is_empty()
3716 && runtime.cell.lifecycle_generation().is_none()
3717 && !runtime.app.has_child_work(&runtime.key) =>
3718 {
3719 Some(LoadedRun { record: loaded.record.clone(), revision: loaded.revision })
3723 }
3724 _ => None,
3725 };
3726 if let Err(error) = runtime.finish_execution(generation, loaded).await
3727 && result.is_ok()
3728 {
3729 result = Err(error.into());
3730 }
3731 }
3732 let _ = sender.send(result);
3733 })
3734 {
3735 if let Some(generation) = generation {
3736 self.runtime.cell.end_publication(generation);
3737 }
3738 return Err(AppError::Closed(
3739 "run publication admission is closed".into(),
3740 ));
3741 }
3742 self.state = SessionState::Publishing { receipt, purpose };
3743 Ok(())
3744 }
3745
3746 fn poll_publication(
3747 &mut self,
3748 cx: &mut Context<'_>,
3749 closing: bool,
3750 ) -> Poll<Result<Option<ExecutionEvent>, AppError>> {
3751 if !matches!(&self.state, SessionState::Publishing { .. }) {
3752 return Poll::Ready(Ok(None));
3753 }
3754 self.expire_sources(None);
3755 let deadline = self.source_deadline(None);
3756 if self.poll_timer(cx, deadline) {
3757 self.expire_sources(None);
3758 }
3759 let commit = matches!(
3760 &self.state,
3761 SessionState::Publishing {
3762 purpose: PublicationPurpose::Commit { .. },
3763 ..
3764 }
3765 );
3766 if commit && let Some(event) = self.poll_native_partial(cx) {
3767 return Poll::Ready(Ok(Some(event)));
3768 }
3769 let SessionState::Publishing { receipt, .. } = &mut self.state else {
3770 unreachable!()
3771 };
3772 let observed = match Pin::new(receipt).poll(cx) {
3773 Poll::Ready(observed) => observed
3774 .map_err(|_| {
3775 RuntimeOperationFailure::from(AppError::Closed(
3776 "run publication observation closed".into(),
3777 ))
3778 })
3779 .and_then(|observed| observed),
3780 Poll::Pending => {
3781 if let Some(event) = self.poll_native_partial(cx) {
3782 return Poll::Ready(Ok(Some(event)));
3783 }
3784 return Poll::Pending;
3785 }
3786 };
3787 let SessionState::Publishing { purpose, .. } =
3788 std::mem::replace(&mut self.state, SessionState::Finished)
3789 else {
3790 unreachable!()
3791 };
3792 let observed = match observed {
3793 Ok(observed) => observed,
3794 Err(failure) => {
3795 if self.generation.is_none() {
3796 if matches!(
3797 purpose,
3798 PublicationPurpose::Load
3799 | PublicationPurpose::Reload
3800 | PublicationPurpose::ObserveChildren
3801 | PublicationPurpose::VerifyChildren
3802 ) {
3803 if self.runtime.schedule_operation_failure(&failure) {
3804 self.observed = None;
3805 }
3806 } else if matches!(purpose, PublicationPurpose::AdmitChildren) {
3807 self.observed = None;
3810 }
3811 }
3812 return Poll::Ready(Err(failure.into_error()));
3813 }
3814 };
3815 let (loaded, staged) = match observed {
3816 SessionReceipt::Checkpoint { loaded, staged } => (loaded, staged),
3817 SessionReceipt::Children {
3818 loaded,
3819 sequence,
3820 children,
3821 verified,
3822 } => {
3823 if closing {
3824 self.state = SessionState::Loaded(loaded);
3825 } else if matches!(purpose, PublicationPurpose::AdmitChildren)
3826 && self.poll_notification(cx)?
3827 {
3828 let deadline = loaded
3829 .record
3830 .checkpoint
3831 .control(sequence)
3832 .map(|control| {
3833 WorkflowRunRuntime::<CS, A, X, C, T, H, E>::remaining(
3834 control.deadline_millis,
3835 )
3836 })
3837 .transpose()?
3838 .and_then(|remaining| tokio::time::Instant::now().checked_add(remaining));
3839 self.publish(
3840 SessionPublication::Reload {
3841 receipt: SessionReceipt::Children {
3842 loaded,
3843 sequence,
3844 children,
3845 verified,
3846 },
3847 deadline,
3848 },
3849 false,
3850 )?;
3851 } else {
3852 self.plan_child_group(loaded, sequence, children, verified)?;
3853 }
3854 return Poll::Ready(Ok(None));
3855 }
3856 };
3857 if matches!(purpose, PublicationPurpose::Commit { .. })
3858 && let Err(error) = self
3859 .runtime
3860 .cell
3861 .lifecycle_observation(loaded.record.checkpoint.sequence, false)
3862 {
3863 self.state = SessionState::Loaded(loaded);
3867 return Poll::Ready(Err(error));
3868 }
3869 if closing
3870 || matches!(
3871 purpose,
3872 PublicationPurpose::Load | PublicationPurpose::Reload
3873 )
3874 {
3875 self.state = SessionState::Loaded(loaded);
3876 return Poll::Ready(Ok(None));
3877 }
3878 self.state = SessionState::Ready;
3879 if let PublicationPurpose::Start {
3880 activation,
3881 context,
3882 remaining,
3883 deadline,
3884 deadline_millis,
3885 } = purpose
3886 {
3887 self.state = if loaded.record.checkpoint.is_terminal()
3888 || loaded.record.checkpoint.is_stopping()
3889 {
3890 self.observed = Some(loaded.record);
3891 SessionState::Finished
3892 } else {
3893 SessionState::ActivationReady {
3894 loaded,
3895 activation,
3896 context,
3897 remaining,
3898 staged,
3899 deadline,
3900 deadline_millis,
3901 }
3902 };
3903 return Poll::Ready(Ok(None));
3904 }
3905 let purpose = match purpose {
3906 PublicationPurpose::Descendants {
3907 context,
3908 lineage,
3909 inputs,
3910 remaining,
3911 ordinary,
3912 } => {
3913 if !loaded.record.checkpoint.is_terminal()
3914 && !loaded.record.checkpoint.is_stopping()
3915 {
3916 let control = loaded
3917 .record
3918 .checkpoint
3919 .control_for(&context.node, &lineage)
3920 .ok_or_else(|| {
3921 AppError::Integration(
3922 "original child input control is unavailable".into(),
3923 )
3924 })?;
3925 let deadline = tokio::time::Instant::now()
3926 .checked_add(WorkflowRunRuntime::<CS, A, X, C, T, H, E>::remaining(
3927 control.deadline_millis,
3928 )?)
3929 .ok_or_else(|| {
3930 AppError::Timeout("child deadline is outside the clock range".into())
3931 })?;
3932 self.children.push(ChildExecution {
3933 sequence: control.sequence,
3934 parent: None,
3935 context,
3936 lineage,
3937 deadline,
3938 state: ChildExecutionState::Awaiting,
3939 inputs,
3940 });
3941 }
3942 if remaining.as_slice().is_empty() {
3943 PublicationPurpose::Advance
3944 } else {
3945 PublicationPurpose::Continue {
3946 remaining,
3947 ordinary,
3948 }
3949 }
3950 }
3951 purpose => purpose,
3952 };
3953 if let PublicationPurpose::Continue {
3954 mut remaining,
3955 ordinary,
3956 } = purpose
3957 {
3958 if loaded.record.checkpoint.is_terminal() || loaded.record.checkpoint.is_stopping() {
3959 self.observed = Some(loaded.record);
3960 self.state = SessionState::Finished;
3961 } else {
3962 let activation = remaining.next().ok_or_else(|| {
3963 AppError::Integration("native activation continuation is empty".into())
3964 })?;
3965 self.begin_activation(loaded, activation, remaining, staged, ordinary)?;
3966 }
3967 return Poll::Ready(Ok(None));
3968 }
3969 if let RunAdmission::Halted { node, result } = &loaded.record.checkpoint.admission {
3970 let event = ExecutionEvent::Halted {
3971 node: node.clone(),
3972 result: result.clone(),
3973 };
3974 self.observed = Some(loaded.record);
3975 self.state = SessionState::Finished;
3976 return Poll::Ready(Ok(Some(event)));
3977 }
3978 if let PublicationPurpose::Commit { ordinary } = purpose {
3979 self.observed = Some(loaded.record.clone());
3980 self.runtime
3981 .cell
3982 .record_execution_boundary(loaded.record.checkpoint.outcomes.len(), ordinary)?;
3983 return Poll::Ready(Ok(Some(ExecutionEvent::Committed {
3984 checkpoint: loaded.record.checkpoint.sequence,
3985 outcomes: loaded.record.checkpoint.outcomes,
3986 })));
3987 }
3988 if matches!(purpose, PublicationPurpose::EndSource) {
3989 self.observed = Some(loaded.record);
3990 self.state = SessionState::Eof;
3991 } else {
3992 self.observed = None;
3993 self.state = SessionState::Loaded(loaded);
3994 }
3995 Poll::Ready(Ok(None))
3996 }
3997}
3998
3999impl<
4000 CS: StateStoreBackend,
4001 A: AgentBackend,
4002 X: ActionBackend,
4003 C: ConditionBackend,
4004 T: StateTyping,
4005 H: LifecycleHooks,
4006 E: ChildExecutor,
4007> Drop for Execution<'_, CS, A, X, C, T, H, E>
4008{
4009 fn drop(&mut self) {
4010 if let Some(generation) = self.generation {
4011 self.runtime.cell.clear_execution_options(Some(generation));
4012 }
4013 let pending_observation = matches!(
4014 &self.state,
4015 SessionState::Publishing { .. } | SessionState::ObservingItem { .. }
4016 );
4017 self.stop_sources();
4018 self.stop_children();
4019 if self.generation.is_none()
4020 && !pending_observation
4021 && matches!(&self.state, SessionState::Sources { .. })
4022 {
4023 let failed = self
4024 .sources
4025 .iter()
4026 .enumerate()
4027 .filter_map(|(index, source)| {
4028 let category = self
4029 .prefetched
4030 .as_ref()
4031 .filter(|(buffered, _)| *buffered == index)
4032 .and_then(|(_, event)| event.as_ref().err().copied())
4033 .or(match source.state {
4034 CallerSourceState::Failed(category) => Some(category),
4035 _ => None,
4036 });
4037 category.map(|category| (index, category))
4038 })
4039 .min_by(|left, right| {
4040 let left = &self.sources[left.0].activation;
4041 let right = &self.sources[right.0].activation;
4042 (&left.node, &left.lineage).cmp(&(&right.node, &right.lineage))
4043 });
4044 if let Some((index, category)) = failed {
4045 let _ = self.stage_source_failure(index, category, false);
4046 }
4047 }
4048 let source_failed = self
4049 .runtime
4050 .cell
4051 .lifecycle_generation()
4052 .is_some_and(|generation| self.runtime.cell.source_failure(generation).is_some());
4053 let record = match &self.state {
4054 SessionState::Loaded(loaded)
4055 | SessionState::Sources { loaded }
4056 | SessionState::ObservingItem { loaded, .. }
4057 | SessionState::ActivationReady { loaded, .. } => Some(&loaded.record),
4058 _ => self.observed.as_ref(),
4059 };
4060 let child_pending = self.runtime.app.has_child_work(&self.runtime.key);
4061 let local_only = !child_pending
4062 && !pending_observation
4063 && !matches!(&self.state, SessionState::Publishing { .. })
4064 && !source_failed
4065 && !self.sources.is_empty()
4066 && record.is_some_and(|record| {
4067 record.checkpoint.staged.is_empty() && record.checkpoint.controls.is_empty()
4068 });
4069 if !child_pending
4070 && !pending_observation
4071 && !matches!(&self.state, SessionState::Publishing { .. })
4072 && !source_failed
4073 && let Some(record) = record
4074 && !self.retains_execution_lifecycle(record)
4075 && (self.generation.is_some()
4076 || record.checkpoint.controls.is_empty()
4077 || record.checkpoint.is_public_outcome())
4078 && !record
4079 .checkpoint
4080 .staged
4081 .iter()
4082 .any(|activation| matches!(activation.outcome, StagedOutcomeRecord::Failed { .. }))
4083 {
4084 let _ = self.runtime.finish_execution_observation_sync(record);
4085 }
4086 if let Some(generation) = self.generation.take()
4087 && self.runtime.cell.drop_execution(generation)
4088 {
4089 if local_only {
4090 if self.runtime.cell.release_execution(generation) {
4091 self.runtime.schedule(ScheduleCause::Reconcile);
4092 }
4093 return;
4094 }
4095 let loaded = if !child_pending
4096 && !pending_observation
4097 && !source_failed
4098 && self.runtime.cell.lifecycle_generation().is_none()
4099 && matches!(&self.state, SessionState::Loaded(loaded)
4100 if (loaded.record.checkpoint.is_paused() || loaded.record.checkpoint.is_stopping())
4101 && loaded.record.checkpoint.controls.is_empty())
4102 {
4103 let SessionState::Loaded(loaded) =
4104 std::mem::replace(&mut self.state, SessionState::Finished)
4105 else {
4106 unreachable!()
4107 };
4108 Some(loaded)
4109 } else {
4110 None
4111 };
4112 let runtime = self.runtime.clone();
4113 self.runtime.app.spawn_run(
4114 self.runtime.key.clone(),
4115 RunTaskKind::Foreground { generation },
4116 None,
4117 async move {
4118 let _ = runtime.finish_execution(generation, loaded).await;
4119 },
4120 );
4121 }
4122 }
4123}
4124
4125impl<
4126 CS: StateStoreBackend,
4127 A: AgentBackend,
4128 X: ActionBackend,
4129 C: ConditionBackend,
4130 T: StateTyping,
4131 H: LifecycleHooks,
4132 E: ChildExecutor,
4133> WorkflowRunRuntime<CS, A, X, C, T, H, E>
4134{
4135 pub(crate) async fn execute_child_group(
4136 &self,
4137 sequence: u64,
4138 generation: Option<u64>,
4139 parent: Arc<RunRecord>,
4140 children: Vec<RunRecord>,
4141 ) -> Result<SessionReceipt, RuntimeOperationFailure> {
4142 if generation.is_some_and(|generation| !self.cell.execution_is_live(generation)) {
4143 return Err(
4144 AppError::Closed("child executor admission generation changed".into()).into(),
4145 );
4146 }
4147 let mut outcome_published = false;
4148 let result = async {
4149 let control = parent.checkpoint.control(sequence).ok_or_else(|| AppError::Conflict("child control is unavailable".into()))?;
4150 let remaining = Self::remaining(control.deadline_millis)?;
4151 if remaining.is_zero() {
4152 outcome_published = true;
4156 let loaded = self.commit_scheduled_transition(control, ControlTransition::Failed {
4157 category: FailureCategory::Timeout, message: "child workflow deadline expired".into(), observation: None,
4158 }).await.ok_or_else(|| AppError::Closed(
4159 "child timeout publication requires reconciliation".into(),
4160 ))?;
4161 return Ok(SessionReceipt::Checkpoint { loaded, staged: Vec::new() });
4162 }
4163 let deadline = tokio::time::Instant::now().checked_add(remaining)
4164 .ok_or_else(|| AppError::Timeout("child deadline is outside the clock range".into()))?;
4165 let current = self.load_execution_observation(None, Some(deadline)).await?
4168 .ok_or_else(|| AppError::NotFound("child parent is unavailable".into()))?;
4169 if current.record.checkpoint.is_stopping() || current.record.checkpoint.is_terminal() {
4170 return Ok(SessionReceipt::Checkpoint { loaded: current, staged: Vec::new() });
4171 }
4172 if !current.record.owns_child_activation(&parent, sequence) {
4173 return Err(AppError::Conflict("child parent reservation changed".into()).into());
4174 }
4175 drop(current);
4176 let context = self.child_context(&parent, sequence)?;
4177 let mut group = ChildExecution {
4178 sequence, parent: Some(Arc::clone(&parent)), context,
4179 lineage: control.lineage.clone(), deadline,
4180 state: ChildExecutionState::Prepared(Vec::new()), inputs: Vec::new(),
4181 };
4182 group.start_local(self, &self.app, ExecutionLane::Worker, children)?;
4183 let context = Arc::clone(&group.context);
4184 let stopped = {
4185 let mut executor = Box::pin(poll_fn(|cx| group.poll_ready(cx, self)));
4186 let mut stopping = Box::pin(self.observe_child_parent(&parent, sequence, true, None));
4187 tokio::select! {
4188 biased;
4189 result = &mut stopping => {
4190 let loaded = result?;
4191 let loaded = if !loaded.record.checkpoint.is_stopping() {
4192 self.retain_source_failure(&context, &control.lineage, FailureCategory::Timeout);
4193 outcome_published = true;
4194 let mut failure = Box::pin(self.commit_scheduled_transition(control, ControlTransition::Failed {
4195 category: FailureCategory::Timeout, message: "child workflow deadline expired".into(), observation: None,
4196 }));
4197 let committed = tokio::select! {
4198 _ = &mut executor => failure.await,
4199 result = &mut failure => result,
4200 };
4201 committed.ok_or_else(|| AppError::Closed(
4202 "child timeout publication requires reconciliation".into(),
4203 ))?
4204 } else { loaded };
4205 Some(loaded)
4206 }
4207 _ = &mut executor => None,
4208 }
4209 };
4210 if let Some(loaded) = stopped {
4211 group.stop(self);
4215 return Ok(SessionReceipt::Checkpoint { loaded, staged: Vec::new() });
4216 }
4217 let ChildExecutionState::Complete(completed) = group.state else {
4218 return Err(AppError::Integration("child worker completed without its local executor".into()).into());
4219 };
4220 let completions = match completed {
4221 Ok(completions) => completions,
4222 Err(category) => {
4223 outcome_published = true;
4224 let loaded = self.commit_scheduled_transition(control, ControlTransition::Failed {
4225 category, message: "child execution failed".into(), observation: None,
4226 }).await.ok_or_else(|| AppError::Closed(
4227 "child failure publication requires reconciliation".into(),
4228 ))?;
4229 return Ok(SessionReceipt::Checkpoint { loaded, staged: Vec::new() });
4230 }
4231 };
4232 let (loaded, children) = self.verify_child_completions(&parent, sequence, completions).await?;
4233 if loaded.record.checkpoint.is_stopping() || loaded.record.checkpoint.is_terminal() {
4234 self.schedule_child_retention(control);
4235 return Ok(SessionReceipt::Checkpoint { loaded, staged: Vec::new() });
4236 }
4237 let transition = self.plan_child_completion(&loaded, control, children)?;
4238 outcome_published = true;
4242 self.publish_child_outcome(control, transition).await
4243 }.await;
4244 match result {
4245 Ok(receipt) => Ok(receipt),
4246 Err(RuntimeOperationFailure::Storage(failure)) => {
4247 self.retain_execution_read_failure(&failure);
4248 self.schedule_after_storage_failure(&failure);
4249 Err(failure.into())
4250 }
4251 Err(RuntimeOperationFailure::Application(error)) => {
4252 if outcome_published {
4253 return Err(error.into());
4254 }
4255 let failure = self
4256 .finish_child_admission(sequence, ChildAdmissionReceipt::Rejected(error))
4257 .await;
4258 let loaded = self
4259 .load_execution_observation(None, None)
4260 .await
4261 .inspect_err(|failure| self.retain_execution_read_failure(failure))?
4262 .ok_or_else(|| AppError::NotFound("child parent is unavailable".into()))?;
4263 if loaded.record.checkpoint.is_stopping() || loaded.record.checkpoint.is_terminal()
4264 {
4265 Ok(SessionReceipt::Checkpoint {
4266 loaded,
4267 staged: Vec::new(),
4268 })
4269 } else {
4270 match failure {
4271 Err(failure) => Err(failure),
4272 Ok(_) => Err(AppError::Integration(
4273 "child failure policy returned admission success".into(),
4274 )
4275 .into()),
4276 }
4277 }
4278 }
4279 }
4280 }
4281
4282 fn schedule_child_retention(&self, control: &ControlRecord) {
4283 let ControlRecordState::Child { admitted, .. } = &control.state else {
4284 return;
4285 };
4286 for member in admitted {
4287 match Self::attach(Arc::clone(&self.app), member.key.clone()) {
4288 Ok(child) => {
4289 child.schedule(ScheduleCause::Reconcile);
4290 }
4291 Err(error) => self.app.tasks_failed(FinalResult::from(&error)),
4292 }
4293 }
4294 }
4295
4296 fn plan_child_completion(
4297 &self,
4298 loaded: &LoadedRun,
4299 control: &ControlRecord,
4300 children: Vec<RunRecord>,
4301 ) -> Result<ControlTransition, AppError> {
4302 let planned = catch_unwind(AssertUnwindSafe(|| {
4303 self.plan_children(loaded, control.sequence, children, true)
4304 }))
4305 .map_err(|_| AppError::Integration("child planning callback panicked".into()))
4306 .and_then(|result| result)
4307 .and_then(|decision| match decision {
4308 BspDecision::Stage(activations) => Ok(ControlTransition::Stage { activations }),
4309 BspDecision::Failed {
4310 category,
4311 message,
4312 observation,
4313 } => Ok(ControlTransition::Failed {
4314 category,
4315 message,
4316 observation,
4317 }),
4318 _ => Err(AppError::Integration(
4319 "completed child group has no final outcome".into(),
4320 )),
4321 });
4322 if !matches!(&planned, Ok(ControlTransition::Stage { activations })
4323 if activations.iter().all(|activation| matches!(activation.outcome, StagedOutcomeRecord::Complete { .. })))
4324 {
4325 self.schedule_child_retention(control);
4328 }
4329 planned
4330 }
4331
4332 async fn publish_child_outcome(
4333 &self,
4334 control: &ControlRecord,
4335 transition: ControlTransition,
4336 ) -> Result<SessionReceipt, RuntimeOperationFailure> {
4337 let completed = matches!(&transition, ControlTransition::Stage { activations }
4341 if activations.iter().all(|activation| matches!(activation.outcome, StagedOutcomeRecord::Complete { .. })));
4342 if let Some(loaded) = self.commit_scheduled_transition(control, transition).await {
4343 return Ok(SessionReceipt::Checkpoint {
4346 loaded,
4347 staged: Vec::new(),
4348 });
4349 }
4350 if completed {
4351 self.schedule_child_retention(control);
4355 }
4356 let loaded = match tokio::time::timeout(
4357 Self::remaining(control.deadline_millis)?,
4358 self.load_for_recovery(),
4359 )
4360 .await
4361 {
4362 Ok(result) => result
4363 .map_err(RuntimeOperationFailure::from)
4364 .and_then(|loaded| {
4365 loaded.ok_or_else(|| {
4366 AppError::NotFound("child parent is unavailable".into()).into()
4367 })
4368 }),
4369 Err(_) => Err(AppError::Timeout("child outcome observation timed out".into()).into()),
4370 };
4371 if !loaded.as_ref().is_ok_and(|loaded| {
4372 loaded.record.checkpoint.is_stopping() || loaded.record.checkpoint.is_terminal()
4373 }) {
4374 self.app.tasks_failed(FinalResult::Integration(
4375 "child outcome publication authority is unresolved".into(),
4376 ));
4377 return Err(loaded.err().unwrap_or_else(|| {
4378 AppError::Closed("child outcome publication requires reconciliation".into()).into()
4379 }));
4380 }
4381 Ok(SessionReceipt::Checkpoint {
4382 loaded: loaded?,
4383 staged: Vec::new(),
4384 })
4385 }
4386
4387 fn retain_source_failure(
4388 &self,
4389 context: &ActionContext,
4390 lineage: &str,
4391 category: FailureCategory,
4392 ) {
4393 let Some(generation) = self.cell.lifecycle_generation() else {
4394 return;
4395 };
4396 match self
4397 .cell
4398 .observe_source_failure(generation, context, lineage, category)
4399 {
4400 Ok(()) | Err(AppError::Closed(_)) => {
4401 }
4404 Err(_) => self.app.tasks_failed(FinalResult::Integration(
4405 "source failure observation authority failed".into(),
4406 )),
4407 }
4408 }
4409
4410 async fn end_source(
4411 &self,
4412 loaded: LoadedRun,
4413 execution: Option<u64>,
4414 ) -> Result<LoadedRun, AppError> {
4415 let generation = self
4416 .cell
4417 .lifecycle_generation()
4418 .ok_or_else(|| AppError::Closed("source cleanup observation is unavailable".into()))?;
4419 let deadline = self.cell.begin_action_cleanup(generation)?;
4420 for context in self.cell.action_observations() {
4423 if context.checkpoint != loaded.record.checkpoint.base_sequence {
4424 continue;
4425 }
4426 let mut observed_activation = None;
4427 for activation in &loaded.record.checkpoint.frontier {
4428 if self
4429 .cell
4430 .action_context(
4431 loaded.record.checkpoint.base_sequence,
4432 &activation.node,
4433 &activation.lineage,
4434 )
4435 .is_some_and(|observed| Arc::ptr_eq(&observed, &context))
4436 {
4437 observed_activation = Some(activation);
4438 break;
4439 }
4440 }
4441 let activation = observed_activation.ok_or_else(|| {
4442 AppError::Integration("source cleanup activation is unavailable".into())
4443 })?;
4444 if let Err(failure) = self
4445 .observe_step_finish(
4446 generation,
4447 &context,
4448 None,
4449 &loaded.record,
4450 loaded
4451 .record
4452 .checkpoint
4453 .control_for(&activation.node, &activation.lineage)
4454 .is_some(),
4455 Some(deadline),
4456 )
4457 .await
4458 {
4459 return Err(self
4460 .fail_hook(&loaded, Some((activation, &context)), failure, Vec::new())
4461 .await);
4462 }
4463 }
4464 let loaded = self
4465 .load_execution_observation(None, Some(deadline))
4466 .await
4467 .map_err(|failure| {
4468 self.retain_execution_read_failure(&failure);
4469 failure.into_error()
4470 })?
4471 .ok_or_else(|| AppError::NotFound("workflow run does not exist".into()))?;
4472 let loaded = if let Some(generation) = execution {
4473 self.settle_execution(generation, Some(loaded)).await?.0
4476 } else if loaded.record.checkpoint.is_stopping() || loaded.record.checkpoint.is_terminal() {
4477 loaded
4478 } else {
4479 let definition = self.app.definition_for(&loaded.record)?;
4480 let discard_staged = loaded.record.checkpoint.rollback_stage_keys(&definition);
4481 self.commit_for_recovery(loaded, ControlTransition::Pause { discard_staged })
4483 .await
4484 .map_err(RuntimeOperationFailure::into_error)?
4485 };
4486 Ok(loaded)
4487 }
4488
4489 pub(super) async fn finish_execution(
4490 &self,
4491 generation: u64,
4492 loaded: Option<LoadedRun>,
4493 ) -> Result<RunRecord, AppError> {
4494 if !self.cell.owns_execution(generation) {
4495 return Err(AppError::Closed(
4496 "run execution generation no longer owns cleanup".into(),
4497 ));
4498 }
4499 let mut changes = self.app.runtime_changes();
4500 while self.app.has_child_work(&self.key) {
4501 changes
4502 .changed()
4503 .await
4504 .map_err(|_| AppError::Closed("child receipt observation closed".into()))?;
4505 }
4506 let settled = self.settle_execution(generation, loaded).await;
4507 if let Err(error) = &settled
4508 && self.cell.source_failure(generation).is_some()
4509 {
4510 if !matches!(error, AppError::Closed(_)) {
4511 self.app.tasks_failed(FinalResult::Integration(
4512 "source failure cleanup authority failed".into(),
4513 ));
4514 }
4515 return settled.map(|(loaded, _)| loaded.record);
4516 }
4517 let retained = settled.as_ref().is_ok_and(|(_, retained)| *retained);
4518 let mut result = settled.map(|(loaded, _)| loaded.record);
4519 let cleanup_pending = result.as_ref().is_ok_and(|record| {
4520 record.checkpoint.is_stopping()
4521 || !record.checkpoint.controls.is_empty()
4522 || record.checkpoint.is_terminal() && !retained
4523 });
4524 if let Ok(record) = &result {
4525 let observed = self
4526 .finish_execution_observation(record, None, None, false)
4527 .await;
4528 if record.checkpoint.is_terminal() && self.cell.child_parent().is_none() {
4529 self.maintain_retention(None).await;
4530 }
4531 if let Err(error) = observed {
4532 result = Err(error);
4533 }
4534 }
4535 self.cell.end_publication(generation);
4536 self.cell.clear_execution_options(Some(generation));
4537 let scheduled = self.cell.release_execution(generation);
4538 if !retained && (cleanup_pending || scheduled) {
4539 self.schedule(ScheduleCause::Reconcile);
4540 }
4541 result
4542 }
4543
4544 pub(super) fn settle_execution(
4545 &self,
4546 generation: u64,
4547 loaded: Option<LoadedRun>,
4548 ) -> Pin<Box<impl Future<Output = Result<(LoadedRun, bool), AppError>> + '_>> {
4549 Box::pin(async move {
4550 if !self.cell.owns_execution(generation) {
4551 return Err(AppError::Closed(
4552 "run execution generation no longer owns cleanup".into(),
4553 ));
4554 }
4555 let mut retained = false;
4556 let result = async {
4557 let mut loaded = match loaded {
4558 Some(loaded) => loaded,
4559 None => {
4560 let deadline = self.cell.begin_action_cleanup(generation)?;
4561 self.load_execution_observation(None, Some(deadline))
4564 .await
4565 .map_err(|failure| {
4566 self.retain_execution_read_failure(&failure);
4567 failure.into_error()
4568 })?
4569 .ok_or_else(|| {
4570 AppError::NotFound("workflow run does not exist".into())
4571 })?
4572 }
4573 };
4574 if let Some((context, lineage, category)) = self.cell.source_failure(generation)
4575 && context.checkpoint == loaded.record.checkpoint.base_sequence
4576 && !loaded.record.checkpoint.is_stopping()
4577 && !loaded.record.checkpoint.is_terminal()
4578 && loaded
4579 .record
4580 .checkpoint
4581 .position
4582 .as_ref()
4583 .is_none_or(|committed| committed.ordinal < context.action_ordinal)
4584 {
4585 let position = ActionPosition {
4586 node: context.node.clone(),
4587 ordinal: context.action_ordinal,
4588 };
4589 let definition = self.app.definition_for(&loaded.record)?;
4590 let (position, category, message) = match loaded
4591 .record
4592 .checkpoint
4593 .preceding_failure(&definition, &position, &lineage)?
4594 {
4595 Some((position, category, message)) => {
4596 (position, category, message.to_owned())
4597 }
4598 None => (position, category, "caller action execution failed".into()),
4599 };
4600 loaded = self
4601 .commit_for_recovery(
4602 loaded,
4603 ControlTransition::Failed {
4604 category,
4605 message,
4606 observation: Some(position),
4607 },
4608 )
4609 .await
4610 .map_err(RuntimeOperationFailure::into_error)?;
4611 }
4612 if !loaded.record.checkpoint.is_terminal()
4613 && !loaded.record.checkpoint.is_stopping()
4614 {
4615 let definition = self.app.definition_for(&loaded.record)?;
4616 let failed = loaded
4617 .record
4618 .checkpoint
4619 .selected_staged(&definition)
4620 .filter_map(|activation| {
4621 if let StagedOutcomeRecord::Failed { category, message } =
4622 &activation.outcome
4623 {
4624 Some((activation, *category, message))
4625 } else {
4626 None
4627 }
4628 })
4629 .min_by(|left, right| {
4630 (&left.0.node, &left.0.lineage).cmp(&(&right.0.node, &right.0.lineage))
4631 });
4632 if let Some((activation, category, message)) = failed {
4633 let message = message.clone();
4634 let observation = loaded.record.checkpoint.proposed_position(
4635 &definition,
4636 &activation.node,
4637 &activation.lineage,
4638 &[],
4639 )?;
4640 loaded = self
4641 .commit_for_recovery(
4642 loaded,
4643 ControlTransition::Failed {
4644 category,
4645 message,
4646 observation: Some(observation),
4647 },
4648 )
4649 .await
4650 .map_err(RuntimeOperationFailure::into_error)?;
4651 }
4652 }
4653 if loaded.record.checkpoint.is_stopping()
4656 && loaded.record.checkpoint.controls.is_empty()
4657 {
4658 loaded = match Box::pin(self.settle_stopping(loaded)).await {
4659 Some(loaded) => loaded,
4660 None => {
4661 self.load_execution_observation(None, None)
4665 .await
4666 .map_err(|failure| {
4667 self.retain_execution_read_failure(&failure);
4668 failure.into_error()
4669 })?
4670 .ok_or_else(|| {
4671 AppError::NotFound("workflow run does not exist".into())
4672 })?
4673 }
4674 };
4675 }
4676 if !loaded.record.checkpoint.staged.is_empty()
4677 && !loaded.record.checkpoint.is_terminal()
4678 && !loaded.record.checkpoint.is_stopping()
4679 {
4680 let definition = self.app.definition_for(&loaded.record)?;
4681 let discard_staged = loaded.record.checkpoint.rollback_stage_keys(&definition);
4682 return self
4683 .commit_for_recovery(loaded, ControlTransition::Pause { discard_staged })
4684 .await
4685 .map_err(RuntimeOperationFailure::into_error);
4686 }
4687 if loaded.record.checkpoint.is_terminal() {
4688 let deadline = if self.cell.lifecycle_generation().is_none() {
4691 Box::pin(self.maintain_retention(Some(loaded))).await;
4692 let deadline = LifecycleEvent::deadline(None);
4695 loaded = self
4696 .load_execution_observation(None, Some(deadline))
4697 .await
4698 .map_err(|failure| {
4699 self.retain_execution_read_failure(&failure);
4700 failure.into_error()
4701 })?
4702 .ok_or_else(|| {
4703 AppError::NotFound("workflow run does not exist".into())
4704 })?;
4705 deadline
4706 } else {
4707 LifecycleEvent::deadline(None)
4708 };
4709 retained = tokio::time::timeout_at(deadline, async {
4710 let mut delay = RECONCILIATION_INITIAL_DELAY;
4711 loop {
4712 match self.app.storage().run_is_active(&loaded.record).await {
4713 Ok(active) => {
4714 self.observe_generation(deadline).await?;
4717 return Ok(!active);
4718 }
4719 Err(failure)
4720 if failure.is_recoverable()
4721 && self.schedule_after_storage_failure(&failure) =>
4722 {
4723 tokio::time::sleep(delay).await;
4724 delay = delay.saturating_mul(2).min(RECONCILIATION_MAX_DELAY);
4725 }
4726 Err(failure) => return Err(failure),
4727 }
4728 }
4729 })
4730 .await
4731 .unwrap_or_else(|_| {
4732 Err(StorageFailure::permanent(AppError::Timeout(
4733 "workflow capacity observation deadline expired".into(),
4734 )))
4735 })
4736 .map_err(|failure| {
4737 self.retain_execution_read_failure(&failure);
4738 failure.into_error()
4739 })?;
4740 } else if loaded.record.checkpoint.is_stopping() {
4741 self.observe_generation(LifecycleEvent::deadline(None))
4744 .await
4745 .map_err(|failure| {
4746 self.retain_execution_read_failure(&failure);
4747 failure.into_error()
4748 })?;
4749 }
4750 Ok(loaded)
4751 }
4752 .await;
4753 result.map(|loaded| (loaded, retained))
4754 })
4755 }
4756}
4757
4758impl<
4759 CS: StateStoreBackend,
4760 A: AgentBackend,
4761 X: ActionBackend,
4762 C: ConditionBackend,
4763 T: StateTyping,
4764 H: LifecycleHooks,
4765 E: ChildExecutor,
4766> WorkflowRunRuntime<CS, A, X, C, T, H, E>
4767{
4768 pub(super) async fn begin_execution_observation(&self) -> Result<u64, WorkflowFailure> {
4769 let owner = self
4770 .cell
4771 .observation_owner()
4772 .map_err(|error| WorkflowFailure::Execution(error.to_string()))?;
4773 let (generation, opening) = self
4774 .cell
4775 .begin_lifecycle(owner)
4776 .map_err(|error| WorkflowFailure::Execution(error.to_string()))?;
4777 if opening {
4778 let deadline = self
4779 .cell
4780 .begin_action_cleanup(generation)
4781 .map_err(|error| WorkflowFailure::Execution(error.to_string()))?;
4782 LifecycleEvent::ExecuteStarted {
4783 run: self.id(),
4784 occurrence: self.occurrence(),
4785 }
4786 .deliver(
4787 self.app.hooks(),
4788 self.cell
4789 .execution_mode()
4790 .map_err(|error| WorkflowFailure::Execution(error.to_string()))?,
4791 deadline,
4792 )
4793 .await?;
4794 self.cell
4795 .opened_lifecycle(generation)
4796 .map_err(|error| WorkflowFailure::Execution(error.to_string()))?;
4797 }
4798 Ok(generation)
4799 }
4800
4801 async fn observe_activation_start(
4802 &self,
4803 loaded: &LoadedRun,
4804 activation: &FrontierRecord,
4805 context: &Arc<ActionContext>,
4806 inputs: &Map<String, Value>,
4807 deadline: tokio::time::Instant,
4808 ) -> Result<(), WorkflowFailure> {
4809 let generation = self.begin_execution_observation().await?;
4810 let definition = self
4811 .app
4812 .definition_for(&loaded.record)
4813 .map_err(|error| WorkflowFailure::Execution(error.to_string()))?;
4814 let Some(Step::Action {
4815 action, revision, ..
4816 }) = definition.node(&activation.node).map(|node| node.step())
4817 else {
4818 return self
4819 .observe_step_start(
4820 generation,
4821 &loaded.record,
4822 context,
4823 &activation.lineage,
4824 inputs,
4825 deadline,
4826 )
4827 .await
4828 .map(|_| ());
4829 };
4830 let binding = self
4831 .app
4832 .action(action, revision)
4833 .map_err(|error| WorkflowFailure::Execution(error.to_string()))?;
4834 let inputs = catch_unwind(AssertUnwindSafe(|| {
4835 binding
4836 .declaration()
4837 .resolve_inputs(inputs, self.cell.execution_mode()?)
4838 .and_then(ActionInputs::into_values)
4839 }))
4840 .map_err(|_| WorkflowFailure::Execution("action declaration panicked".into()))?
4841 .map_err(|error| WorkflowFailure::State(error.to_string()))?;
4842 let context = self
4843 .observe_step_start(
4844 generation,
4845 &loaded.record,
4846 context,
4847 &activation.lineage,
4848 &inputs,
4849 deadline,
4850 )
4851 .await?;
4852 self.observe_stream_start(generation, &context, deadline)
4853 .await
4854 }
4855
4856 pub(super) async fn observe_stream_start(
4857 &self,
4858 generation: u64,
4859 context: &ActionContext,
4860 deadline: tokio::time::Instant,
4861 ) -> Result<(), WorkflowFailure> {
4862 if self.cell.lifecycle_generation() != Some(generation)
4863 || self.cell.execution_was_dropped(generation)
4864 {
4865 return Ok(());
4866 }
4867 LifecycleEvent::StreamStarted {
4868 context,
4869 span: None,
4870 }
4871 .deliver(
4872 self.app.hooks(),
4873 context.mode,
4874 LifecycleEvent::deadline(Some(deadline)),
4875 )
4876 .await?;
4877 self.cell
4878 .open_action_stream(generation, context)
4879 .map_err(|error| WorkflowFailure::Execution(error.to_string()))?;
4880 Ok(())
4881 }
4882
4883 pub(super) async fn observe_stream_item(
4884 &self,
4885 generation: u64,
4886 context: &ActionContext,
4887 result: &Map<String, Value>,
4888 deadline: tokio::time::Instant,
4889 ) -> Result<(), WorkflowFailure> {
4890 if self.cell.lifecycle_generation() != Some(generation)
4891 || self.cell.execution_was_dropped(generation)
4892 {
4893 return Ok(());
4894 }
4895 let index = self
4896 .cell
4897 .action_stream_item(generation, context, false)
4898 .map_err(|error| WorkflowFailure::Execution(error.to_string()))?;
4899 LifecycleEvent::StreamItem {
4900 context,
4901 span: None,
4902 index,
4903 result,
4904 }
4905 .deliver(
4906 self.app.hooks(),
4907 context.mode,
4908 LifecycleEvent::deadline(Some(deadline)),
4909 )
4910 .await?;
4911 self.cell
4912 .action_stream_item(generation, context, true)
4913 .map_err(|error| WorkflowFailure::Execution(error.to_string()))?;
4914 Ok(())
4915 }
4916
4917 async fn observe_source_event(
4918 &self,
4919 context: &ActionContext,
4920 event: ActionEvent,
4921 deadline: tokio::time::Instant,
4922 ) -> Result<Option<Map<String, Value>>, WorkflowFailure> {
4923 let generation = self.cell.lifecycle_generation().ok_or_else(|| {
4924 WorkflowFailure::Execution("source observation generation closed".into())
4925 })?;
4926 if let ActionEvent::Partial { result } = event {
4927 self.observe_stream_item(generation, context, &result, deadline)
4928 .await?;
4929 return Ok(Some(result));
4930 }
4931 if self.cell.execution_was_dropped(generation) {
4932 return Ok(None);
4933 }
4934 event
4935 .validate_span()
4936 .map_err(|error| WorkflowFailure::Execution(error.to_string()))?;
4937 let deadline = LifecycleEvent::deadline(Some(deadline));
4938 match event {
4939 ActionEvent::SpanStarted {
4940 id,
4941 name,
4942 parent,
4943 dependencies,
4944 } => {
4945 let parent_index = self
4946 .cell
4947 .span_parent(generation, context, &id, parent.as_deref())
4948 .map_err(|error| WorkflowFailure::Execution(error.to_string()))?;
4949 LifecycleEvent::SpanStarted {
4950 context,
4951 id: &id,
4952 name: &name,
4953 parent: parent.as_deref(),
4954 dependencies: &dependencies,
4955 }
4956 .deliver(self.app.hooks(), context.mode, deadline)
4957 .await?;
4958 self.cell
4959 .opened_span(generation, context, id, parent_index)
4960 .map_err(|error| WorkflowFailure::Execution(error.to_string()))?;
4961 }
4962 ActionEvent::SpanAttribute {
4963 span,
4964 key,
4965 value,
4966 tags,
4967 } => {
4968 self.cell
4969 .span_attribute(generation, context, span.as_deref())
4970 .map_err(|error| WorkflowFailure::Execution(error.to_string()))?;
4971 LifecycleEvent::SpanAttribute {
4972 context,
4973 span: span.as_deref(),
4974 key: &key,
4975 value: &value,
4976 tags: &tags,
4977 }
4978 .deliver(self.app.hooks(), context.mode, deadline)
4979 .await?;
4980 }
4981 ActionEvent::SpanFinished { id } => {
4982 let id = self
4983 .cell
4984 .take_span(generation, context, Some(&id))
4985 .map_err(|error| WorkflowFailure::Execution(error.to_string()))?
4986 .ok_or_else(|| {
4987 WorkflowFailure::Execution(
4988 "caller span finish target is unavailable".into(),
4989 )
4990 })?;
4991 LifecycleEvent::SpanFinished { context, id: &id }
4992 .deliver(self.app.hooks(), context.mode, deadline)
4993 .await?;
4994 }
4995 ActionEvent::Partial { .. } | ActionEvent::Completed { .. } => unreachable!(),
4996 }
4997 Ok(None)
4998 }
4999
5000 async fn finish_spans(
5001 &self,
5002 generation: u64,
5003 context: &ActionContext,
5004 async_deadline: Option<tokio::time::Instant>,
5005 ) -> Result<(), WorkflowFailure> {
5006 let mut failure = None;
5007 loop {
5008 let id = match self.cell.take_span(generation, context, None) {
5009 Ok(Some(id)) => id,
5010 Ok(None) => break,
5011 Err(error) => {
5012 return Err(
5013 failure.unwrap_or_else(|| WorkflowFailure::Execution(error.to_string()))
5014 );
5015 }
5016 };
5017 let event = LifecycleEvent::SpanFinished { context, id: &id };
5018 let observed =
5019 match async_deadline.filter(|_| !self.cell.execution_was_dropped(generation)) {
5020 Some(deadline) => {
5021 event
5022 .deliver(self.app.hooks(), context.mode, deadline)
5023 .await
5024 }
5025 None => event.deliver_sync(self.app.hooks()),
5026 };
5027 if let Err(next) = observed
5028 && failure.is_none()
5029 {
5030 failure = Some(next);
5031 }
5032 }
5033 failure.map_or(Ok(()), Err)
5034 }
5035
5036 fn finish_spans_sync(
5037 &self,
5038 generation: u64,
5039 context: &ActionContext,
5040 ) -> Result<(), WorkflowFailure> {
5041 let mut failure = None;
5042 loop {
5043 let id = match self.cell.take_span(generation, context, None) {
5044 Ok(Some(id)) => id,
5045 Ok(None) => break,
5046 Err(error) => {
5047 return Err(
5048 failure.unwrap_or_else(|| WorkflowFailure::Execution(error.to_string()))
5049 );
5050 }
5051 };
5052 if let Err(next) =
5053 (LifecycleEvent::SpanFinished { context, id: &id }).deliver_sync(self.app.hooks())
5054 && failure.is_none()
5055 {
5056 failure = Some(next);
5057 }
5058 }
5059 failure.map_or(Ok(()), Err)
5060 }
5061
5062 async fn finish_staged_spans(
5063 &self,
5064 loaded: &LoadedRun,
5065 decision: &BspDecision,
5066 context: Option<&ActionContext>,
5067 ) -> Result<(), AppError> {
5068 let (BspDecision::Stage(staged), Some(context)) = (decision, context) else {
5069 return Ok(());
5070 };
5071 let Some(generation) = self.cell.lifecycle_generation() else {
5072 return Ok(());
5073 };
5074 let Some(staged) = staged.iter().find(|staged| {
5075 self.cell
5076 .action_context(
5077 loaded.record.checkpoint.base_sequence,
5078 &staged.node,
5079 &staged.lineage,
5080 )
5081 .is_some_and(|observed| std::ptr::eq(observed.as_ref(), context))
5082 && matches!(staged.outcome, StagedOutcomeRecord::Complete { .. })
5083 }) else {
5084 return Ok(());
5085 };
5086 let activation = loaded
5087 .record
5088 .checkpoint
5089 .frontier
5090 .iter()
5091 .find(|activation| {
5092 activation.node == staged.node && activation.lineage == staged.lineage
5093 })
5094 .ok_or_else(|| {
5095 AppError::Integration("completed caller activation is unavailable".into())
5096 })?;
5097 if let Err(failure) = self
5100 .finish_spans(generation, context, Some(LifecycleEvent::deadline(None)))
5101 .await
5102 {
5103 return Err(self
5104 .fail_hook(loaded, Some((activation, context)), failure, Vec::new())
5105 .await);
5106 }
5107 Ok(())
5108 }
5109
5110 pub(super) async fn fail_hook(
5111 &self,
5112 loaded: &LoadedRun,
5113 activation: Option<(&FrontierRecord, &ActionContext)>,
5114 failure: WorkflowFailure,
5115 staged: Vec<StagedActivationRecord>,
5116 ) -> AppError {
5117 let error = failure.application_error();
5118 let mut prior = None;
5119 let settled = async {
5120 let generation = self
5121 .cell
5122 .lifecycle_generation()
5123 .ok_or_else(|| AppError::Closed("lifecycle failure cleanup is closed".into()))?;
5124 let deadline = self.cell.begin_action_cleanup(generation)?;
5125 let mut loaded = LoadedRun {
5126 record: loaded.record.clone(),
5127 revision: loaded.revision,
5128 };
5129 if !staged.is_empty() {
5130 loaded = self
5133 .commit_for_recovery(
5134 loaded,
5135 ControlTransition::Stage {
5136 activations: staged,
5137 },
5138 )
5139 .await
5140 .map_err(RuntimeOperationFailure::into_error)?;
5141 }
5142 let definition = self.app.definition_for(&loaded.record)?;
5143 let retained = self
5144 .cell
5145 .lifecycle_generation()
5146 .and_then(|generation| self.cell.source_failure(generation));
5147 let source = retained.as_ref().filter(|(context, lineage, _)| {
5148 context.checkpoint == loaded.record.checkpoint.base_sequence
5149 && activation.is_some_and(|(activation, _)| {
5150 (&context.node, lineage) < (&activation.node, &activation.lineage)
5151 })
5152 });
5153 let (mut observation, lineage, mut category, mut message) = match source {
5157 Some((context, lineage, category)) => {
5158 let message = "caller action execution failed".to_owned();
5159 prior = Some(category.project(message.clone()));
5160 (
5161 Some(ActionPosition {
5162 node: context.node.clone(),
5163 ordinal: context.action_ordinal,
5164 }),
5165 Some(lineage.as_str()),
5166 *category,
5167 message,
5168 )
5169 }
5170 None => (
5171 activation.map(|(_, context)| ActionPosition {
5172 node: context.node.clone(),
5173 ordinal: context.action_ordinal,
5174 }),
5175 activation.map(|(activation, _)| activation.lineage.as_str()),
5176 FailureCategory::from_failure(&failure),
5177 failure.message().to_owned(),
5178 ),
5179 };
5180 if let Some((position, lineage)) = observation.as_ref().zip(lineage)
5181 && let Some((position, preceding, text)) = loaded
5182 .record
5183 .checkpoint
5184 .preceding_failure(&definition, position, lineage)?
5185 {
5186 observation = Some(position);
5187 category = preceding;
5188 message = text.to_owned();
5189 prior = Some(category.project(message.clone()));
5190 }
5191 let mut loaded = self
5192 .commit_for_recovery(
5193 loaded,
5194 ControlTransition::Failed {
5195 category,
5196 message,
5197 observation,
5198 },
5199 )
5200 .await
5201 .map_err(RuntimeOperationFailure::into_error)?;
5202 if !loaded.record.checkpoint.is_terminal() {
5203 loaded = self
5207 .load_execution_observation(None, Some(deadline))
5208 .await
5209 .map_err(|failure| {
5210 self.retain_execution_read_failure(&failure);
5211 failure.into_error()
5212 })?
5213 .ok_or_else(|| AppError::NotFound("workflow run does not exist".into()))?;
5214 if loaded.record.checkpoint.is_stopping()
5215 && loaded.record.checkpoint.controls.is_empty()
5216 {
5217 loaded = match Box::pin(self.settle_stopping(loaded)).await {
5218 Some(loaded) => loaded,
5219 None => self
5220 .load_execution_observation(None, Some(deadline))
5221 .await
5222 .map_err(|failure| {
5223 self.retain_execution_read_failure(&failure);
5224 failure.into_error()
5225 })?
5226 .ok_or_else(|| {
5227 AppError::NotFound("workflow run does not exist".into())
5228 })?,
5229 };
5230 }
5231 }
5232 if loaded.record.checkpoint.is_terminal() {
5233 self.observe_generation(deadline).await.map_err(|failure| {
5236 self.retain_execution_read_failure(&failure);
5237 failure.into_error()
5238 })?;
5239 }
5240 Ok(loaded)
5241 }
5242 .await;
5243 match settled {
5244 Ok(loaded) => {
5245 let record = &loaded.record;
5246 let completion = prior.map(|prior| failure.with_cause(Some(prior)));
5250 let _ = self
5251 .finish_execution_observation(record, completion, None, false)
5252 .await;
5253 if record.checkpoint.is_terminal() && self.cell.child_parent().is_none() {
5254 Box::pin(self.maintain_retention(Some(loaded))).await;
5255 }
5256 }
5257 Err(AppError::Closed(_)) => {}
5258 Err(_) => self.app.tasks_failed(FinalResult::Integration(
5259 "lifecycle failure cleanup authority failed".into(),
5260 )),
5261 }
5262 error
5263 }
5264
5265 async fn observe_step_start(
5266 &self,
5267 generation: u64,
5268 record: &RunRecord,
5269 context: &Arc<ActionContext>,
5270 lineage: &str,
5271 inputs: &Map<String, Value>,
5272 deadline: tokio::time::Instant,
5273 ) -> Result<Arc<ActionContext>, WorkflowFailure> {
5274 if let Some(context) = self
5275 .cell
5276 .action_context(context.checkpoint, &context.node, lineage)
5277 {
5278 return Ok(context);
5279 }
5280 let state = WorkflowState {
5281 values: record.checkpoint.state.clone(),
5282 };
5283 LifecycleEvent::StepStarted {
5284 context,
5285 state: &state,
5286 inputs,
5287 }
5288 .deliver(
5289 self.app.hooks(),
5290 context.mode,
5291 LifecycleEvent::deadline(Some(deadline)),
5292 )
5293 .await?;
5294 self.cell
5295 .observe_action(generation, Arc::clone(context), lineage)
5296 .map_err(|error| WorkflowFailure::Execution(error.to_string()))?;
5297 Ok(Arc::clone(context))
5298 }
5299}
5300
5301impl<
5302 CS: StateStoreBackend,
5303 A: AgentBackend,
5304 X: ActionBackend,
5305 C: ConditionBackend,
5306 T: StateTyping,
5307 H: LifecycleHooks,
5308 E: ChildExecutor,
5309> WorkflowRunRuntime<CS, A, X, C, T, H, E>
5310{
5311 async fn observe_step_finish(
5312 &self,
5313 generation: u64,
5314 context: &ActionContext,
5315 outcome: Option<Result<&ActionOutcome, &WorkflowFailure>>,
5316 record: &RunRecord,
5317 unresolved: bool,
5318 async_deadline: Option<tokio::time::Instant>,
5319 ) -> Result<(), WorkflowFailure> {
5320 let checkpoint = self
5321 .project_history(record)
5322 .map_err(|error| WorkflowFailure::Execution(error.to_string()))?;
5323 let spans = self.finish_spans(generation, context, async_deadline).await;
5324 let Some((context, items)) = self.cell.take_action_observation(generation, context) else {
5325 if !unresolved {
5326 self.cell.retire_closed_observation(generation, context);
5327 }
5328 return spans;
5329 };
5330 let event = LifecycleEvent::StepFinished {
5331 context: &context,
5332 outcome,
5333 checkpoint: &checkpoint,
5334 };
5335 let step = match async_deadline.filter(|_| !self.cell.execution_was_dropped(generation)) {
5336 Some(deadline) => {
5337 event
5338 .deliver(self.app.hooks(), context.mode, deadline)
5339 .await
5340 }
5341 None => event.deliver_sync(self.app.hooks()),
5342 };
5343 let result = spans.and(step);
5344 let result = if let Some(items) = items {
5345 let failure = result
5346 .as_ref()
5347 .err()
5348 .or_else(|| outcome.and_then(Result::err));
5349 let event = LifecycleEvent::StreamFinished {
5350 context: &context,
5351 span: None,
5352 items,
5353 failure,
5354 };
5355 let stream_result = match async_deadline
5358 .filter(|_| result.is_ok() && !self.cell.execution_was_dropped(generation))
5359 {
5360 Some(deadline) => {
5361 event
5362 .deliver(self.app.hooks(), context.mode, deadline)
5363 .await
5364 }
5365 None => event.deliver_sync(self.app.hooks()),
5366 };
5367 result.and(stream_result)
5368 } else {
5369 result
5370 };
5371 self.cell
5372 .finish_action_observation(generation, &context, unresolved);
5373 result
5374 }
5375
5376 fn observe_step_finish_sync(
5377 &self,
5378 generation: u64,
5379 context: &ActionContext,
5380 record: &RunRecord,
5381 unresolved: bool,
5382 ) -> Result<(), WorkflowFailure> {
5383 let checkpoint = self
5384 .project_history(record)
5385 .map_err(|error| WorkflowFailure::Execution(error.to_string()))?;
5386 let spans = self.finish_spans_sync(generation, context);
5387 let Some((context, items)) = self.cell.take_action_observation(generation, context) else {
5388 if !unresolved {
5389 self.cell.retire_closed_observation(generation, context);
5390 }
5391 return spans;
5392 };
5393 let step = LifecycleEvent::StepFinished {
5394 context: &context,
5395 outcome: None,
5396 checkpoint: &checkpoint,
5397 }
5398 .deliver_sync(self.app.hooks());
5399 let result = spans.and(step);
5400 let result = if let Some(items) = items {
5401 let next = LifecycleEvent::StreamFinished {
5402 context: &context,
5403 span: None,
5404 items,
5405 failure: result.as_ref().err(),
5406 }
5407 .deliver_sync(self.app.hooks());
5408 result.and(next)
5409 } else {
5410 result
5411 };
5412 self.cell
5413 .finish_action_observation(generation, &context, unresolved);
5414 result
5415 }
5416
5417 fn finish_execution_observation_sync(&self, record: &RunRecord) -> Result<(), AppError> {
5418 let Some(generation) = self.cell.lifecycle_generation() else {
5419 return self
5420 .cell
5421 .lifecycle_observation(record.checkpoint.sequence, false)
5422 .map(|_| ());
5423 };
5424 let unfinished = record
5425 .checkpoint
5426 .controls
5427 .values()
5428 .filter_map(|control| {
5429 self.cell.action_context(
5430 record.checkpoint.base_sequence,
5431 &control.node,
5432 &control.lineage,
5433 )
5434 })
5435 .collect::<Vec<_>>();
5436 if !self
5437 .cell
5438 .begin_lifecycle_finish(generation, record.checkpoint.sequence)
5439 {
5440 return self
5441 .cell
5442 .lifecycle_observation(record.checkpoint.sequence, false)
5443 .map(|_| ());
5444 }
5445 let mut failure = None;
5446 for context in self.cell.action_observations() {
5447 let unresolved = unfinished
5448 .iter()
5449 .any(|unfinished| Arc::ptr_eq(unfinished, &context));
5450 if let Err(next) =
5451 self.observe_step_finish_sync(generation, &context, record, unresolved)
5452 && failure.is_none()
5453 {
5454 failure = Some(next);
5455 }
5456 }
5457 match self.project(record) {
5458 Ok(snapshot) => {
5459 if let Err(next) = (LifecycleEvent::ExecuteFinished {
5460 snapshot: &snapshot,
5461 failure: failure.as_ref(),
5462 })
5463 .deliver_sync(self.app.hooks())
5464 && failure.is_none()
5465 {
5466 failure = Some(next);
5467 }
5468 }
5469 Err(error) => {
5470 if failure.is_none() {
5471 failure = Some(WorkflowFailure::Execution(error.to_string()));
5472 }
5473 }
5474 }
5475 if record.checkpoint.is_terminal() || record.checkpoint.is_paused() {
5476 self.cell.clear_execution_options(Some(generation));
5477 }
5478 let error = failure.as_ref().map(WorkflowFailure::application_error);
5479 let completed =
5480 self.cell
5481 .complete_lifecycle(generation, record.checkpoint.sequence, failure);
5482 match error {
5483 Some(error) => Err(error),
5484 None => completed,
5485 }
5486 }
5487
5488 pub(super) async fn finish_execution_observation(
5489 &self,
5490 record: &RunRecord,
5491 mut failure: Option<WorkflowFailure>,
5492 deadline: Option<tokio::time::Instant>,
5493 synchronous: bool,
5494 ) -> Result<(), AppError> {
5495 let Some(generation) = self.cell.lifecycle_generation() else {
5496 return self
5497 .cell
5498 .lifecycle_observation(record.checkpoint.sequence, false)
5499 .map(|_| ());
5500 };
5501 let synchronous = synchronous || self.cell.execution_was_dropped(generation);
5502 let unfinished = record
5503 .checkpoint
5504 .controls
5505 .values()
5506 .filter_map(|control| {
5507 self.cell.action_context(
5508 record.checkpoint.base_sequence,
5509 &control.node,
5510 &control.lineage,
5511 )
5512 })
5513 .collect::<Vec<_>>();
5514 if !self
5515 .cell
5516 .begin_lifecycle_finish(generation, record.checkpoint.sequence)
5517 {
5518 return self
5519 .cell
5520 .lifecycle_observation(record.checkpoint.sequence, false)
5521 .map(|_| ());
5522 }
5523 let deadline = match deadline {
5524 Some(deadline) => deadline,
5525 None => self.cell.begin_action_cleanup(generation)?,
5526 };
5527 let primary = match &record.checkpoint.phase {
5528 CheckpointPhaseRecord::Failing {
5529 category, message, ..
5530 }
5531 | CheckpointPhaseRecord::Failed {
5532 category, message, ..
5533 } => Some(category.project(message.clone())),
5534 _ => None,
5535 };
5536 for context in self.cell.action_observations() {
5537 let unresolved = unfinished
5538 .iter()
5539 .any(|unfinished| Arc::ptr_eq(unfinished, &context));
5540 if unresolved && !record.checkpoint.is_stopping() && !synchronous {
5541 continue;
5542 }
5543 let outcome = record.checkpoint.outcomes.iter().find(|outcome| {
5544 outcome.ordinal == context.action_ordinal && outcome.node == context.node
5545 });
5546 let failed = record
5547 .checkpoint
5548 .failed_observation()
5549 .is_some_and(|position| {
5550 position.ordinal == context.action_ordinal && position.node == context.node
5551 });
5552 let observed = outcome
5553 .map(Ok)
5554 .or_else(|| failed.then(|| primary.as_ref().map(Err)).flatten());
5555 if let Err(next) = self
5556 .observe_step_finish(
5557 generation,
5558 &context,
5559 observed,
5560 record,
5561 unresolved,
5562 (!synchronous && failure.is_none()).then_some(deadline),
5563 )
5564 .await
5565 && failure.is_none()
5566 {
5567 failure = Some(next.with_cause(match &record.checkpoint.phase {
5568 CheckpointPhaseRecord::Failing {
5569 category, message, ..
5570 }
5571 | CheckpointPhaseRecord::Failed {
5572 category, message, ..
5573 } => Some(category.project(message.clone())),
5574 _ => None,
5575 }));
5576 }
5577 }
5578 match self.project(record) {
5579 Ok(snapshot) => {
5580 let event = LifecycleEvent::ExecuteFinished {
5581 snapshot: &snapshot,
5582 failure: failure.as_ref().or(primary.as_ref()),
5583 };
5584 let observed = if synchronous || self.cell.execution_was_dropped(generation) {
5585 event.deliver_sync(self.app.hooks())
5586 } else {
5587 event
5588 .deliver(self.app.hooks(), self.cell.execution_mode()?, deadline)
5589 .await
5590 };
5591 if let Err(next) = observed
5592 && failure.is_none()
5593 {
5594 failure = Some(next.with_cause(primary));
5595 }
5596 }
5597 Err(error) => {
5598 if failure.is_none() {
5599 failure = Some(WorkflowFailure::Execution(error.to_string()));
5600 }
5601 }
5602 }
5603 if record.parent.is_some()
5604 && record.checkpoint.is_terminal()
5605 && let Err(next) = self.observe_child_finished(record, deadline).await
5606 && failure.is_none()
5607 {
5608 failure = Some(next);
5609 }
5610 if record.checkpoint.is_terminal() || record.checkpoint.is_paused() {
5611 self.cell.clear_execution_options(Some(generation));
5612 }
5613 let error = failure.as_ref().map(WorkflowFailure::application_error);
5614 let completed =
5615 self.cell
5616 .complete_lifecycle(generation, record.checkpoint.sequence, failure);
5617 match error {
5618 Some(error) => Err(error),
5619 None => completed,
5620 }
5621 }
5622
5623 pub(super) async fn observe_barrier(&self, loaded: LoadedRun) -> Result<LoadedRun, AppError> {
5624 let Some(generation) = self.cell.lifecycle_generation() else {
5625 return Ok(loaded);
5626 };
5627 let record = &loaded.record;
5628 let observations = self.cell.action_observations();
5629 for outcome in &record.checkpoint.outcomes {
5630 let context = observations.iter().find(|context| {
5631 context.action_ordinal == outcome.ordinal && context.node == outcome.node
5632 });
5633 let Some(context) = context else {
5634 continue;
5635 };
5636 let deadline = self.cell.begin_action_cleanup(generation)?;
5637 if let Err(failure) = self
5638 .observe_step_finish(
5639 generation,
5640 context,
5641 Some(Ok(outcome)),
5642 record,
5643 false,
5644 Some(deadline),
5645 )
5646 .await
5647 {
5648 let error = failure.application_error();
5649 let parked = self
5653 .commit_for_recovery(
5654 loaded,
5655 ControlTransition::Pause {
5656 discard_staged: Vec::new(),
5657 },
5658 )
5659 .await
5660 .map_err(RuntimeOperationFailure::into_error);
5661 match parked {
5662 Ok(parked) => {
5663 let _ = self
5664 .finish_execution_observation(
5665 &parked.record,
5666 Some(failure),
5667 Some(deadline),
5668 false,
5669 )
5670 .await;
5671 return Ok(parked);
5672 }
5673 Err(AppError::Closed(_)) => {}
5674 Err(_) => self.app.tasks_failed(FinalResult::Integration(
5675 "post-hook cleanup authority failed".into(),
5676 )),
5677 }
5678 return Err(error);
5679 }
5680 }
5681 Ok(loaded)
5682 }
5683}