1use std::future::{Future, poll_fn};
2use std::panic::{AssertUnwindSafe, catch_unwind};
3use std::pin::Pin;
4use std::sync::Arc;
5use std::task::{Context, Poll};
6use std::time::Duration;
7
8use agents_kit::AgentBackend;
9use futures_util::{FutureExt, Stream};
10use serde_json::{Map, Value};
11use storage_kit::StateStoreBackend;
12use tokio::sync::{OwnedSemaphorePermit, mpsc, oneshot};
13
14use crate::AppError;
15use crate::app::{AppRuntime, DriverCommand, FinalResult, RunTaskKind};
16use crate::storage::StorageFailure;
17use crate::workflow::action::ActionInputs;
18use crate::workflow::definition::{JsonBoundary, Step};
19use crate::workflow::execution::{
20 BspDecision, ExecutionEngine, StagedActivationRecord, StagedOutcomeRecord,
21};
22use crate::workflow::run::{
23 ActionPosition, CheckpointPhaseRecord, ChildAdmissionReceipt, ControlRecord,
24 ControlRecordState, ControlTransition, FailureCategory, FrontierRecord, LoadedRun,
25 MAX_DRIVER_TRANSITIONS, RECONCILIATION_INITIAL_DELAY, RECONCILIATION_MAX_DELAY, RunAdmission,
26 RunRecord, RuntimeOperationFailure, ScheduleCause, WorkflowRunRuntime,
27};
28use crate::workflow::{
29 ActionBackend, ActionContext, ActionEvent, ActionMode, ActionOutcome, ActionRequest,
30 ChildBatch, ChildCompletion, ChildExecutor, ConditionBackend, ExecutionEvent, ExecutionOptions,
31 InlineExecutor, JsonState, LifecycleEvent, LifecycleHooks, NoActions, NoConditions, NoHooks,
32 RunSnapshot, RunStatus, StateTyping, WorkflowFailure, WorkflowRun, WorkflowState,
33};
34
35#[must_use = "execution owns lazy advancement; finish or abort it explicitly"]
49pub struct Execution<
50 'a,
51 CS: StateStoreBackend,
52 A: AgentBackend,
53 X: ActionBackend = NoActions,
54 C: ConditionBackend = NoConditions,
55 T: StateTyping = JsonState,
56 H: LifecycleHooks = NoHooks,
57 E: ChildExecutor = InlineExecutor,
58> {
59 runtime: WorkflowRunRuntime<CS, A, X, C, T, H, E>,
60 app: &'a AppRuntime<CS, A, X, C, T, H, E>,
61 lane: ExecutionLane,
62 admission: Option<OwnedSemaphorePermit>,
63 children: Vec<ChildExecution<'a, ChildFuture<Self>, E>>,
64 wake: Option<Pin<Box<tokio::sync::futures::OwnedNotified>>>,
65 timer: Option<Pin<Box<tokio::time::Sleep>>>,
66 generation: Option<u64>,
67 parent_deadline: Option<tokio::time::Instant>,
68 state: SessionState,
69 sources: Vec<CallerSource<X>>,
70 next_source: usize,
71 native_partials: Option<(mpsc::Sender<ExecutionEvent>, mpsc::Receiver<ExecutionEvent>)>,
72 prefetched: Option<(usize, Result<Option<ActionEvent>, FailureCategory>)>,
73 transitions: usize,
74 observed: Option<RunRecord>,
75}
76
77#[must_use = "execution owns lazy advancement; finish or abort it explicitly"]
86pub struct SyncExecution<
87 'a,
88 CS: StateStoreBackend,
89 A: AgentBackend,
90 X: ActionBackend = NoActions,
91 C: ConditionBackend = NoConditions,
92 T: StateTyping = JsonState,
93 H: LifecycleHooks = NoHooks,
94 E: ChildExecutor = InlineExecutor,
95> {
96 execution: Execution<'a, CS, A, X, C, T, H, E>,
97 runtime: tokio::runtime::Handle,
98}
99
100impl<
101 'a,
102 CS: StateStoreBackend,
103 A: AgentBackend,
104 X: ActionBackend,
105 C: ConditionBackend,
106 T: StateTyping,
107 H: LifecycleHooks,
108 E: ChildExecutor,
109> SyncExecution<'a, CS, A, X, C, T, H, E>
110{
111 pub fn open(
184 run: &'a mut WorkflowRun<CS, A, X, C, T, H, E>,
185 options: ExecutionOptions,
186 ) -> Result<Self, AppError> {
187 Self::ensure_blocking()?;
188 let runtime = run.runtime.app.execution_handle()?;
189 let execution = runtime.block_on(Execution::open(run, options, ActionMode::Synchronous))?;
190 Ok(Self { execution, runtime })
191 }
192
193 fn ensure_blocking() -> Result<(), AppError> {
194 if tokio::runtime::Handle::try_current().is_ok() {
195 return Err(AppError::InvalidInput(
196 "synchronous execution requires a thread outside an entered Tokio runtime".into(),
197 ));
198 }
199 Ok(())
200 }
201
202 pub fn try_next(&mut self) -> Result<Option<ExecutionEvent>, AppError> {
215 Self::ensure_blocking()?;
216 self.runtime.block_on(self.execution.next())
217 }
218
219 pub fn step(&mut self) -> Result<Option<ExecutionEvent>, AppError> {
232 Self::ensure_blocking()?;
233 self.runtime.block_on(self.execution.step())
234 }
235
236 pub fn finish(self) -> Result<RunSnapshot, AppError> {
250 Self::ensure_blocking()?;
251 self.runtime.block_on(self.execution.finish())
252 }
253
254 pub fn abort(self) -> Result<RunSnapshot, AppError> {
268 Self::ensure_blocking()?;
269 self.runtime.block_on(self.execution.abort())
270 }
271}
272
273struct CallerSource<X: ActionBackend> {
274 activation: FrontierRecord,
275 context: Arc<ActionContext>,
276 deadline: tokio::time::Instant,
277 state: CallerSourceState<X>,
278}
279
280enum CallerSourceState<X: ActionBackend> {
281 Polling(Pin<Box<X::Stream>>),
282 Failed(FailureCategory),
283 Ended,
284 Eof,
285}
286
287impl<X: ActionBackend> CallerSource<X> {
288 fn stop(&mut self) -> Result<(), FailureCategory> {
289 let previous = std::mem::replace(&mut self.state, CallerSourceState::Ended);
290 match previous {
291 CallerSourceState::Polling(source) => catch_unwind(AssertUnwindSafe(|| drop(source)))
292 .map_err(|_| FailureCategory::Execution),
293 previous => {
294 self.state = previous;
295 Ok(())
296 }
297 }
298 }
299
300 fn poll(
301 &mut self,
302 context: &mut Context<'_>,
303 ) -> Poll<Result<Option<ActionEvent>, FailureCategory>> {
304 let event = match &mut self.state {
305 CallerSourceState::Polling(source) => {
306 match catch_unwind(AssertUnwindSafe(|| source.as_mut().poll_next(context))) {
307 Ok(Poll::Pending) => return Poll::Pending,
308 Ok(Poll::Ready(Some(Ok(event)))) => Ok(Some(event)),
309 Ok(Poll::Ready(Some(Err(_)))) => Err(FailureCategory::Action),
310 Ok(Poll::Ready(None)) => Ok(None),
311 Err(_) => Err(FailureCategory::Execution),
312 }
313 }
314 CallerSourceState::Failed(category) => {
315 let category = *category;
316 self.state = CallerSourceState::Ended;
317 return Poll::Ready(Err(category));
318 }
319 CallerSourceState::Ended | CallerSourceState::Eof => return Poll::Pending,
320 };
321 Poll::Ready(event)
322 }
323}
324
325#[derive(Clone, Copy, PartialEq)]
326enum ExecutionLane {
327 Caller,
328 Worker,
329}
330
331struct ChildExecution<'a, F, E>
332where
333 F: Future<Output = Result<ChildCompletion, AppError>> + 'a,
334 E: ChildExecutor,
335{
336 sequence: u64,
337 parent: Option<Arc<RunRecord>>,
338 context: Arc<ActionContext>,
339 lineage: String,
340 deadline: tokio::time::Instant,
341 state: ChildExecutionState<'a, F, E>,
342 inputs: Vec<Map<String, Value>>,
343}
344
345enum ChildExecutionState<'a, F, E>
346where
347 F: Future<Output = Result<ChildCompletion, AppError>> + 'a,
348 E: ChildExecutor,
349{
350 Awaiting,
351 Prepared(Vec<RunRecord>),
352 Local(Pin<Box<E::Execution<'a, F>>>),
353 Pipeline(ChildBatch<F>),
354 Worker(oneshot::Receiver<Result<SessionReceipt, RuntimeOperationFailure>>),
355 Receipt(Box<Result<SessionReceipt, RuntimeOperationFailure>>),
356 Complete(Result<Vec<ChildCompletion>, FailureCategory>),
357 Closed,
358}
359
360impl<
361 'a,
362 CS: StateStoreBackend,
363 A: AgentBackend,
364 X: ActionBackend,
365 C: ConditionBackend,
366 T: StateTyping,
367 H: LifecycleHooks,
368 E: ChildExecutor,
369> ChildExecution<'a, ChildFuture<Execution<'a, CS, A, X, C, T, H, E>>, E>
370{
371 fn start_local(
372 &mut self,
373 runtime: &WorkflowRunRuntime<CS, A, X, C, T, H, E>,
374 app: &'a AppRuntime<CS, A, X, C, T, H, E>,
375 lane: ExecutionLane,
376 children: Vec<RunRecord>,
377 ) -> Result<(), AppError> {
378 if tokio::time::Instant::now() >= self.deadline {
379 runtime.retain_source_failure(&self.context, &self.lineage, FailureCategory::Timeout);
380 self.state = ChildExecutionState::Complete(Err(FailureCategory::Timeout));
381 return Ok(());
382 }
383 let parent = self.parent.as_ref().ok_or_else(|| {
384 AppError::Integration("child group parent evidence is unavailable".into())
385 })?;
386 let definition = app.definition_for(parent)?;
387 let control = parent
388 .checkpoint
389 .control(self.sequence)
390 .ok_or_else(|| AppError::Conflict("child control is unavailable".into()))?;
391 let step = definition
392 .node(&control.node)
393 .ok_or_else(|| AppError::Conflict("child definition changed".into()))?
394 .step();
395 let (parallelism, _) = step
396 .child_plan()
397 .ok_or_else(|| AppError::Conflict("child definition changed".into()))?;
398 let slots = Arc::new(tokio::sync::Semaphore::new(parallelism));
399 let notification = runtime.notifier()?;
400 let mut tasks = Vec::with_capacity(children.len());
401 for record in children {
402 record.validate_child(parent, &definition)?;
403 let child = WorkflowRunRuntime::attach_record(Arc::clone(&runtime.app), &record)?;
404 if !record.checkpoint.is_terminal() {
405 child.cell.bind_child(runtime.key.clone())?;
406 child
407 .cell
408 .retain_child_observation(Arc::clone(&self.context))?;
409 }
410 tasks.push(ChildFuture::new(
411 Execution::owned(child, app, lane),
412 Arc::clone(parent),
413 self.sequence,
414 self.deadline,
415 Arc::clone(&slots),
416 Arc::clone(¬ification),
417 ));
418 }
419 let batch = ChildBatch::new(tasks, self.context.mode)?;
420 self.state = if matches!(step, Step::Pipeline { .. }) {
421 ChildExecutionState::Pipeline(batch)
422 } else {
423 match catch_unwind(AssertUnwindSafe(|| app.child_executor().execute(batch))) {
424 Ok(future) => ChildExecutionState::Local(Box::pin(future)),
425 Err(_) => {
426 runtime.retain_source_failure(
427 &self.context,
428 &self.lineage,
429 FailureCategory::Execution,
430 );
431 ChildExecutionState::Complete(Err(FailureCategory::Execution))
432 }
433 }
434 };
435 Ok(())
436 }
437
438 fn stop(&mut self, runtime: &WorkflowRunRuntime<CS, A, X, C, T, H, E>) {
439 let previous = std::mem::replace(&mut self.state, ChildExecutionState::Closed);
440 if let ChildExecutionState::Complete(Err(category)) = &previous {
441 runtime.retain_source_failure(&self.context, &self.lineage, *category);
442 }
443 if catch_unwind(AssertUnwindSafe(|| drop(previous))).is_err() {
444 runtime.retain_source_failure(&self.context, &self.lineage, FailureCategory::Execution);
447 }
448 }
449
450 fn poll_ready(
451 &mut self,
452 cx: &mut Context<'_>,
453 runtime: &WorkflowRunRuntime<CS, A, X, C, T, H, E>,
454 ) -> Poll<()> {
455 if matches!(
456 self.state,
457 ChildExecutionState::Complete(_)
458 | ChildExecutionState::Receipt(_)
459 | ChildExecutionState::Closed
460 ) {
461 return Poll::Ready(());
462 }
463 if matches!(
464 self.state,
465 ChildExecutionState::Awaiting | ChildExecutionState::Prepared(_)
466 ) {
467 return Poll::Pending;
468 }
469 if matches!(self.state, ChildExecutionState::Worker(_)) {
470 return self.poll_receipt(cx);
471 }
472 let result = if tokio::time::Instant::now() >= self.deadline {
473 Err(FailureCategory::Timeout)
474 } else {
475 match catch_unwind(AssertUnwindSafe(|| match &mut self.state {
476 ChildExecutionState::Local(future) => future.as_mut().poll(cx),
477 ChildExecutionState::Pipeline(batch) => Pin::new(batch).poll(cx),
478 _ => unreachable!(),
479 })) {
480 Ok(Poll::Pending) => return Poll::Pending,
481 Ok(Poll::Ready(result)) => result.map_err(|_| FailureCategory::Child),
482 Err(_) => Err(FailureCategory::Execution),
483 }
484 };
485 self.complete(runtime, result);
486 Poll::Ready(())
487 }
488
489 fn poll_receipt(&mut self, cx: &mut Context<'_>) -> Poll<()> {
490 let ChildExecutionState::Worker(receipt) = &mut self.state else {
491 unreachable!()
492 };
493 match Pin::new(receipt).poll(cx) {
494 Poll::Pending => Poll::Pending,
495 Poll::Ready(result) => {
496 self.state = ChildExecutionState::Receipt(Box::new(
497 result
498 .map_err(|_| {
499 RuntimeOperationFailure::from(AppError::Closed(
500 "child executor receipt closed".into(),
501 ))
502 })
503 .and_then(|result| result),
504 ));
505 Poll::Ready(())
506 }
507 }
508 }
509
510 fn complete(
511 &mut self,
512 runtime: &WorkflowRunRuntime<CS, A, X, C, T, H, E>,
513 result: Result<Vec<ChildCompletion>, FailureCategory>,
514 ) {
515 if let Err(category) = result.as_ref() {
516 runtime.retain_source_failure(&self.context, &self.lineage, *category);
517 }
518 let previous = std::mem::replace(&mut self.state, ChildExecutionState::Complete(result));
519 if catch_unwind(AssertUnwindSafe(|| drop(previous))).is_err()
520 && matches!(&self.state, ChildExecutionState::Complete(Ok(_)))
521 {
522 runtime.retain_source_failure(&self.context, &self.lineage, FailureCategory::Execution);
523 self.state = ChildExecutionState::Complete(Err(FailureCategory::Execution));
524 }
525 }
526}
527
528struct ChildFuture<EX> {
529 execution: EX,
530 parent: Arc<RunRecord>,
531 sequence: u64,
532 slots: Arc<tokio::sync::Semaphore>,
533 permit: Option<tokio::sync::OwnedSemaphorePermit>,
534 notification: Arc<tokio::sync::Notify>,
535 wake: Pin<Box<tokio::sync::futures::OwnedNotified>>,
536 state: ChildFutureState,
537}
538
539enum ChildFutureState {
540 Admission,
541 Driving,
542 Reconcile(Option<oneshot::Receiver<Result<LoadedRun, AppError>>>),
543 Retry(tokio::time::Instant),
544 Finishing,
545 Waiting,
546 Finished,
547}
548
549impl<
550 'a,
551 CS: StateStoreBackend,
552 A: AgentBackend,
553 X: ActionBackend,
554 C: ConditionBackend,
555 T: StateTyping,
556 H: LifecycleHooks,
557 E: ChildExecutor,
558> ChildFuture<Execution<'a, CS, A, X, C, T, H, E>>
559{
560 fn new(
561 mut execution: Execution<'a, CS, A, X, C, T, H, E>,
562 parent: Arc<RunRecord>,
563 sequence: u64,
564 deadline: tokio::time::Instant,
565 slots: Arc<tokio::sync::Semaphore>,
566 notification: Arc<tokio::sync::Notify>,
567 ) -> Self {
568 let wake = Box::pin(Arc::clone(¬ification).notified_owned());
569 execution.parent_deadline = Some(deadline);
570 Self {
571 execution,
572 parent,
573 sequence,
574 slots,
575 permit: None,
576 notification,
577 wake,
578 state: ChildFutureState::Admission,
579 }
580 }
581
582 fn poll_reconciliation(&mut self, cx: &mut Context<'_>) -> Poll<Result<(), AppError>> {
583 let ChildFutureState::Reconcile(receipt) = &mut self.state else {
584 unreachable!()
585 };
586 if receipt.is_none() {
587 match &mut self.execution.state {
591 SessionState::Publishing { receipt, .. } => {
592 if Pin::new(receipt).poll(cx).is_pending() {
593 return Poll::Pending;
594 }
595 }
596 SessionState::ObservingItem { receipt, .. } => {
597 if Pin::new(receipt).poll(cx).is_pending() {
598 return Poll::Pending;
599 }
600 }
601 _ => {}
602 }
603 self.execution.state = SessionState::Finished;
604 *receipt = Some(
605 self.execution
606 .runtime
607 .reconcile_child_panic(self.execution.parent_deadline),
608 );
609 }
610 let loaded = match Pin::new(receipt.as_mut().expect("reconciliation was admitted")).poll(cx)
611 {
612 Poll::Pending => return Poll::Pending,
613 Poll::Ready(Ok(result)) => result?,
614 Poll::Ready(Err(_)) => {
615 self.execution
616 .runtime
617 .app
618 .tasks_failed(FinalResult::Integration(
619 "child panic reconciliation receipt closed".into(),
620 ));
621 return Poll::Ready(Err(AppError::Integration(
622 "child panic reconciliation receipt closed".into(),
623 )));
624 }
625 };
626 self.execution.state = SessionState::Loaded(loaded);
627 self.state = ChildFutureState::Driving;
628 Poll::Ready(Ok(()))
629 }
630
631 fn poll_completion(
632 &mut self,
633 cx: &mut Context<'_>,
634 ) -> Poll<Result<Option<ChildCompletion>, AppError>> {
635 match self.execution.poll_publication(cx, false) {
636 Poll::Pending => Poll::Pending,
637 Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
638 Poll::Ready(Ok(Some(_))) => Poll::Ready(Ok(None)),
639 Poll::Ready(Ok(None)) => {
640 let SessionState::Loaded(loaded) =
641 std::mem::replace(&mut self.execution.state, SessionState::Finished)
642 else {
643 return Poll::Ready(Err(AppError::Integration(
644 "child terminal observation is unavailable".into(),
645 )));
646 };
647 if !loaded.record.checkpoint.is_terminal() {
648 self.execution.state = SessionState::Loaded(loaded);
649 self.state = ChildFutureState::Waiting;
650 return Poll::Ready(Ok(None));
651 }
652 let definition = self.execution.runtime.app.definition_for(&self.parent)?;
653 let result = if matches!(
654 loaded.record.checkpoint.phase,
655 CheckpointPhaseRecord::Succeeded
656 ) {
657 ChildCompletion::admitted(loaded.record, &self.parent, &definition)
658 } else {
659 Err(AppError::Integration(
660 "child workflow did not succeed".into(),
661 ))
662 };
663 Poll::Ready(result.map(Some))
664 }
665 }
666 }
667}
668
669impl<
670 CS: StateStoreBackend,
671 A: AgentBackend,
672 X: ActionBackend,
673 C: ConditionBackend,
674 T: StateTyping,
675 H: LifecycleHooks,
676 E: ChildExecutor,
677> Future for ChildFuture<Execution<'_, CS, A, X, C, T, H, E>>
678{
679 type Output = Result<ChildCompletion, AppError>;
680
681 fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
682 let child = self.get_mut();
683 let observed = (|| -> Poll<Result<ChildCompletion, AppError>> {
684 loop {
685 match child.state {
686 ChildFutureState::Admission => {
687 child.wake.as_mut().enable();
688 match Arc::clone(&child.slots).try_acquire_owned() {
689 Ok(permit) => {
690 child.permit = Some(permit);
691 child.state = ChildFutureState::Driving;
692 }
693 Err(tokio::sync::TryAcquireError::NoPermits) => {
694 if child.wake.as_mut().poll(cx).is_pending() {
695 return Poll::Pending;
696 }
697 child.wake =
698 Box::pin(Arc::clone(&child.notification).notified_owned());
699 }
700 Err(tokio::sync::TryAcquireError::Closed) => {
701 return Poll::Ready(Err(AppError::Closed(
702 "child execution capacity is closed".into(),
703 )));
704 }
705 }
706 }
707 ChildFutureState::Reconcile(_) => match child.poll_reconciliation(cx) {
708 Poll::Pending => return Poll::Pending,
709 Poll::Ready(result) => result?,
710 },
711 ChildFutureState::Retry(deadline) => {
712 if !child.execution.poll_timer(cx, Some(deadline)) {
713 return Poll::Pending;
714 }
715 child.execution.state = SessionState::Ready;
716 child.state = ChildFutureState::Driving;
717 }
718 ChildFutureState::Driving => {
719 let observed = catch_unwind(AssertUnwindSafe(|| {
720 child.execution.poll_advance(cx, true)
721 }));
722 match observed {
723 Err(_) => {
724 child.state = ChildFutureState::Reconcile(None);
727 }
728 Ok(Poll::Pending) => return Poll::Pending,
729 Ok(Poll::Ready(Err(error))) => {
730 if let Some(delay) = child.execution.runtime.cell.complete_drive() {
731 child.state = ChildFutureState::Retry(
732 tokio::time::Instant::now() + delay,
733 );
734 } else {
735 return Poll::Ready(Err(error));
736 }
737 }
738 Ok(Poll::Ready(Ok(Some(_)))) => continue,
739 Ok(Poll::Ready(Ok(None))) => {
740 child.execution.publish(
741 SessionPublication::FinishChild {
742 parent: Arc::clone(&child.parent),
743 sequence: child.sequence,
744 },
745 false,
746 )?;
747 child.state = ChildFutureState::Finishing;
748 }
749 }
750 }
751 ChildFutureState::Finishing => match child.poll_completion(cx) {
752 Poll::Pending => return Poll::Pending,
753 Poll::Ready(Err(error)) => return Poll::Ready(Err(error)),
754 Poll::Ready(Ok(Some(completion))) => return Poll::Ready(Ok(completion)),
755 Poll::Ready(Ok(None)) => continue,
756 },
757 ChildFutureState::Waiting => {
758 if !child.execution.poll_notification(cx)? {
761 return Poll::Pending;
762 }
763 child.execution.state = SessionState::Ready;
764 child.state = ChildFutureState::Driving;
765 }
766 ChildFutureState::Finished => {
767 return Poll::Ready(Err(AppError::Conflict(
768 "child future already completed".into(),
769 )));
770 }
771 }
772 }
773 })();
774 if observed.is_ready() {
775 child.release();
776 child.state = ChildFutureState::Finished;
777 }
778 observed
779 }
780}
781
782impl<EX> ChildFuture<EX> {
783 fn release(&mut self) {
784 if self.permit.take().is_some() {
785 self.notification.notify_waiters();
786 }
787 }
788}
789
790impl<EX> Drop for ChildFuture<EX> {
791 fn drop(&mut self) {
792 self.release();
793 }
794}
795
796enum SessionState {
797 Ready,
798 Loaded(LoadedRun),
799 ActivationReady {
800 loaded: LoadedRun,
801 activation: FrontierRecord,
802 context: Arc<ActionContext>,
803 remaining: std::vec::IntoIter<FrontierRecord>,
804 staged: Vec<StagedActivationRecord>,
805 deadline: tokio::time::Instant,
806 deadline_millis: u64,
807 },
808 Sources {
809 loaded: LoadedRun,
810 },
811 ObservingItem {
812 loaded: LoadedRun,
813 index: usize,
814 receipt: oneshot::Receiver<Result<Option<Map<String, Value>>, AppError>>,
815 },
816 Publishing {
817 receipt: oneshot::Receiver<Result<SessionReceipt, RuntimeOperationFailure>>,
818 purpose: PublicationPurpose,
819 },
820 Eof,
821 Finished,
822}
823
824enum PublicationPurpose {
825 Load,
826 Reload,
827 ObserveChildren,
828 AdmitChildren,
829 VerifyChildren,
830 Start {
831 activation: FrontierRecord,
832 context: Arc<ActionContext>,
833 remaining: std::vec::IntoIter<FrontierRecord>,
834 deadline: tokio::time::Instant,
835 deadline_millis: u64,
836 },
837 Continue {
838 remaining: std::vec::IntoIter<FrontierRecord>,
839 ordinary: bool,
840 },
841 Descendants {
842 context: Arc<ActionContext>,
843 lineage: String,
844 inputs: Vec<Map<String, Value>>,
845 remaining: std::vec::IntoIter<FrontierRecord>,
846 ordinary: bool,
847 },
848 Commit {
849 ordinary: bool,
850 },
851 Advance,
852 EndSource,
853}
854
855pub(crate) enum SessionReceipt {
856 Checkpoint {
857 loaded: LoadedRun,
858 staged: Vec<StagedActivationRecord>,
859 },
860 Children {
861 loaded: LoadedRun,
862 sequence: u64,
863 children: Vec<RunRecord>,
864 verified: bool,
865 },
866}
867
868impl SessionReceipt {
869 fn record(&self) -> &RunRecord {
870 match self {
871 Self::Checkpoint { loaded, .. } | Self::Children { loaded, .. } => &loaded.record,
872 }
873 }
874
875 fn with_loaded(self, loaded: LoadedRun) -> Self {
876 match self {
877 Self::Checkpoint { staged, .. } => Self::Checkpoint { loaded, staged },
878 Self::Children {
879 sequence,
880 children,
881 verified,
882 ..
883 } => Self::Children {
884 loaded,
885 sequence,
886 children,
887 verified,
888 },
889 }
890 }
891}
892
893enum SessionPublication {
894 ObserveChildren {
895 loaded: LoadedRun,
896 sequence: u64,
897 },
898 VerifyChildren {
899 parent: Arc<RunRecord>,
900 sequence: u64,
901 completions: Vec<ChildCompletion>,
902 },
903 ChildOutcome {
904 control: ControlRecord,
905 transition: ControlTransition,
906 },
907 FinishChild {
908 parent: Arc<RunRecord>,
909 sequence: u64,
910 },
911 SettleStopping {
912 loaded: LoadedRun,
913 },
914 Load {
915 terminal: Option<u64>,
916 },
917 Reload {
918 receipt: SessionReceipt,
919 deadline: Option<tokio::time::Instant>,
920 },
921 BeginActivation {
922 loaded: LoadedRun,
923 activation: FrontierRecord,
924 context: Arc<ActionContext>,
925 remaining: std::vec::IntoIter<FrontierRecord>,
926 staged: Vec<StagedActivationRecord>,
927 deadline: tokio::time::Instant,
928 deadline_millis: u64,
929 },
930 Decision {
931 loaded: LoadedRun,
932 decision: BspDecision,
933 activation_deadline: Option<u64>,
934 context: Option<Arc<ActionContext>>,
935 staged: Vec<StagedActivationRecord>,
936 remaining: std::vec::IntoIter<FrontierRecord>,
937 },
938 Transition {
939 loaded: LoadedRun,
940 transition: ControlTransition,
941 },
942 Retry {
943 loaded: LoadedRun,
944 },
945 EndSource {
946 loaded: LoadedRun,
947 },
948 Release,
949}
950
951impl<
952 'a,
953 CS: StateStoreBackend,
954 A: AgentBackend,
955 X: ActionBackend,
956 C: ConditionBackend,
957 T: StateTyping,
958 H: LifecycleHooks,
959 E: ChildExecutor,
960> Execution<'a, CS, A, X, C, T, H, E>
961{
962 pub(crate) async fn open(
963 run: &'a mut WorkflowRun<CS, A, X, C, T, H, E>,
964 mut options: ExecutionOptions,
965 mode: ActionMode,
966 ) -> Result<Self, AppError> {
967 let admission = run.runtime.app.reserve_caller()?;
968 if !options.state.values().is_empty() {
969 return Err(AppError::InvalidInput(
970 "existing execution cannot replace initial state".into(),
971 ));
972 }
973 JsonBoundary::validate_inputs(&options.inputs)?;
974 run.runtime.app.ensure_open().await?;
975 let loaded = run.runtime.load().await?;
976 let definition = run.runtime.app.definition_for(&loaded.record)?;
977 let graph = run.runtime.app.graph(&definition);
978 options.resolve_selectors(&graph)?;
979 let pending_controls = !loaded.record.checkpoint.controls.is_empty();
980 let discard_staged = loaded.record.checkpoint.rollback_stage_keys(&definition);
981 let generation = run.runtime.cell.claim_execution(mode)?;
982 run.runtime.cell.configure_execution(generation, options)?;
983 let mut execution = Self {
984 runtime: run.runtime.clone(),
985 app: &run.runtime.app,
986 lane: ExecutionLane::Caller,
987 admission: Some(admission),
988 children: Vec::new(),
989 wake: None,
990 timer: None,
991 generation: Some(generation),
992 parent_deadline: None,
993 state: SessionState::Ready,
994 sources: Vec::new(),
995 next_source: 0,
996 native_partials: None,
997 prefetched: None,
998 transitions: 0,
999 observed: Some(loaded.record.clone()),
1000 };
1001 if loaded.record.checkpoint.can_retry_failed() {
1002 execution.publish(SessionPublication::Retry { loaded }, false)?;
1003 poll_fn(|cx| execution.poll_publication(cx, false)).await?;
1004 } else if !loaded.record.checkpoint.is_terminal()
1005 && (loaded.record.checkpoint.admission != RunAdmission::Prepared
1006 || !discard_staged.is_empty())
1007 {
1008 execution.publish_transition(loaded, ControlTransition::Pause { discard_staged })?;
1009 poll_fn(|cx| execution.poll_publication(cx, false)).await?;
1010 }
1011 drop(execution.admission.take());
1013 execution.runtime.cell.admit_lifecycle(generation)?;
1014 if pending_controls {
1015 execution.runtime.schedule(ScheduleCause::Reconcile);
1016 }
1017 Ok(execution)
1018 }
1019
1020 pub(super) fn background(run: &'a mut WorkflowRun<CS, A, X, C, T, H, E>) -> Self {
1021 Self::owned(run.runtime.clone(), &run.runtime.app, ExecutionLane::Worker)
1022 }
1023
1024 fn owned(
1025 runtime: WorkflowRunRuntime<CS, A, X, C, T, H, E>,
1026 app: &'a AppRuntime<CS, A, X, C, T, H, E>,
1027 lane: ExecutionLane,
1028 ) -> Self {
1029 Self {
1030 runtime,
1031 app,
1032 lane,
1033 admission: None,
1034 children: Vec::new(),
1035 wake: None,
1036 timer: None,
1037 generation: None,
1038 parent_deadline: None,
1039 state: SessionState::Ready,
1040 sources: Vec::new(),
1041 next_source: 0,
1042 native_partials: None,
1043 prefetched: None,
1044 transitions: 0,
1045 observed: None,
1046 }
1047 }
1048
1049 pub async fn next(&mut self) -> Result<Option<ExecutionEvent>, AppError> {
1066 let event = poll_fn(|cx| self.poll_advance(cx, false)).await;
1067 if (matches!(&event, Ok(Some(ExecutionEvent::Committed { .. })))
1068 || matches!(&event, Ok(None)) && matches!(&self.state, SessionState::Eof)
1069 || event.is_err() && self.runtime.cell.lifecycle_generation().is_none())
1070 && let Some(generation) = self.generation
1071 {
1072 self.runtime.cell.returned_boundary(generation);
1073 }
1074 event
1075 }
1076
1077 fn poll_advance(
1078 &mut self,
1079 cx: &mut Context<'_>,
1080 ordinary: bool,
1081 ) -> Poll<Result<Option<ExecutionEvent>, AppError>> {
1082 loop {
1083 if !matches!(&self.state, SessionState::Sources { .. })
1087 || self
1088 .children
1089 .iter()
1090 .any(|group| matches!(group.state, ChildExecutionState::Prepared(_)))
1091 {
1092 self.poll_child_groups(cx)?;
1093 }
1094 match &mut self.state {
1095 SessionState::ObservingItem { .. } => match self.poll_item(cx) {
1096 Poll::Pending => return Poll::Pending,
1097 Poll::Ready(Err(error)) => return Poll::Ready(Err(error)),
1098 Poll::Ready(Ok(Some(event))) if !ordinary => {
1099 return Poll::Ready(Ok(Some(event)));
1100 }
1101 Poll::Ready(Ok(_)) => continue,
1102 },
1103 SessionState::Finished => return Poll::Ready(Ok(None)),
1104 SessionState::Eof => self.state = SessionState::Ready,
1105 SessionState::ActivationReady { .. } => self.activate_ready(ordinary)?,
1106 SessionState::Publishing { .. } => {
1107 match self.poll_publication(cx, false) {
1108 Poll::Pending => return Poll::Pending,
1109 Poll::Ready(Err(error)) => return Poll::Ready(Err(error)),
1110 Poll::Ready(Ok(Some(event))) => {
1111 if ordinary && matches!(event, ExecutionEvent::Partial { .. }) {
1112 continue;
1113 }
1114 return Poll::Ready(Ok(Some(event)));
1115 }
1116 Poll::Ready(Ok(None)) => {}
1117 }
1118 if matches!(&self.state, SessionState::Eof) {
1119 return Poll::Ready(Ok(None));
1120 }
1121 }
1122 SessionState::Ready => {
1123 self.ensure_owned()?;
1124 self.arm_notification()?;
1125 let terminal = self
1126 .observed
1127 .as_ref()
1128 .filter(|record| {
1129 self.generation.is_none() && record.checkpoint.is_terminal()
1130 })
1131 .map(|record| record.checkpoint.sequence);
1132 self.publish(SessionPublication::Load { terminal }, ordinary)?;
1133 }
1134 SessionState::Loaded(_) => {
1135 if self.advance_loaded(ordinary)? {
1136 return Poll::Ready(Ok(None));
1137 }
1138 }
1139 SessionState::Sources { .. } => {
1140 self.start_child_groups(cx)?;
1141 if !matches!(self.state, SessionState::Sources { .. }) {
1142 continue;
1143 }
1144 self.poll_child_groups(cx)?;
1145 if self.publish_child_completion(ordinary, false)? {
1146 continue;
1147 }
1148 if self.prefetched.is_none()
1149 && !self.sources.is_empty()
1150 && self.sources.iter().all(|source| {
1151 matches!(
1152 source.state,
1153 CallerSourceState::Ended | CallerSourceState::Eof
1154 )
1155 })
1156 {
1157 let SessionState::Sources { loaded, .. } = &self.state else {
1158 unreachable!()
1159 };
1160 if !self.retains_external_continuation(&loaded.record) {
1161 self.finish_wave(ordinary)?;
1162 continue;
1163 }
1164 }
1165 if self.prefetched.is_none() {
1166 match self.poll_sources_observation(cx) {
1167 Poll::Pending => return Poll::Pending,
1168 Poll::Ready(Err(error)) => return Poll::Ready(Err(error)),
1169 Poll::Ready(Ok(Some(event))) if !ordinary => {
1170 return Poll::Ready(Ok(Some(event)));
1171 }
1172 Poll::Ready(Ok(Some(_))) => continue,
1173 Poll::Ready(Ok(None)) => {}
1174 }
1175 if !matches!(&self.state, SessionState::Sources { .. }) {
1176 continue;
1177 }
1178 }
1179 if let Some((index, event)) = self.prefetched.take() {
1180 self.consume_source_event(index, event, ordinary)?;
1181 } else {
1182 return Poll::Ready(Ok(None));
1183 }
1184 }
1185 }
1186 }
1187 }
1188
1189 fn advance_loaded(&mut self, ordinary: bool) -> Result<bool, AppError> {
1190 let SessionState::Loaded(loaded) =
1191 std::mem::replace(&mut self.state, SessionState::Finished)
1192 else {
1193 unreachable!()
1194 };
1195 self.ensure_owned()?;
1196 self.runtime.supervise_controls(&loaded.record);
1197 if loaded.record.checkpoint.is_stopping() {
1198 self.stop_children();
1199 self.stop_sources();
1200 self.publish(SessionPublication::SettleStopping { loaded }, ordinary)?;
1201 return Ok(false);
1202 }
1203 if loaded.record.checkpoint.is_terminal() {
1204 self.stop_children();
1205 self.stop_sources();
1206 self.observed = Some(loaded.record);
1207 return Ok(true);
1208 }
1209 self.observed = Some(loaded.record.clone());
1210 if self.children.iter().any(|group| {
1211 matches!(
1212 group.state,
1213 ChildExecutionState::Worker(_) | ChildExecutionState::Receipt(_)
1214 ) && group
1215 .parent
1216 .as_ref()
1217 .is_some_and(|parent| !loaded.record.owns_child_activation(parent, group.sequence))
1218 }) {
1219 self.state = SessionState::Sources { loaded };
1223 return Ok(false);
1224 }
1225 if self.generation.is_none()
1226 && self.parent_deadline.is_none()
1227 && loaded.record.checkpoint.is_paused()
1228 {
1229 self.state = SessionState::Sources { loaded };
1232 return Ok(true);
1233 }
1234 if self.runtime.cell.execution_boundary()? == Some(false) && self.halt_after(&loaded)? {
1235 return Ok(false);
1236 }
1237 if self.transitions >= MAX_DRIVER_TRANSITIONS {
1238 self.publish_decision(
1239 loaded,
1240 BspDecision::Failed {
1241 category: FailureCategory::Limit,
1242 message: "workflow transition bound exhausted".into(),
1243 observation: None,
1244 },
1245 ordinary,
1246 )?;
1247 return Ok(false);
1248 }
1249 let mut decision = match self.runtime.decide_checkpoint(&loaded) {
1250 Ok(decision) => decision,
1251 Err(failure) => {
1252 if self.generation.is_none() && self.runtime.schedule_operation_failure(&failure) {
1253 self.observed = None;
1254 }
1255 return Err(failure.into_error());
1256 }
1257 };
1258 if let BspDecision::Activate(selected) = &mut decision {
1259 selected.retain(|activation| {
1260 !self
1261 .sources
1262 .iter()
1263 .any(|source| source.activation == *activation)
1264 });
1265 if selected.is_empty() {
1266 decision = BspDecision::AwaitControls;
1267 }
1268 }
1269 let boundary = self.runtime.cell.execution_boundary()?;
1270 if boundary.is_some()
1271 && let Some(node) =
1272 decision.pending_node(&self.runtime.cell.execution_options()?.halt_before)
1273 {
1274 let node = node.to_owned();
1275 self.publish_transition(loaded, ControlTransition::Halt { node, result: None })?;
1276 return Ok(false);
1277 }
1278 if boundary == Some(true)
1279 && !matches!(&decision, BspDecision::Routed { .. })
1280 && self.halt_after(&loaded)?
1281 {
1282 return Ok(false);
1283 }
1284 if !matches!(&decision, BspDecision::Routed { .. }) {
1285 self.runtime.cell.clear_execution_boundary()?;
1286 }
1287 match decision {
1288 BspDecision::AwaitControls => {
1289 let pending = loaded
1290 .record
1291 .checkpoint
1292 .controls
1293 .values()
1294 .find(|control| {
1295 matches!(control.state, ControlRecordState::Child { .. })
1296 && !self.children.iter().any(|group| {
1297 group.sequence == control.sequence
1298 && !matches!(group.state, ChildExecutionState::Awaiting)
1299 })
1300 })
1301 .map(|control| control.sequence);
1302 if let Some(sequence) = pending {
1303 self.publish(
1304 SessionPublication::ObserveChildren { loaded, sequence },
1305 ordinary,
1306 )?;
1307 } else {
1308 self.state = SessionState::Sources { loaded };
1309 }
1310 }
1311 BspDecision::Activate(selected) => {
1312 self.transitions += 1;
1313 let mut remaining = selected.into_iter();
1314 let activation = remaining.next().ok_or_else(|| {
1315 AppError::Integration("native selected prefix is empty".into())
1316 })?;
1317 self.begin_activation(loaded, activation, remaining, Vec::new(), ordinary)?;
1318 }
1319 decision => {
1320 self.transitions += 1;
1321 self.publish_decision(loaded, decision, ordinary)?;
1322 }
1323 }
1324 Ok(false)
1325 }
1326
1327 pub async fn step(&mut self) -> Result<Option<ExecutionEvent>, AppError> {
1343 loop {
1344 match poll_fn(|cx| self.poll_advance(cx, true)).await {
1345 Ok(Some(ExecutionEvent::Partial { .. })) => {}
1346 event => {
1347 if (matches!(&event, Ok(Some(ExecutionEvent::Committed { .. })))
1348 || matches!(&event, Ok(None)) && matches!(&self.state, SessionState::Eof)
1349 || event.is_err() && self.runtime.cell.lifecycle_generation().is_none())
1350 && let Some(generation) = self.generation
1351 {
1352 self.runtime.cell.returned_boundary(generation);
1353 }
1354 return event;
1355 }
1356 }
1357 }
1358 }
1359
1360 pub async fn finish(mut self) -> Result<RunSnapshot, AppError> {
1377 if !matches!(&self.state, SessionState::Eof) {
1378 while poll_fn(|cx| self.poll_advance(cx, true)).await?.is_some() {}
1379 }
1380 self.close().await
1381 }
1382
1383 pub async fn abort(mut self) -> Result<RunSnapshot, AppError> {
1444 self.close().await
1445 }
1446
1447 async fn close(&mut self) -> Result<RunSnapshot, AppError> {
1448 self.stop_sources();
1449 self.children.retain(|group| {
1450 !matches!(
1451 group.state,
1452 ChildExecutionState::Awaiting | ChildExecutionState::Prepared(_)
1453 )
1454 });
1455 poll_fn(|cx| -> Poll<Result<(), AppError>> {
1456 loop {
1457 self.poll_child_groups(cx)?;
1458 let deadline = self.source_deadline(None);
1459 if self.poll_timer(cx, deadline) {
1460 self.poll_child_groups(cx)?;
1461 }
1462 if matches!(self.state, SessionState::Publishing { .. }) {
1463 match self.poll_publication(cx, true) {
1464 Poll::Pending => return Poll::Pending,
1465 Poll::Ready(result) => {
1466 result?;
1467 continue;
1468 }
1469 }
1470 }
1471 if let SessionState::ObservingItem { receipt, .. } = &mut self.state {
1472 let observed = match Pin::new(receipt).poll(cx) {
1473 Poll::Pending => return Poll::Pending,
1474 Poll::Ready(result) => result.map_err(|_| {
1475 AppError::Closed("stream observation receipt closed".into())
1476 })?,
1477 };
1478 self.finish_item(observed)?;
1479 continue;
1480 }
1481 if self.children.is_empty() {
1482 return Poll::Ready(Ok(()));
1483 }
1484 if let SessionState::Loaded(_) = &self.state {
1485 let SessionState::Loaded(loaded) =
1486 std::mem::replace(&mut self.state, SessionState::Finished)
1487 else {
1488 unreachable!()
1489 };
1490 self.state = SessionState::Sources { loaded };
1491 }
1492 if matches!(self.state, SessionState::Sources { .. }) {
1493 if self.publish_child_completion(false, true)? {
1494 continue;
1495 }
1496 if self.poll_notification(cx)? {
1499 let SessionState::Sources { loaded } =
1500 std::mem::replace(&mut self.state, SessionState::Finished)
1501 else {
1502 unreachable!()
1503 };
1504 self.publish(
1505 SessionPublication::Reload {
1506 receipt: SessionReceipt::Checkpoint {
1507 loaded,
1508 staged: Vec::new(),
1509 },
1510 deadline,
1511 },
1512 false,
1513 )?;
1514 continue;
1515 }
1516 }
1517 return Poll::Pending;
1518 }
1519 })
1520 .await?;
1521 if !self.sources.is_empty() {
1522 if !matches!(&self.state, SessionState::Sources { .. }) {
1523 let deadline = LifecycleEvent::deadline(self.parent_deadline);
1524 let loaded = if matches!(&self.state, SessionState::Loaded(_)) {
1525 self.runtime
1529 .observe_generation(deadline)
1530 .await
1531 .map_err(|failure| {
1532 self.runtime.retain_execution_read_failure(&failure);
1533 failure.into_error()
1534 })?;
1535 let SessionState::Loaded(loaded) =
1536 std::mem::replace(&mut self.state, SessionState::Finished)
1537 else {
1538 unreachable!()
1539 };
1540 loaded
1541 } else {
1542 self.runtime
1543 .load_execution_observation(None, Some(deadline))
1544 .await
1545 .map_err(|failure| {
1546 self.runtime.retain_execution_read_failure(&failure);
1547 failure.into_error()
1548 })?
1549 .ok_or_else(|| AppError::NotFound("workflow run does not exist".into()))?
1550 };
1551 self.state = SessionState::Sources { loaded };
1552 }
1553 let buffered_failure = self
1554 .prefetched
1555 .take()
1556 .and_then(|(index, event)| event.err().map(|category| (index, category)));
1557 for index in 0..self.sources.len() {
1558 let source = &mut self.sources[index];
1559 let failure = buffered_failure
1560 .filter(|(buffered, _)| *buffered == index)
1561 .map(|(_, category)| category)
1562 .or(match source.state {
1563 CallerSourceState::Failed(category) => Some(category),
1564 _ => None,
1565 });
1566 source.state = CallerSourceState::Ended;
1567 if let Some(category) = failure {
1568 self.stage_source_failure(index, category, false)?;
1569 poll_fn(|cx| self.poll_publication(cx, true)).await?;
1570 self.runtime
1571 .observe_generation(LifecycleEvent::deadline(self.parent_deadline))
1572 .await
1573 .map_err(|failure| {
1574 self.runtime.retain_execution_read_failure(&failure);
1575 failure.into_error()
1576 })?;
1577 let SessionState::Loaded(loaded) =
1578 std::mem::replace(&mut self.state, SessionState::Finished)
1579 else {
1580 unreachable!("closing publication retains its original checkpoint receipt")
1581 };
1582 self.state = SessionState::Sources { loaded };
1583 }
1584 }
1585 let SessionState::Sources { loaded, .. } = &self.state else {
1586 unreachable!()
1587 };
1588 if loaded
1589 .record
1590 .checkpoint
1591 .staged
1592 .iter()
1593 .any(|activation| matches!(activation.outcome, StagedOutcomeRecord::Failed { .. }))
1594 {
1595 self.finish_wave(false)?;
1596 poll_fn(|cx| self.poll_publication(cx, true)).await?;
1597 }
1598 }
1599 if let SessionState::Sources { loaded, .. } =
1600 std::mem::replace(&mut self.state, SessionState::Finished)
1601 {
1602 self.observed = Some(loaded.record);
1603 }
1604 self.sources.clear();
1605 let foreground = self.generation.is_some();
1606 if foreground {
1607 self.publish(SessionPublication::Release, false)?;
1608 poll_fn(|cx| self.poll_publication(cx, true)).await?;
1609 self.state = SessionState::Finished;
1610 self.generation = None;
1611 }
1612 let snapshot = if foreground {
1613 self.runtime.snapshot().await?
1614 } else {
1615 self.runtime.app.ensure_snapshot_read()?;
1616 let terminal = self
1617 .observed
1618 .as_ref()
1619 .filter(|record| record.checkpoint.is_terminal())
1620 .map(|record| record.checkpoint.sequence);
1621 let loaded = self
1622 .runtime
1623 .load_execution_observation(terminal, self.parent_deadline)
1624 .await
1625 .map_err(|failure| {
1626 self.runtime.retain_execution_read_failure(&failure);
1627 failure.into_error()
1628 })?
1629 .ok_or_else(|| AppError::NotFound("workflow run does not exist".into()))?;
1630 let mut deadline = LifecycleEvent::deadline(self.parent_deadline);
1631 if loaded.record.checkpoint.is_terminal()
1632 && let Some(generation) = self.runtime.cell.lifecycle_generation()
1633 {
1634 deadline = self
1635 .runtime
1636 .cell
1637 .begin_action_cleanup(generation)?
1638 .min(deadline);
1639 }
1640 if !self.retains_execution_lifecycle(&loaded.record)
1641 && (loaded.record.checkpoint.controls.is_empty()
1642 || loaded.record.checkpoint.is_public_outcome())
1643 {
1644 self.runtime
1645 .finish_execution_observation(
1646 &loaded.record,
1647 None,
1648 loaded.record.checkpoint.is_terminal().then_some(deadline),
1649 false,
1650 )
1651 .await?;
1652 }
1653 if loaded.record.checkpoint.is_terminal() {
1654 self.runtime
1657 .observe_generation(deadline)
1658 .await
1659 .map_err(|failure| {
1660 self.runtime.retain_execution_read_failure(&failure);
1661 failure.into_error()
1662 })?;
1663 let snapshot = self.runtime.project(&loaded.record)?;
1664 self.runtime.app.ensure_snapshot_read()?;
1665 snapshot
1666 } else {
1667 self.runtime.snapshot().await?
1668 }
1669 };
1670 if !foreground
1671 && matches!(
1672 snapshot.status,
1673 RunStatus::Succeeded
1674 | RunStatus::Cancelled
1675 | RunStatus::Failed(_)
1676 | RunStatus::Halted { .. }
1677 )
1678 {
1679 self.runtime.cell.clear_execution_options(None);
1680 }
1681 if foreground && matches!(snapshot.status, crate::workflow::RunStatus::Running) {
1682 self.runtime.wait().await
1683 } else {
1684 Ok(snapshot)
1685 }
1686 }
1687
1688 fn stop_sources(&mut self) {
1689 for source in &mut self.sources {
1690 let failure = match source.state {
1691 CallerSourceState::Failed(category) => Some(category),
1692 _ => None,
1693 };
1694 let destroyed = source.stop().err();
1695 if let Some(category) = failure.or(destroyed) {
1696 self.runtime.retain_source_failure(
1697 &source.context,
1698 &source.activation.lineage,
1699 category,
1700 );
1701 source.state = CallerSourceState::Failed(category);
1702 }
1703 }
1704 }
1705
1706 fn stop_children(&mut self) {
1707 for group in &mut self.children {
1708 group.stop(&self.runtime);
1709 }
1710 self.children.clear();
1711 }
1712
1713 fn plan_child_group(
1714 &mut self,
1715 loaded: LoadedRun,
1716 sequence: u64,
1717 children: Vec<RunRecord>,
1718 verified: bool,
1719 ) -> Result<(), AppError> {
1720 if loaded.record.checkpoint.is_stopping() || loaded.record.checkpoint.is_terminal() {
1721 self.state = SessionState::Loaded(loaded);
1722 return Ok(());
1723 }
1724 if let Some(group) = self
1725 .children
1726 .iter_mut()
1727 .find(|group| group.sequence == sequence)
1728 && let Some(ControlRecord {
1729 state: ControlRecordState::Child { admitted, .. },
1730 ..
1731 }) = loaded.record.checkpoint.control(sequence)
1732 {
1733 for values in group.inputs.iter_mut().take(admitted.len()) {
1736 values.clear();
1737 }
1738 }
1739 let decision = catch_unwind(AssertUnwindSafe(|| {
1740 self.runtime
1741 .plan_children(&loaded, sequence, children, verified)
1742 }))
1743 .map_err(|_| AppError::Integration("child planning callback panicked".into()))??;
1744 match decision {
1745 BspDecision::ExecuteChildren { sequence, children } => {
1746 let control = loaded
1747 .record
1748 .checkpoint
1749 .control(sequence)
1750 .ok_or_else(|| AppError::Conflict("child control is unavailable".into()))?;
1751 let context = self.runtime.child_context(&loaded.record, sequence)?;
1752 let definition = self.runtime.app.definition_for(&loaded.record)?;
1753 let step = definition
1754 .node(&control.node)
1755 .ok_or_else(|| AppError::Integration("child definition is unavailable".into()))?
1756 .step();
1757 step.validate_mode(context.mode)?;
1758 let deadline = tokio::time::Instant::now()
1759 .checked_add(WorkflowRunRuntime::<CS, A, X, C, T, H, E>::remaining(
1760 control.deadline_millis,
1761 )?)
1762 .ok_or_else(|| {
1763 AppError::Timeout("child deadline is outside the clock range".into())
1764 })?;
1765 if let Some(group) = self
1766 .children
1767 .iter_mut()
1768 .find(|group| group.sequence == sequence)
1769 {
1770 if !matches!(group.state, ChildExecutionState::Awaiting) {
1771 return Err(AppError::Conflict(
1772 "child group already has a polling owner".into(),
1773 ));
1774 }
1775 group.state = ChildExecutionState::Prepared(children);
1776 } else {
1777 self.children.push(ChildExecution {
1778 sequence,
1779 parent: None,
1780 context,
1781 lineage: control.lineage.clone(),
1782 deadline,
1783 state: ChildExecutionState::Prepared(children),
1784 inputs: Vec::new(),
1785 });
1786 }
1787 self.state = SessionState::Loaded(loaded);
1788 Ok(())
1789 }
1790 BspDecision::AwaitControls => {
1791 self.state = SessionState::Loaded(loaded);
1792 Ok(())
1793 }
1794 decision => {
1795 let transition = match decision {
1796 BspDecision::Stage(activations) => ControlTransition::Stage { activations },
1797 BspDecision::Failed {
1798 category,
1799 message,
1800 observation,
1801 } => ControlTransition::Failed {
1802 category,
1803 message,
1804 observation,
1805 },
1806 decision => return self.publish_decision(loaded, decision, false),
1807 };
1808 let control = loaded
1809 .record
1810 .checkpoint
1811 .control(sequence)
1812 .cloned()
1813 .ok_or_else(|| {
1814 AppError::Conflict("completed child control is unavailable".into())
1815 })?;
1816 self.publish(
1817 SessionPublication::ChildOutcome {
1818 control,
1819 transition,
1820 },
1821 false,
1822 )
1823 }
1824 }
1825 }
1826
1827 fn start_child_groups(&mut self, cx: &mut Context<'_>) -> Result<(), AppError> {
1828 if !self
1829 .children
1830 .iter()
1831 .any(|group| matches!(group.state, ChildExecutionState::Prepared(_)))
1832 {
1833 return Ok(());
1834 }
1835 if self.poll_notification(cx)? {
1836 let deadline = self.source_deadline(None);
1837 let SessionState::Sources { loaded } =
1838 std::mem::replace(&mut self.state, SessionState::Finished)
1839 else {
1840 unreachable!()
1841 };
1842 self.publish(
1843 SessionPublication::Reload {
1844 receipt: SessionReceipt::Checkpoint {
1845 loaded,
1846 staged: Vec::new(),
1847 },
1848 deadline,
1849 },
1850 false,
1851 )?;
1852 return Ok(());
1853 }
1854 let SessionState::Sources { loaded, .. } = &self.state else {
1855 return Ok(());
1856 };
1857 let parent = Arc::new(loaded.record.clone());
1858 for index in 0..self.children.len() {
1859 if !matches!(self.children[index].state, ChildExecutionState::Prepared(_)) {
1860 continue;
1861 }
1862 let group = &mut self.children[index];
1863 let ChildExecutionState::Prepared(children) =
1864 std::mem::replace(&mut group.state, ChildExecutionState::Closed)
1865 else {
1866 unreachable!()
1867 };
1868 group.parent = Some(Arc::clone(&parent));
1869 if group.context.mode == ActionMode::Asynchronous && self.lane == ExecutionLane::Caller
1870 {
1871 let (sender, receipt) = oneshot::channel();
1872 if !self.runtime.app.spawn_driver(
1873 self.runtime.key.clone(),
1874 Duration::ZERO,
1875 DriverCommand::Child {
1876 sequence: group.sequence,
1877 generation: self.generation,
1878 parent: Arc::clone(&parent),
1879 children,
1880 receipt: sender,
1881 },
1882 ) {
1883 return Err(AppError::Busy(
1884 "child group already has an accepted owner".into(),
1885 ));
1886 }
1887 group.state = ChildExecutionState::Worker(receipt);
1888 } else {
1889 group.start_local(&self.runtime, self.app, self.lane, children)?;
1890 }
1891 }
1892 Ok(())
1893 }
1894
1895 fn poll_child_groups(&mut self, cx: &mut Context<'_>) -> Result<(), AppError> {
1896 let stopped = match &self.state {
1897 SessionState::Loaded(loaded)
1898 | SessionState::Sources { loaded }
1899 | SessionState::ObservingItem { loaded, .. } => {
1900 loaded.record.checkpoint.is_stopping() || loaded.record.checkpoint.is_terminal()
1901 }
1902 _ => false,
1903 };
1904 if stopped {
1905 self.stop_children();
1906 return Ok(());
1907 }
1908 if let SessionState::Loaded(loaded)
1909 | SessionState::Sources { loaded }
1910 | SessionState::ObservingItem { loaded, .. } = &self.state
1911 {
1912 self.children.retain(|group| {
1913 !matches!(group.state, ChildExecutionState::Awaiting)
1914 || loaded.record.checkpoint.control(group.sequence).is_some()
1915 });
1916 if self.children.iter().any(|group| {
1917 !matches!(
1918 group.state,
1919 ChildExecutionState::Awaiting
1920 | ChildExecutionState::Worker(_)
1921 | ChildExecutionState::Receipt(_)
1922 ) && group.parent.as_ref().is_some_and(|parent| {
1923 !loaded.record.owns_child_activation(parent, group.sequence)
1924 })
1925 }) {
1926 return Err(AppError::Conflict(
1927 "child parent reservation changed".into(),
1928 ));
1929 }
1930 }
1931 let observing_parent = matches!(
1932 self.state,
1933 SessionState::Publishing {
1934 purpose: PublicationPurpose::Reload,
1935 ..
1936 }
1937 );
1938 for group in &mut self.children {
1939 if observing_parent
1940 && tokio::time::Instant::now() >= group.deadline
1941 && matches!(
1942 group.state,
1943 ChildExecutionState::Local(_) | ChildExecutionState::Pipeline(_)
1944 )
1945 {
1946 continue;
1949 }
1950 let _ = group.poll_ready(cx, &self.runtime);
1951 }
1952 Ok(())
1953 }
1954
1955 fn publish_child_completion(
1956 &mut self,
1957 ordinary: bool,
1958 closing: bool,
1959 ) -> Result<bool, AppError> {
1960 let Some(index) = self.children.iter().position(|group| {
1961 matches!(
1962 group.state,
1963 ChildExecutionState::Complete(_) | ChildExecutionState::Receipt(_)
1964 )
1965 }) else {
1966 return Ok(false);
1967 };
1968 let group = &mut self.children[index];
1969 let sequence = group.sequence;
1970 let parent = group.parent.take();
1971 let context = Arc::clone(&group.context);
1972 let lineage = group.lineage.clone();
1973 let completed = std::mem::replace(&mut group.state, ChildExecutionState::Awaiting);
1974 if closing {
1975 self.children.remove(index);
1976 }
1977 if let ChildExecutionState::Receipt(result) = completed {
1978 match (*result).map_err(RuntimeOperationFailure::into_error)? {
1979 SessionReceipt::Children {
1980 loaded,
1981 sequence,
1982 children,
1983 verified: true,
1984 } => {
1985 if closing {
1986 self.state = SessionState::Loaded(loaded);
1987 } else {
1988 self.plan_child_group(loaded, sequence, children, true)?;
1989 }
1990 }
1991 SessionReceipt::Checkpoint { loaded, .. } => {
1992 self.state = SessionState::Loaded(loaded)
1993 }
1994 SessionReceipt::Children {
1995 verified: false, ..
1996 } => {
1997 return Err(AppError::Integration(
1998 "child worker returned an unverified execution result".into(),
1999 ));
2000 }
2001 }
2002 return Ok(true);
2003 }
2004 let ChildExecutionState::Complete(result) = completed else {
2005 unreachable!()
2006 };
2007 let SessionState::Sources { loaded } =
2008 std::mem::replace(&mut self.state, SessionState::Finished)
2009 else {
2010 unreachable!()
2011 };
2012 match result {
2013 Ok(completions) => self.publish(
2014 SessionPublication::VerifyChildren {
2015 parent: parent.ok_or_else(|| {
2016 AppError::Integration("child group parent evidence is unavailable".into())
2017 })?,
2018 sequence,
2019 completions,
2020 },
2021 ordinary,
2022 )?,
2023 Err(category) => {
2024 let activation = loaded
2025 .record
2026 .checkpoint
2027 .frontier
2028 .iter()
2029 .find(|activation| {
2030 activation.node == context.node && activation.lineage == lineage
2031 })
2032 .cloned()
2033 .ok_or_else(|| AppError::Conflict("child activation changed".into()))?;
2034 self.publish_failure(loaded, &activation, &context, category)?;
2035 }
2036 }
2037 Ok(true)
2038 }
2039
2040 fn retains_external_continuation(&self, record: &RunRecord) -> bool {
2041 !self.children.is_empty()
2042 || self.runtime.app.has_child_work(&self.runtime.key)
2043 || record.checkpoint.controls.values().any(|control| {
2044 matches!(&control.state, ControlRecordState::Attempt(_))
2045 && self
2046 .runtime
2047 .app
2048 .has_control_work(&self.runtime.key, control.sequence)
2049 })
2050 }
2051
2052 fn retains_execution_lifecycle(&self, record: &RunRecord) -> bool {
2053 self.retains_execution_work(record)
2054 || record
2055 .checkpoint
2056 .controls
2057 .values()
2058 .any(|control| matches!(control.state, ControlRecordState::Inputs { .. }))
2059 }
2060
2061 fn retains_execution_work(&self, record: &RunRecord) -> bool {
2062 self.sources.iter().any(|source| {
2063 matches!(
2064 source.state,
2065 CallerSourceState::Polling(_) | CallerSourceState::Failed(_)
2066 )
2067 }) || self.prefetched.is_some()
2068 || self.retains_external_continuation(record)
2069 || record.checkpoint.controls.values().any(|control| {
2070 matches!(control.state, ControlRecordState::Sleep)
2071 && self
2072 .runtime
2073 .app
2074 .has_control_work(&self.runtime.key, control.sequence)
2075 })
2076 }
2077
2078 fn ensure_owned(&self) -> Result<(), AppError> {
2079 self.runtime.app.ensure_runtime_owned()?;
2080 if self
2081 .generation
2082 .is_some_and(|generation| !self.runtime.cell.execution_is_live(generation))
2083 {
2084 return Err(AppError::Closed(
2085 "run execution generation is closed".into(),
2086 ));
2087 }
2088 Ok(())
2089 }
2090
2091 fn source_deadline(&self, excluded: Option<usize>) -> Option<tokio::time::Instant> {
2092 self.sources
2093 .iter()
2094 .enumerate()
2095 .filter_map(|(index, source)| {
2096 (Some(index) != excluded && matches!(source.state, CallerSourceState::Polling(_)))
2097 .then_some(source.deadline)
2098 })
2099 .chain(self.children.iter().filter_map(|group| {
2100 (!matches!(
2101 group.state,
2102 ChildExecutionState::Complete(_) | ChildExecutionState::Receipt(_)
2103 ))
2104 .then_some(group.deadline)
2105 }))
2106 .min()
2107 }
2108
2109 fn expire_sources(&mut self, excluded: Option<usize>) {
2110 let now = tokio::time::Instant::now();
2111 for (index, source) in self.sources.iter_mut().enumerate() {
2112 if Some(index) == excluded
2113 || now < source.deadline
2114 || !matches!(source.state, CallerSourceState::Polling(_))
2115 {
2116 continue;
2117 }
2118 self.runtime.retain_source_failure(
2119 &source.context,
2120 &source.activation.lineage,
2121 FailureCategory::Timeout,
2122 );
2123 let _ = source.stop();
2124 if self
2125 .prefetched
2126 .as_ref()
2127 .is_some_and(|(buffered, _)| *buffered == index)
2128 {
2129 self.prefetched = Some((index, Err(FailureCategory::Timeout)));
2130 } else {
2131 source.state = CallerSourceState::Failed(FailureCategory::Timeout);
2132 }
2133 }
2134 }
2135
2136 fn poll_sources(&mut self, context: &mut Context<'_>, excluded: Option<usize>) {
2137 self.expire_sources(excluded);
2138 if self.prefetched.is_some() {
2139 return;
2140 }
2141 let count = self.sources.len();
2142 for offset in 0..count {
2143 let index = (self.next_source + offset) % count;
2144 if Some(index) == excluded {
2145 continue;
2146 }
2147 if let Poll::Ready(mut event) = self.sources[index].poll(context) {
2148 let source = &mut self.sources[index];
2149 if let Err(category) = &event {
2150 self.runtime.retain_source_failure(
2153 &source.context,
2154 &source.activation.lineage,
2155 *category,
2156 );
2157 }
2158 if matches!(
2159 &event,
2160 Ok(Some(ActionEvent::Completed { .. })) | Ok(None) | Err(_)
2161 ) {
2162 let destroyed = source.stop();
2163 if event.is_ok()
2164 && let Err(category) = destroyed
2165 {
2166 self.runtime.retain_source_failure(
2167 &source.context,
2168 &source.activation.lineage,
2169 category,
2170 );
2171 event = Err(category);
2172 }
2173 if matches!(&event, Ok(None)) {
2174 source.state = CallerSourceState::Eof;
2175 }
2176 }
2177 self.next_source = (index + 1) % count;
2178 self.prefetched = Some((index, event));
2179 return;
2180 }
2181 }
2182 }
2183
2184 fn arm_notification(&mut self) -> Result<(), AppError> {
2185 if self.wake.is_none() {
2186 self.wake = Some(Box::pin(self.runtime.notifier()?.notified_owned()));
2187 }
2188 let wake = self.wake.as_mut().unwrap();
2189 wake.as_mut().enable();
2190 Ok(())
2193 }
2194
2195 fn poll_notification(&mut self, cx: &mut Context<'_>) -> Result<bool, AppError> {
2196 self.arm_notification()?;
2197 let ready = self.wake.as_mut().unwrap().as_mut().poll(cx).is_ready();
2198 if ready {
2199 self.wake = None;
2200 }
2201 Ok(ready)
2202 }
2203
2204 fn poll_timer(&mut self, cx: &mut Context<'_>, deadline: Option<tokio::time::Instant>) -> bool {
2205 let Some(deadline) = deadline else {
2206 self.timer = None;
2207 return false;
2208 };
2209 let timer = self
2210 .timer
2211 .get_or_insert_with(|| Box::pin(tokio::time::sleep_until(deadline)));
2212 if timer.deadline() != deadline {
2213 timer.as_mut().reset(deadline);
2214 }
2215 timer.as_mut().poll(cx).is_ready()
2216 }
2217
2218 fn poll_native_partial(&mut self, cx: &mut Context<'_>) -> Option<ExecutionEvent> {
2219 let Some((_, receiver)) = &mut self.native_partials else {
2220 return None;
2221 };
2222 match receiver.poll_recv(cx) {
2223 Poll::Ready(Some(event)) => Some(event),
2224 Poll::Ready(None) => {
2225 self.native_partials = None;
2226 None
2227 }
2228 Poll::Pending => None,
2229 }
2230 }
2231
2232 fn poll_sources_observation(
2233 &mut self,
2234 cx: &mut Context<'_>,
2235 ) -> Poll<Result<Option<ExecutionEvent>, AppError>> {
2236 self.ensure_owned()?;
2237 self.expire_sources(None);
2238 let deadline = self.source_deadline(None);
2239 if self.poll_timer(cx, deadline) {
2240 self.expire_sources(None);
2241 }
2242 let pending_failure = self
2243 .sources
2244 .iter()
2245 .any(|source| matches!(source.state, CallerSourceState::Failed(_)));
2246 if !pending_failure && self.poll_notification(cx)? {
2247 let SessionState::Sources { loaded } =
2248 std::mem::replace(&mut self.state, SessionState::Finished)
2249 else {
2250 unreachable!()
2251 };
2252 self.publish(
2253 SessionPublication::Reload {
2254 receipt: SessionReceipt::Checkpoint {
2255 loaded,
2256 staged: Vec::new(),
2257 },
2258 deadline,
2259 },
2260 false,
2261 )?;
2262 return Poll::Ready(Ok(None));
2263 }
2264 self.poll_sources(cx, None);
2265 if let Some(event) = self.poll_native_partial(cx) {
2266 return Poll::Ready(Ok(Some(event)));
2267 }
2268 if self.prefetched.is_some() {
2269 return Poll::Ready(Ok(None));
2270 }
2271 let SessionState::Sources { loaded, .. } = &self.state else {
2272 unreachable!()
2273 };
2274 let known_eof = self
2275 .sources
2276 .iter()
2277 .any(|source| matches!(source.state, CallerSourceState::Eof));
2278 if (self.generation.is_some() || !self.retains_execution_work(&loaded.record))
2279 && !known_eof
2280 && !self.children.iter().any(|group| {
2281 matches!(
2282 group.state,
2283 ChildExecutionState::Complete(_) | ChildExecutionState::Receipt(_)
2284 )
2285 })
2286 && loaded.record.checkpoint.is_actionable()
2287 {
2288 return Poll::Ready(Ok(None));
2289 }
2290 Poll::Pending
2291 }
2292
2293 fn poll_item(
2294 &mut self,
2295 cx: &mut Context<'_>,
2296 ) -> Poll<Result<Option<ExecutionEvent>, AppError>> {
2297 let SessionState::ObservingItem { index, .. } = &self.state else {
2298 unreachable!()
2299 };
2300 let index = *index;
2301 self.expire_sources(Some(index));
2302 let deadline = self.source_deadline(Some(index));
2303 if self.poll_timer(cx, deadline) {
2304 self.expire_sources(Some(index));
2305 }
2306 let SessionState::ObservingItem { receipt, .. } = &mut self.state else {
2307 unreachable!()
2308 };
2309 if let Poll::Ready(result) = Pin::new(receipt).poll(cx) {
2310 let result = result
2311 .map_err(|_| AppError::Closed("stream observation receipt closed".into()))
2312 .and_then(|result| result);
2313 return Poll::Ready(self.finish_item(result).map(|result| {
2314 result.map(|result| ExecutionEvent::Partial {
2315 node: self.sources[index].activation.node.clone(),
2316 result,
2317 })
2318 }));
2319 }
2320 self.poll_sources(cx, Some(index));
2321 if let Some(event) = self.poll_native_partial(cx) {
2322 return Poll::Ready(Ok(Some(event)));
2323 }
2324 Poll::Pending
2325 }
2326
2327 fn finish_item(
2328 &mut self,
2329 result: Result<Option<Map<String, Value>>, AppError>,
2330 ) -> Result<Option<Map<String, Value>>, AppError> {
2331 let SessionState::ObservingItem {
2332 loaded, receipt, ..
2333 } = std::mem::replace(&mut self.state, SessionState::Finished)
2334 else {
2335 unreachable!()
2336 };
2337 drop(receipt);
2338 if result.is_err() {
2339 self.observed = Some(loaded.record);
2340 for source in &mut self.sources {
2341 let _ = source.stop();
2342 }
2343 } else {
2344 self.state = SessionState::Sources { loaded };
2345 }
2346 result
2347 }
2348
2349 fn consume_source_event(
2350 &mut self,
2351 index: usize,
2352 event: Result<Option<ActionEvent>, FailureCategory>,
2353 ordinary: bool,
2354 ) -> Result<(), AppError> {
2355 match event {
2356 Ok(Some(ActionEvent::Partial { result })) => {
2357 if JsonBoundary::validate_size(&result).is_err() {
2358 let _ = self.sources[index].stop();
2359 return self.stage_source_failure(index, FailureCategory::State, ordinary);
2360 }
2361 self.publish_item(index, ActionEvent::Partial { result })
2362 }
2363 Ok(Some(
2364 event @ (ActionEvent::SpanStarted { .. }
2365 | ActionEvent::SpanAttribute { .. }
2366 | ActionEvent::SpanFinished { .. }),
2367 )) => self.publish_item(index, event),
2368 Ok(Some(ActionEvent::Completed { result, mut writes })) => {
2369 let SessionState::Sources { loaded, .. } = &self.state else {
2370 unreachable!()
2371 };
2372 let source = &self.sources[index];
2373 let staged = catch_unwind(AssertUnwindSafe(|| {
2374 let definition = self.runtime.app.definition_for(&loaded.record)?;
2375 if let Some(Step::Action {
2376 action,
2377 revision,
2378 project_result: true,
2379 }) = definition
2380 .node(&source.activation.node)
2381 .map(|node| node.step())
2382 {
2383 writes.insert(
2384 0,
2385 self.runtime
2386 .app
2387 .action(action, revision)?
2388 .declaration()
2389 .result_projection(&result),
2390 );
2391 }
2392 StagedActivationRecord::action(
2393 &source.activation,
2394 &loaded.record.checkpoint.state,
2395 &writes,
2396 result,
2397 self.runtime.app.typing(),
2398 )
2399 }));
2400 match staged {
2401 Ok(Ok(staged)) => self.stage_source(index, staged, ordinary),
2402 Ok(Err(_)) => {
2403 self.stage_source_failure(index, FailureCategory::State, ordinary)
2404 }
2405 Err(_) => {
2406 self.stage_source_failure(index, FailureCategory::Execution, ordinary)
2407 }
2408 }
2409 }
2410 Ok(None) => Ok(()),
2411 Err(category) => self.stage_source_failure(index, category, ordinary),
2412 }
2413 }
2414
2415 fn stage_source_failure(
2416 &mut self,
2417 index: usize,
2418 category: FailureCategory,
2419 ordinary: bool,
2420 ) -> Result<(), AppError> {
2421 let source = &self.sources[index];
2422 self.runtime
2423 .retain_source_failure(&source.context, &source.activation.lineage, category);
2424 let staged = StagedActivationRecord::failed(
2425 source.activation.node.clone(),
2426 source.activation.lineage.clone(),
2427 category,
2428 "caller action execution failed".into(),
2429 );
2430 self.stage_source(index, staged, ordinary)
2431 }
2432
2433 fn stage_source(
2434 &mut self,
2435 index: usize,
2436 staged: StagedActivationRecord,
2437 ordinary: bool,
2438 ) -> Result<(), AppError> {
2439 let SessionState::Sources { loaded } =
2440 std::mem::replace(&mut self.state, SessionState::Finished)
2441 else {
2442 unreachable!()
2443 };
2444 self.observed = Some(loaded.record.clone());
2445 self.publish(
2446 SessionPublication::Decision {
2447 loaded,
2448 decision: BspDecision::Stage(vec![staged]),
2449 activation_deadline: None,
2450 context: Some(Arc::clone(&self.sources[index].context)),
2451 staged: Vec::new(),
2452 remaining: Vec::new().into_iter(),
2453 },
2454 ordinary,
2455 )
2456 }
2457
2458 fn finish_wave(&mut self, ordinary: bool) -> Result<(), AppError> {
2459 let SessionState::Sources { loaded } =
2460 std::mem::replace(&mut self.state, SessionState::Finished)
2461 else {
2462 unreachable!()
2463 };
2464 let definition = self.runtime.app.definition_for(&loaded.record)?;
2465 let failed = loaded
2466 .record
2467 .checkpoint
2468 .selected_staged(&definition)
2469 .filter_map(|activation| {
2470 if let StagedOutcomeRecord::Failed { category, message } = &activation.outcome {
2471 Some((activation, *category, message))
2472 } else {
2473 None
2474 }
2475 })
2476 .min_by(|left, right| {
2477 (&left.0.node, &left.0.lineage).cmp(&(&right.0.node, &right.0.lineage))
2478 });
2479 if let Some((activation, category, message)) = failed {
2480 let message = message.clone();
2481 let observation = match self.sources.iter().find(|source| {
2482 source.activation.node == activation.node
2483 && source.activation.lineage == activation.lineage
2484 }) {
2485 Some(source) => ActionPosition {
2486 node: source.context.node.clone(),
2487 ordinal: source.context.action_ordinal,
2488 },
2489 None => loaded.record.checkpoint.proposed_position(
2490 &definition,
2491 &activation.node,
2492 &activation.lineage,
2493 &[],
2494 )?,
2495 };
2496 self.sources.clear();
2497 self.publish_decision(
2498 loaded,
2499 BspDecision::Failed {
2500 category,
2501 message,
2502 observation: Some(observation),
2503 },
2504 ordinary,
2505 )
2506 } else if self
2507 .sources
2508 .iter()
2509 .any(|source| matches!(source.state, CallerSourceState::Eof))
2510 {
2511 self.sources.clear();
2512 self.publish(SessionPublication::EndSource { loaded }, ordinary)
2513 } else {
2514 self.sources.clear();
2515 self.observed = Some(loaded.record);
2516 self.state = SessionState::Ready;
2517 Ok(())
2518 }
2519 }
2520
2521 fn begin_activation(
2522 &mut self,
2523 loaded: LoadedRun,
2524 activation: FrontierRecord,
2525 remaining: std::vec::IntoIter<FrontierRecord>,
2526 staged: Vec<StagedActivationRecord>,
2527 ordinary: bool,
2528 ) -> Result<(), AppError> {
2529 self.ensure_owned()?;
2530 let definition = self.runtime.app.definition_for(&loaded.record)?;
2531 if let Some(node) = definition.node(&activation.node) {
2532 node.step()
2533 .validate_mode(self.runtime.cell.execution_mode()?)?;
2534 }
2535 let (started, deadline_millis) = WorkflowRunRuntime::<CS, A, X, C, T, H, E>::node_deadline(
2536 &definition,
2537 &activation.node,
2538 )?;
2539 let deadline = tokio::time::Instant::now()
2540 .checked_add(Duration::from_millis(deadline_millis - started))
2541 .ok_or_else(|| {
2542 AppError::Timeout("activation deadline is outside the clock range".into())
2543 })?;
2544 let context = match self.runtime.cell.action_context(
2545 loaded.record.checkpoint.base_sequence,
2546 &activation.node,
2547 &activation.lineage,
2548 ) {
2549 Some(context) => context,
2550 None => Arc::new(loaded.record.action_context(
2551 &definition,
2552 &activation,
2553 &staged,
2554 self.runtime.cell.execution_mode()?,
2555 )?),
2556 };
2557 if loaded.record.checkpoint.loses_join_first(
2558 &definition,
2559 &activation.node,
2560 &activation.lineage,
2561 ) {
2562 self.state = SessionState::ActivationReady {
2563 loaded,
2564 activation,
2565 context,
2566 remaining,
2567 staged,
2568 deadline,
2569 deadline_millis,
2570 };
2571 return Ok(());
2572 }
2573 if let Some(Step::Action {
2574 action, revision, ..
2575 }) = definition.node(&activation.node).map(|node| node.step())
2576 {
2577 let binding = self.runtime.app.action(action, revision)?;
2578 let validated = catch_unwind(AssertUnwindSafe(|| {
2579 binding.declaration().validate_reads(&WorkflowState {
2580 values: loaded.record.checkpoint.state.clone(),
2581 })?;
2582 binding.declaration().resolve_inputs(
2583 &self.runtime.cell.execution_options()?.inputs,
2584 self.runtime.cell.execution_mode()?,
2585 )
2586 }));
2587 match validated {
2588 Ok(Err(AppError::InvalidInput(message))) if self.generation.is_none() => {
2589 return self.publish_decision(
2590 loaded,
2591 BspDecision::Failed {
2592 category: FailureCategory::Execution,
2593 message,
2594 observation: None,
2595 },
2596 ordinary,
2597 );
2598 }
2599 Ok(Ok(ActionInputs::Required(required))) => {
2600 return self.publish(
2601 SessionPublication::Decision {
2602 loaded,
2603 decision: BspDecision::Inputs {
2604 node: activation.node,
2605 lineage: activation.lineage,
2606 required,
2607 },
2608 activation_deadline: Some(deadline_millis),
2609 context: None,
2610 staged,
2611 remaining,
2612 },
2613 ordinary,
2614 );
2615 }
2616 Ok(result) => {
2617 result?;
2618 }
2619 Err(_) => {
2620 return self.publish_failure(
2621 loaded,
2622 &activation,
2623 &context,
2624 FailureCategory::Execution,
2625 );
2626 }
2627 }
2628 }
2629 self.publish(
2630 SessionPublication::BeginActivation {
2631 loaded,
2632 activation,
2633 context,
2634 remaining,
2635 staged,
2636 deadline,
2637 deadline_millis,
2638 },
2639 ordinary,
2640 )
2641 }
2642
2643 fn activate_ready(&mut self, ordinary: bool) -> Result<(), AppError> {
2644 self.ensure_owned()?;
2645 let SessionState::ActivationReady {
2646 loaded,
2647 activation,
2648 context,
2649 mut remaining,
2650 mut staged,
2651 deadline,
2652 deadline_millis,
2653 } = std::mem::replace(&mut self.state, SessionState::Finished)
2654 else {
2655 unreachable!()
2656 };
2657 let definition = self.runtime.app.definition_for(&loaded.record)?;
2658 let skipped = loaded.record.checkpoint.loses_join_first(
2659 &definition,
2660 &activation.node,
2661 &activation.lineage,
2662 );
2663 if !skipped
2664 && matches!(
2665 definition.node(&activation.node).map(|node| node.step()),
2666 Some(Step::Action { .. })
2667 )
2668 {
2669 self.construct_source(&loaded, activation, context, deadline)?;
2670 return match remaining.next() {
2671 Some(activation) => {
2672 self.begin_activation(loaded, activation, remaining, staged, ordinary)
2673 }
2674 None if staged.is_empty() => {
2675 self.observed = None;
2676 self.state = SessionState::Sources { loaded };
2677 Ok(())
2678 }
2679 None => self.publish_decision(loaded, BspDecision::Stage(staged), ordinary),
2680 };
2681 }
2682 if !skipped && tokio::time::Instant::now() >= deadline {
2683 return self.publish_failure(loaded, &activation, &context, FailureCategory::Timeout);
2684 }
2685 let engine = ExecutionEngine::new(
2686 &definition,
2687 self.runtime.app.typing(),
2688 self.runtime.app.conditions(),
2689 );
2690 let decision = match catch_unwind(AssertUnwindSafe(|| {
2691 let decision = engine.activate(
2692 &loaded.record.checkpoint,
2693 &activation,
2694 loaded
2695 .record
2696 .parent
2697 .as_ref()
2698 .map_or(0, |parent| parent.depth),
2699 |workflow, revision| {
2700 let revision = match revision {
2701 Some(revision) => revision,
2702 None => loaded.record.child_revision(&workflow)?.to_owned(),
2703 };
2704 self.runtime.app.definition_revision(&workflow, &revision)?;
2705 Ok((workflow, revision))
2706 },
2707 )?;
2708 if let BspDecision::Descendants { inputs, .. } = &decision {
2709 let step = definition
2710 .node(&activation.node)
2711 .ok_or_else(|| {
2712 AppError::Integration("child declaration is unavailable".into())
2713 })?
2714 .step();
2715 for task in inputs {
2716 let child = self
2717 .runtime
2718 .app
2719 .definition_revision(&task.workflow, &task.revision)?;
2720 step.validate_child_binding(&definition, &child, &task.input)?;
2721 }
2722 }
2723 Ok(decision)
2724 })) {
2725 Ok(Ok(decision)) => decision,
2726 Ok(Err(error)) => BspDecision::Failed {
2727 category: FailureCategory::Evaluation,
2728 message: match error {
2729 AppError::InvalidInput(message) => message,
2730 _ => "workflow evaluation failed".into(),
2731 },
2732 observation: Some(loaded.record.checkpoint.proposed_position(
2733 &definition,
2734 &activation.node,
2735 &activation.lineage,
2736 &staged,
2737 )?),
2738 },
2739 Err(_) => BspDecision::Failed {
2740 category: FailureCategory::Execution,
2741 message: "workflow callback panicked".into(),
2742 observation: Some(loaded.record.checkpoint.proposed_position(
2743 &definition,
2744 &activation.node,
2745 &activation.lineage,
2746 &staged,
2747 )?),
2748 },
2749 };
2750 match decision {
2751 BspDecision::Stage(mut activations) => {
2752 staged.append(&mut activations);
2753 match remaining.next() {
2754 Some(activation) => {
2755 self.begin_activation(loaded, activation, remaining, staged, ordinary)
2756 }
2757 None => self.publish_decision(loaded, BspDecision::Stage(staged), ordinary),
2758 }
2759 }
2760 decision => self.publish(
2761 SessionPublication::Decision {
2762 loaded,
2763 decision,
2764 activation_deadline: Some(deadline_millis),
2765 context: Some(context),
2766 staged,
2767 remaining,
2768 },
2769 ordinary,
2770 ),
2771 }
2772 }
2773
2774 fn construct_source(
2775 &mut self,
2776 loaded: &LoadedRun,
2777 activation: FrontierRecord,
2778 context: Arc<ActionContext>,
2779 deadline: tokio::time::Instant,
2780 ) -> Result<(), AppError> {
2781 if tokio::time::Instant::now() >= deadline {
2782 self.runtime.retain_source_failure(
2783 &context,
2784 &activation.lineage,
2785 FailureCategory::Timeout,
2786 );
2787 self.sources.push(CallerSource {
2788 activation,
2789 context,
2790 deadline,
2791 state: CallerSourceState::Failed(FailureCategory::Timeout),
2792 });
2793 return Ok(());
2794 }
2795 let definition = self.runtime.app.definition_for(&loaded.record)?;
2796 let Some(Step::Action {
2797 action, revision, ..
2798 }) = definition.node(&activation.node).map(|node| node.step())
2799 else {
2800 return Err(AppError::Integration(
2801 "action decision has no exact declaration".into(),
2802 ));
2803 };
2804 let binding = self.runtime.app.action(action, revision)?;
2805 let state = WorkflowState {
2806 values: loaded.record.checkpoint.state.clone(),
2807 };
2808 let constructed = catch_unwind(AssertUnwindSafe(|| {
2809 binding.declaration().validate_reads(&state)?;
2810 let inputs = binding
2811 .declaration()
2812 .resolve_inputs(
2813 &self.runtime.cell.execution_options()?.inputs,
2814 self.runtime.cell.execution_mode()?,
2815 )?
2816 .into_values()?;
2817 Ok::<_, AppError>(binding.start(ActionRequest {
2818 state,
2819 inputs,
2820 context: context.as_ref().clone(),
2821 }))
2822 }));
2823 let state = match constructed {
2824 Ok(Ok(source)) => CallerSourceState::Polling(Box::pin(source)),
2825 Ok(Err(error)) => return Err(error),
2826 Err(_) => CallerSourceState::Failed(FailureCategory::Execution),
2827 };
2828 if let CallerSourceState::Failed(category) = &state {
2829 self.runtime
2830 .retain_source_failure(&context, &activation.lineage, *category);
2831 }
2832 self.sources.push(CallerSource {
2833 activation,
2834 context,
2835 deadline,
2836 state,
2837 });
2838 self.expire_sources(None);
2839 Ok(())
2840 }
2841
2842 fn halt_after(&mut self, loaded: &LoadedRun) -> Result<bool, AppError> {
2843 let outcomes = &loaded.record.checkpoint.outcomes;
2844 let Some(last) = outcomes.last() else {
2845 return Ok(false);
2846 };
2847 let options = self.runtime.cell.execution_options()?;
2848 let selected = outcomes
2849 .iter()
2850 .find(|outcome| options.halt_after.iter().any(|node| node == &outcome.node));
2851 let outcome = match selected {
2852 Some(outcome) => outcome,
2853 None if self.runtime.cell.halt_after(&last.node)? => last,
2854 None => return Ok(false),
2855 };
2856 self.publish_transition(
2857 LoadedRun {
2858 record: loaded.record.clone(),
2859 revision: loaded.revision,
2860 },
2861 ControlTransition::Halt {
2862 node: outcome.node.clone(),
2863 result: Some(outcome.result.clone()),
2864 },
2865 )?;
2866 Ok(true)
2867 }
2868
2869 fn publish_item(&mut self, index: usize, event: ActionEvent) -> Result<(), AppError> {
2870 let SessionState::Sources { loaded } = &self.state else {
2871 return Err(AppError::Integration(
2872 "stream item has no source owner".into(),
2873 ));
2874 };
2875 let source = &self.sources[index];
2876 let context = Arc::clone(&source.context);
2877 let deadline = source.deadline;
2878 let generation = self.generation;
2879 if let Some(generation) = generation {
2880 self.runtime.cell.begin_publication(generation)?;
2881 }
2882 let runtime = self.runtime.clone();
2883 let loaded = LoadedRun {
2884 record: loaded.record.clone(),
2885 revision: loaded.revision,
2886 };
2887 let activation = source.activation.clone();
2888 let (sender, receipt) = oneshot::channel();
2889 let kind = generation.map_or(RunTaskKind::Operation, |generation| {
2890 RunTaskKind::Foreground { generation }
2891 });
2892 if !self
2893 .runtime
2894 .app
2895 .spawn_run(self.runtime.key.clone(), kind, None, async move {
2896 let mut result = match runtime
2899 .observe_source_event(&context, event, deadline)
2900 .await
2901 {
2902 Ok(result) => Ok(result),
2903 Err(failure) => Err(runtime
2904 .fail_hook(&loaded, Some((&activation, &context)), failure, Vec::new())
2905 .await),
2906 };
2907 if let Some(generation) = generation
2908 && runtime.cell.end_publication(generation)
2909 && let Err(error) = runtime.finish_execution(generation, None).await
2910 && result.is_ok()
2911 {
2912 result = Err(error);
2913 }
2914 let _ = sender.send(result);
2915 })
2916 {
2917 if let Some(generation) = generation {
2918 self.runtime.cell.end_publication(generation);
2919 }
2920 return Err(AppError::Closed(
2921 "stream observation admission closed".into(),
2922 ));
2923 }
2924 let SessionState::Sources { loaded } =
2925 std::mem::replace(&mut self.state, SessionState::Finished)
2926 else {
2927 unreachable!()
2928 };
2929 self.state = SessionState::ObservingItem {
2930 loaded,
2931 index,
2932 receipt,
2933 };
2934 Ok(())
2935 }
2936
2937 fn publish_failure(
2938 &mut self,
2939 loaded: LoadedRun,
2940 activation: &FrontierRecord,
2941 context: &ActionContext,
2942 category: FailureCategory,
2943 ) -> Result<(), AppError> {
2944 let definition = self.runtime.app.definition_for(&loaded.record)?;
2945 let position = ActionPosition {
2946 node: context.node.clone(),
2947 ordinal: context.action_ordinal,
2948 };
2949 let (position, category, message) = match loaded.record.checkpoint.preceding_failure(
2950 &definition,
2951 &position,
2952 &activation.lineage,
2953 )? {
2954 Some((position, category, message)) => (position, category, message.to_owned()),
2955 None => (position, category, "caller action execution failed".into()),
2956 };
2957 self.publish_decision(
2958 loaded,
2959 BspDecision::Failed {
2960 category,
2961 message,
2962 observation: Some(position),
2963 },
2964 false,
2965 )
2966 }
2967
2968 fn publish_decision(
2969 &mut self,
2970 loaded: LoadedRun,
2971 decision: BspDecision,
2972 ordinary: bool,
2973 ) -> Result<(), AppError> {
2974 self.publish(
2975 SessionPublication::Decision {
2976 loaded,
2977 decision,
2978 activation_deadline: None,
2979 context: None,
2980 staged: Vec::new(),
2981 remaining: Vec::new().into_iter(),
2982 },
2983 ordinary,
2984 )
2985 }
2986
2987 fn publish_transition(
2988 &mut self,
2989 loaded: LoadedRun,
2990 transition: ControlTransition,
2991 ) -> Result<(), AppError> {
2992 self.publish(SessionPublication::Transition { loaded, transition }, false)
2993 }
2994
2995 fn publish(
2996 &mut self,
2997 mut publication: SessionPublication,
2998 ordinary: bool,
2999 ) -> Result<(), AppError> {
3000 let mut staged_prefix = Vec::new();
3001 let mut admitted_inputs = Vec::new();
3002 if let SessionPublication::Decision {
3003 loaded,
3004 decision: BspDecision::AdmitChildren { sequence, children },
3005 ..
3006 } = &publication
3007 && let Some(group) = self
3008 .children
3009 .iter()
3010 .find(|group| group.sequence == *sequence)
3011 {
3012 let original = loaded
3013 .record
3014 .checkpoint
3015 .control(*sequence)
3016 .and_then(|control| match &control.state {
3017 ControlRecordState::Child { admitted, .. } => Some(admitted.len()),
3018 _ => None,
3019 })
3020 .ok_or_else(|| {
3021 AppError::Integration("child admission evidence is unavailable".into())
3022 })?;
3023 if !group.inputs.is_empty() {
3024 for index in original..children.len() {
3025 let values = group.inputs.get(index).ok_or_else(|| {
3026 AppError::Integration("child runtime input membership changed".into())
3027 })?;
3028 admitted_inputs.push(values.clone());
3031 }
3032 }
3033 }
3034 let purpose = match &mut publication {
3035 SessionPublication::Decision {
3036 decision:
3037 BspDecision::Descendants {
3038 lineage,
3039 runtime_inputs,
3040 ..
3041 },
3042 context,
3043 remaining,
3044 ..
3045 } => PublicationPurpose::Descendants {
3046 context: Arc::clone(context.as_ref().ok_or_else(|| {
3047 AppError::Integration(
3048 "child runtime inputs have no original action context".into(),
3049 )
3050 })?),
3051 lineage: lineage.clone(),
3052 inputs: std::mem::take(runtime_inputs),
3053 remaining: std::mem::take(remaining),
3054 ordinary,
3055 },
3056 SessionPublication::Load { .. } => PublicationPurpose::Load,
3057 SessionPublication::Reload { .. } => PublicationPurpose::Reload,
3058 SessionPublication::ObserveChildren { .. } => PublicationPurpose::ObserveChildren,
3059 SessionPublication::VerifyChildren { .. } => PublicationPurpose::VerifyChildren,
3060 SessionPublication::FinishChild { .. } | SessionPublication::SettleStopping { .. } => {
3061 PublicationPurpose::Load
3062 }
3063 SessionPublication::Decision {
3064 decision: BspDecision::AdmitChildren { .. },
3065 ..
3066 } => PublicationPurpose::AdmitChildren,
3067 SessionPublication::BeginActivation {
3068 activation,
3069 context,
3070 remaining,
3071 staged,
3072 deadline,
3073 deadline_millis,
3074 ..
3075 } => {
3076 staged_prefix = std::mem::take(staged);
3079 PublicationPurpose::Start {
3080 activation: activation.clone(),
3081 context: Arc::clone(context),
3082 remaining: std::mem::take(remaining),
3083 deadline: *deadline,
3084 deadline_millis: *deadline_millis,
3085 }
3086 }
3087 SessionPublication::Decision { remaining, .. } if !remaining.as_slice().is_empty() => {
3088 PublicationPurpose::Continue {
3089 remaining: std::mem::take(remaining),
3090 ordinary,
3091 }
3092 }
3093 SessionPublication::Decision {
3094 decision: BspDecision::Barrier { .. },
3095 ..
3096 } => PublicationPurpose::Commit { ordinary },
3097 SessionPublication::EndSource { .. } => PublicationPurpose::EndSource,
3098 _ => PublicationPurpose::Advance,
3099 };
3100 if let SessionPublication::Decision {
3101 decision: BspDecision::Agent(request),
3102 ..
3103 } = &mut publication
3104 && request.is_streaming()
3105 {
3106 let (sender, _) = self.native_partials.get_or_insert_with(|| mpsc::channel(1));
3107 request.observe_partials(sender.clone());
3108 }
3109 let generation = self.generation;
3110 if let Some(generation) = generation {
3111 self.runtime.cell.begin_publication(generation)?;
3112 if matches!(&publication, SessionPublication::Release) {
3113 self.runtime.cell.revoke_execution(generation);
3114 }
3115 }
3116 let child_observation = match &publication {
3117 SessionPublication::Reload { receipt, .. } => receipt
3118 .record()
3119 .checkpoint
3120 .controls
3121 .values()
3122 .filter(|control| {
3123 matches!(control.state, ControlRecordState::Child { .. })
3124 && !self.children.iter().any(|group| {
3125 group.sequence == control.sequence
3126 && matches!(
3127 group.state,
3128 ChildExecutionState::Worker(_)
3129 | ChildExecutionState::Receipt(_)
3130 )
3131 })
3132 })
3133 .min_by_key(|control| control.deadline_millis)
3134 .map(|control| control.sequence),
3135 _ => None,
3136 };
3137 let options = self.runtime.cell.execution_options()?;
3138 let parent_deadline = self.parent_deadline;
3139 let reconcile_read = generation.is_none()
3140 && self.runtime.cell.child_parent().is_some()
3141 && matches!(
3142 &publication,
3143 SessionPublication::Load { .. }
3144 | SessionPublication::Reload { .. }
3145 | SessionPublication::ObserveChildren { .. }
3146 | SessionPublication::VerifyChildren { .. }
3147 );
3148 let execution_read = matches!(
3149 &publication,
3150 SessionPublication::Load { .. }
3151 | SessionPublication::Reload { .. }
3152 | SessionPublication::ObserveChildren { .. }
3153 | SessionPublication::FinishChild { .. }
3154 | SessionPublication::BeginActivation { .. }
3155 );
3156 let runtime = self.runtime.clone();
3157 let (sender, receipt) = oneshot::channel();
3158 let kind = generation.map_or(RunTaskKind::Operation, |generation| {
3159 RunTaskKind::Foreground { generation }
3160 });
3161 if !self
3162 .runtime
3163 .app
3164 .spawn_run(self.runtime.key.clone(), kind, self.admission.take(), async move {
3165 let inputs = &options.inputs;
3166 let observed = AssertUnwindSafe(async {
3167 match publication {
3168 SessionPublication::ObserveChildren { loaded, sequence } => async {
3169 let (loaded, children) = runtime.observe_children(loaded, sequence).await?;
3170 Ok(SessionReceipt::Children { loaded, sequence, children, verified: false })
3171 }.await,
3172 SessionPublication::VerifyChildren { parent, sequence, completions } => {
3173 match runtime.verify_child_completions(&parent, sequence, completions).await {
3174 Ok((loaded, children)) => {
3175 let control = parent.checkpoint.control(sequence).ok_or_else(|| {
3176 AppError::Conflict("verified child control is unavailable".into())
3177 })?;
3178 let complete = matches!(&control.state,
3179 ControlRecordState::Child { inputs, .. } if children.len() == inputs.len());
3180 if loaded.record.checkpoint.is_stopping() || loaded.record.checkpoint.is_terminal() || !complete {
3181 runtime.schedule_child_retention(control);
3182 Ok(SessionReceipt::Children { loaded, sequence, children, verified: true })
3183 } else {
3184 let transition = runtime.plan_child_completion(&loaded, control, children)?;
3188 runtime.publish_child_outcome(control, transition).await
3189 }
3190 },
3191 Err(RuntimeOperationFailure::Application(error)) => runtime
3192 .finish_child_admission(sequence, ChildAdmissionReceipt::Rejected(error))
3193 .await
3194 .map(|(loaded, children)| SessionReceipt::Children {
3195 loaded, sequence, children, verified: false,
3196 }),
3197 Err(failure) => Err(failure),
3198 }
3199 },
3200 SessionPublication::Decision { loaded, decision: BspDecision::AdmitChildren { sequence, children }, .. } => {
3201 let receipt = runtime.register_children(loaded, sequence, children, admitted_inputs).await;
3202 runtime.finish_child_admission(sequence, receipt).await.map(|(loaded, children)| {
3203 SessionReceipt::Children { loaded, sequence, children, verified: false }
3204 })
3205 },
3206 SessionPublication::ChildOutcome { control, transition } => {
3207 runtime.publish_child_outcome(&control, transition).await
3208 },
3209 SessionPublication::FinishChild { parent, sequence } => async {
3210 let loaded = runtime.load_execution_observation(None, parent_deadline).await?
3211 .ok_or_else(|| AppError::NotFound("admitted child is unavailable".into()))?;
3212 let definition = runtime.app.definition_for(&parent)?;
3213 loaded.record.validate_child(&parent, &definition)?;
3214 if loaded.record.parent.as_ref().is_none_or(|relation| relation.control_sequence != sequence) {
3215 return Err(AppError::Conflict("child completion group changed".into()).into());
3216 }
3217 if loaded.record.checkpoint.is_stopping()
3218 || loaded.record.checkpoint.is_terminal()
3219 || !loaded.record.checkpoint.controls.values().any(|control| {
3220 matches!(control.state, ControlRecordState::Inputs { .. })
3221 })
3222 {
3223 runtime.finish_execution_observation(&loaded.record, None, None, false).await?;
3224 runtime.cell.lifecycle_observation(loaded.record.checkpoint.sequence, false)?;
3225 }
3226 Ok(SessionReceipt::Checkpoint { loaded, staged: Vec::new() })
3227 }.await,
3228 SessionPublication::SettleStopping { loaded } => async {
3229 if let Some(loaded) = Box::pin(runtime.settle_stopping(loaded)).await {
3230 let observed = runtime.finish_execution_observation(&loaded.record, None, None, false).await;
3231 if runtime.cell.child_parent().is_none() {
3232 runtime.maintain_retention(None).await;
3233 }
3234 observed?;
3235 }
3236 let loaded = runtime.load_execution_observation(None, parent_deadline).await
3240 .inspect_err(|failure| runtime.retain_execution_read_failure(failure))?
3241 .ok_or_else(|| AppError::NotFound("workflow run does not exist".into()))?;
3242 Ok(SessionReceipt::Checkpoint { loaded, staged: Vec::new() })
3243 }.await,
3244 SessionPublication::Load { terminal } => async {
3245 let loaded = runtime.load_execution_observation(terminal, parent_deadline).await?
3246 .ok_or_else(|| AppError::NotFound("workflow run does not exist".into()))?;
3247 if !loaded.record.checkpoint.is_stopping()
3248 && !loaded.record.checkpoint.is_terminal()
3249 && loaded.record.checkpoint.controls.values().any(|control| {
3250 matches!(control.state, ControlRecordState::Inputs { .. })
3251 })
3252 && let Err(failure) = runtime.begin_execution_observation().await
3253 {
3254 return Err(runtime.fail_hook(&loaded, None, failure, Vec::new()).await.into());
3255 }
3256 Ok(SessionReceipt::Checkpoint { loaded, staged: Vec::new() })
3257 }.await,
3258 SessionPublication::Reload { receipt, deadline } => {
3259 let observed = tokio::select! {
3260 biased;
3261 _ = async { match deadline {
3262 Some(deadline) => tokio::time::sleep_until(deadline).await,
3263 None => std::future::pending().await,
3264 }}, if child_observation.is_none() && parent_deadline.is_none() => Ok(None),
3265 observed = async {
3266 let observed = match child_observation {
3271 Some(sequence) => runtime.observe_child_parent(receipt.record(), sequence, false, None).await?,
3272 None => runtime.load_execution_observation(None, deadline.into_iter().chain(parent_deadline).min()).await?
3273 .ok_or_else(|| AppError::NotFound("workflow run does not exist".into()))?,
3274 };
3275 Ok::<_, RuntimeOperationFailure>(Some(observed))
3276 } => observed,
3277 };
3278 observed.map(|observed| match observed {
3279 Some(loaded) => receipt.with_loaded(loaded),
3280 None => receipt,
3281 })
3282 },
3283 SessionPublication::BeginActivation { loaded, activation, context, deadline, .. } => async {
3284 if let Err(failure) = runtime.observe_activation_start(&loaded, &activation, &context, inputs, deadline).await {
3285 return Err(runtime.fail_hook(&loaded, Some((&activation, &context)), failure, std::mem::take(&mut staged_prefix)).await.into());
3286 }
3287 let current = runtime.load_execution_observation(
3288 None, parent_deadline.into_iter().chain(Some(deadline)).min(),
3289 ).await?.ok_or_else(|| AppError::NotFound("workflow run does not exist".into()))?;
3290 if !(current.record.checkpoint.is_terminal()
3291 || current.record.checkpoint.is_stopping()
3292 || current.record.same_run(&loaded.record)
3293 && current.record.checkpoint.base_sequence == loaded.record.checkpoint.base_sequence
3294 && current.record.checkpoint.frontier.iter().any(|candidate| candidate == &activation)
3295 && current.record.checkpoint.control_for(&activation.node, &activation.lineage).is_none()
3296 && !current.record.checkpoint.staged.iter().any(|staged| staged.node == activation.node && staged.lineage == activation.lineage))
3297 {
3298 return Err(AppError::Conflict("action pre-hook authority changed".into()).into());
3299 }
3300 Ok(SessionReceipt::Checkpoint { loaded: current, staged: std::mem::take(&mut staged_prefix) })
3301 }.await,
3302 publication => async {
3303 match publication {
3304 SessionPublication::Load { .. } | SessionPublication::Reload { .. }
3305 | SessionPublication::ObserveChildren { .. } | SessionPublication::VerifyChildren { .. }
3306 | SessionPublication::ChildOutcome { .. } | SessionPublication::FinishChild { .. } | SessionPublication::SettleStopping { .. }
3307 | SessionPublication::BeginActivation { .. } => unreachable!(),
3308 SessionPublication::Decision { loaded, decision, activation_deadline, context, staged, .. } => {
3309 if matches!(decision, BspDecision::Inputs { .. })
3310 && let Err(failure) = runtime.begin_execution_observation().await
3311 {
3312 return Err(runtime.fail_hook(&loaded, None, failure, staged).await);
3313 }
3314 runtime.finish_staged_spans(&loaded, &decision, context.as_deref()).await?;
3315 runtime.apply_decision(loaded, decision, activation_deadline, staged).await
3316 }
3317 SessionPublication::Transition { loaded, transition } => {
3318 let loaded = runtime.commit_for_recovery(loaded, transition).await
3319 .map_err(RuntimeOperationFailure::into_error)?;
3320 runtime.observe_generation(LifecycleEvent::deadline(parent_deadline)).await
3324 .map_err(|failure| {
3325 runtime.retain_execution_read_failure(&failure);
3326 failure.into_error()
3327 })?;
3328 Ok(loaded)
3329 }
3330 SessionPublication::Retry { loaded } => {
3331 let loaded = runtime.readmit_failed(loaded).await?;
3332 runtime.observe_generation(LifecycleEvent::deadline(parent_deadline)).await
3333 .map_err(|failure| {
3334 runtime.retain_execution_read_failure(&failure);
3335 failure.into_error()
3336 })?;
3337 Ok(loaded)
3338 }
3339 SessionPublication::EndSource { loaded } => {
3340 let loaded = runtime.end_source(loaded, generation).await?;
3341 runtime.observe_generation(LifecycleEvent::deadline(parent_deadline)).await
3342 .map_err(|failure| {
3343 runtime.retain_execution_read_failure(&failure);
3344 failure.into_error()
3345 })?;
3346 Ok(loaded)
3347 }
3348 SessionPublication::Release => {
3349 if let Some(generation) = generation { runtime.finish_execution(generation, None).await?; }
3350 runtime.load_execution_observation(None, parent_deadline).await
3351 .map_err(|failure| {
3352 runtime.retain_execution_read_failure(&failure);
3353 failure.into_error()
3354 })?
3355 .ok_or_else(|| AppError::NotFound("workflow run does not exist".into()))
3356 }
3357 }
3358 }.await.map(|loaded| SessionReceipt::Checkpoint { loaded, staged: std::mem::take(&mut staged_prefix) })
3359 .map_err(RuntimeOperationFailure::from),
3360 }
3361 }).catch_unwind().await;
3362 let mut result: Result<SessionReceipt, RuntimeOperationFailure> = match observed {
3363 Ok(result) => result,
3364 Err(payload) if reconcile_read => {
3365 drop(payload);
3366 match runtime.reconcile_child_panic(parent_deadline).await {
3367 Ok(result) => result.map(|loaded| SessionReceipt::Checkpoint {
3368 loaded, staged: Vec::new(),
3369 }).map_err(RuntimeOperationFailure::from),
3370 Err(_) => {
3371 runtime.app.tasks_failed(FinalResult::Integration(
3372 "child read panic reconciliation receipt closed".into(),
3373 ));
3374 Err(AppError::Integration("child read panic reconciliation receipt closed".into()).into())
3375 }
3376 }
3377 }
3378 Err(payload) => std::panic::resume_unwind(payload),
3379 };
3380 if execution_read
3381 && let Err(RuntimeOperationFailure::Storage(failure)) = &result
3382 {
3383 runtime.retain_execution_read_failure(failure);
3384 }
3385 if let Some(generation) = generation
3386 && runtime.cell.end_publication(generation)
3387 {
3388 let loaded = match &result {
3389 Ok(SessionReceipt::Checkpoint { loaded, .. })
3390 if (loaded.record.checkpoint.is_paused() || loaded.record.checkpoint.is_stopping())
3391 && loaded.record.checkpoint.controls.is_empty()
3392 && runtime.cell.lifecycle_generation().is_none()
3393 && !runtime.app.has_child_work(&runtime.key) =>
3394 {
3395 Some(LoadedRun { record: loaded.record.clone(), revision: loaded.revision })
3399 }
3400 _ => None,
3401 };
3402 if let Err(error) = runtime.finish_execution(generation, loaded).await
3403 && result.is_ok()
3404 {
3405 result = Err(error.into());
3406 }
3407 }
3408 let _ = sender.send(result);
3409 })
3410 {
3411 if let Some(generation) = generation {
3412 self.runtime.cell.end_publication(generation);
3413 }
3414 return Err(AppError::Closed(
3415 "run publication admission is closed".into(),
3416 ));
3417 }
3418 self.state = SessionState::Publishing { receipt, purpose };
3419 Ok(())
3420 }
3421
3422 fn poll_publication(
3423 &mut self,
3424 cx: &mut Context<'_>,
3425 closing: bool,
3426 ) -> Poll<Result<Option<ExecutionEvent>, AppError>> {
3427 if !matches!(&self.state, SessionState::Publishing { .. }) {
3428 return Poll::Ready(Ok(None));
3429 }
3430 self.expire_sources(None);
3431 let deadline = self.source_deadline(None);
3432 if self.poll_timer(cx, deadline) {
3433 self.expire_sources(None);
3434 }
3435 let commit = matches!(
3436 &self.state,
3437 SessionState::Publishing {
3438 purpose: PublicationPurpose::Commit { .. },
3439 ..
3440 }
3441 );
3442 if commit && let Some(event) = self.poll_native_partial(cx) {
3443 return Poll::Ready(Ok(Some(event)));
3444 }
3445 let SessionState::Publishing { receipt, .. } = &mut self.state else {
3446 unreachable!()
3447 };
3448 let observed = match Pin::new(receipt).poll(cx) {
3449 Poll::Ready(observed) => observed
3450 .map_err(|_| {
3451 RuntimeOperationFailure::from(AppError::Closed(
3452 "run publication observation closed".into(),
3453 ))
3454 })
3455 .and_then(|observed| observed),
3456 Poll::Pending => {
3457 if let Some(event) = self.poll_native_partial(cx) {
3458 return Poll::Ready(Ok(Some(event)));
3459 }
3460 return Poll::Pending;
3461 }
3462 };
3463 let SessionState::Publishing { purpose, .. } =
3464 std::mem::replace(&mut self.state, SessionState::Finished)
3465 else {
3466 unreachable!()
3467 };
3468 let observed = match observed {
3469 Ok(observed) => observed,
3470 Err(failure) => {
3471 if self.generation.is_none() {
3472 if matches!(
3473 purpose,
3474 PublicationPurpose::Load
3475 | PublicationPurpose::Reload
3476 | PublicationPurpose::ObserveChildren
3477 | PublicationPurpose::VerifyChildren
3478 ) {
3479 if self.runtime.schedule_operation_failure(&failure) {
3480 self.observed = None;
3481 }
3482 } else if matches!(purpose, PublicationPurpose::AdmitChildren) {
3483 self.observed = None;
3486 }
3487 }
3488 return Poll::Ready(Err(failure.into_error()));
3489 }
3490 };
3491 let (loaded, staged) = match observed {
3492 SessionReceipt::Checkpoint { loaded, staged } => (loaded, staged),
3493 SessionReceipt::Children {
3494 loaded,
3495 sequence,
3496 children,
3497 verified,
3498 } => {
3499 if closing {
3500 self.state = SessionState::Loaded(loaded);
3501 } else if matches!(purpose, PublicationPurpose::AdmitChildren)
3502 && self.poll_notification(cx)?
3503 {
3504 let deadline = loaded
3505 .record
3506 .checkpoint
3507 .control(sequence)
3508 .map(|control| {
3509 WorkflowRunRuntime::<CS, A, X, C, T, H, E>::remaining(
3510 control.deadline_millis,
3511 )
3512 })
3513 .transpose()?
3514 .and_then(|remaining| tokio::time::Instant::now().checked_add(remaining));
3515 self.publish(
3516 SessionPublication::Reload {
3517 receipt: SessionReceipt::Children {
3518 loaded,
3519 sequence,
3520 children,
3521 verified,
3522 },
3523 deadline,
3524 },
3525 false,
3526 )?;
3527 } else {
3528 self.plan_child_group(loaded, sequence, children, verified)?;
3529 }
3530 return Poll::Ready(Ok(None));
3531 }
3532 };
3533 if matches!(purpose, PublicationPurpose::Commit { .. })
3534 && let Err(error) = self
3535 .runtime
3536 .cell
3537 .lifecycle_observation(loaded.record.checkpoint.sequence, false)
3538 {
3539 self.state = SessionState::Loaded(loaded);
3543 return Poll::Ready(Err(error));
3544 }
3545 if closing
3546 || matches!(
3547 purpose,
3548 PublicationPurpose::Load | PublicationPurpose::Reload
3549 )
3550 {
3551 self.state = SessionState::Loaded(loaded);
3552 return Poll::Ready(Ok(None));
3553 }
3554 self.state = SessionState::Ready;
3555 if let PublicationPurpose::Start {
3556 activation,
3557 context,
3558 remaining,
3559 deadline,
3560 deadline_millis,
3561 } = purpose
3562 {
3563 self.state = if loaded.record.checkpoint.is_terminal()
3564 || loaded.record.checkpoint.is_stopping()
3565 {
3566 self.observed = Some(loaded.record);
3567 SessionState::Finished
3568 } else {
3569 SessionState::ActivationReady {
3570 loaded,
3571 activation,
3572 context,
3573 remaining,
3574 staged,
3575 deadline,
3576 deadline_millis,
3577 }
3578 };
3579 return Poll::Ready(Ok(None));
3580 }
3581 let purpose = match purpose {
3582 PublicationPurpose::Descendants {
3583 context,
3584 lineage,
3585 inputs,
3586 remaining,
3587 ordinary,
3588 } => {
3589 if !loaded.record.checkpoint.is_terminal()
3590 && !loaded.record.checkpoint.is_stopping()
3591 {
3592 let control = loaded
3593 .record
3594 .checkpoint
3595 .control_for(&context.node, &lineage)
3596 .ok_or_else(|| {
3597 AppError::Integration(
3598 "original child input control is unavailable".into(),
3599 )
3600 })?;
3601 let deadline = tokio::time::Instant::now()
3602 .checked_add(WorkflowRunRuntime::<CS, A, X, C, T, H, E>::remaining(
3603 control.deadline_millis,
3604 )?)
3605 .ok_or_else(|| {
3606 AppError::Timeout("child deadline is outside the clock range".into())
3607 })?;
3608 self.children.push(ChildExecution {
3609 sequence: control.sequence,
3610 parent: None,
3611 context,
3612 lineage,
3613 deadline,
3614 state: ChildExecutionState::Awaiting,
3615 inputs,
3616 });
3617 }
3618 if remaining.as_slice().is_empty() {
3619 PublicationPurpose::Advance
3620 } else {
3621 PublicationPurpose::Continue {
3622 remaining,
3623 ordinary,
3624 }
3625 }
3626 }
3627 purpose => purpose,
3628 };
3629 if let PublicationPurpose::Continue {
3630 mut remaining,
3631 ordinary,
3632 } = purpose
3633 {
3634 if loaded.record.checkpoint.is_terminal() || loaded.record.checkpoint.is_stopping() {
3635 self.observed = Some(loaded.record);
3636 self.state = SessionState::Finished;
3637 } else {
3638 let activation = remaining.next().ok_or_else(|| {
3639 AppError::Integration("native activation continuation is empty".into())
3640 })?;
3641 self.begin_activation(loaded, activation, remaining, staged, ordinary)?;
3642 }
3643 return Poll::Ready(Ok(None));
3644 }
3645 if let RunAdmission::Halted { node, result } = &loaded.record.checkpoint.admission {
3646 let event = ExecutionEvent::Halted {
3647 node: node.clone(),
3648 result: result.clone(),
3649 };
3650 self.observed = Some(loaded.record);
3651 self.state = SessionState::Finished;
3652 return Poll::Ready(Ok(Some(event)));
3653 }
3654 if let PublicationPurpose::Commit { ordinary } = purpose {
3655 self.observed = Some(loaded.record.clone());
3656 self.runtime
3657 .cell
3658 .record_execution_boundary(loaded.record.checkpoint.outcomes.len(), ordinary)?;
3659 return Poll::Ready(Ok(Some(ExecutionEvent::Committed {
3660 checkpoint: loaded.record.checkpoint.sequence,
3661 outcomes: loaded.record.checkpoint.outcomes,
3662 })));
3663 }
3664 if matches!(purpose, PublicationPurpose::EndSource) {
3665 self.observed = Some(loaded.record);
3666 self.state = SessionState::Eof;
3667 } else {
3668 self.observed = None;
3669 self.state = SessionState::Loaded(loaded);
3670 }
3671 Poll::Ready(Ok(None))
3672 }
3673}
3674
3675impl<
3676 CS: StateStoreBackend,
3677 A: AgentBackend,
3678 X: ActionBackend,
3679 C: ConditionBackend,
3680 T: StateTyping,
3681 H: LifecycleHooks,
3682 E: ChildExecutor,
3683> Drop for Execution<'_, CS, A, X, C, T, H, E>
3684{
3685 fn drop(&mut self) {
3686 if let Some(generation) = self.generation {
3687 self.runtime.cell.clear_execution_options(Some(generation));
3688 }
3689 let pending_observation = matches!(
3690 &self.state,
3691 SessionState::Publishing { .. } | SessionState::ObservingItem { .. }
3692 );
3693 self.stop_sources();
3694 self.stop_children();
3695 if self.generation.is_none()
3696 && !pending_observation
3697 && matches!(&self.state, SessionState::Sources { .. })
3698 {
3699 let failed = self
3700 .sources
3701 .iter()
3702 .enumerate()
3703 .filter_map(|(index, source)| {
3704 let category = self
3705 .prefetched
3706 .as_ref()
3707 .filter(|(buffered, _)| *buffered == index)
3708 .and_then(|(_, event)| event.as_ref().err().copied())
3709 .or(match source.state {
3710 CallerSourceState::Failed(category) => Some(category),
3711 _ => None,
3712 });
3713 category.map(|category| (index, category))
3714 })
3715 .min_by(|left, right| {
3716 let left = &self.sources[left.0].activation;
3717 let right = &self.sources[right.0].activation;
3718 (&left.node, &left.lineage).cmp(&(&right.node, &right.lineage))
3719 });
3720 if let Some((index, category)) = failed {
3721 let _ = self.stage_source_failure(index, category, false);
3722 }
3723 }
3724 let source_failed = self
3725 .runtime
3726 .cell
3727 .lifecycle_generation()
3728 .is_some_and(|generation| self.runtime.cell.source_failure(generation).is_some());
3729 let record = match &self.state {
3730 SessionState::Loaded(loaded)
3731 | SessionState::Sources { loaded }
3732 | SessionState::ObservingItem { loaded, .. }
3733 | SessionState::ActivationReady { loaded, .. } => Some(&loaded.record),
3734 _ => self.observed.as_ref(),
3735 };
3736 let child_pending = self.runtime.app.has_child_work(&self.runtime.key);
3737 let local_only = !child_pending
3738 && !pending_observation
3739 && !matches!(&self.state, SessionState::Publishing { .. })
3740 && !source_failed
3741 && !self.sources.is_empty()
3742 && record.is_some_and(|record| {
3743 record.checkpoint.staged.is_empty() && record.checkpoint.controls.is_empty()
3744 });
3745 if !child_pending
3746 && !pending_observation
3747 && !matches!(&self.state, SessionState::Publishing { .. })
3748 && !source_failed
3749 && let Some(record) = record
3750 && !self.retains_execution_lifecycle(record)
3751 && (self.generation.is_some()
3752 || record.checkpoint.controls.is_empty()
3753 || record.checkpoint.is_public_outcome())
3754 && !record
3755 .checkpoint
3756 .staged
3757 .iter()
3758 .any(|activation| matches!(activation.outcome, StagedOutcomeRecord::Failed { .. }))
3759 {
3760 let _ = self.runtime.finish_execution_observation_sync(record);
3761 }
3762 if let Some(generation) = self.generation.take()
3763 && self.runtime.cell.drop_execution(generation)
3764 {
3765 if local_only {
3766 if self.runtime.cell.release_execution(generation) {
3767 self.runtime.schedule(ScheduleCause::Reconcile);
3768 }
3769 return;
3770 }
3771 let loaded = if !child_pending
3772 && !pending_observation
3773 && !source_failed
3774 && self.runtime.cell.lifecycle_generation().is_none()
3775 && matches!(&self.state, SessionState::Loaded(loaded)
3776 if (loaded.record.checkpoint.is_paused() || loaded.record.checkpoint.is_stopping())
3777 && loaded.record.checkpoint.controls.is_empty())
3778 {
3779 let SessionState::Loaded(loaded) =
3780 std::mem::replace(&mut self.state, SessionState::Finished)
3781 else {
3782 unreachable!()
3783 };
3784 Some(loaded)
3785 } else {
3786 None
3787 };
3788 let runtime = self.runtime.clone();
3789 self.runtime.app.spawn_run(
3790 self.runtime.key.clone(),
3791 RunTaskKind::Foreground { generation },
3792 None,
3793 async move {
3794 let _ = runtime.finish_execution(generation, loaded).await;
3795 },
3796 );
3797 }
3798 }
3799}
3800
3801impl<
3802 CS: StateStoreBackend,
3803 A: AgentBackend,
3804 X: ActionBackend,
3805 C: ConditionBackend,
3806 T: StateTyping,
3807 H: LifecycleHooks,
3808 E: ChildExecutor,
3809> WorkflowRunRuntime<CS, A, X, C, T, H, E>
3810{
3811 pub(crate) async fn execute_child_group(
3812 &self,
3813 sequence: u64,
3814 generation: Option<u64>,
3815 parent: Arc<RunRecord>,
3816 children: Vec<RunRecord>,
3817 ) -> Result<SessionReceipt, RuntimeOperationFailure> {
3818 if generation.is_some_and(|generation| !self.cell.execution_is_live(generation)) {
3819 return Err(
3820 AppError::Closed("child executor admission generation changed".into()).into(),
3821 );
3822 }
3823 let mut outcome_published = false;
3824 let result = async {
3825 let control = parent.checkpoint.control(sequence).ok_or_else(|| AppError::Conflict("child control is unavailable".into()))?;
3826 let remaining = Self::remaining(control.deadline_millis)?;
3827 if remaining.is_zero() {
3828 outcome_published = true;
3832 let loaded = self.commit_scheduled_transition(control, ControlTransition::Failed {
3833 category: FailureCategory::Timeout, message: "child workflow deadline expired".into(), observation: None,
3834 }).await.ok_or_else(|| AppError::Closed(
3835 "child timeout publication requires reconciliation".into(),
3836 ))?;
3837 return Ok(SessionReceipt::Checkpoint { loaded, staged: Vec::new() });
3838 }
3839 let deadline = tokio::time::Instant::now().checked_add(remaining)
3840 .ok_or_else(|| AppError::Timeout("child deadline is outside the clock range".into()))?;
3841 let current = self.load_execution_observation(None, Some(deadline)).await?
3844 .ok_or_else(|| AppError::NotFound("child parent is unavailable".into()))?;
3845 if current.record.checkpoint.is_stopping() || current.record.checkpoint.is_terminal() {
3846 return Ok(SessionReceipt::Checkpoint { loaded: current, staged: Vec::new() });
3847 }
3848 if !current.record.owns_child_activation(&parent, sequence) {
3849 return Err(AppError::Conflict("child parent reservation changed".into()).into());
3850 }
3851 drop(current);
3852 let context = self.child_context(&parent, sequence)?;
3853 let mut group = ChildExecution {
3854 sequence, parent: Some(Arc::clone(&parent)), context,
3855 lineage: control.lineage.clone(), deadline,
3856 state: ChildExecutionState::Prepared(Vec::new()), inputs: Vec::new(),
3857 };
3858 group.start_local(self, &self.app, ExecutionLane::Worker, children)?;
3859 let context = Arc::clone(&group.context);
3860 let stopped = {
3861 let mut executor = Box::pin(poll_fn(|cx| group.poll_ready(cx, self)));
3862 let mut stopping = Box::pin(self.observe_child_parent(&parent, sequence, true, None));
3863 tokio::select! {
3864 biased;
3865 result = &mut stopping => {
3866 let loaded = result?;
3867 let loaded = if !loaded.record.checkpoint.is_stopping() {
3868 self.retain_source_failure(&context, &control.lineage, FailureCategory::Timeout);
3869 outcome_published = true;
3870 let mut failure = Box::pin(self.commit_scheduled_transition(control, ControlTransition::Failed {
3871 category: FailureCategory::Timeout, message: "child workflow deadline expired".into(), observation: None,
3872 }));
3873 let committed = tokio::select! {
3874 _ = &mut executor => failure.await,
3875 result = &mut failure => result,
3876 };
3877 committed.ok_or_else(|| AppError::Closed(
3878 "child timeout publication requires reconciliation".into(),
3879 ))?
3880 } else { loaded };
3881 Some(loaded)
3882 }
3883 _ = &mut executor => None,
3884 }
3885 };
3886 if let Some(loaded) = stopped {
3887 group.stop(self);
3891 return Ok(SessionReceipt::Checkpoint { loaded, staged: Vec::new() });
3892 }
3893 let ChildExecutionState::Complete(completed) = group.state else {
3894 return Err(AppError::Integration("child worker completed without its local executor".into()).into());
3895 };
3896 let completions = match completed {
3897 Ok(completions) => completions,
3898 Err(category) => {
3899 outcome_published = true;
3900 let loaded = self.commit_scheduled_transition(control, ControlTransition::Failed {
3901 category, message: "child execution failed".into(), observation: None,
3902 }).await.ok_or_else(|| AppError::Closed(
3903 "child failure publication requires reconciliation".into(),
3904 ))?;
3905 return Ok(SessionReceipt::Checkpoint { loaded, staged: Vec::new() });
3906 }
3907 };
3908 let (loaded, children) = self.verify_child_completions(&parent, sequence, completions).await?;
3909 if loaded.record.checkpoint.is_stopping() || loaded.record.checkpoint.is_terminal() {
3910 self.schedule_child_retention(control);
3911 return Ok(SessionReceipt::Checkpoint { loaded, staged: Vec::new() });
3912 }
3913 let transition = self.plan_child_completion(&loaded, control, children)?;
3914 outcome_published = true;
3918 self.publish_child_outcome(control, transition).await
3919 }.await;
3920 match result {
3921 Ok(receipt) => Ok(receipt),
3922 Err(RuntimeOperationFailure::Storage(failure)) => {
3923 self.retain_execution_read_failure(&failure);
3924 self.schedule_after_storage_failure(&failure);
3925 Err(failure.into())
3926 }
3927 Err(RuntimeOperationFailure::Application(error)) => {
3928 if outcome_published {
3929 return Err(error.into());
3930 }
3931 let failure = self
3932 .finish_child_admission(sequence, ChildAdmissionReceipt::Rejected(error))
3933 .await;
3934 let loaded = self
3935 .load_execution_observation(None, None)
3936 .await
3937 .inspect_err(|failure| self.retain_execution_read_failure(failure))?
3938 .ok_or_else(|| AppError::NotFound("child parent is unavailable".into()))?;
3939 if loaded.record.checkpoint.is_stopping() || loaded.record.checkpoint.is_terminal()
3940 {
3941 Ok(SessionReceipt::Checkpoint {
3942 loaded,
3943 staged: Vec::new(),
3944 })
3945 } else {
3946 match failure {
3947 Err(failure) => Err(failure),
3948 Ok(_) => Err(AppError::Integration(
3949 "child failure policy returned admission success".into(),
3950 )
3951 .into()),
3952 }
3953 }
3954 }
3955 }
3956 }
3957
3958 fn schedule_child_retention(&self, control: &ControlRecord) {
3959 let ControlRecordState::Child { admitted, .. } = &control.state else {
3960 return;
3961 };
3962 for member in admitted {
3963 match Self::attach(Arc::clone(&self.app), member.key.clone()) {
3964 Ok(child) => {
3965 child.schedule(ScheduleCause::Reconcile);
3966 }
3967 Err(error) => self.app.tasks_failed(FinalResult::from(&error)),
3968 }
3969 }
3970 }
3971
3972 fn plan_child_completion(
3973 &self,
3974 loaded: &LoadedRun,
3975 control: &ControlRecord,
3976 children: Vec<RunRecord>,
3977 ) -> Result<ControlTransition, AppError> {
3978 let planned = catch_unwind(AssertUnwindSafe(|| {
3979 self.plan_children(loaded, control.sequence, children, true)
3980 }))
3981 .map_err(|_| AppError::Integration("child planning callback panicked".into()))
3982 .and_then(|result| result)
3983 .and_then(|decision| match decision {
3984 BspDecision::Stage(activations) => Ok(ControlTransition::Stage { activations }),
3985 BspDecision::Failed {
3986 category,
3987 message,
3988 observation,
3989 } => Ok(ControlTransition::Failed {
3990 category,
3991 message,
3992 observation,
3993 }),
3994 _ => Err(AppError::Integration(
3995 "completed child group has no final outcome".into(),
3996 )),
3997 });
3998 if !matches!(&planned, Ok(ControlTransition::Stage { activations })
3999 if activations.iter().all(|activation| matches!(activation.outcome, StagedOutcomeRecord::Complete { .. })))
4000 {
4001 self.schedule_child_retention(control);
4004 }
4005 planned
4006 }
4007
4008 async fn publish_child_outcome(
4009 &self,
4010 control: &ControlRecord,
4011 transition: ControlTransition,
4012 ) -> Result<SessionReceipt, RuntimeOperationFailure> {
4013 let completed = matches!(&transition, ControlTransition::Stage { activations }
4017 if activations.iter().all(|activation| matches!(activation.outcome, StagedOutcomeRecord::Complete { .. })));
4018 if let Some(loaded) = self.commit_scheduled_transition(control, transition).await {
4019 return Ok(SessionReceipt::Checkpoint {
4022 loaded,
4023 staged: Vec::new(),
4024 });
4025 }
4026 if completed {
4027 self.schedule_child_retention(control);
4031 }
4032 let loaded = match tokio::time::timeout(
4033 Self::remaining(control.deadline_millis)?,
4034 self.load_for_recovery(),
4035 )
4036 .await
4037 {
4038 Ok(result) => result
4039 .map_err(RuntimeOperationFailure::from)
4040 .and_then(|loaded| {
4041 loaded.ok_or_else(|| {
4042 AppError::NotFound("child parent is unavailable".into()).into()
4043 })
4044 }),
4045 Err(_) => Err(AppError::Timeout("child outcome observation timed out".into()).into()),
4046 };
4047 if !loaded.as_ref().is_ok_and(|loaded| {
4048 loaded.record.checkpoint.is_stopping() || loaded.record.checkpoint.is_terminal()
4049 }) {
4050 self.app.tasks_failed(FinalResult::Integration(
4051 "child outcome publication authority is unresolved".into(),
4052 ));
4053 return Err(loaded.err().unwrap_or_else(|| {
4054 AppError::Closed("child outcome publication requires reconciliation".into()).into()
4055 }));
4056 }
4057 Ok(SessionReceipt::Checkpoint {
4058 loaded: loaded?,
4059 staged: Vec::new(),
4060 })
4061 }
4062
4063 fn retain_source_failure(
4064 &self,
4065 context: &ActionContext,
4066 lineage: &str,
4067 category: FailureCategory,
4068 ) {
4069 let Some(generation) = self.cell.lifecycle_generation() else {
4070 return;
4071 };
4072 match self
4073 .cell
4074 .observe_source_failure(generation, context, lineage, category)
4075 {
4076 Ok(()) | Err(AppError::Closed(_)) => {
4077 }
4080 Err(_) => self.app.tasks_failed(FinalResult::Integration(
4081 "source failure observation authority failed".into(),
4082 )),
4083 }
4084 }
4085
4086 async fn end_source(
4087 &self,
4088 loaded: LoadedRun,
4089 execution: Option<u64>,
4090 ) -> Result<LoadedRun, AppError> {
4091 let generation = self
4092 .cell
4093 .lifecycle_generation()
4094 .ok_or_else(|| AppError::Closed("source cleanup observation is unavailable".into()))?;
4095 let deadline = self.cell.begin_action_cleanup(generation)?;
4096 for context in self.cell.action_observations() {
4099 if context.checkpoint != loaded.record.checkpoint.base_sequence {
4100 continue;
4101 }
4102 let mut observed_activation = None;
4103 for activation in &loaded.record.checkpoint.frontier {
4104 if self
4105 .cell
4106 .action_context(
4107 loaded.record.checkpoint.base_sequence,
4108 &activation.node,
4109 &activation.lineage,
4110 )
4111 .is_some_and(|observed| Arc::ptr_eq(&observed, &context))
4112 {
4113 observed_activation = Some(activation);
4114 break;
4115 }
4116 }
4117 let activation = observed_activation.ok_or_else(|| {
4118 AppError::Integration("source cleanup activation is unavailable".into())
4119 })?;
4120 if let Err(failure) = self
4121 .observe_step_finish(
4122 generation,
4123 &context,
4124 None,
4125 &loaded.record,
4126 loaded
4127 .record
4128 .checkpoint
4129 .control_for(&activation.node, &activation.lineage)
4130 .is_some(),
4131 Some(deadline),
4132 )
4133 .await
4134 {
4135 return Err(self
4136 .fail_hook(&loaded, Some((activation, &context)), failure, Vec::new())
4137 .await);
4138 }
4139 }
4140 let loaded = self
4141 .load_execution_observation(None, Some(deadline))
4142 .await
4143 .map_err(|failure| {
4144 self.retain_execution_read_failure(&failure);
4145 failure.into_error()
4146 })?
4147 .ok_or_else(|| AppError::NotFound("workflow run does not exist".into()))?;
4148 let loaded = if let Some(generation) = execution {
4149 self.settle_execution(generation, Some(loaded)).await?.0
4152 } else if loaded.record.checkpoint.is_stopping() || loaded.record.checkpoint.is_terminal() {
4153 loaded
4154 } else {
4155 let definition = self.app.definition_for(&loaded.record)?;
4156 let discard_staged = loaded.record.checkpoint.rollback_stage_keys(&definition);
4157 self.commit_for_recovery(loaded, ControlTransition::Pause { discard_staged })
4159 .await
4160 .map_err(RuntimeOperationFailure::into_error)?
4161 };
4162 Ok(loaded)
4163 }
4164
4165 pub(super) async fn finish_execution(
4166 &self,
4167 generation: u64,
4168 loaded: Option<LoadedRun>,
4169 ) -> Result<RunRecord, AppError> {
4170 if !self.cell.owns_execution(generation) {
4171 return Err(AppError::Closed(
4172 "run execution generation no longer owns cleanup".into(),
4173 ));
4174 }
4175 let mut changes = self.app.runtime_changes();
4176 while self.app.has_child_work(&self.key) {
4177 changes
4178 .changed()
4179 .await
4180 .map_err(|_| AppError::Closed("child receipt observation closed".into()))?;
4181 }
4182 let settled = self.settle_execution(generation, loaded).await;
4183 if let Err(error) = &settled
4184 && self.cell.source_failure(generation).is_some()
4185 {
4186 if !matches!(error, AppError::Closed(_)) {
4187 self.app.tasks_failed(FinalResult::Integration(
4188 "source failure cleanup authority failed".into(),
4189 ));
4190 }
4191 return settled.map(|(loaded, _)| loaded.record);
4192 }
4193 let retained = settled.as_ref().is_ok_and(|(_, retained)| *retained);
4194 let mut result = settled.map(|(loaded, _)| loaded.record);
4195 let cleanup_pending = result.as_ref().is_ok_and(|record| {
4196 record.checkpoint.is_stopping()
4197 || !record.checkpoint.controls.is_empty()
4198 || record.checkpoint.is_terminal() && !retained
4199 });
4200 if let Ok(record) = &result {
4201 let observed = self
4202 .finish_execution_observation(record, None, None, false)
4203 .await;
4204 if record.checkpoint.is_terminal() && self.cell.child_parent().is_none() {
4205 self.maintain_retention(None).await;
4206 }
4207 if let Err(error) = observed {
4208 result = Err(error);
4209 }
4210 }
4211 self.cell.end_publication(generation);
4212 self.cell.clear_execution_options(Some(generation));
4213 let scheduled = self.cell.release_execution(generation);
4214 if !retained && (cleanup_pending || scheduled) {
4215 self.schedule(ScheduleCause::Reconcile);
4216 }
4217 result
4218 }
4219
4220 pub(super) fn settle_execution(
4221 &self,
4222 generation: u64,
4223 loaded: Option<LoadedRun>,
4224 ) -> Pin<Box<impl Future<Output = Result<(LoadedRun, bool), AppError>> + '_>> {
4225 Box::pin(async move {
4226 if !self.cell.owns_execution(generation) {
4227 return Err(AppError::Closed(
4228 "run execution generation no longer owns cleanup".into(),
4229 ));
4230 }
4231 let mut retained = false;
4232 let result = async {
4233 let mut loaded = match loaded {
4234 Some(loaded) => loaded,
4235 None => {
4236 let deadline = self.cell.begin_action_cleanup(generation)?;
4237 self.load_execution_observation(None, Some(deadline))
4240 .await
4241 .map_err(|failure| {
4242 self.retain_execution_read_failure(&failure);
4243 failure.into_error()
4244 })?
4245 .ok_or_else(|| {
4246 AppError::NotFound("workflow run does not exist".into())
4247 })?
4248 }
4249 };
4250 if let Some((context, lineage, category)) = self.cell.source_failure(generation)
4251 && context.checkpoint == loaded.record.checkpoint.base_sequence
4252 && !loaded.record.checkpoint.is_stopping()
4253 && !loaded.record.checkpoint.is_terminal()
4254 && loaded
4255 .record
4256 .checkpoint
4257 .position
4258 .as_ref()
4259 .is_none_or(|committed| committed.ordinal < context.action_ordinal)
4260 {
4261 let position = ActionPosition {
4262 node: context.node.clone(),
4263 ordinal: context.action_ordinal,
4264 };
4265 let definition = self.app.definition_for(&loaded.record)?;
4266 let (position, category, message) = match loaded
4267 .record
4268 .checkpoint
4269 .preceding_failure(&definition, &position, &lineage)?
4270 {
4271 Some((position, category, message)) => {
4272 (position, category, message.to_owned())
4273 }
4274 None => (position, category, "caller action execution failed".into()),
4275 };
4276 loaded = self
4277 .commit_for_recovery(
4278 loaded,
4279 ControlTransition::Failed {
4280 category,
4281 message,
4282 observation: Some(position),
4283 },
4284 )
4285 .await
4286 .map_err(RuntimeOperationFailure::into_error)?;
4287 }
4288 if !loaded.record.checkpoint.is_terminal()
4289 && !loaded.record.checkpoint.is_stopping()
4290 {
4291 let definition = self.app.definition_for(&loaded.record)?;
4292 let failed = loaded
4293 .record
4294 .checkpoint
4295 .selected_staged(&definition)
4296 .filter_map(|activation| {
4297 if let StagedOutcomeRecord::Failed { category, message } =
4298 &activation.outcome
4299 {
4300 Some((activation, *category, message))
4301 } else {
4302 None
4303 }
4304 })
4305 .min_by(|left, right| {
4306 (&left.0.node, &left.0.lineage).cmp(&(&right.0.node, &right.0.lineage))
4307 });
4308 if let Some((activation, category, message)) = failed {
4309 let message = message.clone();
4310 let observation = loaded.record.checkpoint.proposed_position(
4311 &definition,
4312 &activation.node,
4313 &activation.lineage,
4314 &[],
4315 )?;
4316 loaded = self
4317 .commit_for_recovery(
4318 loaded,
4319 ControlTransition::Failed {
4320 category,
4321 message,
4322 observation: Some(observation),
4323 },
4324 )
4325 .await
4326 .map_err(RuntimeOperationFailure::into_error)?;
4327 }
4328 }
4329 if loaded.record.checkpoint.is_stopping()
4332 && loaded.record.checkpoint.controls.is_empty()
4333 {
4334 loaded = match Box::pin(self.settle_stopping(loaded)).await {
4335 Some(loaded) => loaded,
4336 None => {
4337 self.load_execution_observation(None, None)
4341 .await
4342 .map_err(|failure| {
4343 self.retain_execution_read_failure(&failure);
4344 failure.into_error()
4345 })?
4346 .ok_or_else(|| {
4347 AppError::NotFound("workflow run does not exist".into())
4348 })?
4349 }
4350 };
4351 }
4352 if !loaded.record.checkpoint.staged.is_empty()
4353 && !loaded.record.checkpoint.is_terminal()
4354 && !loaded.record.checkpoint.is_stopping()
4355 {
4356 let definition = self.app.definition_for(&loaded.record)?;
4357 let discard_staged = loaded.record.checkpoint.rollback_stage_keys(&definition);
4358 return self
4359 .commit_for_recovery(loaded, ControlTransition::Pause { discard_staged })
4360 .await
4361 .map_err(RuntimeOperationFailure::into_error);
4362 }
4363 if loaded.record.checkpoint.is_terminal() {
4364 let deadline = if self.cell.lifecycle_generation().is_none() {
4367 Box::pin(self.maintain_retention(Some(loaded))).await;
4368 let deadline = LifecycleEvent::deadline(None);
4371 loaded = self
4372 .load_execution_observation(None, Some(deadline))
4373 .await
4374 .map_err(|failure| {
4375 self.retain_execution_read_failure(&failure);
4376 failure.into_error()
4377 })?
4378 .ok_or_else(|| {
4379 AppError::NotFound("workflow run does not exist".into())
4380 })?;
4381 deadline
4382 } else {
4383 LifecycleEvent::deadline(None)
4384 };
4385 retained = tokio::time::timeout_at(deadline, async {
4386 let mut delay = RECONCILIATION_INITIAL_DELAY;
4387 loop {
4388 match self.app.storage().run_is_active(&loaded.record).await {
4389 Ok(active) => {
4390 self.observe_generation(deadline).await?;
4393 return Ok(!active);
4394 }
4395 Err(failure)
4396 if failure.is_recoverable()
4397 && self.schedule_after_storage_failure(&failure) =>
4398 {
4399 tokio::time::sleep(delay).await;
4400 delay = delay.saturating_mul(2).min(RECONCILIATION_MAX_DELAY);
4401 }
4402 Err(failure) => return Err(failure),
4403 }
4404 }
4405 })
4406 .await
4407 .unwrap_or_else(|_| {
4408 Err(StorageFailure::permanent(AppError::Timeout(
4409 "workflow capacity observation deadline expired".into(),
4410 )))
4411 })
4412 .map_err(|failure| {
4413 self.retain_execution_read_failure(&failure);
4414 failure.into_error()
4415 })?;
4416 } else if loaded.record.checkpoint.is_stopping() {
4417 self.observe_generation(LifecycleEvent::deadline(None))
4420 .await
4421 .map_err(|failure| {
4422 self.retain_execution_read_failure(&failure);
4423 failure.into_error()
4424 })?;
4425 }
4426 Ok(loaded)
4427 }
4428 .await;
4429 result.map(|loaded| (loaded, retained))
4430 })
4431 }
4432}
4433
4434impl<
4435 CS: StateStoreBackend,
4436 A: AgentBackend,
4437 X: ActionBackend,
4438 C: ConditionBackend,
4439 T: StateTyping,
4440 H: LifecycleHooks,
4441 E: ChildExecutor,
4442> WorkflowRunRuntime<CS, A, X, C, T, H, E>
4443{
4444 pub(super) async fn begin_execution_observation(&self) -> Result<u64, WorkflowFailure> {
4445 let owner = self
4446 .cell
4447 .observation_owner()
4448 .map_err(|error| WorkflowFailure::Execution(error.to_string()))?;
4449 let (generation, opening) = self
4450 .cell
4451 .begin_lifecycle(owner)
4452 .map_err(|error| WorkflowFailure::Execution(error.to_string()))?;
4453 if opening {
4454 let deadline = self
4455 .cell
4456 .begin_action_cleanup(generation)
4457 .map_err(|error| WorkflowFailure::Execution(error.to_string()))?;
4458 LifecycleEvent::ExecuteStarted {
4459 run: self.id(),
4460 occurrence: self.occurrence(),
4461 }
4462 .deliver(
4463 self.app.hooks(),
4464 self.cell
4465 .execution_mode()
4466 .map_err(|error| WorkflowFailure::Execution(error.to_string()))?,
4467 deadline,
4468 )
4469 .await?;
4470 self.cell
4471 .opened_lifecycle(generation)
4472 .map_err(|error| WorkflowFailure::Execution(error.to_string()))?;
4473 }
4474 Ok(generation)
4475 }
4476
4477 async fn observe_activation_start(
4478 &self,
4479 loaded: &LoadedRun,
4480 activation: &FrontierRecord,
4481 context: &Arc<ActionContext>,
4482 inputs: &Map<String, Value>,
4483 deadline: tokio::time::Instant,
4484 ) -> Result<(), WorkflowFailure> {
4485 let generation = self.begin_execution_observation().await?;
4486 let definition = self
4487 .app
4488 .definition_for(&loaded.record)
4489 .map_err(|error| WorkflowFailure::Execution(error.to_string()))?;
4490 let Some(Step::Action {
4491 action, revision, ..
4492 }) = definition.node(&activation.node).map(|node| node.step())
4493 else {
4494 return self
4495 .observe_step_start(
4496 generation,
4497 &loaded.record,
4498 context,
4499 &activation.lineage,
4500 inputs,
4501 deadline,
4502 )
4503 .await
4504 .map(|_| ());
4505 };
4506 let binding = self
4507 .app
4508 .action(action, revision)
4509 .map_err(|error| WorkflowFailure::Execution(error.to_string()))?;
4510 let inputs = catch_unwind(AssertUnwindSafe(|| {
4511 binding
4512 .declaration()
4513 .resolve_inputs(inputs, self.cell.execution_mode()?)
4514 .and_then(ActionInputs::into_values)
4515 }))
4516 .map_err(|_| WorkflowFailure::Execution("action declaration panicked".into()))?
4517 .map_err(|error| WorkflowFailure::State(error.to_string()))?;
4518 let context = self
4519 .observe_step_start(
4520 generation,
4521 &loaded.record,
4522 context,
4523 &activation.lineage,
4524 &inputs,
4525 deadline,
4526 )
4527 .await?;
4528 self.observe_stream_start(generation, &context, deadline)
4529 .await
4530 }
4531
4532 pub(super) async fn observe_stream_start(
4533 &self,
4534 generation: u64,
4535 context: &ActionContext,
4536 deadline: tokio::time::Instant,
4537 ) -> Result<(), WorkflowFailure> {
4538 if self.cell.lifecycle_generation() != Some(generation)
4539 || self.cell.execution_was_dropped(generation)
4540 {
4541 return Ok(());
4542 }
4543 LifecycleEvent::StreamStarted {
4544 context,
4545 span: None,
4546 }
4547 .deliver(
4548 self.app.hooks(),
4549 context.mode,
4550 LifecycleEvent::deadline(Some(deadline)),
4551 )
4552 .await?;
4553 self.cell
4554 .open_action_stream(generation, context)
4555 .map_err(|error| WorkflowFailure::Execution(error.to_string()))?;
4556 Ok(())
4557 }
4558
4559 pub(super) async fn observe_stream_item(
4560 &self,
4561 generation: u64,
4562 context: &ActionContext,
4563 result: &Map<String, Value>,
4564 deadline: tokio::time::Instant,
4565 ) -> Result<(), WorkflowFailure> {
4566 if self.cell.lifecycle_generation() != Some(generation)
4567 || self.cell.execution_was_dropped(generation)
4568 {
4569 return Ok(());
4570 }
4571 let index = self
4572 .cell
4573 .action_stream_item(generation, context, false)
4574 .map_err(|error| WorkflowFailure::Execution(error.to_string()))?;
4575 LifecycleEvent::StreamItem {
4576 context,
4577 span: None,
4578 index,
4579 result,
4580 }
4581 .deliver(
4582 self.app.hooks(),
4583 context.mode,
4584 LifecycleEvent::deadline(Some(deadline)),
4585 )
4586 .await?;
4587 self.cell
4588 .action_stream_item(generation, context, true)
4589 .map_err(|error| WorkflowFailure::Execution(error.to_string()))?;
4590 Ok(())
4591 }
4592
4593 async fn observe_source_event(
4594 &self,
4595 context: &ActionContext,
4596 event: ActionEvent,
4597 deadline: tokio::time::Instant,
4598 ) -> Result<Option<Map<String, Value>>, WorkflowFailure> {
4599 let generation = self.cell.lifecycle_generation().ok_or_else(|| {
4600 WorkflowFailure::Execution("source observation generation closed".into())
4601 })?;
4602 if let ActionEvent::Partial { result } = event {
4603 self.observe_stream_item(generation, context, &result, deadline)
4604 .await?;
4605 return Ok(Some(result));
4606 }
4607 if self.cell.execution_was_dropped(generation) {
4608 return Ok(None);
4609 }
4610 event
4611 .validate_span()
4612 .map_err(|error| WorkflowFailure::Execution(error.to_string()))?;
4613 let deadline = LifecycleEvent::deadline(Some(deadline));
4614 match event {
4615 ActionEvent::SpanStarted {
4616 id,
4617 name,
4618 parent,
4619 dependencies,
4620 } => {
4621 let parent_index = self
4622 .cell
4623 .span_parent(generation, context, &id, parent.as_deref())
4624 .map_err(|error| WorkflowFailure::Execution(error.to_string()))?;
4625 LifecycleEvent::SpanStarted {
4626 context,
4627 id: &id,
4628 name: &name,
4629 parent: parent.as_deref(),
4630 dependencies: &dependencies,
4631 }
4632 .deliver(self.app.hooks(), context.mode, deadline)
4633 .await?;
4634 self.cell
4635 .opened_span(generation, context, id, parent_index)
4636 .map_err(|error| WorkflowFailure::Execution(error.to_string()))?;
4637 }
4638 ActionEvent::SpanAttribute {
4639 span,
4640 key,
4641 value,
4642 tags,
4643 } => {
4644 self.cell
4645 .span_attribute(generation, context, span.as_deref())
4646 .map_err(|error| WorkflowFailure::Execution(error.to_string()))?;
4647 LifecycleEvent::SpanAttribute {
4648 context,
4649 span: span.as_deref(),
4650 key: &key,
4651 value: &value,
4652 tags: &tags,
4653 }
4654 .deliver(self.app.hooks(), context.mode, deadline)
4655 .await?;
4656 }
4657 ActionEvent::SpanFinished { id } => {
4658 let id = self
4659 .cell
4660 .take_span(generation, context, Some(&id))
4661 .map_err(|error| WorkflowFailure::Execution(error.to_string()))?
4662 .ok_or_else(|| {
4663 WorkflowFailure::Execution(
4664 "caller span finish target is unavailable".into(),
4665 )
4666 })?;
4667 LifecycleEvent::SpanFinished { context, id: &id }
4668 .deliver(self.app.hooks(), context.mode, deadline)
4669 .await?;
4670 }
4671 ActionEvent::Partial { .. } | ActionEvent::Completed { .. } => unreachable!(),
4672 }
4673 Ok(None)
4674 }
4675
4676 async fn finish_spans(
4677 &self,
4678 generation: u64,
4679 context: &ActionContext,
4680 async_deadline: Option<tokio::time::Instant>,
4681 ) -> Result<(), WorkflowFailure> {
4682 let mut failure = None;
4683 loop {
4684 let id = match self.cell.take_span(generation, context, None) {
4685 Ok(Some(id)) => id,
4686 Ok(None) => break,
4687 Err(error) => {
4688 return Err(
4689 failure.unwrap_or_else(|| WorkflowFailure::Execution(error.to_string()))
4690 );
4691 }
4692 };
4693 let event = LifecycleEvent::SpanFinished { context, id: &id };
4694 let observed =
4695 match async_deadline.filter(|_| !self.cell.execution_was_dropped(generation)) {
4696 Some(deadline) => {
4697 event
4698 .deliver(self.app.hooks(), context.mode, deadline)
4699 .await
4700 }
4701 None => event.deliver_sync(self.app.hooks()),
4702 };
4703 if let Err(next) = observed
4704 && failure.is_none()
4705 {
4706 failure = Some(next);
4707 }
4708 }
4709 failure.map_or(Ok(()), Err)
4710 }
4711
4712 fn finish_spans_sync(
4713 &self,
4714 generation: u64,
4715 context: &ActionContext,
4716 ) -> Result<(), WorkflowFailure> {
4717 let mut failure = None;
4718 loop {
4719 let id = match self.cell.take_span(generation, context, None) {
4720 Ok(Some(id)) => id,
4721 Ok(None) => break,
4722 Err(error) => {
4723 return Err(
4724 failure.unwrap_or_else(|| WorkflowFailure::Execution(error.to_string()))
4725 );
4726 }
4727 };
4728 if let Err(next) =
4729 (LifecycleEvent::SpanFinished { context, id: &id }).deliver_sync(self.app.hooks())
4730 && failure.is_none()
4731 {
4732 failure = Some(next);
4733 }
4734 }
4735 failure.map_or(Ok(()), Err)
4736 }
4737
4738 async fn finish_staged_spans(
4739 &self,
4740 loaded: &LoadedRun,
4741 decision: &BspDecision,
4742 context: Option<&ActionContext>,
4743 ) -> Result<(), AppError> {
4744 let (BspDecision::Stage(staged), Some(context)) = (decision, context) else {
4745 return Ok(());
4746 };
4747 let Some(generation) = self.cell.lifecycle_generation() else {
4748 return Ok(());
4749 };
4750 let Some(staged) = staged.iter().find(|staged| {
4751 self.cell
4752 .action_context(
4753 loaded.record.checkpoint.base_sequence,
4754 &staged.node,
4755 &staged.lineage,
4756 )
4757 .is_some_and(|observed| std::ptr::eq(observed.as_ref(), context))
4758 && matches!(staged.outcome, StagedOutcomeRecord::Complete { .. })
4759 }) else {
4760 return Ok(());
4761 };
4762 let activation = loaded
4763 .record
4764 .checkpoint
4765 .frontier
4766 .iter()
4767 .find(|activation| {
4768 activation.node == staged.node && activation.lineage == staged.lineage
4769 })
4770 .ok_or_else(|| {
4771 AppError::Integration("completed caller activation is unavailable".into())
4772 })?;
4773 if let Err(failure) = self
4776 .finish_spans(generation, context, Some(LifecycleEvent::deadline(None)))
4777 .await
4778 {
4779 return Err(self
4780 .fail_hook(loaded, Some((activation, context)), failure, Vec::new())
4781 .await);
4782 }
4783 Ok(())
4784 }
4785
4786 pub(super) async fn fail_hook(
4787 &self,
4788 loaded: &LoadedRun,
4789 activation: Option<(&FrontierRecord, &ActionContext)>,
4790 failure: WorkflowFailure,
4791 staged: Vec<StagedActivationRecord>,
4792 ) -> AppError {
4793 let error = failure.application_error();
4794 let mut prior = None;
4795 let settled = async {
4796 let generation = self
4797 .cell
4798 .lifecycle_generation()
4799 .ok_or_else(|| AppError::Closed("lifecycle failure cleanup is closed".into()))?;
4800 let deadline = self.cell.begin_action_cleanup(generation)?;
4801 let mut loaded = LoadedRun {
4802 record: loaded.record.clone(),
4803 revision: loaded.revision,
4804 };
4805 if !staged.is_empty() {
4806 loaded = self
4809 .commit_for_recovery(
4810 loaded,
4811 ControlTransition::Stage {
4812 activations: staged,
4813 },
4814 )
4815 .await
4816 .map_err(RuntimeOperationFailure::into_error)?;
4817 }
4818 let definition = self.app.definition_for(&loaded.record)?;
4819 let retained = self
4820 .cell
4821 .lifecycle_generation()
4822 .and_then(|generation| self.cell.source_failure(generation));
4823 let source = retained.as_ref().filter(|(context, lineage, _)| {
4824 context.checkpoint == loaded.record.checkpoint.base_sequence
4825 && activation.is_some_and(|(activation, _)| {
4826 (&context.node, lineage) < (&activation.node, &activation.lineage)
4827 })
4828 });
4829 let (mut observation, lineage, mut category, mut message) = match source {
4833 Some((context, lineage, category)) => {
4834 let message = "caller action execution failed".to_owned();
4835 prior = Some(category.project(message.clone()));
4836 (
4837 Some(ActionPosition {
4838 node: context.node.clone(),
4839 ordinal: context.action_ordinal,
4840 }),
4841 Some(lineage.as_str()),
4842 *category,
4843 message,
4844 )
4845 }
4846 None => (
4847 activation.map(|(_, context)| ActionPosition {
4848 node: context.node.clone(),
4849 ordinal: context.action_ordinal,
4850 }),
4851 activation.map(|(activation, _)| activation.lineage.as_str()),
4852 FailureCategory::from_failure(&failure),
4853 failure.message().to_owned(),
4854 ),
4855 };
4856 if let Some((position, lineage)) = observation.as_ref().zip(lineage)
4857 && let Some((position, preceding, text)) = loaded
4858 .record
4859 .checkpoint
4860 .preceding_failure(&definition, position, lineage)?
4861 {
4862 observation = Some(position);
4863 category = preceding;
4864 message = text.to_owned();
4865 prior = Some(category.project(message.clone()));
4866 }
4867 let mut loaded = self
4868 .commit_for_recovery(
4869 loaded,
4870 ControlTransition::Failed {
4871 category,
4872 message,
4873 observation,
4874 },
4875 )
4876 .await
4877 .map_err(RuntimeOperationFailure::into_error)?;
4878 if !loaded.record.checkpoint.is_terminal() {
4879 loaded = self
4883 .load_execution_observation(None, Some(deadline))
4884 .await
4885 .map_err(|failure| {
4886 self.retain_execution_read_failure(&failure);
4887 failure.into_error()
4888 })?
4889 .ok_or_else(|| AppError::NotFound("workflow run does not exist".into()))?;
4890 if loaded.record.checkpoint.is_stopping()
4891 && loaded.record.checkpoint.controls.is_empty()
4892 {
4893 loaded = match Box::pin(self.settle_stopping(loaded)).await {
4894 Some(loaded) => loaded,
4895 None => self
4896 .load_execution_observation(None, Some(deadline))
4897 .await
4898 .map_err(|failure| {
4899 self.retain_execution_read_failure(&failure);
4900 failure.into_error()
4901 })?
4902 .ok_or_else(|| {
4903 AppError::NotFound("workflow run does not exist".into())
4904 })?,
4905 };
4906 }
4907 }
4908 if loaded.record.checkpoint.is_terminal() {
4909 self.observe_generation(deadline).await.map_err(|failure| {
4912 self.retain_execution_read_failure(&failure);
4913 failure.into_error()
4914 })?;
4915 }
4916 Ok(loaded)
4917 }
4918 .await;
4919 match settled {
4920 Ok(loaded) => {
4921 let record = &loaded.record;
4922 let completion = prior.map(|prior| failure.with_cause(Some(prior)));
4926 let _ = self
4927 .finish_execution_observation(record, completion, None, false)
4928 .await;
4929 if record.checkpoint.is_terminal() && self.cell.child_parent().is_none() {
4930 Box::pin(self.maintain_retention(Some(loaded))).await;
4931 }
4932 }
4933 Err(AppError::Closed(_)) => {}
4934 Err(_) => self.app.tasks_failed(FinalResult::Integration(
4935 "lifecycle failure cleanup authority failed".into(),
4936 )),
4937 }
4938 error
4939 }
4940
4941 async fn observe_step_start(
4942 &self,
4943 generation: u64,
4944 record: &RunRecord,
4945 context: &Arc<ActionContext>,
4946 lineage: &str,
4947 inputs: &Map<String, Value>,
4948 deadline: tokio::time::Instant,
4949 ) -> Result<Arc<ActionContext>, WorkflowFailure> {
4950 if let Some(context) = self
4951 .cell
4952 .action_context(context.checkpoint, &context.node, lineage)
4953 {
4954 return Ok(context);
4955 }
4956 let state = WorkflowState {
4957 values: record.checkpoint.state.clone(),
4958 };
4959 LifecycleEvent::StepStarted {
4960 context,
4961 state: &state,
4962 inputs,
4963 }
4964 .deliver(
4965 self.app.hooks(),
4966 context.mode,
4967 LifecycleEvent::deadline(Some(deadline)),
4968 )
4969 .await?;
4970 self.cell
4971 .observe_action(generation, Arc::clone(context), lineage)
4972 .map_err(|error| WorkflowFailure::Execution(error.to_string()))?;
4973 Ok(Arc::clone(context))
4974 }
4975}
4976
4977impl<
4978 CS: StateStoreBackend,
4979 A: AgentBackend,
4980 X: ActionBackend,
4981 C: ConditionBackend,
4982 T: StateTyping,
4983 H: LifecycleHooks,
4984 E: ChildExecutor,
4985> WorkflowRunRuntime<CS, A, X, C, T, H, E>
4986{
4987 async fn observe_step_finish(
4988 &self,
4989 generation: u64,
4990 context: &ActionContext,
4991 outcome: Option<Result<&ActionOutcome, &WorkflowFailure>>,
4992 record: &RunRecord,
4993 unresolved: bool,
4994 async_deadline: Option<tokio::time::Instant>,
4995 ) -> Result<(), WorkflowFailure> {
4996 let checkpoint = self
4997 .project_history(record)
4998 .map_err(|error| WorkflowFailure::Execution(error.to_string()))?;
4999 let spans = self.finish_spans(generation, context, async_deadline).await;
5000 let Some((context, items)) = self.cell.take_action_observation(generation, context) else {
5001 if !unresolved {
5002 self.cell.retire_closed_observation(generation, context);
5003 }
5004 return spans;
5005 };
5006 let event = LifecycleEvent::StepFinished {
5007 context: &context,
5008 outcome,
5009 checkpoint: &checkpoint,
5010 };
5011 let step = match async_deadline.filter(|_| !self.cell.execution_was_dropped(generation)) {
5012 Some(deadline) => {
5013 event
5014 .deliver(self.app.hooks(), context.mode, deadline)
5015 .await
5016 }
5017 None => event.deliver_sync(self.app.hooks()),
5018 };
5019 let result = spans.and(step);
5020 let result = if let Some(items) = items {
5021 let failure = result
5022 .as_ref()
5023 .err()
5024 .or_else(|| outcome.and_then(Result::err));
5025 let event = LifecycleEvent::StreamFinished {
5026 context: &context,
5027 span: None,
5028 items,
5029 failure,
5030 };
5031 let stream_result = match async_deadline
5034 .filter(|_| result.is_ok() && !self.cell.execution_was_dropped(generation))
5035 {
5036 Some(deadline) => {
5037 event
5038 .deliver(self.app.hooks(), context.mode, deadline)
5039 .await
5040 }
5041 None => event.deliver_sync(self.app.hooks()),
5042 };
5043 result.and(stream_result)
5044 } else {
5045 result
5046 };
5047 self.cell
5048 .finish_action_observation(generation, &context, unresolved);
5049 result
5050 }
5051
5052 fn observe_step_finish_sync(
5053 &self,
5054 generation: u64,
5055 context: &ActionContext,
5056 record: &RunRecord,
5057 unresolved: bool,
5058 ) -> Result<(), WorkflowFailure> {
5059 let checkpoint = self
5060 .project_history(record)
5061 .map_err(|error| WorkflowFailure::Execution(error.to_string()))?;
5062 let spans = self.finish_spans_sync(generation, context);
5063 let Some((context, items)) = self.cell.take_action_observation(generation, context) else {
5064 if !unresolved {
5065 self.cell.retire_closed_observation(generation, context);
5066 }
5067 return spans;
5068 };
5069 let step = LifecycleEvent::StepFinished {
5070 context: &context,
5071 outcome: None,
5072 checkpoint: &checkpoint,
5073 }
5074 .deliver_sync(self.app.hooks());
5075 let result = spans.and(step);
5076 let result = if let Some(items) = items {
5077 let next = LifecycleEvent::StreamFinished {
5078 context: &context,
5079 span: None,
5080 items,
5081 failure: result.as_ref().err(),
5082 }
5083 .deliver_sync(self.app.hooks());
5084 result.and(next)
5085 } else {
5086 result
5087 };
5088 self.cell
5089 .finish_action_observation(generation, &context, unresolved);
5090 result
5091 }
5092
5093 fn finish_execution_observation_sync(&self, record: &RunRecord) -> Result<(), AppError> {
5094 let Some(generation) = self.cell.lifecycle_generation() else {
5095 return self
5096 .cell
5097 .lifecycle_observation(record.checkpoint.sequence, false)
5098 .map(|_| ());
5099 };
5100 let unfinished = record
5101 .checkpoint
5102 .controls
5103 .values()
5104 .filter_map(|control| {
5105 self.cell.action_context(
5106 record.checkpoint.base_sequence,
5107 &control.node,
5108 &control.lineage,
5109 )
5110 })
5111 .collect::<Vec<_>>();
5112 if !self
5113 .cell
5114 .begin_lifecycle_finish(generation, record.checkpoint.sequence)
5115 {
5116 return self
5117 .cell
5118 .lifecycle_observation(record.checkpoint.sequence, false)
5119 .map(|_| ());
5120 }
5121 let mut failure = None;
5122 for context in self.cell.action_observations() {
5123 let unresolved = unfinished
5124 .iter()
5125 .any(|unfinished| Arc::ptr_eq(unfinished, &context));
5126 if let Err(next) =
5127 self.observe_step_finish_sync(generation, &context, record, unresolved)
5128 && failure.is_none()
5129 {
5130 failure = Some(next);
5131 }
5132 }
5133 match self.project(record) {
5134 Ok(snapshot) => {
5135 if let Err(next) = (LifecycleEvent::ExecuteFinished {
5136 snapshot: &snapshot,
5137 failure: failure.as_ref(),
5138 })
5139 .deliver_sync(self.app.hooks())
5140 && failure.is_none()
5141 {
5142 failure = Some(next);
5143 }
5144 }
5145 Err(error) => {
5146 if failure.is_none() {
5147 failure = Some(WorkflowFailure::Execution(error.to_string()));
5148 }
5149 }
5150 }
5151 if record.checkpoint.is_terminal() || record.checkpoint.is_paused() {
5152 self.cell.clear_execution_options(Some(generation));
5153 }
5154 let error = failure.as_ref().map(WorkflowFailure::application_error);
5155 let completed =
5156 self.cell
5157 .complete_lifecycle(generation, record.checkpoint.sequence, failure);
5158 match error {
5159 Some(error) => Err(error),
5160 None => completed,
5161 }
5162 }
5163
5164 pub(super) async fn finish_execution_observation(
5165 &self,
5166 record: &RunRecord,
5167 mut failure: Option<WorkflowFailure>,
5168 deadline: Option<tokio::time::Instant>,
5169 synchronous: bool,
5170 ) -> Result<(), AppError> {
5171 let Some(generation) = self.cell.lifecycle_generation() else {
5172 return self
5173 .cell
5174 .lifecycle_observation(record.checkpoint.sequence, false)
5175 .map(|_| ());
5176 };
5177 let synchronous = synchronous || self.cell.execution_was_dropped(generation);
5178 let unfinished = record
5179 .checkpoint
5180 .controls
5181 .values()
5182 .filter_map(|control| {
5183 self.cell.action_context(
5184 record.checkpoint.base_sequence,
5185 &control.node,
5186 &control.lineage,
5187 )
5188 })
5189 .collect::<Vec<_>>();
5190 if !self
5191 .cell
5192 .begin_lifecycle_finish(generation, record.checkpoint.sequence)
5193 {
5194 return self
5195 .cell
5196 .lifecycle_observation(record.checkpoint.sequence, false)
5197 .map(|_| ());
5198 }
5199 let deadline = match deadline {
5200 Some(deadline) => deadline,
5201 None => self.cell.begin_action_cleanup(generation)?,
5202 };
5203 let primary = match &record.checkpoint.phase {
5204 CheckpointPhaseRecord::Failing {
5205 category, message, ..
5206 }
5207 | CheckpointPhaseRecord::Failed {
5208 category, message, ..
5209 } => Some(category.project(message.clone())),
5210 _ => None,
5211 };
5212 for context in self.cell.action_observations() {
5213 let unresolved = unfinished
5214 .iter()
5215 .any(|unfinished| Arc::ptr_eq(unfinished, &context));
5216 if unresolved && !record.checkpoint.is_stopping() && !synchronous {
5217 continue;
5218 }
5219 let outcome = record.checkpoint.outcomes.iter().find(|outcome| {
5220 outcome.ordinal == context.action_ordinal && outcome.node == context.node
5221 });
5222 let failed = record
5223 .checkpoint
5224 .failed_observation()
5225 .is_some_and(|position| {
5226 position.ordinal == context.action_ordinal && position.node == context.node
5227 });
5228 let observed = outcome
5229 .map(Ok)
5230 .or_else(|| failed.then(|| primary.as_ref().map(Err)).flatten());
5231 if let Err(next) = self
5232 .observe_step_finish(
5233 generation,
5234 &context,
5235 observed,
5236 record,
5237 unresolved,
5238 (!synchronous && failure.is_none()).then_some(deadline),
5239 )
5240 .await
5241 && failure.is_none()
5242 {
5243 failure = Some(next.with_cause(match &record.checkpoint.phase {
5244 CheckpointPhaseRecord::Failing {
5245 category, message, ..
5246 }
5247 | CheckpointPhaseRecord::Failed {
5248 category, message, ..
5249 } => Some(category.project(message.clone())),
5250 _ => None,
5251 }));
5252 }
5253 }
5254 match self.project(record) {
5255 Ok(snapshot) => {
5256 let event = LifecycleEvent::ExecuteFinished {
5257 snapshot: &snapshot,
5258 failure: failure.as_ref().or(primary.as_ref()),
5259 };
5260 let observed = if synchronous || self.cell.execution_was_dropped(generation) {
5261 event.deliver_sync(self.app.hooks())
5262 } else {
5263 event
5264 .deliver(self.app.hooks(), self.cell.execution_mode()?, deadline)
5265 .await
5266 };
5267 if let Err(next) = observed
5268 && failure.is_none()
5269 {
5270 failure = Some(next.with_cause(primary));
5271 }
5272 }
5273 Err(error) => {
5274 if failure.is_none() {
5275 failure = Some(WorkflowFailure::Execution(error.to_string()));
5276 }
5277 }
5278 }
5279 if record.parent.is_some()
5280 && record.checkpoint.is_terminal()
5281 && let Err(next) = self.observe_child_finished(record, deadline).await
5282 && failure.is_none()
5283 {
5284 failure = Some(next);
5285 }
5286 if record.checkpoint.is_terminal() || record.checkpoint.is_paused() {
5287 self.cell.clear_execution_options(Some(generation));
5288 }
5289 let error = failure.as_ref().map(WorkflowFailure::application_error);
5290 let completed =
5291 self.cell
5292 .complete_lifecycle(generation, record.checkpoint.sequence, failure);
5293 match error {
5294 Some(error) => Err(error),
5295 None => completed,
5296 }
5297 }
5298
5299 pub(super) async fn observe_barrier(&self, loaded: LoadedRun) -> Result<LoadedRun, AppError> {
5300 let Some(generation) = self.cell.lifecycle_generation() else {
5301 return Ok(loaded);
5302 };
5303 let record = &loaded.record;
5304 let observations = self.cell.action_observations();
5305 for outcome in &record.checkpoint.outcomes {
5306 let context = observations.iter().find(|context| {
5307 context.action_ordinal == outcome.ordinal && context.node == outcome.node
5308 });
5309 let Some(context) = context else {
5310 continue;
5311 };
5312 let deadline = self.cell.begin_action_cleanup(generation)?;
5313 if let Err(failure) = self
5314 .observe_step_finish(
5315 generation,
5316 context,
5317 Some(Ok(outcome)),
5318 record,
5319 false,
5320 Some(deadline),
5321 )
5322 .await
5323 {
5324 let error = failure.application_error();
5325 let parked = self
5329 .commit_for_recovery(
5330 loaded,
5331 ControlTransition::Pause {
5332 discard_staged: Vec::new(),
5333 },
5334 )
5335 .await
5336 .map_err(RuntimeOperationFailure::into_error);
5337 match parked {
5338 Ok(parked) => {
5339 let _ = self
5340 .finish_execution_observation(
5341 &parked.record,
5342 Some(failure),
5343 Some(deadline),
5344 false,
5345 )
5346 .await;
5347 return Ok(parked);
5348 }
5349 Err(AppError::Closed(_)) => {}
5350 Err(_) => self.app.tasks_failed(FinalResult::Integration(
5351 "post-hook cleanup authority failed".into(),
5352 )),
5353 }
5354 return Err(error);
5355 }
5356 }
5357 Ok(loaded)
5358 }
5359}