Skip to main content

app_kit/workflow/run/
session.rs

1use std::future::{Future, poll_fn};
2use std::panic::{AssertUnwindSafe, catch_unwind};
3use std::pin::Pin;
4use std::sync::Arc;
5use std::task::{Context, Poll};
6use std::time::Duration;
7
8use agents_kit::AgentBackend;
9use futures_util::{FutureExt, Stream};
10use serde_json::{Map, Value};
11use storage_kit::StateStoreBackend;
12use tokio::sync::{OwnedSemaphorePermit, mpsc, oneshot};
13
14use crate::AppError;
15use crate::app::{AppRuntime, DriverCommand, FinalResult, RunTaskKind};
16use crate::storage::StorageFailure;
17use crate::workflow::action::ActionInputs;
18use crate::workflow::definition::{JsonBoundary, Step};
19use crate::workflow::execution::{
20    BspDecision, ExecutionEngine, StagedActivationRecord, StagedOutcomeRecord,
21};
22use crate::workflow::run::{
23    ActionPosition, CheckpointPhaseRecord, ChildAdmissionReceipt, ControlRecord,
24    ControlRecordState, ControlTransition, FailureCategory, FrontierRecord, LoadedRun,
25    MAX_DRIVER_TRANSITIONS, RECONCILIATION_INITIAL_DELAY, RECONCILIATION_MAX_DELAY, RunAdmission,
26    RunRecord, RuntimeOperationFailure, ScheduleCause, WorkflowRunRuntime,
27};
28use crate::workflow::{
29    ActionBackend, ActionContext, ActionEvent, ActionMode, ActionOutcome, ActionRequest,
30    ChildBatch, ChildCompletion, ChildExecutor, ConditionBackend, ExecutionEvent, ExecutionOptions,
31    InlineExecutor, JsonState, LifecycleEvent, LifecycleHooks, NoActions, NoConditions, NoHooks,
32    RunSnapshot, RunStatus, StateTyping, WorkflowFailure, WorkflowRun, WorkflowState,
33};
34
35/// Exclusive lazy advancement whose caller source stays on the polling task.
36///
37/// Dropping an operation future preserves the source and any admitted write on
38/// this owner. Dropping the owner stops source polling and retains tracked write
39/// reconciliation before another execution can acquire the run.
40///
41/// Before native commit admission, owner drop discards uncommitted state and
42/// revokes later advancement. After admission, App retains native commit authority
43/// and reconciles the original receipt; observation loss cannot promise rollback.
44/// This does not undo caller Action I/O. Prefer explicit finish/abort while the
45/// runtime is driven; ordinary run and history observers never own cancellation.
46///
47/// See [the explicit execution workflow](crate::workflow#execute-observe-restore-and-delete).
48#[must_use = "execution owns lazy advancement; finish or abort it explicitly"]
49pub struct Execution<
50    'a,
51    CS: StateStoreBackend,
52    A: AgentBackend,
53    X: ActionBackend = NoActions,
54    C: ConditionBackend = NoConditions,
55    T: StateTyping = JsonState,
56    H: LifecycleHooks = NoHooks,
57    E: ChildExecutor = InlineExecutor,
58> {
59    runtime: WorkflowRunRuntime<CS, A, X, C, T, H, E>,
60    app: &'a AppRuntime<CS, A, X, C, T, H, E>,
61    lane: ExecutionLane,
62    admission: Option<OwnedSemaphorePermit>,
63    children: Vec<ChildExecution<'a, ChildFuture<Self>, E>>,
64    wake: Option<Pin<Box<tokio::sync::futures::OwnedNotified>>>,
65    timer: Option<Pin<Box<tokio::time::Sleep>>>,
66    generation: Option<u64>,
67    parent_deadline: Option<tokio::time::Instant>,
68    state: SessionState,
69    sources: Vec<CallerSource<X>>,
70    next_source: usize,
71    native_partials: Option<(mpsc::Sender<ExecutionEvent>, mpsc::Receiver<ExecutionEvent>)>,
72    prefetched: Option<(usize, Result<Option<ActionEvent>, FailureCategory>)>,
73    transitions: usize,
74    observed: Option<RunRecord>,
75}
76
77/// Blocking advancement of the same caller-owned execution source.
78///
79/// Calls require a thread outside any entered Tokio runtime context. The App's
80/// captured runtime and caller-provided lower runtimes must remain alive and
81/// driven elsewhere. Source polling and destruction remain on the caller thread;
82/// dropping this owner delegates nonblocking revocation to [`Execution`].
83///
84/// See [the synchronous execution workflow](SyncExecution::open).
85#[must_use = "execution owns lazy advancement; finish or abort it explicitly"]
86pub struct SyncExecution<
87    'a,
88    CS: StateStoreBackend,
89    A: AgentBackend,
90    X: ActionBackend = NoActions,
91    C: ConditionBackend = NoConditions,
92    T: StateTyping = JsonState,
93    H: LifecycleHooks = NoHooks,
94    E: ChildExecutor = InlineExecutor,
95> {
96    execution: Execution<'a, CS, A, X, C, T, H, E>,
97    runtime: tokio::runtime::Handle,
98}
99
100impl<
101    'a,
102    CS: StateStoreBackend,
103    A: AgentBackend,
104    X: ActionBackend,
105    C: ConditionBackend,
106    T: StateTyping,
107    H: LifecycleHooks,
108    E: ChildExecutor,
109> SyncExecution<'a, CS, A, X, C, T, H, E>
110{
111    /// Acquires lazy synchronous advancement on a thread outside Tokio.
112    ///
113    /// The App's captured runtime must remain alive and driven elsewhere.
114    /// Acquisition validates options before callbacks or source construction.
115    /// The returned owner borrows the run exclusively through its last use.
116    ///
117    /// A live synchronous owner prevents overlapping access to the run:
118    ///
119    /// ```compile_fail
120    /// use agents_kit::AgentBackend;
121    /// use app_kit::{AppError, workflow::{ExecutionOptions, RunSnapshot, SyncExecution, WorkflowRun}};
122    /// use storage_kit::StateStoreBackend;
123    /// fn overlapping<CS: StateStoreBackend, A: AgentBackend>(
124    /// run: &mut WorkflowRun<CS, A>,
125    /// ) -> Result<RunSnapshot, AppError> {
126    /// let execution = SyncExecution::open(run, ExecutionOptions::default())?;
127    /// let _ = run.id();
128    /// execution.abort()
129    /// }
130    /// ```
131    ///
132    /// # Errors
133    ///
134    /// Returns `InvalidInput` inside an entered Tokio runtime context before blocking.
135    /// Otherwise propagates the corresponding asynchronous execution errors, including
136    /// stale/revoked authority, invalid options, durable failures and timeouts.
137    /// The captured runtime and lower runtimes must stay alive and driven elsewhere.
138    ///
139    /// # Example
140    ///
141    /// ```
142    /// use agents_kit::Agent;
143    /// use app_kit::{App, AppConfig, Shutdown, WorkspaceConfig};
144    /// use app_kit::workflow::{ExecutionOptions, SyncExecution, WorkflowDefinition, WorkflowState};
145    /// use storage_kit::{StateStore, state::config::{LanceStateStoreConfig, StateStoreConfig}};
146    /// let runtime = tokio::runtime::Builder::new_multi_thread()
147    ///     .worker_threads(2).enable_all().build()?;
148    /// # let store = runtime.block_on(StateStore::connect(
149    /// #     StateStoreConfig::lance(LanceStateStoreConfig::memory()),
150    /// # ))?;
151    /// # let definition = WorkflowDefinition::parse(r#"{
152    /// #     "schema":"app-kit.workflow/v2", "id":"counter", "revision":"r1",
153    /// #     "state":{"count":0}, "required":["count"],
154    /// #     "reducers":{"count":"reject_concurrent"}, "entry":"increment",
155    /// #     "nodes":[{"id":"increment","step":{"kind":"state","mutations":[
156    /// #         {"kind":"increment","target":"count","value":{"kind":"literal","value":1}}
157    /// #     ]}}], "routes":{}
158    /// # }"#)?;
159    /// # let config = AppConfig::<_, Agent>::new(store, Vec::new(), vec![definition]);
160    /// # let mut app = runtime.block_on(App::open(config))?;
161    /// # let outcome = (|| -> Result<_, Box<dyn std::error::Error>> {
162    /// # let mut run = runtime.block_on(async {
163    /// #     let workspace = app.create_workspace(WorkspaceConfig {
164    /// #         name: Some("local counter".into()), root: std::env::current_dir()?,
165    /// #     }).await?;
166    /// #     let workflow = workspace.workflow("counter".parse()?).await?;
167    /// #     Ok::<_, Box<dyn std::error::Error>>(workflow.prepare(WorkflowState::default()).await?)
168    /// # })?;
169    /// // Workers keep the runtime driven while this thread stays outside Tokio.
170    /// let execution = SyncExecution::open(&mut run, ExecutionOptions::default())?;
171    /// let snapshot = execution.finish()?;
172    /// # Ok(snapshot)
173    /// # })();
174    /// # let cleanup = runtime.block_on(app.shutdown(Shutdown::CancelAndDrain {
175    /// #     timeout: std::time::Duration::from_secs(5),
176    /// # }));
177    /// # if let Err(error) = &cleanup { eprintln!("App shutdown: {error}"); }
178    /// # let snapshot = outcome?;
179    /// # cleanup?;
180    /// assert_eq!(snapshot.state.values()["count"], 1);
181    /// # Ok::<(), Box<dyn std::error::Error>>(())
182    /// ```
183    pub fn open(
184        run: &'a mut WorkflowRun<CS, A, X, C, T, H, E>,
185        options: ExecutionOptions,
186    ) -> Result<Self, AppError> {
187        Self::ensure_blocking()?;
188        let runtime = run.runtime.app.execution_handle()?;
189        let execution = runtime.block_on(Execution::open(run, options, ActionMode::Synchronous))?;
190        Ok(Self { execution, runtime })
191    }
192
193    fn ensure_blocking() -> Result<(), AppError> {
194        if tokio::runtime::Handle::try_current().is_ok() {
195            return Err(AppError::InvalidInput(
196                "synchronous execution requires a thread outside an entered Tokio runtime".into(),
197            ));
198        }
199        Ok(())
200    }
201
202    /// Blocks until one partial result or committed boundary is available.
203    ///
204    /// An entered runtime context is rejected before advancing this owner.
205    ///
206    /// # Errors
207    ///
208    /// Returns `InvalidInput` inside an entered Tokio runtime context before blocking.
209    /// Otherwise propagates the corresponding asynchronous execution errors, including
210    /// stale/revoked authority, invalid options, durable failures and timeouts.
211    /// The captured runtime and lower runtimes must stay alive and driven elsewhere.
212    ///
213    /// See [the synchronous execution workflow](SyncExecution::open).
214    pub fn try_next(&mut self) -> Result<Option<ExecutionEvent>, AppError> {
215        Self::ensure_blocking()?;
216        self.runtime.block_on(self.execution.next())
217    }
218
219    /// Drains partial results and blocks until one committed boundary.
220    ///
221    /// An entered runtime context is rejected before advancing this owner.
222    ///
223    /// # Errors
224    ///
225    /// Returns `InvalidInput` inside an entered Tokio runtime context before blocking.
226    /// Otherwise propagates the corresponding asynchronous execution errors, including
227    /// stale/revoked authority, invalid options, durable failures and timeouts.
228    /// The captured runtime and lower runtimes must stay alive and driven elsewhere.
229    ///
230    /// See [the synchronous execution workflow](SyncExecution::open).
231    pub fn step(&mut self) -> Result<Option<ExecutionEvent>, AppError> {
232        Self::ensure_blocking()?;
233        self.runtime.block_on(self.execution.step())
234    }
235
236    /// Drains execution to a halt or exhaustion and finalizes its observation.
237    ///
238    /// Rejection inside an entered runtime drops this consumed owner; existing
239    /// execution cleanup retains any previously admitted publication.
240    ///
241    /// # Errors
242    ///
243    /// Returns `InvalidInput` inside an entered Tokio runtime context before blocking.
244    /// Otherwise propagates the corresponding asynchronous execution errors, including
245    /// stale/revoked authority, invalid options, durable failures and timeouts.
246    /// The captured runtime and lower runtimes must stay alive and driven elsewhere.
247    ///
248    /// See [the synchronous execution workflow](SyncExecution::open).
249    pub fn finish(self) -> Result<RunSnapshot, AppError> {
250        Self::ensure_blocking()?;
251        self.runtime.block_on(self.execution.finish())
252    }
253
254    /// Finalizes the current uncommitted boundary without cancelling the run.
255    ///
256    /// Rejection inside an entered runtime drops this consumed owner; it does
257    /// not dispatch a new operation or discard accepted publication ownership.
258    ///
259    /// # Errors
260    ///
261    /// Returns `InvalidInput` inside an entered Tokio runtime context before blocking.
262    /// Otherwise propagates the corresponding asynchronous execution errors, including
263    /// stale/revoked authority, invalid options, durable failures and timeouts.
264    /// The captured runtime and lower runtimes must stay alive and driven elsewhere.
265    ///
266    /// See [the synchronous execution workflow](SyncExecution::open).
267    pub fn abort(self) -> Result<RunSnapshot, AppError> {
268        Self::ensure_blocking()?;
269        self.runtime.block_on(self.execution.abort())
270    }
271}
272
273struct CallerSource<X: ActionBackend> {
274    activation: FrontierRecord,
275    context: Arc<ActionContext>,
276    deadline: tokio::time::Instant,
277    state: CallerSourceState<X>,
278}
279
280enum CallerSourceState<X: ActionBackend> {
281    Polling(Pin<Box<X::Stream>>),
282    Failed(FailureCategory),
283    Ended,
284    Eof,
285}
286
287impl<X: ActionBackend> CallerSource<X> {
288    fn stop(&mut self) -> Result<(), FailureCategory> {
289        let previous = std::mem::replace(&mut self.state, CallerSourceState::Ended);
290        match previous {
291            CallerSourceState::Polling(source) => catch_unwind(AssertUnwindSafe(|| drop(source)))
292                .map_err(|_| FailureCategory::Execution),
293            previous => {
294                self.state = previous;
295                Ok(())
296            }
297        }
298    }
299
300    fn poll(
301        &mut self,
302        context: &mut Context<'_>,
303    ) -> Poll<Result<Option<ActionEvent>, FailureCategory>> {
304        let event = match &mut self.state {
305            CallerSourceState::Polling(source) => {
306                match catch_unwind(AssertUnwindSafe(|| source.as_mut().poll_next(context))) {
307                    Ok(Poll::Pending) => return Poll::Pending,
308                    Ok(Poll::Ready(Some(Ok(event)))) => Ok(Some(event)),
309                    Ok(Poll::Ready(Some(Err(_)))) => Err(FailureCategory::Action),
310                    Ok(Poll::Ready(None)) => Ok(None),
311                    Err(_) => Err(FailureCategory::Execution),
312                }
313            }
314            CallerSourceState::Failed(category) => {
315                let category = *category;
316                self.state = CallerSourceState::Ended;
317                return Poll::Ready(Err(category));
318            }
319            CallerSourceState::Ended | CallerSourceState::Eof => return Poll::Pending,
320        };
321        Poll::Ready(event)
322    }
323}
324
325#[derive(Clone, Copy, PartialEq)]
326enum ExecutionLane {
327    Caller,
328    Worker,
329}
330
331struct ChildExecution<'a, F, E>
332where
333    F: Future<Output = Result<ChildCompletion, AppError>> + 'a,
334    E: ChildExecutor,
335{
336    sequence: u64,
337    parent: Option<Arc<RunRecord>>,
338    context: Arc<ActionContext>,
339    lineage: String,
340    deadline: tokio::time::Instant,
341    state: ChildExecutionState<'a, F, E>,
342    inputs: Vec<Map<String, Value>>,
343}
344
345enum ChildExecutionState<'a, F, E>
346where
347    F: Future<Output = Result<ChildCompletion, AppError>> + 'a,
348    E: ChildExecutor,
349{
350    Awaiting,
351    Prepared(Vec<RunRecord>),
352    Local(Pin<Box<E::Execution<'a, F>>>),
353    Pipeline(ChildBatch<F>),
354    Worker(oneshot::Receiver<Result<SessionReceipt, RuntimeOperationFailure>>),
355    Receipt(Box<Result<SessionReceipt, RuntimeOperationFailure>>),
356    Complete(Result<Vec<ChildCompletion>, FailureCategory>),
357    Closed,
358}
359
360impl<
361    'a,
362    CS: StateStoreBackend,
363    A: AgentBackend,
364    X: ActionBackend,
365    C: ConditionBackend,
366    T: StateTyping,
367    H: LifecycleHooks,
368    E: ChildExecutor,
369> ChildExecution<'a, ChildFuture<Execution<'a, CS, A, X, C, T, H, E>>, E>
370{
371    fn start_local(
372        &mut self,
373        runtime: &WorkflowRunRuntime<CS, A, X, C, T, H, E>,
374        app: &'a AppRuntime<CS, A, X, C, T, H, E>,
375        lane: ExecutionLane,
376        children: Vec<RunRecord>,
377    ) -> Result<(), AppError> {
378        if tokio::time::Instant::now() >= self.deadline {
379            runtime.retain_source_failure(&self.context, &self.lineage, FailureCategory::Timeout);
380            self.state = ChildExecutionState::Complete(Err(FailureCategory::Timeout));
381            return Ok(());
382        }
383        let parent = self.parent.as_ref().ok_or_else(|| {
384            AppError::Integration("child group parent evidence is unavailable".into())
385        })?;
386        let definition = app.definition_for(parent)?;
387        let control = parent
388            .checkpoint
389            .control(self.sequence)
390            .ok_or_else(|| AppError::Conflict("child control is unavailable".into()))?;
391        let step = definition
392            .node(&control.node)
393            .ok_or_else(|| AppError::Conflict("child definition changed".into()))?
394            .step();
395        let (parallelism, _) = step
396            .child_plan()
397            .ok_or_else(|| AppError::Conflict("child definition changed".into()))?;
398        let slots = Arc::new(tokio::sync::Semaphore::new(parallelism));
399        let notification = runtime.notifier()?;
400        let mut tasks = Vec::with_capacity(children.len());
401        for record in children {
402            record.validate_child(parent, &definition)?;
403            let child = WorkflowRunRuntime::attach_record(Arc::clone(&runtime.app), &record)?;
404            if !record.checkpoint.is_terminal() {
405                child.cell.bind_child(runtime.key.clone())?;
406                child
407                    .cell
408                    .retain_child_observation(Arc::clone(&self.context))?;
409            }
410            tasks.push(ChildFuture::new(
411                Execution::owned(child, app, lane),
412                Arc::clone(parent),
413                self.sequence,
414                self.deadline,
415                Arc::clone(&slots),
416                Arc::clone(&notification),
417            ));
418        }
419        let batch = ChildBatch::new(tasks, self.context.mode)?;
420        self.state = if matches!(step, Step::Pipeline { .. }) {
421            ChildExecutionState::Pipeline(batch)
422        } else {
423            match catch_unwind(AssertUnwindSafe(|| app.child_executor().execute(batch))) {
424                Ok(future) => ChildExecutionState::Local(Box::pin(future)),
425                Err(_) => {
426                    runtime.retain_source_failure(
427                        &self.context,
428                        &self.lineage,
429                        FailureCategory::Execution,
430                    );
431                    ChildExecutionState::Complete(Err(FailureCategory::Execution))
432                }
433            }
434        };
435        Ok(())
436    }
437
438    fn stop(&mut self, runtime: &WorkflowRunRuntime<CS, A, X, C, T, H, E>) {
439        let previous = std::mem::replace(&mut self.state, ChildExecutionState::Closed);
440        if let ChildExecutionState::Complete(Err(category)) = &previous {
441            runtime.retain_source_failure(&self.context, &self.lineage, *category);
442        }
443        if catch_unwind(AssertUnwindSafe(|| drop(previous))).is_err() {
444            // An already retained source/E error remains primary in the
445            // original observation. This only supplies a destructor fallback.
446            runtime.retain_source_failure(&self.context, &self.lineage, FailureCategory::Execution);
447        }
448    }
449
450    fn poll_ready(
451        &mut self,
452        cx: &mut Context<'_>,
453        runtime: &WorkflowRunRuntime<CS, A, X, C, T, H, E>,
454    ) -> Poll<()> {
455        if matches!(
456            self.state,
457            ChildExecutionState::Complete(_)
458                | ChildExecutionState::Receipt(_)
459                | ChildExecutionState::Closed
460        ) {
461            return Poll::Ready(());
462        }
463        if matches!(
464            self.state,
465            ChildExecutionState::Awaiting | ChildExecutionState::Prepared(_)
466        ) {
467            return Poll::Pending;
468        }
469        if matches!(self.state, ChildExecutionState::Worker(_)) {
470            return self.poll_receipt(cx);
471        }
472        let result = if tokio::time::Instant::now() >= self.deadline {
473            Err(FailureCategory::Timeout)
474        } else {
475            match catch_unwind(AssertUnwindSafe(|| match &mut self.state {
476                ChildExecutionState::Local(future) => future.as_mut().poll(cx),
477                ChildExecutionState::Pipeline(batch) => Pin::new(batch).poll(cx),
478                _ => unreachable!(),
479            })) {
480                Ok(Poll::Pending) => return Poll::Pending,
481                Ok(Poll::Ready(result)) => result.map_err(|_| FailureCategory::Child),
482                Err(_) => Err(FailureCategory::Execution),
483            }
484        };
485        self.complete(runtime, result);
486        Poll::Ready(())
487    }
488
489    fn poll_receipt(&mut self, cx: &mut Context<'_>) -> Poll<()> {
490        let ChildExecutionState::Worker(receipt) = &mut self.state else {
491            unreachable!()
492        };
493        match Pin::new(receipt).poll(cx) {
494            Poll::Pending => Poll::Pending,
495            Poll::Ready(result) => {
496                self.state = ChildExecutionState::Receipt(Box::new(
497                    result
498                        .map_err(|_| {
499                            RuntimeOperationFailure::from(AppError::Closed(
500                                "child executor receipt closed".into(),
501                            ))
502                        })
503                        .and_then(|result| result),
504                ));
505                Poll::Ready(())
506            }
507        }
508    }
509
510    fn complete(
511        &mut self,
512        runtime: &WorkflowRunRuntime<CS, A, X, C, T, H, E>,
513        result: Result<Vec<ChildCompletion>, FailureCategory>,
514    ) {
515        if let Err(category) = result.as_ref() {
516            runtime.retain_source_failure(&self.context, &self.lineage, *category);
517        }
518        let previous = std::mem::replace(&mut self.state, ChildExecutionState::Complete(result));
519        if catch_unwind(AssertUnwindSafe(|| drop(previous))).is_err()
520            && matches!(&self.state, ChildExecutionState::Complete(Ok(_)))
521        {
522            runtime.retain_source_failure(&self.context, &self.lineage, FailureCategory::Execution);
523            self.state = ChildExecutionState::Complete(Err(FailureCategory::Execution));
524        }
525    }
526}
527
528struct ChildFuture<EX> {
529    execution: EX,
530    parent: Arc<RunRecord>,
531    sequence: u64,
532    slots: Arc<tokio::sync::Semaphore>,
533    permit: Option<tokio::sync::OwnedSemaphorePermit>,
534    notification: Arc<tokio::sync::Notify>,
535    wake: Pin<Box<tokio::sync::futures::OwnedNotified>>,
536    state: ChildFutureState,
537}
538
539enum ChildFutureState {
540    Admission,
541    Driving,
542    Reconcile(Option<oneshot::Receiver<Result<LoadedRun, AppError>>>),
543    Retry(tokio::time::Instant),
544    Finishing,
545    Waiting,
546    Finished,
547}
548
549impl<
550    'a,
551    CS: StateStoreBackend,
552    A: AgentBackend,
553    X: ActionBackend,
554    C: ConditionBackend,
555    T: StateTyping,
556    H: LifecycleHooks,
557    E: ChildExecutor,
558> ChildFuture<Execution<'a, CS, A, X, C, T, H, E>>
559{
560    fn new(
561        mut execution: Execution<'a, CS, A, X, C, T, H, E>,
562        parent: Arc<RunRecord>,
563        sequence: u64,
564        deadline: tokio::time::Instant,
565        slots: Arc<tokio::sync::Semaphore>,
566        notification: Arc<tokio::sync::Notify>,
567    ) -> Self {
568        let wake = Box::pin(Arc::clone(&notification).notified_owned());
569        execution.parent_deadline = Some(deadline);
570        Self {
571            execution,
572            parent,
573            sequence,
574            slots,
575            permit: None,
576            notification,
577            wake,
578            state: ChildFutureState::Admission,
579        }
580    }
581
582    fn poll_reconciliation(&mut self, cx: &mut Context<'_>) -> Poll<Result<(), AppError>> {
583        let ChildFutureState::Reconcile(receipt) = &mut self.state else {
584            unreachable!()
585        };
586        if receipt.is_none() {
587            // Finish an already admitted publication before sampling
588            // failure authority. Its older result must not restart
589            // caller work after the panic is reconciled.
590            match &mut self.execution.state {
591                SessionState::Publishing { receipt, .. } => {
592                    if Pin::new(receipt).poll(cx).is_pending() {
593                        return Poll::Pending;
594                    }
595                }
596                SessionState::ObservingItem { receipt, .. } => {
597                    if Pin::new(receipt).poll(cx).is_pending() {
598                        return Poll::Pending;
599                    }
600                }
601                _ => {}
602            }
603            self.execution.state = SessionState::Finished;
604            *receipt = Some(
605                self.execution
606                    .runtime
607                    .reconcile_child_panic(self.execution.parent_deadline),
608            );
609        }
610        let loaded = match Pin::new(receipt.as_mut().expect("reconciliation was admitted")).poll(cx)
611        {
612            Poll::Pending => return Poll::Pending,
613            Poll::Ready(Ok(result)) => result?,
614            Poll::Ready(Err(_)) => {
615                self.execution
616                    .runtime
617                    .app
618                    .tasks_failed(FinalResult::Integration(
619                        "child panic reconciliation receipt closed".into(),
620                    ));
621                return Poll::Ready(Err(AppError::Integration(
622                    "child panic reconciliation receipt closed".into(),
623                )));
624            }
625        };
626        self.execution.state = SessionState::Loaded(loaded);
627        self.state = ChildFutureState::Driving;
628        Poll::Ready(Ok(()))
629    }
630
631    fn poll_completion(
632        &mut self,
633        cx: &mut Context<'_>,
634    ) -> Poll<Result<Option<ChildCompletion>, AppError>> {
635        match self.execution.poll_publication(cx, false) {
636            Poll::Pending => Poll::Pending,
637            Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
638            Poll::Ready(Ok(Some(_))) => Poll::Ready(Ok(None)),
639            Poll::Ready(Ok(None)) => {
640                let SessionState::Loaded(loaded) =
641                    std::mem::replace(&mut self.execution.state, SessionState::Finished)
642                else {
643                    return Poll::Ready(Err(AppError::Integration(
644                        "child terminal observation is unavailable".into(),
645                    )));
646                };
647                if !loaded.record.checkpoint.is_terminal() {
648                    self.execution.state = SessionState::Loaded(loaded);
649                    self.state = ChildFutureState::Waiting;
650                    return Poll::Ready(Ok(None));
651                }
652                let definition = self.execution.runtime.app.definition_for(&self.parent)?;
653                let result = if matches!(
654                    loaded.record.checkpoint.phase,
655                    CheckpointPhaseRecord::Succeeded
656                ) {
657                    ChildCompletion::admitted(loaded.record, &self.parent, &definition)
658                } else {
659                    Err(AppError::Integration(
660                        "child workflow did not succeed".into(),
661                    ))
662                };
663                Poll::Ready(result.map(Some))
664            }
665        }
666    }
667}
668
669impl<
670    CS: StateStoreBackend,
671    A: AgentBackend,
672    X: ActionBackend,
673    C: ConditionBackend,
674    T: StateTyping,
675    H: LifecycleHooks,
676    E: ChildExecutor,
677> Future for ChildFuture<Execution<'_, CS, A, X, C, T, H, E>>
678{
679    type Output = Result<ChildCompletion, AppError>;
680
681    fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
682        let child = self.get_mut();
683        let observed = (|| -> Poll<Result<ChildCompletion, AppError>> {
684            loop {
685                match child.state {
686                    ChildFutureState::Admission => {
687                        child.wake.as_mut().enable();
688                        match Arc::clone(&child.slots).try_acquire_owned() {
689                            Ok(permit) => {
690                                child.permit = Some(permit);
691                                child.state = ChildFutureState::Driving;
692                            }
693                            Err(tokio::sync::TryAcquireError::NoPermits) => {
694                                if child.wake.as_mut().poll(cx).is_pending() {
695                                    return Poll::Pending;
696                                }
697                                child.wake =
698                                    Box::pin(Arc::clone(&child.notification).notified_owned());
699                            }
700                            Err(tokio::sync::TryAcquireError::Closed) => {
701                                return Poll::Ready(Err(AppError::Closed(
702                                    "child execution capacity is closed".into(),
703                                )));
704                            }
705                        }
706                    }
707                    ChildFutureState::Reconcile(_) => match child.poll_reconciliation(cx) {
708                        Poll::Pending => return Poll::Pending,
709                        Poll::Ready(result) => result?,
710                    },
711                    ChildFutureState::Retry(deadline) => {
712                        if !child.execution.poll_timer(cx, Some(deadline)) {
713                            return Poll::Pending;
714                        }
715                        child.execution.state = SessionState::Ready;
716                        child.state = ChildFutureState::Driving;
717                    }
718                    ChildFutureState::Driving => {
719                        let observed = catch_unwind(AssertUnwindSafe(|| {
720                            child.execution.poll_advance(cx, true)
721                        }));
722                        match observed {
723                            Err(_) => {
724                                // Unrelated wakes cannot re-admit this child before
725                                // its original failure publication is authoritative.
726                                child.state = ChildFutureState::Reconcile(None);
727                            }
728                            Ok(Poll::Pending) => return Poll::Pending,
729                            Ok(Poll::Ready(Err(error))) => {
730                                if let Some(delay) = child.execution.runtime.cell.complete_drive() {
731                                    child.state = ChildFutureState::Retry(
732                                        tokio::time::Instant::now() + delay,
733                                    );
734                                } else {
735                                    return Poll::Ready(Err(error));
736                                }
737                            }
738                            Ok(Poll::Ready(Ok(Some(_)))) => continue,
739                            Ok(Poll::Ready(Ok(None))) => {
740                                child.execution.publish(
741                                    SessionPublication::FinishChild {
742                                        parent: Arc::clone(&child.parent),
743                                        sequence: child.sequence,
744                                    },
745                                    false,
746                                )?;
747                                child.state = ChildFutureState::Finishing;
748                            }
749                        }
750                    }
751                    ChildFutureState::Finishing => match child.poll_completion(cx) {
752                        Poll::Pending => return Poll::Pending,
753                        Poll::Ready(Err(error)) => return Poll::Ready(Err(error)),
754                        Poll::Ready(Ok(Some(completion))) => return Poll::Ready(Ok(completion)),
755                        Poll::Ready(Ok(None)) => continue,
756                    },
757                    ChildFutureState::Waiting => {
758                        // A child's public Input can pause its lifecycle, but the
759                        // same admitted batch future remains owned by its parent.
760                        if !child.execution.poll_notification(cx)? {
761                            return Poll::Pending;
762                        }
763                        child.execution.state = SessionState::Ready;
764                        child.state = ChildFutureState::Driving;
765                    }
766                    ChildFutureState::Finished => {
767                        return Poll::Ready(Err(AppError::Conflict(
768                            "child future already completed".into(),
769                        )));
770                    }
771                }
772            }
773        })();
774        if observed.is_ready() {
775            child.release();
776            child.state = ChildFutureState::Finished;
777        }
778        observed
779    }
780}
781
782impl<EX> ChildFuture<EX> {
783    fn release(&mut self) {
784        if self.permit.take().is_some() {
785            self.notification.notify_waiters();
786        }
787    }
788}
789
790impl<EX> Drop for ChildFuture<EX> {
791    fn drop(&mut self) {
792        self.release();
793    }
794}
795
796enum SessionState {
797    Ready,
798    Loaded(LoadedRun),
799    ActivationReady {
800        loaded: LoadedRun,
801        activation: FrontierRecord,
802        context: Arc<ActionContext>,
803        remaining: std::vec::IntoIter<FrontierRecord>,
804        staged: Vec<StagedActivationRecord>,
805        deadline: tokio::time::Instant,
806        deadline_millis: u64,
807    },
808    Sources {
809        loaded: LoadedRun,
810    },
811    ObservingItem {
812        loaded: LoadedRun,
813        index: usize,
814        receipt: oneshot::Receiver<Result<Option<Map<String, Value>>, AppError>>,
815    },
816    Publishing {
817        receipt: oneshot::Receiver<Result<SessionReceipt, RuntimeOperationFailure>>,
818        purpose: PublicationPurpose,
819    },
820    Eof,
821    Finished,
822}
823
824enum PublicationPurpose {
825    Load,
826    Reload,
827    ObserveChildren,
828    AdmitChildren,
829    VerifyChildren,
830    Start {
831        activation: FrontierRecord,
832        context: Arc<ActionContext>,
833        remaining: std::vec::IntoIter<FrontierRecord>,
834        deadline: tokio::time::Instant,
835        deadline_millis: u64,
836    },
837    Continue {
838        remaining: std::vec::IntoIter<FrontierRecord>,
839        ordinary: bool,
840    },
841    Descendants {
842        context: Arc<ActionContext>,
843        lineage: String,
844        inputs: Vec<Map<String, Value>>,
845        remaining: std::vec::IntoIter<FrontierRecord>,
846        ordinary: bool,
847    },
848    Commit {
849        ordinary: bool,
850    },
851    Advance,
852    EndSource,
853}
854
855pub(crate) enum SessionReceipt {
856    Checkpoint {
857        loaded: LoadedRun,
858        staged: Vec<StagedActivationRecord>,
859    },
860    Children {
861        loaded: LoadedRun,
862        sequence: u64,
863        children: Vec<RunRecord>,
864        verified: bool,
865    },
866}
867
868impl SessionReceipt {
869    fn record(&self) -> &RunRecord {
870        match self {
871            Self::Checkpoint { loaded, .. } | Self::Children { loaded, .. } => &loaded.record,
872        }
873    }
874
875    fn with_loaded(self, loaded: LoadedRun) -> Self {
876        match self {
877            Self::Checkpoint { staged, .. } => Self::Checkpoint { loaded, staged },
878            Self::Children {
879                sequence,
880                children,
881                verified,
882                ..
883            } => Self::Children {
884                loaded,
885                sequence,
886                children,
887                verified,
888            },
889        }
890    }
891}
892
893enum SessionPublication {
894    ObserveChildren {
895        loaded: LoadedRun,
896        sequence: u64,
897    },
898    VerifyChildren {
899        parent: Arc<RunRecord>,
900        sequence: u64,
901        completions: Vec<ChildCompletion>,
902    },
903    ChildOutcome {
904        control: ControlRecord,
905        transition: ControlTransition,
906    },
907    FinishChild {
908        parent: Arc<RunRecord>,
909        sequence: u64,
910    },
911    SettleStopping {
912        loaded: LoadedRun,
913    },
914    Load {
915        terminal: Option<u64>,
916    },
917    Reload {
918        receipt: SessionReceipt,
919        deadline: Option<tokio::time::Instant>,
920    },
921    BeginActivation {
922        loaded: LoadedRun,
923        activation: FrontierRecord,
924        context: Arc<ActionContext>,
925        remaining: std::vec::IntoIter<FrontierRecord>,
926        staged: Vec<StagedActivationRecord>,
927        deadline: tokio::time::Instant,
928        deadline_millis: u64,
929    },
930    Decision {
931        loaded: LoadedRun,
932        decision: BspDecision,
933        activation_deadline: Option<u64>,
934        context: Option<Arc<ActionContext>>,
935        staged: Vec<StagedActivationRecord>,
936        remaining: std::vec::IntoIter<FrontierRecord>,
937    },
938    Transition {
939        loaded: LoadedRun,
940        transition: ControlTransition,
941    },
942    Retry {
943        loaded: LoadedRun,
944    },
945    EndSource {
946        loaded: LoadedRun,
947    },
948    Release,
949}
950
951impl<
952    'a,
953    CS: StateStoreBackend,
954    A: AgentBackend,
955    X: ActionBackend,
956    C: ConditionBackend,
957    T: StateTyping,
958    H: LifecycleHooks,
959    E: ChildExecutor,
960> Execution<'a, CS, A, X, C, T, H, E>
961{
962    pub(crate) async fn open(
963        run: &'a mut WorkflowRun<CS, A, X, C, T, H, E>,
964        mut options: ExecutionOptions,
965        mode: ActionMode,
966    ) -> Result<Self, AppError> {
967        let admission = run.runtime.app.reserve_caller()?;
968        if !options.state.values().is_empty() {
969            return Err(AppError::InvalidInput(
970                "existing execution cannot replace initial state".into(),
971            ));
972        }
973        JsonBoundary::validate_inputs(&options.inputs)?;
974        run.runtime.app.ensure_open().await?;
975        let loaded = run.runtime.load().await?;
976        let definition = run.runtime.app.definition_for(&loaded.record)?;
977        let graph = run.runtime.app.graph(&definition);
978        options.resolve_selectors(&graph)?;
979        let pending_controls = !loaded.record.checkpoint.controls.is_empty();
980        let discard_staged = loaded.record.checkpoint.rollback_stage_keys(&definition);
981        let generation = run.runtime.cell.claim_execution(mode)?;
982        run.runtime.cell.configure_execution(generation, options)?;
983        let mut execution = Self {
984            runtime: run.runtime.clone(),
985            app: &run.runtime.app,
986            lane: ExecutionLane::Caller,
987            admission: Some(admission),
988            children: Vec::new(),
989            wake: None,
990            timer: None,
991            generation: Some(generation),
992            parent_deadline: None,
993            state: SessionState::Ready,
994            sources: Vec::new(),
995            next_source: 0,
996            native_partials: None,
997            prefetched: None,
998            transitions: 0,
999            observed: Some(loaded.record.clone()),
1000        };
1001        if loaded.record.checkpoint.can_retry_failed() {
1002            execution.publish(SessionPublication::Retry { loaded }, false)?;
1003            poll_fn(|cx| execution.poll_publication(cx, false)).await?;
1004        } else if !loaded.record.checkpoint.is_terminal()
1005            && (loaded.record.checkpoint.admission != RunAdmission::Prepared
1006                || !discard_staged.is_empty())
1007        {
1008            execution.publish_transition(loaded, ControlTransition::Pause { discard_staged })?;
1009            poll_fn(|cx| execution.poll_publication(cx, false)).await?;
1010        }
1011        // Acquisition is bounded; a returned lazy execution owns no caller slot.
1012        drop(execution.admission.take());
1013        execution.runtime.cell.admit_lifecycle(generation)?;
1014        if pending_controls {
1015            execution.runtime.schedule(ScheduleCause::Reconcile);
1016        }
1017        Ok(execution)
1018    }
1019
1020    pub(super) fn background(run: &'a mut WorkflowRun<CS, A, X, C, T, H, E>) -> Self {
1021        Self::owned(run.runtime.clone(), &run.runtime.app, ExecutionLane::Worker)
1022    }
1023
1024    fn owned(
1025        runtime: WorkflowRunRuntime<CS, A, X, C, T, H, E>,
1026        app: &'a AppRuntime<CS, A, X, C, T, H, E>,
1027        lane: ExecutionLane,
1028    ) -> Self {
1029        Self {
1030            runtime,
1031            app,
1032            lane,
1033            admission: None,
1034            children: Vec::new(),
1035            wake: None,
1036            timer: None,
1037            generation: None,
1038            parent_deadline: None,
1039            state: SessionState::Ready,
1040            sources: Vec::new(),
1041            next_source: 0,
1042            native_partials: None,
1043            prefetched: None,
1044            transitions: 0,
1045            observed: None,
1046        }
1047    }
1048
1049    /// Advances lazily to one partial result or committed boundary.
1050    ///
1051    /// Partials are transient; only Committed events publish state/outcomes. Streaming
1052    /// iteration checks halt-after-prior before halt-before-next. A retained halt is
1053    /// not reclassified by a later step/finish call. EOF without Completed commits
1054    /// no action and ends this observation without automatically retrying it.
1055    ///
1056    /// # Errors
1057    ///
1058    /// Returns `Closed` or `Conflict` when execution authority is revoked or stale,
1059    /// `Integration` for unavailable durable/lower evidence, and `Timeout` for expired
1060    /// observation. Invalid execution input may return `InvalidInput`; accepted action
1061    /// failures are retained in the returned run state. Cleanup preserves primary
1062    /// failures and does not detach an admitted native commit.
1063    ///
1064    /// See [the explicit execution workflow](crate::workflow#execute-observe-restore-and-delete).
1065    pub async fn next(&mut self) -> Result<Option<ExecutionEvent>, AppError> {
1066        let event = poll_fn(|cx| self.poll_advance(cx, false)).await;
1067        if (matches!(&event, Ok(Some(ExecutionEvent::Committed { .. })))
1068            || matches!(&event, Ok(None)) && matches!(&self.state, SessionState::Eof)
1069            || event.is_err() && self.runtime.cell.lifecycle_generation().is_none())
1070            && let Some(generation) = self.generation
1071        {
1072            self.runtime.cell.returned_boundary(generation);
1073        }
1074        event
1075    }
1076
1077    fn poll_advance(
1078        &mut self,
1079        cx: &mut Context<'_>,
1080        ordinary: bool,
1081    ) -> Poll<Result<Option<ExecutionEvent>, AppError>> {
1082        loop {
1083            // Sources polls once after starting newly prepared groups. Keep
1084            // its pre-start authority check, but do not double-poll pending
1085            // descendants at every nested level.
1086            if !matches!(&self.state, SessionState::Sources { .. })
1087                || self
1088                    .children
1089                    .iter()
1090                    .any(|group| matches!(group.state, ChildExecutionState::Prepared(_)))
1091            {
1092                self.poll_child_groups(cx)?;
1093            }
1094            match &mut self.state {
1095                SessionState::ObservingItem { .. } => match self.poll_item(cx) {
1096                    Poll::Pending => return Poll::Pending,
1097                    Poll::Ready(Err(error)) => return Poll::Ready(Err(error)),
1098                    Poll::Ready(Ok(Some(event))) if !ordinary => {
1099                        return Poll::Ready(Ok(Some(event)));
1100                    }
1101                    Poll::Ready(Ok(_)) => continue,
1102                },
1103                SessionState::Finished => return Poll::Ready(Ok(None)),
1104                SessionState::Eof => self.state = SessionState::Ready,
1105                SessionState::ActivationReady { .. } => self.activate_ready(ordinary)?,
1106                SessionState::Publishing { .. } => {
1107                    match self.poll_publication(cx, false) {
1108                        Poll::Pending => return Poll::Pending,
1109                        Poll::Ready(Err(error)) => return Poll::Ready(Err(error)),
1110                        Poll::Ready(Ok(Some(event))) => {
1111                            if ordinary && matches!(event, ExecutionEvent::Partial { .. }) {
1112                                continue;
1113                            }
1114                            return Poll::Ready(Ok(Some(event)));
1115                        }
1116                        Poll::Ready(Ok(None)) => {}
1117                    }
1118                    if matches!(&self.state, SessionState::Eof) {
1119                        return Poll::Ready(Ok(None));
1120                    }
1121                }
1122                SessionState::Ready => {
1123                    self.ensure_owned()?;
1124                    self.arm_notification()?;
1125                    let terminal = self
1126                        .observed
1127                        .as_ref()
1128                        .filter(|record| {
1129                            self.generation.is_none() && record.checkpoint.is_terminal()
1130                        })
1131                        .map(|record| record.checkpoint.sequence);
1132                    self.publish(SessionPublication::Load { terminal }, ordinary)?;
1133                }
1134                SessionState::Loaded(_) => {
1135                    if self.advance_loaded(ordinary)? {
1136                        return Poll::Ready(Ok(None));
1137                    }
1138                }
1139                SessionState::Sources { .. } => {
1140                    self.start_child_groups(cx)?;
1141                    if !matches!(self.state, SessionState::Sources { .. }) {
1142                        continue;
1143                    }
1144                    self.poll_child_groups(cx)?;
1145                    if self.publish_child_completion(ordinary, false)? {
1146                        continue;
1147                    }
1148                    if self.prefetched.is_none()
1149                        && !self.sources.is_empty()
1150                        && self.sources.iter().all(|source| {
1151                            matches!(
1152                                source.state,
1153                                CallerSourceState::Ended | CallerSourceState::Eof
1154                            )
1155                        })
1156                    {
1157                        let SessionState::Sources { loaded, .. } = &self.state else {
1158                            unreachable!()
1159                        };
1160                        if !self.retains_external_continuation(&loaded.record) {
1161                            self.finish_wave(ordinary)?;
1162                            continue;
1163                        }
1164                    }
1165                    if self.prefetched.is_none() {
1166                        match self.poll_sources_observation(cx) {
1167                            Poll::Pending => return Poll::Pending,
1168                            Poll::Ready(Err(error)) => return Poll::Ready(Err(error)),
1169                            Poll::Ready(Ok(Some(event))) if !ordinary => {
1170                                return Poll::Ready(Ok(Some(event)));
1171                            }
1172                            Poll::Ready(Ok(Some(_))) => continue,
1173                            Poll::Ready(Ok(None)) => {}
1174                        }
1175                        if !matches!(&self.state, SessionState::Sources { .. }) {
1176                            continue;
1177                        }
1178                    }
1179                    if let Some((index, event)) = self.prefetched.take() {
1180                        self.consume_source_event(index, event, ordinary)?;
1181                    } else {
1182                        return Poll::Ready(Ok(None));
1183                    }
1184                }
1185            }
1186        }
1187    }
1188
1189    fn advance_loaded(&mut self, ordinary: bool) -> Result<bool, AppError> {
1190        let SessionState::Loaded(loaded) =
1191            std::mem::replace(&mut self.state, SessionState::Finished)
1192        else {
1193            unreachable!()
1194        };
1195        self.ensure_owned()?;
1196        self.runtime.supervise_controls(&loaded.record);
1197        if loaded.record.checkpoint.is_stopping() {
1198            self.stop_children();
1199            self.stop_sources();
1200            self.publish(SessionPublication::SettleStopping { loaded }, ordinary)?;
1201            return Ok(false);
1202        }
1203        if loaded.record.checkpoint.is_terminal() {
1204            self.stop_children();
1205            self.stop_sources();
1206            self.observed = Some(loaded.record);
1207            return Ok(true);
1208        }
1209        self.observed = Some(loaded.record.clone());
1210        if self.children.iter().any(|group| {
1211            matches!(
1212                group.state,
1213                ChildExecutionState::Worker(_) | ChildExecutionState::Receipt(_)
1214            ) && group
1215                .parent
1216                .as_ref()
1217                .is_some_and(|parent| !loaded.record.owns_child_activation(parent, group.sequence))
1218        }) {
1219            // A worker may have staged its result before its final
1220            // receipt arrives. Keep that original receipt ahead of
1221            // parent evaluation, halting and barrier publication.
1222            self.state = SessionState::Sources { loaded };
1223            return Ok(false);
1224        }
1225        if self.generation.is_none()
1226            && self.parent_deadline.is_none()
1227            && loaded.record.checkpoint.is_paused()
1228        {
1229            // Background cleanup may finish already admitted children, but
1230            // their receipt cannot authorize a later prepared parent barrier.
1231            self.state = SessionState::Sources { loaded };
1232            return Ok(true);
1233        }
1234        if self.runtime.cell.execution_boundary()? == Some(false) && self.halt_after(&loaded)? {
1235            return Ok(false);
1236        }
1237        if self.transitions >= MAX_DRIVER_TRANSITIONS {
1238            self.publish_decision(
1239                loaded,
1240                BspDecision::Failed {
1241                    category: FailureCategory::Limit,
1242                    message: "workflow transition bound exhausted".into(),
1243                    observation: None,
1244                },
1245                ordinary,
1246            )?;
1247            return Ok(false);
1248        }
1249        let mut decision = match self.runtime.decide_checkpoint(&loaded) {
1250            Ok(decision) => decision,
1251            Err(failure) => {
1252                if self.generation.is_none() && self.runtime.schedule_operation_failure(&failure) {
1253                    self.observed = None;
1254                }
1255                return Err(failure.into_error());
1256            }
1257        };
1258        if let BspDecision::Activate(selected) = &mut decision {
1259            selected.retain(|activation| {
1260                !self
1261                    .sources
1262                    .iter()
1263                    .any(|source| source.activation == *activation)
1264            });
1265            if selected.is_empty() {
1266                decision = BspDecision::AwaitControls;
1267            }
1268        }
1269        let boundary = self.runtime.cell.execution_boundary()?;
1270        if boundary.is_some()
1271            && let Some(node) =
1272                decision.pending_node(&self.runtime.cell.execution_options()?.halt_before)
1273        {
1274            let node = node.to_owned();
1275            self.publish_transition(loaded, ControlTransition::Halt { node, result: None })?;
1276            return Ok(false);
1277        }
1278        if boundary == Some(true)
1279            && !matches!(&decision, BspDecision::Routed { .. })
1280            && self.halt_after(&loaded)?
1281        {
1282            return Ok(false);
1283        }
1284        if !matches!(&decision, BspDecision::Routed { .. }) {
1285            self.runtime.cell.clear_execution_boundary()?;
1286        }
1287        match decision {
1288            BspDecision::AwaitControls => {
1289                let pending = loaded
1290                    .record
1291                    .checkpoint
1292                    .controls
1293                    .values()
1294                    .find(|control| {
1295                        matches!(control.state, ControlRecordState::Child { .. })
1296                            && !self.children.iter().any(|group| {
1297                                group.sequence == control.sequence
1298                                    && !matches!(group.state, ChildExecutionState::Awaiting)
1299                            })
1300                    })
1301                    .map(|control| control.sequence);
1302                if let Some(sequence) = pending {
1303                    self.publish(
1304                        SessionPublication::ObserveChildren { loaded, sequence },
1305                        ordinary,
1306                    )?;
1307                } else {
1308                    self.state = SessionState::Sources { loaded };
1309                }
1310            }
1311            BspDecision::Activate(selected) => {
1312                self.transitions += 1;
1313                let mut remaining = selected.into_iter();
1314                let activation = remaining.next().ok_or_else(|| {
1315                    AppError::Integration("native selected prefix is empty".into())
1316                })?;
1317                self.begin_activation(loaded, activation, remaining, Vec::new(), ordinary)?;
1318            }
1319            decision => {
1320                self.transitions += 1;
1321                self.publish_decision(loaded, decision, ordinary)?;
1322            }
1323        }
1324        Ok(false)
1325    }
1326
1327    /// Drains transient partials through one committed boundary.
1328    ///
1329    /// Consumes partials until a committed boundary or halt. Ordinary iteration
1330    /// checks halt-before-next before halt-after-prior. Only a committed action
1331    /// advances its ordinal; control checkpoints and partials do not.
1332    ///
1333    /// # Errors
1334    ///
1335    /// Returns `Closed` or `Conflict` when execution authority is revoked or stale,
1336    /// `Integration` for unavailable durable/lower evidence, and `Timeout` for expired
1337    /// observation. Invalid execution input may return `InvalidInput`; accepted action
1338    /// failures are retained in the returned run state. Cleanup preserves primary
1339    /// failures and does not detach an admitted native commit.
1340    ///
1341    /// See [the explicit execution workflow](crate::workflow#execute-observe-restore-and-delete).
1342    pub async fn step(&mut self) -> Result<Option<ExecutionEvent>, AppError> {
1343        loop {
1344            match poll_fn(|cx| self.poll_advance(cx, true)).await {
1345                Ok(Some(ExecutionEvent::Partial { .. })) => {}
1346                event => {
1347                    if (matches!(&event, Ok(Some(ExecutionEvent::Committed { .. })))
1348                        || matches!(&event, Ok(None)) && matches!(&self.state, SessionState::Eof)
1349                        || event.is_err() && self.runtime.cell.lifecycle_generation().is_none())
1350                        && let Some(generation) = self.generation
1351                    {
1352                        self.runtime.cell.returned_boundary(generation);
1353                    }
1354                    return event;
1355                }
1356            }
1357        }
1358    }
1359
1360    /// Drains this execution to exhaustion, a halt or an actionable control.
1361    ///
1362    /// Consumes this owner and drains advancement until halt, actionable parking, EOF
1363    /// or exhaustion, then finalizes observation. A returned snapshot need not be
1364    /// Succeeded: inspect status and interactions. Accepted child/native cleanup
1365    /// retains its original authority and deadlines.
1366    ///
1367    /// # Errors
1368    ///
1369    /// Returns `Closed` or `Conflict` when execution authority is revoked or stale,
1370    /// `Integration` for unavailable durable/lower evidence, and `Timeout` for expired
1371    /// observation. Invalid execution input may return `InvalidInput`; accepted action
1372    /// failures are retained in the returned run state. Cleanup preserves primary
1373    /// failures and does not detach an admitted native commit.
1374    ///
1375    /// See [the explicit execution workflow](crate::workflow#execute-observe-restore-and-delete).
1376    pub async fn finish(mut self) -> Result<RunSnapshot, AppError> {
1377        if !matches!(&self.state, SessionState::Eof) {
1378            while poll_fn(|cx| self.poll_advance(cx, true)).await?.is_some() {}
1379        }
1380        self.close().await
1381    }
1382
1383    /// Discards uncommitted caller work after joining admitted publication.
1384    ///
1385    /// Consumes this owner and finalizes its uncommitted boundary without requesting
1386    /// durable run cancellation. An already admitted native publication retains its
1387    /// commit authority; abort is not a guarantee that external work was rolled back.
1388    ///
1389    /// # Errors
1390    ///
1391    /// Returns `Closed` or `Conflict` when execution authority is revoked or stale,
1392    /// `Integration` for unavailable durable/lower evidence, and `Timeout` for expired
1393    /// observation. Invalid execution input may return `InvalidInput`; accepted action
1394    /// failures are retained in the returned run state. Cleanup preserves primary
1395    /// failures and does not detach an admitted native commit.
1396    ///
1397    /// # Example
1398    ///
1399    /// ```
1400    /// # use agents_kit::Agent;
1401    /// # use app_kit::{App, AppConfig, Shutdown, WorkspaceConfig};
1402    /// # use app_kit::workflow::{CheckpointSelector, ExecutionEvent, ExecutionOptions, RunSelector,
1403    /// #     RunStatus, WorkflowDefinition, WorkflowState};
1404    /// # use storage_kit::StateStore;
1405    /// # use storage_kit::state::config::{LanceStateStoreConfig, StateStoreConfig};
1406    /// #
1407    /// # let runtime = tokio::runtime::Builder::new_current_thread().enable_all().build()?;
1408    /// # runtime.block_on(async {
1409    /// # let store = StateStore::connect(StateStoreConfig::Lance(LanceStateStoreConfig::memory())).await?;
1410    /// # let definition = WorkflowDefinition::parse(r#"{
1411    /// #     "schema":"app-kit.workflow/v2", "id":"counter", "revision":"r1",
1412    /// #     "state":{"count":0}, "required":["count"],
1413    /// #     "reducers":{"count":"reject_concurrent"}, "entry":"increment",
1414    /// #     "nodes":[{"id":"increment","step":{"kind":"state","mutations":[
1415    /// #         {"kind":"increment","target":"count","value":{"kind":"literal","value":1}}
1416    /// #     ]}}], "routes":{}
1417    /// # }"#)?;
1418    /// # let config = AppConfig::<_, Agent>::new(store, Vec::new(), vec![definition]);
1419    /// # let mut app = App::open(config).await?;
1420    /// # let outcome = async {
1421    /// #     let mut workspace = app.create_workspace(WorkspaceConfig {
1422    /// #         name: Some("local counter".into()), root: std::env::current_dir()?,
1423    /// #     }).await?;
1424    /// #     let workflow = workspace.workflow("counter".parse()?).await?;
1425    /// #     let mut run = workflow.prepare(WorkflowState::default()).await?;
1426    ///     let execution = run.execution(ExecutionOptions::default()).await?;
1427    ///     let parked = execution.abort().await?;
1428    ///     assert_eq!(parked.state.values()["count"], 0);
1429    ///     assert!(parked.last_action_ordinal.is_none());
1430    ///     let completed = run.execution(ExecutionOptions::default()).await?.finish().await?;
1431    ///     assert_eq!(completed.state.values()["count"], 1);
1432    ///     #
1433    /// #     Ok::<(), Box<dyn std::error::Error>>(())
1434    /// # }.await;
1435    /// # let cleanup = app.shutdown(Shutdown::CancelAndDrain { timeout: std::time::Duration::from_secs(5) }).await;
1436    /// # if let Err(error) = &cleanup { eprintln!("App shutdown: {error}"); }
1437    /// # outcome?;
1438    /// # cleanup?;
1439    /// # Ok::<(), Box<dyn std::error::Error>>(())
1440    /// # })?;
1441    /// # Ok::<(), Box<dyn std::error::Error>>(())
1442    /// ```
1443    pub async fn abort(mut self) -> Result<RunSnapshot, AppError> {
1444        self.close().await
1445    }
1446
1447    async fn close(&mut self) -> Result<RunSnapshot, AppError> {
1448        self.stop_sources();
1449        self.children.retain(|group| {
1450            !matches!(
1451                group.state,
1452                ChildExecutionState::Awaiting | ChildExecutionState::Prepared(_)
1453            )
1454        });
1455        poll_fn(|cx| -> Poll<Result<(), AppError>> {
1456            loop {
1457                self.poll_child_groups(cx)?;
1458                let deadline = self.source_deadline(None);
1459                if self.poll_timer(cx, deadline) {
1460                    self.poll_child_groups(cx)?;
1461                }
1462                if matches!(self.state, SessionState::Publishing { .. }) {
1463                    match self.poll_publication(cx, true) {
1464                        Poll::Pending => return Poll::Pending,
1465                        Poll::Ready(result) => {
1466                            result?;
1467                            continue;
1468                        }
1469                    }
1470                }
1471                if let SessionState::ObservingItem { receipt, .. } = &mut self.state {
1472                    let observed = match Pin::new(receipt).poll(cx) {
1473                        Poll::Pending => return Poll::Pending,
1474                        Poll::Ready(result) => result.map_err(|_| {
1475                            AppError::Closed("stream observation receipt closed".into())
1476                        })?,
1477                    };
1478                    self.finish_item(observed)?;
1479                    continue;
1480                }
1481                if self.children.is_empty() {
1482                    return Poll::Ready(Ok(()));
1483                }
1484                if let SessionState::Loaded(_) = &self.state {
1485                    let SessionState::Loaded(loaded) =
1486                        std::mem::replace(&mut self.state, SessionState::Finished)
1487                    else {
1488                        unreachable!()
1489                    };
1490                    self.state = SessionState::Sources { loaded };
1491                }
1492                if matches!(self.state, SessionState::Sources { .. }) {
1493                    if self.publish_child_completion(false, true)? {
1494                        continue;
1495                    }
1496                    // Observe cancellation under the original receipt/deadline;
1497                    // no new group or source can be constructed during close.
1498                    if self.poll_notification(cx)? {
1499                        let SessionState::Sources { loaded } =
1500                            std::mem::replace(&mut self.state, SessionState::Finished)
1501                        else {
1502                            unreachable!()
1503                        };
1504                        self.publish(
1505                            SessionPublication::Reload {
1506                                receipt: SessionReceipt::Checkpoint {
1507                                    loaded,
1508                                    staged: Vec::new(),
1509                                },
1510                                deadline,
1511                            },
1512                            false,
1513                        )?;
1514                        continue;
1515                    }
1516                }
1517                return Poll::Pending;
1518            }
1519        })
1520        .await?;
1521        if !self.sources.is_empty() {
1522            if !matches!(&self.state, SessionState::Sources { .. }) {
1523                let deadline = LifecycleEvent::deadline(self.parent_deadline);
1524                let loaded = if matches!(&self.state, SessionState::Loaded(_)) {
1525                    // Closing an already-admitted publication kept its exact
1526                    // receipt. Preserve the generation fence without decoding
1527                    // that same checkpoint again.
1528                    self.runtime
1529                        .observe_generation(deadline)
1530                        .await
1531                        .map_err(|failure| {
1532                            self.runtime.retain_execution_read_failure(&failure);
1533                            failure.into_error()
1534                        })?;
1535                    let SessionState::Loaded(loaded) =
1536                        std::mem::replace(&mut self.state, SessionState::Finished)
1537                    else {
1538                        unreachable!()
1539                    };
1540                    loaded
1541                } else {
1542                    self.runtime
1543                        .load_execution_observation(None, Some(deadline))
1544                        .await
1545                        .map_err(|failure| {
1546                            self.runtime.retain_execution_read_failure(&failure);
1547                            failure.into_error()
1548                        })?
1549                        .ok_or_else(|| AppError::NotFound("workflow run does not exist".into()))?
1550                };
1551                self.state = SessionState::Sources { loaded };
1552            }
1553            let buffered_failure = self
1554                .prefetched
1555                .take()
1556                .and_then(|(index, event)| event.err().map(|category| (index, category)));
1557            for index in 0..self.sources.len() {
1558                let source = &mut self.sources[index];
1559                let failure = buffered_failure
1560                    .filter(|(buffered, _)| *buffered == index)
1561                    .map(|(_, category)| category)
1562                    .or(match source.state {
1563                        CallerSourceState::Failed(category) => Some(category),
1564                        _ => None,
1565                    });
1566                source.state = CallerSourceState::Ended;
1567                if let Some(category) = failure {
1568                    self.stage_source_failure(index, category, false)?;
1569                    poll_fn(|cx| self.poll_publication(cx, true)).await?;
1570                    self.runtime
1571                        .observe_generation(LifecycleEvent::deadline(self.parent_deadline))
1572                        .await
1573                        .map_err(|failure| {
1574                            self.runtime.retain_execution_read_failure(&failure);
1575                            failure.into_error()
1576                        })?;
1577                    let SessionState::Loaded(loaded) =
1578                        std::mem::replace(&mut self.state, SessionState::Finished)
1579                    else {
1580                        unreachable!("closing publication retains its original checkpoint receipt")
1581                    };
1582                    self.state = SessionState::Sources { loaded };
1583                }
1584            }
1585            let SessionState::Sources { loaded, .. } = &self.state else {
1586                unreachable!()
1587            };
1588            if loaded
1589                .record
1590                .checkpoint
1591                .staged
1592                .iter()
1593                .any(|activation| matches!(activation.outcome, StagedOutcomeRecord::Failed { .. }))
1594            {
1595                self.finish_wave(false)?;
1596                poll_fn(|cx| self.poll_publication(cx, true)).await?;
1597            }
1598        }
1599        if let SessionState::Sources { loaded, .. } =
1600            std::mem::replace(&mut self.state, SessionState::Finished)
1601        {
1602            self.observed = Some(loaded.record);
1603        }
1604        self.sources.clear();
1605        let foreground = self.generation.is_some();
1606        if foreground {
1607            self.publish(SessionPublication::Release, false)?;
1608            poll_fn(|cx| self.poll_publication(cx, true)).await?;
1609            self.state = SessionState::Finished;
1610            self.generation = None;
1611        }
1612        let snapshot = if foreground {
1613            self.runtime.snapshot().await?
1614        } else {
1615            self.runtime.app.ensure_snapshot_read()?;
1616            let terminal = self
1617                .observed
1618                .as_ref()
1619                .filter(|record| record.checkpoint.is_terminal())
1620                .map(|record| record.checkpoint.sequence);
1621            let loaded = self
1622                .runtime
1623                .load_execution_observation(terminal, self.parent_deadline)
1624                .await
1625                .map_err(|failure| {
1626                    self.runtime.retain_execution_read_failure(&failure);
1627                    failure.into_error()
1628                })?
1629                .ok_or_else(|| AppError::NotFound("workflow run does not exist".into()))?;
1630            let mut deadline = LifecycleEvent::deadline(self.parent_deadline);
1631            if loaded.record.checkpoint.is_terminal()
1632                && let Some(generation) = self.runtime.cell.lifecycle_generation()
1633            {
1634                deadline = self
1635                    .runtime
1636                    .cell
1637                    .begin_action_cleanup(generation)?
1638                    .min(deadline);
1639            }
1640            if !self.retains_execution_lifecycle(&loaded.record)
1641                && (loaded.record.checkpoint.controls.is_empty()
1642                    || loaded.record.checkpoint.is_public_outcome())
1643            {
1644                self.runtime
1645                    .finish_execution_observation(
1646                        &loaded.record,
1647                        None,
1648                        loaded.record.checkpoint.is_terminal().then_some(deadline),
1649                        false,
1650                    )
1651                    .await?;
1652            }
1653            if loaded.record.checkpoint.is_terminal() {
1654                // Recheck the generation after awaited hooks, while retaining
1655                // this terminal receipt instead of reading its run head again.
1656                self.runtime
1657                    .observe_generation(deadline)
1658                    .await
1659                    .map_err(|failure| {
1660                        self.runtime.retain_execution_read_failure(&failure);
1661                        failure.into_error()
1662                    })?;
1663                let snapshot = self.runtime.project(&loaded.record)?;
1664                self.runtime.app.ensure_snapshot_read()?;
1665                snapshot
1666            } else {
1667                self.runtime.snapshot().await?
1668            }
1669        };
1670        if !foreground
1671            && matches!(
1672                snapshot.status,
1673                RunStatus::Succeeded
1674                    | RunStatus::Cancelled
1675                    | RunStatus::Failed(_)
1676                    | RunStatus::Halted { .. }
1677            )
1678        {
1679            self.runtime.cell.clear_execution_options(None);
1680        }
1681        if foreground && matches!(snapshot.status, crate::workflow::RunStatus::Running) {
1682            self.runtime.wait().await
1683        } else {
1684            Ok(snapshot)
1685        }
1686    }
1687
1688    fn stop_sources(&mut self) {
1689        for source in &mut self.sources {
1690            let failure = match source.state {
1691                CallerSourceState::Failed(category) => Some(category),
1692                _ => None,
1693            };
1694            let destroyed = source.stop().err();
1695            if let Some(category) = failure.or(destroyed) {
1696                self.runtime.retain_source_failure(
1697                    &source.context,
1698                    &source.activation.lineage,
1699                    category,
1700                );
1701                source.state = CallerSourceState::Failed(category);
1702            }
1703        }
1704    }
1705
1706    fn stop_children(&mut self) {
1707        for group in &mut self.children {
1708            group.stop(&self.runtime);
1709        }
1710        self.children.clear();
1711    }
1712
1713    fn plan_child_group(
1714        &mut self,
1715        loaded: LoadedRun,
1716        sequence: u64,
1717        children: Vec<RunRecord>,
1718        verified: bool,
1719    ) -> Result<(), AppError> {
1720        if loaded.record.checkpoint.is_stopping() || loaded.record.checkpoint.is_terminal() {
1721            self.state = SessionState::Loaded(loaded);
1722            return Ok(());
1723        }
1724        if let Some(group) = self
1725            .children
1726            .iter_mut()
1727            .find(|group| group.sequence == sequence)
1728            && let Some(ControlRecord {
1729                state: ControlRecordState::Child { admitted, .. },
1730                ..
1731            }) = loaded.record.checkpoint.control(sequence)
1732        {
1733            // Only confirmed native admission transfers these maps to the
1734            // child's concrete execution options. Retry retains the original.
1735            for values in group.inputs.iter_mut().take(admitted.len()) {
1736                values.clear();
1737            }
1738        }
1739        let decision = catch_unwind(AssertUnwindSafe(|| {
1740            self.runtime
1741                .plan_children(&loaded, sequence, children, verified)
1742        }))
1743        .map_err(|_| AppError::Integration("child planning callback panicked".into()))??;
1744        match decision {
1745            BspDecision::ExecuteChildren { sequence, children } => {
1746                let control = loaded
1747                    .record
1748                    .checkpoint
1749                    .control(sequence)
1750                    .ok_or_else(|| AppError::Conflict("child control is unavailable".into()))?;
1751                let context = self.runtime.child_context(&loaded.record, sequence)?;
1752                let definition = self.runtime.app.definition_for(&loaded.record)?;
1753                let step = definition
1754                    .node(&control.node)
1755                    .ok_or_else(|| AppError::Integration("child definition is unavailable".into()))?
1756                    .step();
1757                step.validate_mode(context.mode)?;
1758                let deadline = tokio::time::Instant::now()
1759                    .checked_add(WorkflowRunRuntime::<CS, A, X, C, T, H, E>::remaining(
1760                        control.deadline_millis,
1761                    )?)
1762                    .ok_or_else(|| {
1763                        AppError::Timeout("child deadline is outside the clock range".into())
1764                    })?;
1765                if let Some(group) = self
1766                    .children
1767                    .iter_mut()
1768                    .find(|group| group.sequence == sequence)
1769                {
1770                    if !matches!(group.state, ChildExecutionState::Awaiting) {
1771                        return Err(AppError::Conflict(
1772                            "child group already has a polling owner".into(),
1773                        ));
1774                    }
1775                    group.state = ChildExecutionState::Prepared(children);
1776                } else {
1777                    self.children.push(ChildExecution {
1778                        sequence,
1779                        parent: None,
1780                        context,
1781                        lineage: control.lineage.clone(),
1782                        deadline,
1783                        state: ChildExecutionState::Prepared(children),
1784                        inputs: Vec::new(),
1785                    });
1786                }
1787                self.state = SessionState::Loaded(loaded);
1788                Ok(())
1789            }
1790            BspDecision::AwaitControls => {
1791                self.state = SessionState::Loaded(loaded);
1792                Ok(())
1793            }
1794            decision => {
1795                let transition = match decision {
1796                    BspDecision::Stage(activations) => ControlTransition::Stage { activations },
1797                    BspDecision::Failed {
1798                        category,
1799                        message,
1800                        observation,
1801                    } => ControlTransition::Failed {
1802                        category,
1803                        message,
1804                        observation,
1805                    },
1806                    decision => return self.publish_decision(loaded, decision, false),
1807                };
1808                let control = loaded
1809                    .record
1810                    .checkpoint
1811                    .control(sequence)
1812                    .cloned()
1813                    .ok_or_else(|| {
1814                        AppError::Conflict("completed child control is unavailable".into())
1815                    })?;
1816                self.publish(
1817                    SessionPublication::ChildOutcome {
1818                        control,
1819                        transition,
1820                    },
1821                    false,
1822                )
1823            }
1824        }
1825    }
1826
1827    fn start_child_groups(&mut self, cx: &mut Context<'_>) -> Result<(), AppError> {
1828        if !self
1829            .children
1830            .iter()
1831            .any(|group| matches!(group.state, ChildExecutionState::Prepared(_)))
1832        {
1833            return Ok(());
1834        }
1835        if self.poll_notification(cx)? {
1836            let deadline = self.source_deadline(None);
1837            let SessionState::Sources { loaded } =
1838                std::mem::replace(&mut self.state, SessionState::Finished)
1839            else {
1840                unreachable!()
1841            };
1842            self.publish(
1843                SessionPublication::Reload {
1844                    receipt: SessionReceipt::Checkpoint {
1845                        loaded,
1846                        staged: Vec::new(),
1847                    },
1848                    deadline,
1849                },
1850                false,
1851            )?;
1852            return Ok(());
1853        }
1854        let SessionState::Sources { loaded, .. } = &self.state else {
1855            return Ok(());
1856        };
1857        let parent = Arc::new(loaded.record.clone());
1858        for index in 0..self.children.len() {
1859            if !matches!(self.children[index].state, ChildExecutionState::Prepared(_)) {
1860                continue;
1861            }
1862            let group = &mut self.children[index];
1863            let ChildExecutionState::Prepared(children) =
1864                std::mem::replace(&mut group.state, ChildExecutionState::Closed)
1865            else {
1866                unreachable!()
1867            };
1868            group.parent = Some(Arc::clone(&parent));
1869            if group.context.mode == ActionMode::Asynchronous && self.lane == ExecutionLane::Caller
1870            {
1871                let (sender, receipt) = oneshot::channel();
1872                if !self.runtime.app.spawn_driver(
1873                    self.runtime.key.clone(),
1874                    Duration::ZERO,
1875                    DriverCommand::Child {
1876                        sequence: group.sequence,
1877                        generation: self.generation,
1878                        parent: Arc::clone(&parent),
1879                        children,
1880                        receipt: sender,
1881                    },
1882                ) {
1883                    return Err(AppError::Busy(
1884                        "child group already has an accepted owner".into(),
1885                    ));
1886                }
1887                group.state = ChildExecutionState::Worker(receipt);
1888            } else {
1889                group.start_local(&self.runtime, self.app, self.lane, children)?;
1890            }
1891        }
1892        Ok(())
1893    }
1894
1895    fn poll_child_groups(&mut self, cx: &mut Context<'_>) -> Result<(), AppError> {
1896        let stopped = match &self.state {
1897            SessionState::Loaded(loaded)
1898            | SessionState::Sources { loaded }
1899            | SessionState::ObservingItem { loaded, .. } => {
1900                loaded.record.checkpoint.is_stopping() || loaded.record.checkpoint.is_terminal()
1901            }
1902            _ => false,
1903        };
1904        if stopped {
1905            self.stop_children();
1906            return Ok(());
1907        }
1908        if let SessionState::Loaded(loaded)
1909        | SessionState::Sources { loaded }
1910        | SessionState::ObservingItem { loaded, .. } = &self.state
1911        {
1912            self.children.retain(|group| {
1913                !matches!(group.state, ChildExecutionState::Awaiting)
1914                    || loaded.record.checkpoint.control(group.sequence).is_some()
1915            });
1916            if self.children.iter().any(|group| {
1917                !matches!(
1918                    group.state,
1919                    ChildExecutionState::Awaiting
1920                        | ChildExecutionState::Worker(_)
1921                        | ChildExecutionState::Receipt(_)
1922                ) && group.parent.as_ref().is_some_and(|parent| {
1923                    !loaded.record.owns_child_activation(parent, group.sequence)
1924                })
1925            }) {
1926                return Err(AppError::Conflict(
1927                    "child parent reservation changed".into(),
1928                ));
1929            }
1930        }
1931        let observing_parent = matches!(
1932            self.state,
1933            SessionState::Publishing {
1934                purpose: PublicationPurpose::Reload,
1935                ..
1936            }
1937        );
1938        for group in &mut self.children {
1939            if observing_parent
1940                && tokio::time::Instant::now() >= group.deadline
1941                && matches!(
1942                    group.state,
1943                    ChildExecutionState::Local(_) | ChildExecutionState::Pipeline(_)
1944                )
1945            {
1946                // The outstanding authority read owns deadline disposition.
1947                // Keep the original executor until that receipt or fatal cleanup.
1948                continue;
1949            }
1950            let _ = group.poll_ready(cx, &self.runtime);
1951        }
1952        Ok(())
1953    }
1954
1955    fn publish_child_completion(
1956        &mut self,
1957        ordinary: bool,
1958        closing: bool,
1959    ) -> Result<bool, AppError> {
1960        let Some(index) = self.children.iter().position(|group| {
1961            matches!(
1962                group.state,
1963                ChildExecutionState::Complete(_) | ChildExecutionState::Receipt(_)
1964            )
1965        }) else {
1966            return Ok(false);
1967        };
1968        let group = &mut self.children[index];
1969        let sequence = group.sequence;
1970        let parent = group.parent.take();
1971        let context = Arc::clone(&group.context);
1972        let lineage = group.lineage.clone();
1973        let completed = std::mem::replace(&mut group.state, ChildExecutionState::Awaiting);
1974        if closing {
1975            self.children.remove(index);
1976        }
1977        if let ChildExecutionState::Receipt(result) = completed {
1978            match (*result).map_err(RuntimeOperationFailure::into_error)? {
1979                SessionReceipt::Children {
1980                    loaded,
1981                    sequence,
1982                    children,
1983                    verified: true,
1984                } => {
1985                    if closing {
1986                        self.state = SessionState::Loaded(loaded);
1987                    } else {
1988                        self.plan_child_group(loaded, sequence, children, true)?;
1989                    }
1990                }
1991                SessionReceipt::Checkpoint { loaded, .. } => {
1992                    self.state = SessionState::Loaded(loaded)
1993                }
1994                SessionReceipt::Children {
1995                    verified: false, ..
1996                } => {
1997                    return Err(AppError::Integration(
1998                        "child worker returned an unverified execution result".into(),
1999                    ));
2000                }
2001            }
2002            return Ok(true);
2003        }
2004        let ChildExecutionState::Complete(result) = completed else {
2005            unreachable!()
2006        };
2007        let SessionState::Sources { loaded } =
2008            std::mem::replace(&mut self.state, SessionState::Finished)
2009        else {
2010            unreachable!()
2011        };
2012        match result {
2013            Ok(completions) => self.publish(
2014                SessionPublication::VerifyChildren {
2015                    parent: parent.ok_or_else(|| {
2016                        AppError::Integration("child group parent evidence is unavailable".into())
2017                    })?,
2018                    sequence,
2019                    completions,
2020                },
2021                ordinary,
2022            )?,
2023            Err(category) => {
2024                let activation = loaded
2025                    .record
2026                    .checkpoint
2027                    .frontier
2028                    .iter()
2029                    .find(|activation| {
2030                        activation.node == context.node && activation.lineage == lineage
2031                    })
2032                    .cloned()
2033                    .ok_or_else(|| AppError::Conflict("child activation changed".into()))?;
2034                self.publish_failure(loaded, &activation, &context, category)?;
2035            }
2036        }
2037        Ok(true)
2038    }
2039
2040    fn retains_external_continuation(&self, record: &RunRecord) -> bool {
2041        !self.children.is_empty()
2042            || self.runtime.app.has_child_work(&self.runtime.key)
2043            || record.checkpoint.controls.values().any(|control| {
2044                matches!(&control.state, ControlRecordState::Attempt(_))
2045                    && self
2046                        .runtime
2047                        .app
2048                        .has_control_work(&self.runtime.key, control.sequence)
2049            })
2050    }
2051
2052    fn retains_execution_lifecycle(&self, record: &RunRecord) -> bool {
2053        self.retains_execution_work(record)
2054            || record
2055                .checkpoint
2056                .controls
2057                .values()
2058                .any(|control| matches!(control.state, ControlRecordState::Inputs { .. }))
2059    }
2060
2061    fn retains_execution_work(&self, record: &RunRecord) -> bool {
2062        self.sources.iter().any(|source| {
2063            matches!(
2064                source.state,
2065                CallerSourceState::Polling(_) | CallerSourceState::Failed(_)
2066            )
2067        }) || self.prefetched.is_some()
2068            || self.retains_external_continuation(record)
2069            || record.checkpoint.controls.values().any(|control| {
2070                matches!(control.state, ControlRecordState::Sleep)
2071                    && self
2072                        .runtime
2073                        .app
2074                        .has_control_work(&self.runtime.key, control.sequence)
2075            })
2076    }
2077
2078    fn ensure_owned(&self) -> Result<(), AppError> {
2079        self.runtime.app.ensure_runtime_owned()?;
2080        if self
2081            .generation
2082            .is_some_and(|generation| !self.runtime.cell.execution_is_live(generation))
2083        {
2084            return Err(AppError::Closed(
2085                "run execution generation is closed".into(),
2086            ));
2087        }
2088        Ok(())
2089    }
2090
2091    fn source_deadline(&self, excluded: Option<usize>) -> Option<tokio::time::Instant> {
2092        self.sources
2093            .iter()
2094            .enumerate()
2095            .filter_map(|(index, source)| {
2096                (Some(index) != excluded && matches!(source.state, CallerSourceState::Polling(_)))
2097                    .then_some(source.deadline)
2098            })
2099            .chain(self.children.iter().filter_map(|group| {
2100                (!matches!(
2101                    group.state,
2102                    ChildExecutionState::Complete(_) | ChildExecutionState::Receipt(_)
2103                ))
2104                .then_some(group.deadline)
2105            }))
2106            .min()
2107    }
2108
2109    fn expire_sources(&mut self, excluded: Option<usize>) {
2110        let now = tokio::time::Instant::now();
2111        for (index, source) in self.sources.iter_mut().enumerate() {
2112            if Some(index) == excluded
2113                || now < source.deadline
2114                || !matches!(source.state, CallerSourceState::Polling(_))
2115            {
2116                continue;
2117            }
2118            self.runtime.retain_source_failure(
2119                &source.context,
2120                &source.activation.lineage,
2121                FailureCategory::Timeout,
2122            );
2123            let _ = source.stop();
2124            if self
2125                .prefetched
2126                .as_ref()
2127                .is_some_and(|(buffered, _)| *buffered == index)
2128            {
2129                self.prefetched = Some((index, Err(FailureCategory::Timeout)));
2130            } else {
2131                source.state = CallerSourceState::Failed(FailureCategory::Timeout);
2132            }
2133        }
2134    }
2135
2136    fn poll_sources(&mut self, context: &mut Context<'_>, excluded: Option<usize>) {
2137        self.expire_sources(excluded);
2138        if self.prefetched.is_some() {
2139            return;
2140        }
2141        let count = self.sources.len();
2142        for offset in 0..count {
2143            let index = (self.next_source + offset) % count;
2144            if Some(index) == excluded {
2145                continue;
2146            }
2147            if let Poll::Ready(mut event) = self.sources[index].poll(context) {
2148                let source = &mut self.sources[index];
2149                if let Err(category) = &event {
2150                    // Preserve sampled failure before invoking the destructor:
2151                    // the original sibling callback may settle during Drop.
2152                    self.runtime.retain_source_failure(
2153                        &source.context,
2154                        &source.activation.lineage,
2155                        *category,
2156                    );
2157                }
2158                if matches!(
2159                    &event,
2160                    Ok(Some(ActionEvent::Completed { .. })) | Ok(None) | Err(_)
2161                ) {
2162                    let destroyed = source.stop();
2163                    if event.is_ok()
2164                        && let Err(category) = destroyed
2165                    {
2166                        self.runtime.retain_source_failure(
2167                            &source.context,
2168                            &source.activation.lineage,
2169                            category,
2170                        );
2171                        event = Err(category);
2172                    }
2173                    if matches!(&event, Ok(None)) {
2174                        source.state = CallerSourceState::Eof;
2175                    }
2176                }
2177                self.next_source = (index + 1) % count;
2178                self.prefetched = Some((index, event));
2179                return;
2180            }
2181        }
2182    }
2183
2184    fn arm_notification(&mut self) -> Result<(), AppError> {
2185        if self.wake.is_none() {
2186            self.wake = Some(Box::pin(self.runtime.notifier()?.notified_owned()));
2187        }
2188        let wake = self.wake.as_mut().unwrap();
2189        wake.as_mut().enable();
2190        // Retain registration before admitting reads; polling installs the
2191        // current task waker without recreating this notification future.
2192        Ok(())
2193    }
2194
2195    fn poll_notification(&mut self, cx: &mut Context<'_>) -> Result<bool, AppError> {
2196        self.arm_notification()?;
2197        let ready = self.wake.as_mut().unwrap().as_mut().poll(cx).is_ready();
2198        if ready {
2199            self.wake = None;
2200        }
2201        Ok(ready)
2202    }
2203
2204    fn poll_timer(&mut self, cx: &mut Context<'_>, deadline: Option<tokio::time::Instant>) -> bool {
2205        let Some(deadline) = deadline else {
2206            self.timer = None;
2207            return false;
2208        };
2209        let timer = self
2210            .timer
2211            .get_or_insert_with(|| Box::pin(tokio::time::sleep_until(deadline)));
2212        if timer.deadline() != deadline {
2213            timer.as_mut().reset(deadline);
2214        }
2215        timer.as_mut().poll(cx).is_ready()
2216    }
2217
2218    fn poll_native_partial(&mut self, cx: &mut Context<'_>) -> Option<ExecutionEvent> {
2219        let Some((_, receiver)) = &mut self.native_partials else {
2220            return None;
2221        };
2222        match receiver.poll_recv(cx) {
2223            Poll::Ready(Some(event)) => Some(event),
2224            Poll::Ready(None) => {
2225                self.native_partials = None;
2226                None
2227            }
2228            Poll::Pending => None,
2229        }
2230    }
2231
2232    fn poll_sources_observation(
2233        &mut self,
2234        cx: &mut Context<'_>,
2235    ) -> Poll<Result<Option<ExecutionEvent>, AppError>> {
2236        self.ensure_owned()?;
2237        self.expire_sources(None);
2238        let deadline = self.source_deadline(None);
2239        if self.poll_timer(cx, deadline) {
2240            self.expire_sources(None);
2241        }
2242        let pending_failure = self
2243            .sources
2244            .iter()
2245            .any(|source| matches!(source.state, CallerSourceState::Failed(_)));
2246        if !pending_failure && self.poll_notification(cx)? {
2247            let SessionState::Sources { loaded } =
2248                std::mem::replace(&mut self.state, SessionState::Finished)
2249            else {
2250                unreachable!()
2251            };
2252            self.publish(
2253                SessionPublication::Reload {
2254                    receipt: SessionReceipt::Checkpoint {
2255                        loaded,
2256                        staged: Vec::new(),
2257                    },
2258                    deadline,
2259                },
2260                false,
2261            )?;
2262            return Poll::Ready(Ok(None));
2263        }
2264        self.poll_sources(cx, None);
2265        if let Some(event) = self.poll_native_partial(cx) {
2266            return Poll::Ready(Ok(Some(event)));
2267        }
2268        if self.prefetched.is_some() {
2269            return Poll::Ready(Ok(None));
2270        }
2271        let SessionState::Sources { loaded, .. } = &self.state else {
2272            unreachable!()
2273        };
2274        let known_eof = self
2275            .sources
2276            .iter()
2277            .any(|source| matches!(source.state, CallerSourceState::Eof));
2278        if (self.generation.is_some() || !self.retains_execution_work(&loaded.record))
2279            && !known_eof
2280            && !self.children.iter().any(|group| {
2281                matches!(
2282                    group.state,
2283                    ChildExecutionState::Complete(_) | ChildExecutionState::Receipt(_)
2284                )
2285            })
2286            && loaded.record.checkpoint.is_actionable()
2287        {
2288            return Poll::Ready(Ok(None));
2289        }
2290        Poll::Pending
2291    }
2292
2293    fn poll_item(
2294        &mut self,
2295        cx: &mut Context<'_>,
2296    ) -> Poll<Result<Option<ExecutionEvent>, AppError>> {
2297        let SessionState::ObservingItem { index, .. } = &self.state else {
2298            unreachable!()
2299        };
2300        let index = *index;
2301        self.expire_sources(Some(index));
2302        let deadline = self.source_deadline(Some(index));
2303        if self.poll_timer(cx, deadline) {
2304            self.expire_sources(Some(index));
2305        }
2306        let SessionState::ObservingItem { receipt, .. } = &mut self.state else {
2307            unreachable!()
2308        };
2309        if let Poll::Ready(result) = Pin::new(receipt).poll(cx) {
2310            let result = result
2311                .map_err(|_| AppError::Closed("stream observation receipt closed".into()))
2312                .and_then(|result| result);
2313            return Poll::Ready(self.finish_item(result).map(|result| {
2314                result.map(|result| ExecutionEvent::Partial {
2315                    node: self.sources[index].activation.node.clone(),
2316                    result,
2317                })
2318            }));
2319        }
2320        self.poll_sources(cx, Some(index));
2321        if let Some(event) = self.poll_native_partial(cx) {
2322            return Poll::Ready(Ok(Some(event)));
2323        }
2324        Poll::Pending
2325    }
2326
2327    fn finish_item(
2328        &mut self,
2329        result: Result<Option<Map<String, Value>>, AppError>,
2330    ) -> Result<Option<Map<String, Value>>, AppError> {
2331        let SessionState::ObservingItem {
2332            loaded, receipt, ..
2333        } = std::mem::replace(&mut self.state, SessionState::Finished)
2334        else {
2335            unreachable!()
2336        };
2337        drop(receipt);
2338        if result.is_err() {
2339            self.observed = Some(loaded.record);
2340            for source in &mut self.sources {
2341                let _ = source.stop();
2342            }
2343        } else {
2344            self.state = SessionState::Sources { loaded };
2345        }
2346        result
2347    }
2348
2349    fn consume_source_event(
2350        &mut self,
2351        index: usize,
2352        event: Result<Option<ActionEvent>, FailureCategory>,
2353        ordinary: bool,
2354    ) -> Result<(), AppError> {
2355        match event {
2356            Ok(Some(ActionEvent::Partial { result })) => {
2357                if JsonBoundary::validate_size(&result).is_err() {
2358                    let _ = self.sources[index].stop();
2359                    return self.stage_source_failure(index, FailureCategory::State, ordinary);
2360                }
2361                self.publish_item(index, ActionEvent::Partial { result })
2362            }
2363            Ok(Some(
2364                event @ (ActionEvent::SpanStarted { .. }
2365                | ActionEvent::SpanAttribute { .. }
2366                | ActionEvent::SpanFinished { .. }),
2367            )) => self.publish_item(index, event),
2368            Ok(Some(ActionEvent::Completed { result, mut writes })) => {
2369                let SessionState::Sources { loaded, .. } = &self.state else {
2370                    unreachable!()
2371                };
2372                let source = &self.sources[index];
2373                let staged = catch_unwind(AssertUnwindSafe(|| {
2374                    let definition = self.runtime.app.definition_for(&loaded.record)?;
2375                    if let Some(Step::Action {
2376                        action,
2377                        revision,
2378                        project_result: true,
2379                    }) = definition
2380                        .node(&source.activation.node)
2381                        .map(|node| node.step())
2382                    {
2383                        writes.insert(
2384                            0,
2385                            self.runtime
2386                                .app
2387                                .action(action, revision)?
2388                                .declaration()
2389                                .result_projection(&result),
2390                        );
2391                    }
2392                    StagedActivationRecord::action(
2393                        &source.activation,
2394                        &loaded.record.checkpoint.state,
2395                        &writes,
2396                        result,
2397                        self.runtime.app.typing(),
2398                    )
2399                }));
2400                match staged {
2401                    Ok(Ok(staged)) => self.stage_source(index, staged, ordinary),
2402                    Ok(Err(_)) => {
2403                        self.stage_source_failure(index, FailureCategory::State, ordinary)
2404                    }
2405                    Err(_) => {
2406                        self.stage_source_failure(index, FailureCategory::Execution, ordinary)
2407                    }
2408                }
2409            }
2410            Ok(None) => Ok(()),
2411            Err(category) => self.stage_source_failure(index, category, ordinary),
2412        }
2413    }
2414
2415    fn stage_source_failure(
2416        &mut self,
2417        index: usize,
2418        category: FailureCategory,
2419        ordinary: bool,
2420    ) -> Result<(), AppError> {
2421        let source = &self.sources[index];
2422        self.runtime
2423            .retain_source_failure(&source.context, &source.activation.lineage, category);
2424        let staged = StagedActivationRecord::failed(
2425            source.activation.node.clone(),
2426            source.activation.lineage.clone(),
2427            category,
2428            "caller action execution failed".into(),
2429        );
2430        self.stage_source(index, staged, ordinary)
2431    }
2432
2433    fn stage_source(
2434        &mut self,
2435        index: usize,
2436        staged: StagedActivationRecord,
2437        ordinary: bool,
2438    ) -> Result<(), AppError> {
2439        let SessionState::Sources { loaded } =
2440            std::mem::replace(&mut self.state, SessionState::Finished)
2441        else {
2442            unreachable!()
2443        };
2444        self.observed = Some(loaded.record.clone());
2445        self.publish(
2446            SessionPublication::Decision {
2447                loaded,
2448                decision: BspDecision::Stage(vec![staged]),
2449                activation_deadline: None,
2450                context: Some(Arc::clone(&self.sources[index].context)),
2451                staged: Vec::new(),
2452                remaining: Vec::new().into_iter(),
2453            },
2454            ordinary,
2455        )
2456    }
2457
2458    fn finish_wave(&mut self, ordinary: bool) -> Result<(), AppError> {
2459        let SessionState::Sources { loaded } =
2460            std::mem::replace(&mut self.state, SessionState::Finished)
2461        else {
2462            unreachable!()
2463        };
2464        let definition = self.runtime.app.definition_for(&loaded.record)?;
2465        let failed = loaded
2466            .record
2467            .checkpoint
2468            .selected_staged(&definition)
2469            .filter_map(|activation| {
2470                if let StagedOutcomeRecord::Failed { category, message } = &activation.outcome {
2471                    Some((activation, *category, message))
2472                } else {
2473                    None
2474                }
2475            })
2476            .min_by(|left, right| {
2477                (&left.0.node, &left.0.lineage).cmp(&(&right.0.node, &right.0.lineage))
2478            });
2479        if let Some((activation, category, message)) = failed {
2480            let message = message.clone();
2481            let observation = match self.sources.iter().find(|source| {
2482                source.activation.node == activation.node
2483                    && source.activation.lineage == activation.lineage
2484            }) {
2485                Some(source) => ActionPosition {
2486                    node: source.context.node.clone(),
2487                    ordinal: source.context.action_ordinal,
2488                },
2489                None => loaded.record.checkpoint.proposed_position(
2490                    &definition,
2491                    &activation.node,
2492                    &activation.lineage,
2493                    &[],
2494                )?,
2495            };
2496            self.sources.clear();
2497            self.publish_decision(
2498                loaded,
2499                BspDecision::Failed {
2500                    category,
2501                    message,
2502                    observation: Some(observation),
2503                },
2504                ordinary,
2505            )
2506        } else if self
2507            .sources
2508            .iter()
2509            .any(|source| matches!(source.state, CallerSourceState::Eof))
2510        {
2511            self.sources.clear();
2512            self.publish(SessionPublication::EndSource { loaded }, ordinary)
2513        } else {
2514            self.sources.clear();
2515            self.observed = Some(loaded.record);
2516            self.state = SessionState::Ready;
2517            Ok(())
2518        }
2519    }
2520
2521    fn begin_activation(
2522        &mut self,
2523        loaded: LoadedRun,
2524        activation: FrontierRecord,
2525        remaining: std::vec::IntoIter<FrontierRecord>,
2526        staged: Vec<StagedActivationRecord>,
2527        ordinary: bool,
2528    ) -> Result<(), AppError> {
2529        self.ensure_owned()?;
2530        let definition = self.runtime.app.definition_for(&loaded.record)?;
2531        if let Some(node) = definition.node(&activation.node) {
2532            node.step()
2533                .validate_mode(self.runtime.cell.execution_mode()?)?;
2534        }
2535        let (started, deadline_millis) = WorkflowRunRuntime::<CS, A, X, C, T, H, E>::node_deadline(
2536            &definition,
2537            &activation.node,
2538        )?;
2539        let deadline = tokio::time::Instant::now()
2540            .checked_add(Duration::from_millis(deadline_millis - started))
2541            .ok_or_else(|| {
2542                AppError::Timeout("activation deadline is outside the clock range".into())
2543            })?;
2544        let context = match self.runtime.cell.action_context(
2545            loaded.record.checkpoint.base_sequence,
2546            &activation.node,
2547            &activation.lineage,
2548        ) {
2549            Some(context) => context,
2550            None => Arc::new(loaded.record.action_context(
2551                &definition,
2552                &activation,
2553                &staged,
2554                self.runtime.cell.execution_mode()?,
2555            )?),
2556        };
2557        if loaded.record.checkpoint.loses_join_first(
2558            &definition,
2559            &activation.node,
2560            &activation.lineage,
2561        ) {
2562            self.state = SessionState::ActivationReady {
2563                loaded,
2564                activation,
2565                context,
2566                remaining,
2567                staged,
2568                deadline,
2569                deadline_millis,
2570            };
2571            return Ok(());
2572        }
2573        if let Some(Step::Action {
2574            action, revision, ..
2575        }) = definition.node(&activation.node).map(|node| node.step())
2576        {
2577            let binding = self.runtime.app.action(action, revision)?;
2578            let validated = catch_unwind(AssertUnwindSafe(|| {
2579                binding.declaration().validate_reads(&WorkflowState {
2580                    values: loaded.record.checkpoint.state.clone(),
2581                })?;
2582                binding.declaration().resolve_inputs(
2583                    &self.runtime.cell.execution_options()?.inputs,
2584                    self.runtime.cell.execution_mode()?,
2585                )
2586            }));
2587            match validated {
2588                Ok(Err(AppError::InvalidInput(message))) if self.generation.is_none() => {
2589                    return self.publish_decision(
2590                        loaded,
2591                        BspDecision::Failed {
2592                            category: FailureCategory::Execution,
2593                            message,
2594                            observation: None,
2595                        },
2596                        ordinary,
2597                    );
2598                }
2599                Ok(Ok(ActionInputs::Required(required))) => {
2600                    return self.publish(
2601                        SessionPublication::Decision {
2602                            loaded,
2603                            decision: BspDecision::Inputs {
2604                                node: activation.node,
2605                                lineage: activation.lineage,
2606                                required,
2607                            },
2608                            activation_deadline: Some(deadline_millis),
2609                            context: None,
2610                            staged,
2611                            remaining,
2612                        },
2613                        ordinary,
2614                    );
2615                }
2616                Ok(result) => {
2617                    result?;
2618                }
2619                Err(_) => {
2620                    return self.publish_failure(
2621                        loaded,
2622                        &activation,
2623                        &context,
2624                        FailureCategory::Execution,
2625                    );
2626                }
2627            }
2628        }
2629        self.publish(
2630            SessionPublication::BeginActivation {
2631                loaded,
2632                activation,
2633                context,
2634                remaining,
2635                staged,
2636                deadline,
2637                deadline_millis,
2638            },
2639            ordinary,
2640        )
2641    }
2642
2643    fn activate_ready(&mut self, ordinary: bool) -> Result<(), AppError> {
2644        self.ensure_owned()?;
2645        let SessionState::ActivationReady {
2646            loaded,
2647            activation,
2648            context,
2649            mut remaining,
2650            mut staged,
2651            deadline,
2652            deadline_millis,
2653        } = std::mem::replace(&mut self.state, SessionState::Finished)
2654        else {
2655            unreachable!()
2656        };
2657        let definition = self.runtime.app.definition_for(&loaded.record)?;
2658        let skipped = loaded.record.checkpoint.loses_join_first(
2659            &definition,
2660            &activation.node,
2661            &activation.lineage,
2662        );
2663        if !skipped
2664            && matches!(
2665                definition.node(&activation.node).map(|node| node.step()),
2666                Some(Step::Action { .. })
2667            )
2668        {
2669            self.construct_source(&loaded, activation, context, deadline)?;
2670            return match remaining.next() {
2671                Some(activation) => {
2672                    self.begin_activation(loaded, activation, remaining, staged, ordinary)
2673                }
2674                None if staged.is_empty() => {
2675                    self.observed = None;
2676                    self.state = SessionState::Sources { loaded };
2677                    Ok(())
2678                }
2679                None => self.publish_decision(loaded, BspDecision::Stage(staged), ordinary),
2680            };
2681        }
2682        if !skipped && tokio::time::Instant::now() >= deadline {
2683            return self.publish_failure(loaded, &activation, &context, FailureCategory::Timeout);
2684        }
2685        let engine = ExecutionEngine::new(
2686            &definition,
2687            self.runtime.app.typing(),
2688            self.runtime.app.conditions(),
2689        );
2690        let decision = match catch_unwind(AssertUnwindSafe(|| {
2691            let decision = engine.activate(
2692                &loaded.record.checkpoint,
2693                &activation,
2694                loaded
2695                    .record
2696                    .parent
2697                    .as_ref()
2698                    .map_or(0, |parent| parent.depth),
2699                |workflow, revision| {
2700                    let revision = match revision {
2701                        Some(revision) => revision,
2702                        None => loaded.record.child_revision(&workflow)?.to_owned(),
2703                    };
2704                    self.runtime.app.definition_revision(&workflow, &revision)?;
2705                    Ok((workflow, revision))
2706                },
2707            )?;
2708            if let BspDecision::Descendants { inputs, .. } = &decision {
2709                let step = definition
2710                    .node(&activation.node)
2711                    .ok_or_else(|| {
2712                        AppError::Integration("child declaration is unavailable".into())
2713                    })?
2714                    .step();
2715                for task in inputs {
2716                    let child = self
2717                        .runtime
2718                        .app
2719                        .definition_revision(&task.workflow, &task.revision)?;
2720                    step.validate_child_binding(&definition, &child, &task.input)?;
2721                }
2722            }
2723            Ok(decision)
2724        })) {
2725            Ok(Ok(decision)) => decision,
2726            Ok(Err(error)) => BspDecision::Failed {
2727                category: FailureCategory::Evaluation,
2728                message: match error {
2729                    AppError::InvalidInput(message) => message,
2730                    _ => "workflow evaluation failed".into(),
2731                },
2732                observation: Some(loaded.record.checkpoint.proposed_position(
2733                    &definition,
2734                    &activation.node,
2735                    &activation.lineage,
2736                    &staged,
2737                )?),
2738            },
2739            Err(_) => BspDecision::Failed {
2740                category: FailureCategory::Execution,
2741                message: "workflow callback panicked".into(),
2742                observation: Some(loaded.record.checkpoint.proposed_position(
2743                    &definition,
2744                    &activation.node,
2745                    &activation.lineage,
2746                    &staged,
2747                )?),
2748            },
2749        };
2750        match decision {
2751            BspDecision::Stage(mut activations) => {
2752                staged.append(&mut activations);
2753                match remaining.next() {
2754                    Some(activation) => {
2755                        self.begin_activation(loaded, activation, remaining, staged, ordinary)
2756                    }
2757                    None => self.publish_decision(loaded, BspDecision::Stage(staged), ordinary),
2758                }
2759            }
2760            decision => self.publish(
2761                SessionPublication::Decision {
2762                    loaded,
2763                    decision,
2764                    activation_deadline: Some(deadline_millis),
2765                    context: Some(context),
2766                    staged,
2767                    remaining,
2768                },
2769                ordinary,
2770            ),
2771        }
2772    }
2773
2774    fn construct_source(
2775        &mut self,
2776        loaded: &LoadedRun,
2777        activation: FrontierRecord,
2778        context: Arc<ActionContext>,
2779        deadline: tokio::time::Instant,
2780    ) -> Result<(), AppError> {
2781        if tokio::time::Instant::now() >= deadline {
2782            self.runtime.retain_source_failure(
2783                &context,
2784                &activation.lineage,
2785                FailureCategory::Timeout,
2786            );
2787            self.sources.push(CallerSource {
2788                activation,
2789                context,
2790                deadline,
2791                state: CallerSourceState::Failed(FailureCategory::Timeout),
2792            });
2793            return Ok(());
2794        }
2795        let definition = self.runtime.app.definition_for(&loaded.record)?;
2796        let Some(Step::Action {
2797            action, revision, ..
2798        }) = definition.node(&activation.node).map(|node| node.step())
2799        else {
2800            return Err(AppError::Integration(
2801                "action decision has no exact declaration".into(),
2802            ));
2803        };
2804        let binding = self.runtime.app.action(action, revision)?;
2805        let state = WorkflowState {
2806            values: loaded.record.checkpoint.state.clone(),
2807        };
2808        let constructed = catch_unwind(AssertUnwindSafe(|| {
2809            binding.declaration().validate_reads(&state)?;
2810            let inputs = binding
2811                .declaration()
2812                .resolve_inputs(
2813                    &self.runtime.cell.execution_options()?.inputs,
2814                    self.runtime.cell.execution_mode()?,
2815                )?
2816                .into_values()?;
2817            Ok::<_, AppError>(binding.start(ActionRequest {
2818                state,
2819                inputs,
2820                context: context.as_ref().clone(),
2821            }))
2822        }));
2823        let state = match constructed {
2824            Ok(Ok(source)) => CallerSourceState::Polling(Box::pin(source)),
2825            Ok(Err(error)) => return Err(error),
2826            Err(_) => CallerSourceState::Failed(FailureCategory::Execution),
2827        };
2828        if let CallerSourceState::Failed(category) = &state {
2829            self.runtime
2830                .retain_source_failure(&context, &activation.lineage, *category);
2831        }
2832        self.sources.push(CallerSource {
2833            activation,
2834            context,
2835            deadline,
2836            state,
2837        });
2838        self.expire_sources(None);
2839        Ok(())
2840    }
2841
2842    fn halt_after(&mut self, loaded: &LoadedRun) -> Result<bool, AppError> {
2843        let outcomes = &loaded.record.checkpoint.outcomes;
2844        let Some(last) = outcomes.last() else {
2845            return Ok(false);
2846        };
2847        let options = self.runtime.cell.execution_options()?;
2848        let selected = outcomes
2849            .iter()
2850            .find(|outcome| options.halt_after.iter().any(|node| node == &outcome.node));
2851        let outcome = match selected {
2852            Some(outcome) => outcome,
2853            None if self.runtime.cell.halt_after(&last.node)? => last,
2854            None => return Ok(false),
2855        };
2856        self.publish_transition(
2857            LoadedRun {
2858                record: loaded.record.clone(),
2859                revision: loaded.revision,
2860            },
2861            ControlTransition::Halt {
2862                node: outcome.node.clone(),
2863                result: Some(outcome.result.clone()),
2864            },
2865        )?;
2866        Ok(true)
2867    }
2868
2869    fn publish_item(&mut self, index: usize, event: ActionEvent) -> Result<(), AppError> {
2870        let SessionState::Sources { loaded } = &self.state else {
2871            return Err(AppError::Integration(
2872                "stream item has no source owner".into(),
2873            ));
2874        };
2875        let source = &self.sources[index];
2876        let context = Arc::clone(&source.context);
2877        let deadline = source.deadline;
2878        let generation = self.generation;
2879        if let Some(generation) = generation {
2880            self.runtime.cell.begin_publication(generation)?;
2881        }
2882        let runtime = self.runtime.clone();
2883        let loaded = LoadedRun {
2884            record: loaded.record.clone(),
2885            revision: loaded.revision,
2886        };
2887        let activation = source.activation.clone();
2888        let (sender, receipt) = oneshot::channel();
2889        let kind = generation.map_or(RunTaskKind::Operation, |generation| {
2890            RunTaskKind::Foreground { generation }
2891        });
2892        if !self
2893            .runtime
2894            .app
2895            .spawn_run(self.runtime.key.clone(), kind, None, async move {
2896                // The admitted observer owns its failure even if the caller
2897                // drops the source or never polls this receipt again.
2898                let mut result = match runtime
2899                    .observe_source_event(&context, event, deadline)
2900                    .await
2901                {
2902                    Ok(result) => Ok(result),
2903                    Err(failure) => Err(runtime
2904                        .fail_hook(&loaded, Some((&activation, &context)), failure, Vec::new())
2905                        .await),
2906                };
2907                if let Some(generation) = generation
2908                    && runtime.cell.end_publication(generation)
2909                    && let Err(error) = runtime.finish_execution(generation, None).await
2910                    && result.is_ok()
2911                {
2912                    result = Err(error);
2913                }
2914                let _ = sender.send(result);
2915            })
2916        {
2917            if let Some(generation) = generation {
2918                self.runtime.cell.end_publication(generation);
2919            }
2920            return Err(AppError::Closed(
2921                "stream observation admission closed".into(),
2922            ));
2923        }
2924        let SessionState::Sources { loaded } =
2925            std::mem::replace(&mut self.state, SessionState::Finished)
2926        else {
2927            unreachable!()
2928        };
2929        self.state = SessionState::ObservingItem {
2930            loaded,
2931            index,
2932            receipt,
2933        };
2934        Ok(())
2935    }
2936
2937    fn publish_failure(
2938        &mut self,
2939        loaded: LoadedRun,
2940        activation: &FrontierRecord,
2941        context: &ActionContext,
2942        category: FailureCategory,
2943    ) -> Result<(), AppError> {
2944        let definition = self.runtime.app.definition_for(&loaded.record)?;
2945        let position = ActionPosition {
2946            node: context.node.clone(),
2947            ordinal: context.action_ordinal,
2948        };
2949        let (position, category, message) = match loaded.record.checkpoint.preceding_failure(
2950            &definition,
2951            &position,
2952            &activation.lineage,
2953        )? {
2954            Some((position, category, message)) => (position, category, message.to_owned()),
2955            None => (position, category, "caller action execution failed".into()),
2956        };
2957        self.publish_decision(
2958            loaded,
2959            BspDecision::Failed {
2960                category,
2961                message,
2962                observation: Some(position),
2963            },
2964            false,
2965        )
2966    }
2967
2968    fn publish_decision(
2969        &mut self,
2970        loaded: LoadedRun,
2971        decision: BspDecision,
2972        ordinary: bool,
2973    ) -> Result<(), AppError> {
2974        self.publish(
2975            SessionPublication::Decision {
2976                loaded,
2977                decision,
2978                activation_deadline: None,
2979                context: None,
2980                staged: Vec::new(),
2981                remaining: Vec::new().into_iter(),
2982            },
2983            ordinary,
2984        )
2985    }
2986
2987    fn publish_transition(
2988        &mut self,
2989        loaded: LoadedRun,
2990        transition: ControlTransition,
2991    ) -> Result<(), AppError> {
2992        self.publish(SessionPublication::Transition { loaded, transition }, false)
2993    }
2994
2995    fn publish(
2996        &mut self,
2997        mut publication: SessionPublication,
2998        ordinary: bool,
2999    ) -> Result<(), AppError> {
3000        let mut staged_prefix = Vec::new();
3001        let mut admitted_inputs = Vec::new();
3002        if let SessionPublication::Decision {
3003            loaded,
3004            decision: BspDecision::AdmitChildren { sequence, children },
3005            ..
3006        } = &publication
3007            && let Some(group) = self
3008                .children
3009                .iter()
3010                .find(|group| group.sequence == *sequence)
3011        {
3012            let original = loaded
3013                .record
3014                .checkpoint
3015                .control(*sequence)
3016                .and_then(|control| match &control.state {
3017                    ControlRecordState::Child { admitted, .. } => Some(admitted.len()),
3018                    _ => None,
3019                })
3020                .ok_or_else(|| {
3021                    AppError::Integration("child admission evidence is unavailable".into())
3022                })?;
3023            if !group.inputs.is_empty() {
3024                for index in original..children.len() {
3025                    let values = group.inputs.get(index).ok_or_else(|| {
3026                        AppError::Integration("child runtime input membership changed".into())
3027                    })?;
3028                    // The original group retains this bounded map until its
3029                    // native admitted prefix is confirmed by the receipt.
3030                    admitted_inputs.push(values.clone());
3031                }
3032            }
3033        }
3034        let purpose = match &mut publication {
3035            SessionPublication::Decision {
3036                decision:
3037                    BspDecision::Descendants {
3038                        lineage,
3039                        runtime_inputs,
3040                        ..
3041                    },
3042                context,
3043                remaining,
3044                ..
3045            } => PublicationPurpose::Descendants {
3046                context: Arc::clone(context.as_ref().ok_or_else(|| {
3047                    AppError::Integration(
3048                        "child runtime inputs have no original action context".into(),
3049                    )
3050                })?),
3051                lineage: lineage.clone(),
3052                inputs: std::mem::take(runtime_inputs),
3053                remaining: std::mem::take(remaining),
3054                ordinary,
3055            },
3056            SessionPublication::Load { .. } => PublicationPurpose::Load,
3057            SessionPublication::Reload { .. } => PublicationPurpose::Reload,
3058            SessionPublication::ObserveChildren { .. } => PublicationPurpose::ObserveChildren,
3059            SessionPublication::VerifyChildren { .. } => PublicationPurpose::VerifyChildren,
3060            SessionPublication::FinishChild { .. } | SessionPublication::SettleStopping { .. } => {
3061                PublicationPurpose::Load
3062            }
3063            SessionPublication::Decision {
3064                decision: BspDecision::AdmitChildren { .. },
3065                ..
3066            } => PublicationPurpose::AdmitChildren,
3067            SessionPublication::BeginActivation {
3068                activation,
3069                context,
3070                remaining,
3071                staged,
3072                deadline,
3073                deadline_millis,
3074                ..
3075            } => {
3076                // The same receipt returns this prefix after successful hooks.
3077                // A rejected hook first preserves its native failure evidence.
3078                staged_prefix = std::mem::take(staged);
3079                PublicationPurpose::Start {
3080                    activation: activation.clone(),
3081                    context: Arc::clone(context),
3082                    remaining: std::mem::take(remaining),
3083                    deadline: *deadline,
3084                    deadline_millis: *deadline_millis,
3085                }
3086            }
3087            SessionPublication::Decision { remaining, .. } if !remaining.as_slice().is_empty() => {
3088                PublicationPurpose::Continue {
3089                    remaining: std::mem::take(remaining),
3090                    ordinary,
3091                }
3092            }
3093            SessionPublication::Decision {
3094                decision: BspDecision::Barrier { .. },
3095                ..
3096            } => PublicationPurpose::Commit { ordinary },
3097            SessionPublication::EndSource { .. } => PublicationPurpose::EndSource,
3098            _ => PublicationPurpose::Advance,
3099        };
3100        if let SessionPublication::Decision {
3101            decision: BspDecision::Agent(request),
3102            ..
3103        } = &mut publication
3104            && request.is_streaming()
3105        {
3106            let (sender, _) = self.native_partials.get_or_insert_with(|| mpsc::channel(1));
3107            request.observe_partials(sender.clone());
3108        }
3109        let generation = self.generation;
3110        if let Some(generation) = generation {
3111            self.runtime.cell.begin_publication(generation)?;
3112            if matches!(&publication, SessionPublication::Release) {
3113                self.runtime.cell.revoke_execution(generation);
3114            }
3115        }
3116        let child_observation = match &publication {
3117            SessionPublication::Reload { receipt, .. } => receipt
3118                .record()
3119                .checkpoint
3120                .controls
3121                .values()
3122                .filter(|control| {
3123                    matches!(control.state, ControlRecordState::Child { .. })
3124                        && !self.children.iter().any(|group| {
3125                            group.sequence == control.sequence
3126                                && matches!(
3127                                    group.state,
3128                                    ChildExecutionState::Worker(_)
3129                                        | ChildExecutionState::Receipt(_)
3130                                )
3131                        })
3132                })
3133                .min_by_key(|control| control.deadline_millis)
3134                .map(|control| control.sequence),
3135            _ => None,
3136        };
3137        let options = self.runtime.cell.execution_options()?;
3138        let parent_deadline = self.parent_deadline;
3139        let reconcile_read = generation.is_none()
3140            && self.runtime.cell.child_parent().is_some()
3141            && matches!(
3142                &publication,
3143                SessionPublication::Load { .. }
3144                    | SessionPublication::Reload { .. }
3145                    | SessionPublication::ObserveChildren { .. }
3146                    | SessionPublication::VerifyChildren { .. }
3147            );
3148        let execution_read = matches!(
3149            &publication,
3150            SessionPublication::Load { .. }
3151                | SessionPublication::Reload { .. }
3152                | SessionPublication::ObserveChildren { .. }
3153                | SessionPublication::FinishChild { .. }
3154                | SessionPublication::BeginActivation { .. }
3155        );
3156        let runtime = self.runtime.clone();
3157        let (sender, receipt) = oneshot::channel();
3158        let kind = generation.map_or(RunTaskKind::Operation, |generation| {
3159            RunTaskKind::Foreground { generation }
3160        });
3161        if !self
3162            .runtime
3163            .app
3164            .spawn_run(self.runtime.key.clone(), kind, self.admission.take(), async move {
3165                let inputs = &options.inputs;
3166                let observed = AssertUnwindSafe(async {
3167                    match publication {
3168                    SessionPublication::ObserveChildren { loaded, sequence } => async {
3169                        let (loaded, children) = runtime.observe_children(loaded, sequence).await?;
3170                        Ok(SessionReceipt::Children { loaded, sequence, children, verified: false })
3171                    }.await,
3172                    SessionPublication::VerifyChildren { parent, sequence, completions } => {
3173                        match runtime.verify_child_completions(&parent, sequence, completions).await {
3174                            Ok((loaded, children)) => {
3175                                let control = parent.checkpoint.control(sequence).ok_or_else(|| {
3176                                    AppError::Conflict("verified child control is unavailable".into())
3177                                })?;
3178                                let complete = matches!(&control.state,
3179                                    ControlRecordState::Child { inputs, .. } if children.len() == inputs.len());
3180                                if loaded.record.checkpoint.is_stopping() || loaded.record.checkpoint.is_terminal() || !complete {
3181                                    runtime.schedule_child_retention(control);
3182                                    Ok(SessionReceipt::Children { loaded, sequence, children, verified: true })
3183                                } else {
3184                                    // Keep final collection and Stage in this accepted
3185                                    // publication even when its local waiter is dropped.
3186                                    // A partial Pipeline remains caller-owned above.
3187                                    let transition = runtime.plan_child_completion(&loaded, control, children)?;
3188                                    runtime.publish_child_outcome(control, transition).await
3189                                }
3190                            },
3191                            Err(RuntimeOperationFailure::Application(error)) => runtime
3192                                .finish_child_admission(sequence, ChildAdmissionReceipt::Rejected(error))
3193                                .await
3194                                .map(|(loaded, children)| SessionReceipt::Children {
3195                                    loaded, sequence, children, verified: false,
3196                                }),
3197                            Err(failure) => Err(failure),
3198                        }
3199                    },
3200                    SessionPublication::Decision { loaded, decision: BspDecision::AdmitChildren { sequence, children }, .. } => {
3201                        let receipt = runtime.register_children(loaded, sequence, children, admitted_inputs).await;
3202                        runtime.finish_child_admission(sequence, receipt).await.map(|(loaded, children)| {
3203                            SessionReceipt::Children { loaded, sequence, children, verified: false }
3204                        })
3205                    },
3206                    SessionPublication::ChildOutcome { control, transition } => {
3207                        runtime.publish_child_outcome(&control, transition).await
3208                    },
3209                    SessionPublication::FinishChild { parent, sequence } => async {
3210                        let loaded = runtime.load_execution_observation(None, parent_deadline).await?
3211                            .ok_or_else(|| AppError::NotFound("admitted child is unavailable".into()))?;
3212                        let definition = runtime.app.definition_for(&parent)?;
3213                        loaded.record.validate_child(&parent, &definition)?;
3214                        if loaded.record.parent.as_ref().is_none_or(|relation| relation.control_sequence != sequence) {
3215                            return Err(AppError::Conflict("child completion group changed".into()).into());
3216                        }
3217                        if loaded.record.checkpoint.is_stopping()
3218                            || loaded.record.checkpoint.is_terminal()
3219                            || !loaded.record.checkpoint.controls.values().any(|control| {
3220                                matches!(control.state, ControlRecordState::Inputs { .. })
3221                            })
3222                        {
3223                            runtime.finish_execution_observation(&loaded.record, None, None, false).await?;
3224                            runtime.cell.lifecycle_observation(loaded.record.checkpoint.sequence, false)?;
3225                        }
3226                        Ok(SessionReceipt::Checkpoint { loaded, staged: Vec::new() })
3227                    }.await,
3228                    SessionPublication::SettleStopping { loaded } => async {
3229                        if let Some(loaded) = Box::pin(runtime.settle_stopping(loaded)).await {
3230                            let observed = runtime.finish_execution_observation(&loaded.record, None, None, false).await;
3231                            if runtime.cell.child_parent().is_none() {
3232                                runtime.maintain_retention(None).await;
3233                            }
3234                            observed?;
3235                        }
3236                        // Retention may have advanced the head revision after
3237                        // the terminal checkpoint. Observe that current authority
3238                        // in one finite pass after the native receipt returns.
3239                        let loaded = runtime.load_execution_observation(None, parent_deadline).await
3240                            .inspect_err(|failure| runtime.retain_execution_read_failure(failure))?
3241                            .ok_or_else(|| AppError::NotFound("workflow run does not exist".into()))?;
3242                        Ok(SessionReceipt::Checkpoint { loaded, staged: Vec::new() })
3243                    }.await,
3244                    SessionPublication::Load { terminal } => async {
3245                        let loaded = runtime.load_execution_observation(terminal, parent_deadline).await?
3246                            .ok_or_else(|| AppError::NotFound("workflow run does not exist".into()))?;
3247                        if !loaded.record.checkpoint.is_stopping()
3248                            && !loaded.record.checkpoint.is_terminal()
3249                            && loaded.record.checkpoint.controls.values().any(|control| {
3250                                matches!(control.state, ControlRecordState::Inputs { .. })
3251                            })
3252                            && let Err(failure) = runtime.begin_execution_observation().await
3253                        {
3254                            return Err(runtime.fail_hook(&loaded, None, failure, Vec::new()).await.into());
3255                        }
3256                        Ok(SessionReceipt::Checkpoint { loaded, staged: Vec::new() })
3257                    }.await,
3258                    SessionPublication::Reload { receipt, deadline } => {
3259                        let observed = tokio::select! {
3260                            biased;
3261                            _ = async { match deadline {
3262                                Some(deadline) => tokio::time::sleep_until(deadline).await,
3263                                None => std::future::pending().await,
3264                            }}, if child_observation.is_none() && parent_deadline.is_none() => Ok(None),
3265                            observed = async {
3266                                // Child authority retains its own sampled deadline;
3267                                // it cannot fall back to a stale receipt on timeout.
3268                                // The original worker retains its own exact child
3269                                // watchdog and final receipt after staging its result.
3270                                let observed = match child_observation {
3271                                    Some(sequence) => runtime.observe_child_parent(receipt.record(), sequence, false, None).await?,
3272                                    None => runtime.load_execution_observation(None, deadline.into_iter().chain(parent_deadline).min()).await?
3273                                        .ok_or_else(|| AppError::NotFound("workflow run does not exist".into()))?,
3274                                };
3275                                Ok::<_, RuntimeOperationFailure>(Some(observed))
3276                            } => observed,
3277                        };
3278                        observed.map(|observed| match observed {
3279                            Some(loaded) => receipt.with_loaded(loaded),
3280                            None => receipt,
3281                        })
3282                    },
3283                    SessionPublication::BeginActivation { loaded, activation, context, deadline, .. } => async {
3284                        if let Err(failure) = runtime.observe_activation_start(&loaded, &activation, &context, inputs, deadline).await {
3285                            return Err(runtime.fail_hook(&loaded, Some((&activation, &context)), failure, std::mem::take(&mut staged_prefix)).await.into());
3286                        }
3287                        let current = runtime.load_execution_observation(
3288                            None, parent_deadline.into_iter().chain(Some(deadline)).min(),
3289                        ).await?.ok_or_else(|| AppError::NotFound("workflow run does not exist".into()))?;
3290                        if !(current.record.checkpoint.is_terminal()
3291                            || current.record.checkpoint.is_stopping()
3292                            || current.record.same_run(&loaded.record)
3293                                && current.record.checkpoint.base_sequence == loaded.record.checkpoint.base_sequence
3294                                && current.record.checkpoint.frontier.iter().any(|candidate| candidate == &activation)
3295                                && current.record.checkpoint.control_for(&activation.node, &activation.lineage).is_none()
3296                                && !current.record.checkpoint.staged.iter().any(|staged| staged.node == activation.node && staged.lineage == activation.lineage))
3297                        {
3298                            return Err(AppError::Conflict("action pre-hook authority changed".into()).into());
3299                        }
3300                        Ok(SessionReceipt::Checkpoint { loaded: current, staged: std::mem::take(&mut staged_prefix) })
3301                    }.await,
3302                    publication => async {
3303                    match publication {
3304                        SessionPublication::Load { .. } | SessionPublication::Reload { .. }
3305                        | SessionPublication::ObserveChildren { .. } | SessionPublication::VerifyChildren { .. }
3306                        | SessionPublication::ChildOutcome { .. } | SessionPublication::FinishChild { .. } | SessionPublication::SettleStopping { .. }
3307                        | SessionPublication::BeginActivation { .. } => unreachable!(),
3308                        SessionPublication::Decision { loaded, decision, activation_deadline, context, staged, .. } => {
3309                            if matches!(decision, BspDecision::Inputs { .. })
3310                                && let Err(failure) = runtime.begin_execution_observation().await
3311                            {
3312                                return Err(runtime.fail_hook(&loaded, None, failure, staged).await);
3313                            }
3314                            runtime.finish_staged_spans(&loaded, &decision, context.as_deref()).await?;
3315                            runtime.apply_decision(loaded, decision, activation_deadline, staged).await
3316                        }
3317                        SessionPublication::Transition { loaded, transition } => {
3318                            let loaded = runtime.commit_for_recovery(loaded, transition).await
3319                                .map_err(RuntimeOperationFailure::into_error)?;
3320                            // The native checkpoint remains this publication's
3321                            // exact result. Only its App generation needs a
3322                            // finite post-receipt observation before returning.
3323                            runtime.observe_generation(LifecycleEvent::deadline(parent_deadline)).await
3324                                .map_err(|failure| {
3325                                    runtime.retain_execution_read_failure(&failure);
3326                                    failure.into_error()
3327                                })?;
3328                            Ok(loaded)
3329                        }
3330                        SessionPublication::Retry { loaded } => {
3331                            let loaded = runtime.readmit_failed(loaded).await?;
3332                            runtime.observe_generation(LifecycleEvent::deadline(parent_deadline)).await
3333                                .map_err(|failure| {
3334                                    runtime.retain_execution_read_failure(&failure);
3335                                    failure.into_error()
3336                                })?;
3337                            Ok(loaded)
3338                        }
3339                        SessionPublication::EndSource { loaded } => {
3340                            let loaded = runtime.end_source(loaded, generation).await?;
3341                            runtime.observe_generation(LifecycleEvent::deadline(parent_deadline)).await
3342                                .map_err(|failure| {
3343                                    runtime.retain_execution_read_failure(&failure);
3344                                    failure.into_error()
3345                                })?;
3346                            Ok(loaded)
3347                        }
3348                        SessionPublication::Release => {
3349                            if let Some(generation) = generation { runtime.finish_execution(generation, None).await?; }
3350                            runtime.load_execution_observation(None, parent_deadline).await
3351                                .map_err(|failure| {
3352                                    runtime.retain_execution_read_failure(&failure);
3353                                    failure.into_error()
3354                                })?
3355                                .ok_or_else(|| AppError::NotFound("workflow run does not exist".into()))
3356                        }
3357                    }
3358                    }.await.map(|loaded| SessionReceipt::Checkpoint { loaded, staged: std::mem::take(&mut staged_prefix) })
3359                        .map_err(RuntimeOperationFailure::from),
3360                    }
3361                }).catch_unwind().await;
3362                let mut result: Result<SessionReceipt, RuntimeOperationFailure> = match observed {
3363                    Ok(result) => result,
3364                    Err(payload) if reconcile_read => {
3365                        drop(payload);
3366                        match runtime.reconcile_child_panic(parent_deadline).await {
3367                            Ok(result) => result.map(|loaded| SessionReceipt::Checkpoint {
3368                                loaded, staged: Vec::new(),
3369                            }).map_err(RuntimeOperationFailure::from),
3370                            Err(_) => {
3371                                runtime.app.tasks_failed(FinalResult::Integration(
3372                                    "child read panic reconciliation receipt closed".into(),
3373                                ));
3374                                Err(AppError::Integration("child read panic reconciliation receipt closed".into()).into())
3375                            }
3376                        }
3377                    }
3378                    Err(payload) => std::panic::resume_unwind(payload),
3379                };
3380                if execution_read
3381                    && let Err(RuntimeOperationFailure::Storage(failure)) = &result
3382                {
3383                    runtime.retain_execution_read_failure(failure);
3384                }
3385                if let Some(generation) = generation
3386                    && runtime.cell.end_publication(generation)
3387                {
3388                    let loaded = match &result {
3389                        Ok(SessionReceipt::Checkpoint { loaded, .. })
3390                            if (loaded.record.checkpoint.is_paused() || loaded.record.checkpoint.is_stopping())
3391                                && loaded.record.checkpoint.controls.is_empty()
3392                                && runtime.cell.lifecycle_generation().is_none()
3393                                && !runtime.app.has_child_work(&runtime.key) =>
3394                        {
3395                            // The publication receipt remains available to a
3396                            // consuming close waiter while cleanup borrows its
3397                            // already-settled native authority.
3398                            Some(LoadedRun { record: loaded.record.clone(), revision: loaded.revision })
3399                        }
3400                        _ => None,
3401                    };
3402                    if let Err(error) = runtime.finish_execution(generation, loaded).await
3403                        && result.is_ok()
3404                    {
3405                        result = Err(error.into());
3406                    }
3407                }
3408                let _ = sender.send(result);
3409            })
3410        {
3411            if let Some(generation) = generation {
3412                self.runtime.cell.end_publication(generation);
3413            }
3414            return Err(AppError::Closed(
3415                "run publication admission is closed".into(),
3416            ));
3417        }
3418        self.state = SessionState::Publishing { receipt, purpose };
3419        Ok(())
3420    }
3421
3422    fn poll_publication(
3423        &mut self,
3424        cx: &mut Context<'_>,
3425        closing: bool,
3426    ) -> Poll<Result<Option<ExecutionEvent>, AppError>> {
3427        if !matches!(&self.state, SessionState::Publishing { .. }) {
3428            return Poll::Ready(Ok(None));
3429        }
3430        self.expire_sources(None);
3431        let deadline = self.source_deadline(None);
3432        if self.poll_timer(cx, deadline) {
3433            self.expire_sources(None);
3434        }
3435        let commit = matches!(
3436            &self.state,
3437            SessionState::Publishing {
3438                purpose: PublicationPurpose::Commit { .. },
3439                ..
3440            }
3441        );
3442        if commit && let Some(event) = self.poll_native_partial(cx) {
3443            return Poll::Ready(Ok(Some(event)));
3444        }
3445        let SessionState::Publishing { receipt, .. } = &mut self.state else {
3446            unreachable!()
3447        };
3448        let observed = match Pin::new(receipt).poll(cx) {
3449            Poll::Ready(observed) => observed
3450                .map_err(|_| {
3451                    RuntimeOperationFailure::from(AppError::Closed(
3452                        "run publication observation closed".into(),
3453                    ))
3454                })
3455                .and_then(|observed| observed),
3456            Poll::Pending => {
3457                if let Some(event) = self.poll_native_partial(cx) {
3458                    return Poll::Ready(Ok(Some(event)));
3459                }
3460                return Poll::Pending;
3461            }
3462        };
3463        let SessionState::Publishing { purpose, .. } =
3464            std::mem::replace(&mut self.state, SessionState::Finished)
3465        else {
3466            unreachable!()
3467        };
3468        let observed = match observed {
3469            Ok(observed) => observed,
3470            Err(failure) => {
3471                if self.generation.is_none() {
3472                    if matches!(
3473                        purpose,
3474                        PublicationPurpose::Load
3475                            | PublicationPurpose::Reload
3476                            | PublicationPurpose::ObserveChildren
3477                            | PublicationPurpose::VerifyChildren
3478                    ) {
3479                        if self.runtime.schedule_operation_failure(&failure) {
3480                            self.observed = None;
3481                        }
3482                    } else if matches!(purpose, PublicationPurpose::AdmitChildren) {
3483                        // The original admission result already ran its exact
3484                        // retry/failure policy. Do not spend its budget twice.
3485                        self.observed = None;
3486                    }
3487                }
3488                return Poll::Ready(Err(failure.into_error()));
3489            }
3490        };
3491        let (loaded, staged) = match observed {
3492            SessionReceipt::Checkpoint { loaded, staged } => (loaded, staged),
3493            SessionReceipt::Children {
3494                loaded,
3495                sequence,
3496                children,
3497                verified,
3498            } => {
3499                if closing {
3500                    self.state = SessionState::Loaded(loaded);
3501                } else if matches!(purpose, PublicationPurpose::AdmitChildren)
3502                    && self.poll_notification(cx)?
3503                {
3504                    let deadline = loaded
3505                        .record
3506                        .checkpoint
3507                        .control(sequence)
3508                        .map(|control| {
3509                            WorkflowRunRuntime::<CS, A, X, C, T, H, E>::remaining(
3510                                control.deadline_millis,
3511                            )
3512                        })
3513                        .transpose()?
3514                        .and_then(|remaining| tokio::time::Instant::now().checked_add(remaining));
3515                    self.publish(
3516                        SessionPublication::Reload {
3517                            receipt: SessionReceipt::Children {
3518                                loaded,
3519                                sequence,
3520                                children,
3521                                verified,
3522                            },
3523                            deadline,
3524                        },
3525                        false,
3526                    )?;
3527                } else {
3528                    self.plan_child_group(loaded, sequence, children, verified)?;
3529                }
3530                return Poll::Ready(Ok(None));
3531            }
3532        };
3533        if matches!(purpose, PublicationPurpose::Commit { .. })
3534            && let Err(error) = self
3535                .runtime
3536                .cell
3537                .lifecycle_observation(loaded.record.checkpoint.sequence, false)
3538        {
3539            // A failed post-hook may already have paused this exact native
3540            // barrier. Keep its receipt for owning cleanup before projecting
3541            // the original retained lifecycle failure to the caller.
3542            self.state = SessionState::Loaded(loaded);
3543            return Poll::Ready(Err(error));
3544        }
3545        if closing
3546            || matches!(
3547                purpose,
3548                PublicationPurpose::Load | PublicationPurpose::Reload
3549            )
3550        {
3551            self.state = SessionState::Loaded(loaded);
3552            return Poll::Ready(Ok(None));
3553        }
3554        self.state = SessionState::Ready;
3555        if let PublicationPurpose::Start {
3556            activation,
3557            context,
3558            remaining,
3559            deadline,
3560            deadline_millis,
3561        } = purpose
3562        {
3563            self.state = if loaded.record.checkpoint.is_terminal()
3564                || loaded.record.checkpoint.is_stopping()
3565            {
3566                self.observed = Some(loaded.record);
3567                SessionState::Finished
3568            } else {
3569                SessionState::ActivationReady {
3570                    loaded,
3571                    activation,
3572                    context,
3573                    remaining,
3574                    staged,
3575                    deadline,
3576                    deadline_millis,
3577                }
3578            };
3579            return Poll::Ready(Ok(None));
3580        }
3581        let purpose = match purpose {
3582            PublicationPurpose::Descendants {
3583                context,
3584                lineage,
3585                inputs,
3586                remaining,
3587                ordinary,
3588            } => {
3589                if !loaded.record.checkpoint.is_terminal()
3590                    && !loaded.record.checkpoint.is_stopping()
3591                {
3592                    let control = loaded
3593                        .record
3594                        .checkpoint
3595                        .control_for(&context.node, &lineage)
3596                        .ok_or_else(|| {
3597                            AppError::Integration(
3598                                "original child input control is unavailable".into(),
3599                            )
3600                        })?;
3601                    let deadline = tokio::time::Instant::now()
3602                        .checked_add(WorkflowRunRuntime::<CS, A, X, C, T, H, E>::remaining(
3603                            control.deadline_millis,
3604                        )?)
3605                        .ok_or_else(|| {
3606                            AppError::Timeout("child deadline is outside the clock range".into())
3607                        })?;
3608                    self.children.push(ChildExecution {
3609                        sequence: control.sequence,
3610                        parent: None,
3611                        context,
3612                        lineage,
3613                        deadline,
3614                        state: ChildExecutionState::Awaiting,
3615                        inputs,
3616                    });
3617                }
3618                if remaining.as_slice().is_empty() {
3619                    PublicationPurpose::Advance
3620                } else {
3621                    PublicationPurpose::Continue {
3622                        remaining,
3623                        ordinary,
3624                    }
3625                }
3626            }
3627            purpose => purpose,
3628        };
3629        if let PublicationPurpose::Continue {
3630            mut remaining,
3631            ordinary,
3632        } = purpose
3633        {
3634            if loaded.record.checkpoint.is_terminal() || loaded.record.checkpoint.is_stopping() {
3635                self.observed = Some(loaded.record);
3636                self.state = SessionState::Finished;
3637            } else {
3638                let activation = remaining.next().ok_or_else(|| {
3639                    AppError::Integration("native activation continuation is empty".into())
3640                })?;
3641                self.begin_activation(loaded, activation, remaining, staged, ordinary)?;
3642            }
3643            return Poll::Ready(Ok(None));
3644        }
3645        if let RunAdmission::Halted { node, result } = &loaded.record.checkpoint.admission {
3646            let event = ExecutionEvent::Halted {
3647                node: node.clone(),
3648                result: result.clone(),
3649            };
3650            self.observed = Some(loaded.record);
3651            self.state = SessionState::Finished;
3652            return Poll::Ready(Ok(Some(event)));
3653        }
3654        if let PublicationPurpose::Commit { ordinary } = purpose {
3655            self.observed = Some(loaded.record.clone());
3656            self.runtime
3657                .cell
3658                .record_execution_boundary(loaded.record.checkpoint.outcomes.len(), ordinary)?;
3659            return Poll::Ready(Ok(Some(ExecutionEvent::Committed {
3660                checkpoint: loaded.record.checkpoint.sequence,
3661                outcomes: loaded.record.checkpoint.outcomes,
3662            })));
3663        }
3664        if matches!(purpose, PublicationPurpose::EndSource) {
3665            self.observed = Some(loaded.record);
3666            self.state = SessionState::Eof;
3667        } else {
3668            self.observed = None;
3669            self.state = SessionState::Loaded(loaded);
3670        }
3671        Poll::Ready(Ok(None))
3672    }
3673}
3674
3675impl<
3676    CS: StateStoreBackend,
3677    A: AgentBackend,
3678    X: ActionBackend,
3679    C: ConditionBackend,
3680    T: StateTyping,
3681    H: LifecycleHooks,
3682    E: ChildExecutor,
3683> Drop for Execution<'_, CS, A, X, C, T, H, E>
3684{
3685    fn drop(&mut self) {
3686        if let Some(generation) = self.generation {
3687            self.runtime.cell.clear_execution_options(Some(generation));
3688        }
3689        let pending_observation = matches!(
3690            &self.state,
3691            SessionState::Publishing { .. } | SessionState::ObservingItem { .. }
3692        );
3693        self.stop_sources();
3694        self.stop_children();
3695        if self.generation.is_none()
3696            && !pending_observation
3697            && matches!(&self.state, SessionState::Sources { .. })
3698        {
3699            let failed = self
3700                .sources
3701                .iter()
3702                .enumerate()
3703                .filter_map(|(index, source)| {
3704                    let category = self
3705                        .prefetched
3706                        .as_ref()
3707                        .filter(|(buffered, _)| *buffered == index)
3708                        .and_then(|(_, event)| event.as_ref().err().copied())
3709                        .or(match source.state {
3710                            CallerSourceState::Failed(category) => Some(category),
3711                            _ => None,
3712                        });
3713                    category.map(|category| (index, category))
3714                })
3715                .min_by(|left, right| {
3716                    let left = &self.sources[left.0].activation;
3717                    let right = &self.sources[right.0].activation;
3718                    (&left.node, &left.lineage).cmp(&(&right.node, &right.lineage))
3719                });
3720            if let Some((index, category)) = failed {
3721                let _ = self.stage_source_failure(index, category, false);
3722            }
3723        }
3724        let source_failed = self
3725            .runtime
3726            .cell
3727            .lifecycle_generation()
3728            .is_some_and(|generation| self.runtime.cell.source_failure(generation).is_some());
3729        let record = match &self.state {
3730            SessionState::Loaded(loaded)
3731            | SessionState::Sources { loaded }
3732            | SessionState::ObservingItem { loaded, .. }
3733            | SessionState::ActivationReady { loaded, .. } => Some(&loaded.record),
3734            _ => self.observed.as_ref(),
3735        };
3736        let child_pending = self.runtime.app.has_child_work(&self.runtime.key);
3737        let local_only = !child_pending
3738            && !pending_observation
3739            && !matches!(&self.state, SessionState::Publishing { .. })
3740            && !source_failed
3741            && !self.sources.is_empty()
3742            && record.is_some_and(|record| {
3743                record.checkpoint.staged.is_empty() && record.checkpoint.controls.is_empty()
3744            });
3745        if !child_pending
3746            && !pending_observation
3747            && !matches!(&self.state, SessionState::Publishing { .. })
3748            && !source_failed
3749            && let Some(record) = record
3750            && !self.retains_execution_lifecycle(record)
3751            && (self.generation.is_some()
3752                || record.checkpoint.controls.is_empty()
3753                || record.checkpoint.is_public_outcome())
3754            && !record
3755                .checkpoint
3756                .staged
3757                .iter()
3758                .any(|activation| matches!(activation.outcome, StagedOutcomeRecord::Failed { .. }))
3759        {
3760            let _ = self.runtime.finish_execution_observation_sync(record);
3761        }
3762        if let Some(generation) = self.generation.take()
3763            && self.runtime.cell.drop_execution(generation)
3764        {
3765            if local_only {
3766                if self.runtime.cell.release_execution(generation) {
3767                    self.runtime.schedule(ScheduleCause::Reconcile);
3768                }
3769                return;
3770            }
3771            let loaded = if !child_pending
3772                && !pending_observation
3773                && !source_failed
3774                && self.runtime.cell.lifecycle_generation().is_none()
3775                && matches!(&self.state, SessionState::Loaded(loaded)
3776                    if (loaded.record.checkpoint.is_paused() || loaded.record.checkpoint.is_stopping())
3777                        && loaded.record.checkpoint.controls.is_empty())
3778            {
3779                let SessionState::Loaded(loaded) =
3780                    std::mem::replace(&mut self.state, SessionState::Finished)
3781                else {
3782                    unreachable!()
3783                };
3784                Some(loaded)
3785            } else {
3786                None
3787            };
3788            let runtime = self.runtime.clone();
3789            self.runtime.app.spawn_run(
3790                self.runtime.key.clone(),
3791                RunTaskKind::Foreground { generation },
3792                None,
3793                async move {
3794                    let _ = runtime.finish_execution(generation, loaded).await;
3795                },
3796            );
3797        }
3798    }
3799}
3800
3801impl<
3802    CS: StateStoreBackend,
3803    A: AgentBackend,
3804    X: ActionBackend,
3805    C: ConditionBackend,
3806    T: StateTyping,
3807    H: LifecycleHooks,
3808    E: ChildExecutor,
3809> WorkflowRunRuntime<CS, A, X, C, T, H, E>
3810{
3811    pub(crate) async fn execute_child_group(
3812        &self,
3813        sequence: u64,
3814        generation: Option<u64>,
3815        parent: Arc<RunRecord>,
3816        children: Vec<RunRecord>,
3817    ) -> Result<SessionReceipt, RuntimeOperationFailure> {
3818        if generation.is_some_and(|generation| !self.cell.execution_is_live(generation)) {
3819            return Err(
3820                AppError::Closed("child executor admission generation changed".into()).into(),
3821            );
3822        }
3823        let mut outcome_published = false;
3824        let result = async {
3825            let control = parent.checkpoint.control(sequence).ok_or_else(|| AppError::Conflict("child control is unavailable".into()))?;
3826            let remaining = Self::remaining(control.deadline_millis)?;
3827            if remaining.is_zero() {
3828                // Queue time belongs to the original activation. Publish its
3829                // fenced Timeout without constructing E or starting a read
3830                // against an already exhausted observation budget.
3831                outcome_published = true;
3832                let loaded = self.commit_scheduled_transition(control, ControlTransition::Failed {
3833                    category: FailureCategory::Timeout, message: "child workflow deadline expired".into(), observation: None,
3834                }).await.ok_or_else(|| AppError::Closed(
3835                    "child timeout publication requires reconciliation".into(),
3836                ))?;
3837                return Ok(SessionReceipt::Checkpoint { loaded, staged: Vec::new() });
3838            }
3839            let deadline = tokio::time::Instant::now().checked_add(remaining)
3840                .ok_or_else(|| AppError::Timeout("child deadline is outside the clock range".into()))?;
3841            // Retain the original ordered child records. Only parent authority
3842            // needs a fresh observation after waiting in the worker queue.
3843            let current = self.load_execution_observation(None, Some(deadline)).await?
3844                .ok_or_else(|| AppError::NotFound("child parent is unavailable".into()))?;
3845            if current.record.checkpoint.is_stopping() || current.record.checkpoint.is_terminal() {
3846                return Ok(SessionReceipt::Checkpoint { loaded: current, staged: Vec::new() });
3847            }
3848            if !current.record.owns_child_activation(&parent, sequence) {
3849                return Err(AppError::Conflict("child parent reservation changed".into()).into());
3850            }
3851            drop(current);
3852            let context = self.child_context(&parent, sequence)?;
3853            let mut group = ChildExecution {
3854                sequence, parent: Some(Arc::clone(&parent)), context,
3855                lineage: control.lineage.clone(), deadline,
3856                state: ChildExecutionState::Prepared(Vec::new()), inputs: Vec::new(),
3857            };
3858            group.start_local(self, &self.app, ExecutionLane::Worker, children)?;
3859            let context = Arc::clone(&group.context);
3860            let stopped = {
3861                let mut executor = Box::pin(poll_fn(|cx| group.poll_ready(cx, self)));
3862                let mut stopping = Box::pin(self.observe_child_parent(&parent, sequence, true, None));
3863                tokio::select! {
3864                    biased;
3865                    result = &mut stopping => {
3866                        let loaded = result?;
3867                        let loaded = if !loaded.record.checkpoint.is_stopping() {
3868                            self.retain_source_failure(&context, &control.lineage, FailureCategory::Timeout);
3869                            outcome_published = true;
3870                            let mut failure = Box::pin(self.commit_scheduled_transition(control, ControlTransition::Failed {
3871                                category: FailureCategory::Timeout, message: "child workflow deadline expired".into(), observation: None,
3872                            }));
3873                            let committed = tokio::select! {
3874                                _ = &mut executor => failure.await,
3875                                result = &mut failure => result,
3876                            };
3877                            committed.ok_or_else(|| AppError::Closed(
3878                                "child timeout publication requires reconciliation".into(),
3879                            ))?
3880                        } else { loaded };
3881                        Some(loaded)
3882                    }
3883                    _ = &mut executor => None,
3884                }
3885            };
3886            if let Some(loaded) = stopped {
3887                // The original parent observer or native timeout publication
3888                // already supplied authority. E destruction does not require
3889                // another head read before returning that exact receipt.
3890                group.stop(self);
3891                return Ok(SessionReceipt::Checkpoint { loaded, staged: Vec::new() });
3892            }
3893            let ChildExecutionState::Complete(completed) = group.state else {
3894                return Err(AppError::Integration("child worker completed without its local executor".into()).into());
3895            };
3896            let completions = match completed {
3897                Ok(completions) => completions,
3898                Err(category) => {
3899                    outcome_published = true;
3900                    let loaded = self.commit_scheduled_transition(control, ControlTransition::Failed {
3901                        category, message: "child execution failed".into(), observation: None,
3902                    }).await.ok_or_else(|| AppError::Closed(
3903                        "child failure publication requires reconciliation".into(),
3904                    ))?;
3905                    return Ok(SessionReceipt::Checkpoint { loaded, staged: Vec::new() });
3906                }
3907            };
3908            let (loaded, children) = self.verify_child_completions(&parent, sequence, completions).await?;
3909            if loaded.record.checkpoint.is_stopping() || loaded.record.checkpoint.is_terminal() {
3910                self.schedule_child_retention(control);
3911                return Ok(SessionReceipt::Checkpoint { loaded, staged: Vec::new() });
3912            }
3913            let transition = self.plan_child_completion(&loaded, control, children)?;
3914            // The original worker publishes only its fenced stage before
3915            // sending. Dropping either a pending or queued caller receipt
3916            // cannot lose completed children or permit another E invocation.
3917            outcome_published = true;
3918            self.publish_child_outcome(control, transition).await
3919        }.await;
3920        match result {
3921            Ok(receipt) => Ok(receipt),
3922            Err(RuntimeOperationFailure::Storage(failure)) => {
3923                self.retain_execution_read_failure(&failure);
3924                self.schedule_after_storage_failure(&failure);
3925                Err(failure.into())
3926            }
3927            Err(RuntimeOperationFailure::Application(error)) => {
3928                if outcome_published {
3929                    return Err(error.into());
3930                }
3931                let failure = self
3932                    .finish_child_admission(sequence, ChildAdmissionReceipt::Rejected(error))
3933                    .await;
3934                let loaded = self
3935                    .load_execution_observation(None, None)
3936                    .await
3937                    .inspect_err(|failure| self.retain_execution_read_failure(failure))?
3938                    .ok_or_else(|| AppError::NotFound("child parent is unavailable".into()))?;
3939                if loaded.record.checkpoint.is_stopping() || loaded.record.checkpoint.is_terminal()
3940                {
3941                    Ok(SessionReceipt::Checkpoint {
3942                        loaded,
3943                        staged: Vec::new(),
3944                    })
3945                } else {
3946                    match failure {
3947                        Err(failure) => Err(failure),
3948                        Ok(_) => Err(AppError::Integration(
3949                            "child failure policy returned admission success".into(),
3950                        )
3951                        .into()),
3952                    }
3953                }
3954            }
3955        }
3956    }
3957
3958    fn schedule_child_retention(&self, control: &ControlRecord) {
3959        let ControlRecordState::Child { admitted, .. } = &control.state else {
3960            return;
3961        };
3962        for member in admitted {
3963            match Self::attach(Arc::clone(&self.app), member.key.clone()) {
3964                Ok(child) => {
3965                    child.schedule(ScheduleCause::Reconcile);
3966                }
3967                Err(error) => self.app.tasks_failed(FinalResult::from(&error)),
3968            }
3969        }
3970    }
3971
3972    fn plan_child_completion(
3973        &self,
3974        loaded: &LoadedRun,
3975        control: &ControlRecord,
3976        children: Vec<RunRecord>,
3977    ) -> Result<ControlTransition, AppError> {
3978        let planned = catch_unwind(AssertUnwindSafe(|| {
3979            self.plan_children(loaded, control.sequence, children, true)
3980        }))
3981        .map_err(|_| AppError::Integration("child planning callback panicked".into()))
3982        .and_then(|result| result)
3983        .and_then(|decision| match decision {
3984            BspDecision::Stage(activations) => Ok(ControlTransition::Stage { activations }),
3985            BspDecision::Failed {
3986                category,
3987                message,
3988                observation,
3989            } => Ok(ControlTransition::Failed {
3990                category,
3991                message,
3992                observation,
3993            }),
3994            _ => Err(AppError::Integration(
3995                "completed child group has no final outcome".into(),
3996            )),
3997        });
3998        if !matches!(&planned, Ok(ControlTransition::Stage { activations })
3999            if activations.iter().all(|activation| matches!(activation.outcome, StagedOutcomeRecord::Complete { .. })))
4000        {
4001            // Failed children, output validation, expired deadlines and planning
4002            // errors keep their existing independent terminal cleanup.
4003            self.schedule_child_retention(control);
4004        }
4005        planned
4006    }
4007
4008    async fn publish_child_outcome(
4009        &self,
4010        control: &ControlRecord,
4011        transition: ControlTransition,
4012    ) -> Result<SessionReceipt, RuntimeOperationFailure> {
4013        // Retain the verified result under its original Child control while
4014        // unrelated sibling resolutions advance the parent revision. Both
4015        // local and worker execution keep this one publication receipt.
4016        let completed = matches!(&transition, ControlTransition::Stage { activations }
4017            if activations.iter().all(|activation| matches!(activation.outcome, StagedOutcomeRecord::Complete { .. })));
4018        if let Some(loaded) = self.commit_scheduled_transition(control, transition).await {
4019            // The native commit owner already retained Applied authority or
4020            // reconciled Unknown. Return that exact receipt and revision.
4021            return Ok(SessionReceipt::Checkpoint {
4022                loaded,
4023                staged: Vec::new(),
4024            });
4025        }
4026        if completed {
4027            // No applied Stage receipt owns the batch. The original exact
4028            // terminal children retain independent cleanup, including a stop
4029            // that won before this publication or unresolved native authority.
4030            self.schedule_child_retention(control);
4031        }
4032        let loaded = match tokio::time::timeout(
4033            Self::remaining(control.deadline_millis)?,
4034            self.load_for_recovery(),
4035        )
4036        .await
4037        {
4038            Ok(result) => result
4039                .map_err(RuntimeOperationFailure::from)
4040                .and_then(|loaded| {
4041                    loaded.ok_or_else(|| {
4042                        AppError::NotFound("child parent is unavailable".into()).into()
4043                    })
4044                }),
4045            Err(_) => Err(AppError::Timeout("child outcome observation timed out".into()).into()),
4046        };
4047        if !loaded.as_ref().is_ok_and(|loaded| {
4048            loaded.record.checkpoint.is_stopping() || loaded.record.checkpoint.is_terminal()
4049        }) {
4050            self.app.tasks_failed(FinalResult::Integration(
4051                "child outcome publication authority is unresolved".into(),
4052            ));
4053            return Err(loaded.err().unwrap_or_else(|| {
4054                AppError::Closed("child outcome publication requires reconciliation".into()).into()
4055            }));
4056        }
4057        Ok(SessionReceipt::Checkpoint {
4058            loaded: loaded?,
4059            staged: Vec::new(),
4060        })
4061    }
4062
4063    fn retain_source_failure(
4064        &self,
4065        context: &ActionContext,
4066        lineage: &str,
4067        category: FailureCategory,
4068    ) {
4069        let Some(generation) = self.cell.lifecycle_generation() else {
4070            return;
4071        };
4072        match self
4073            .cell
4074            .observe_source_failure(generation, context, lineage, category)
4075        {
4076            Ok(()) | Err(AppError::Closed(_)) => {
4077                // Completed guards have already projected authoritative truth.
4078                // A later source destructor cannot reopen that observation.
4079            }
4080            Err(_) => self.app.tasks_failed(FinalResult::Integration(
4081                "source failure observation authority failed".into(),
4082            )),
4083        }
4084    }
4085
4086    async fn end_source(
4087        &self,
4088        loaded: LoadedRun,
4089        execution: Option<u64>,
4090    ) -> Result<LoadedRun, AppError> {
4091        let generation = self
4092            .cell
4093            .lifecycle_generation()
4094            .ok_or_else(|| AppError::Closed("source cleanup observation is unavailable".into()))?;
4095        let deadline = self.cell.begin_action_cleanup(generation)?;
4096        // EOF rolls back the complete uncommitted wave. Close its original
4097        // guards before discarding any evidence a failing hook may need.
4098        for context in self.cell.action_observations() {
4099            if context.checkpoint != loaded.record.checkpoint.base_sequence {
4100                continue;
4101            }
4102            let mut observed_activation = None;
4103            for activation in &loaded.record.checkpoint.frontier {
4104                if self
4105                    .cell
4106                    .action_context(
4107                        loaded.record.checkpoint.base_sequence,
4108                        &activation.node,
4109                        &activation.lineage,
4110                    )
4111                    .is_some_and(|observed| Arc::ptr_eq(&observed, &context))
4112                {
4113                    observed_activation = Some(activation);
4114                    break;
4115                }
4116            }
4117            let activation = observed_activation.ok_or_else(|| {
4118                AppError::Integration("source cleanup activation is unavailable".into())
4119            })?;
4120            if let Err(failure) = self
4121                .observe_step_finish(
4122                    generation,
4123                    &context,
4124                    None,
4125                    &loaded.record,
4126                    loaded
4127                        .record
4128                        .checkpoint
4129                        .control_for(&activation.node, &activation.lineage)
4130                        .is_some(),
4131                    Some(deadline),
4132                )
4133                .await
4134            {
4135                return Err(self
4136                    .fail_hook(&loaded, Some((activation, &context)), failure, Vec::new())
4137                    .await);
4138            }
4139        }
4140        let loaded = self
4141            .load_execution_observation(None, Some(deadline))
4142            .await
4143            .map_err(|failure| {
4144                self.retain_execution_read_failure(&failure);
4145                failure.into_error()
4146            })?
4147            .ok_or_else(|| AppError::NotFound("workflow run does not exist".into()))?;
4148        let loaded = if let Some(generation) = execution {
4149            // Keep this Execution's lease and options: its caller must first
4150            // observe the retained None before choosing retry or finalization.
4151            self.settle_execution(generation, Some(loaded)).await?.0
4152        } else if loaded.record.checkpoint.is_stopping() || loaded.record.checkpoint.is_terminal() {
4153            loaded
4154        } else {
4155            let definition = self.app.definition_for(&loaded.record)?;
4156            let discard_staged = loaded.record.checkpoint.rollback_stage_keys(&definition);
4157            // The original EndSource operation owns the native pause receipt.
4158            self.commit_for_recovery(loaded, ControlTransition::Pause { discard_staged })
4159                .await
4160                .map_err(RuntimeOperationFailure::into_error)?
4161        };
4162        Ok(loaded)
4163    }
4164
4165    pub(super) async fn finish_execution(
4166        &self,
4167        generation: u64,
4168        loaded: Option<LoadedRun>,
4169    ) -> Result<RunRecord, AppError> {
4170        if !self.cell.owns_execution(generation) {
4171            return Err(AppError::Closed(
4172                "run execution generation no longer owns cleanup".into(),
4173            ));
4174        }
4175        let mut changes = self.app.runtime_changes();
4176        while self.app.has_child_work(&self.key) {
4177            changes
4178                .changed()
4179                .await
4180                .map_err(|_| AppError::Closed("child receipt observation closed".into()))?;
4181        }
4182        let settled = self.settle_execution(generation, loaded).await;
4183        if let Err(error) = &settled
4184            && self.cell.source_failure(generation).is_some()
4185        {
4186            if !matches!(error, AppError::Closed(_)) {
4187                self.app.tasks_failed(FinalResult::Integration(
4188                    "source failure cleanup authority failed".into(),
4189                ));
4190            }
4191            return settled.map(|(loaded, _)| loaded.record);
4192        }
4193        let retained = settled.as_ref().is_ok_and(|(_, retained)| *retained);
4194        let mut result = settled.map(|(loaded, _)| loaded.record);
4195        let cleanup_pending = result.as_ref().is_ok_and(|record| {
4196            record.checkpoint.is_stopping()
4197                || !record.checkpoint.controls.is_empty()
4198                || record.checkpoint.is_terminal() && !retained
4199        });
4200        if let Ok(record) = &result {
4201            let observed = self
4202                .finish_execution_observation(record, None, None, false)
4203                .await;
4204            if record.checkpoint.is_terminal() && self.cell.child_parent().is_none() {
4205                self.maintain_retention(None).await;
4206            }
4207            if let Err(error) = observed {
4208                result = Err(error);
4209            }
4210        }
4211        self.cell.end_publication(generation);
4212        self.cell.clear_execution_options(Some(generation));
4213        let scheduled = self.cell.release_execution(generation);
4214        if !retained && (cleanup_pending || scheduled) {
4215            self.schedule(ScheduleCause::Reconcile);
4216        }
4217        result
4218    }
4219
4220    pub(super) fn settle_execution(
4221        &self,
4222        generation: u64,
4223        loaded: Option<LoadedRun>,
4224    ) -> Pin<Box<impl Future<Output = Result<(LoadedRun, bool), AppError>> + '_>> {
4225        Box::pin(async move {
4226            if !self.cell.owns_execution(generation) {
4227                return Err(AppError::Closed(
4228                    "run execution generation no longer owns cleanup".into(),
4229                ));
4230            }
4231            let mut retained = false;
4232            let result = async {
4233                let mut loaded = match loaded {
4234                    Some(loaded) => loaded,
4235                    None => {
4236                        let deadline = self.cell.begin_action_cleanup(generation)?;
4237                        // Pure settlement reads and unfinished guards retain
4238                        // the same originally sampled cleanup budget.
4239                        self.load_execution_observation(None, Some(deadline))
4240                            .await
4241                            .map_err(|failure| {
4242                                self.retain_execution_read_failure(&failure);
4243                                failure.into_error()
4244                            })?
4245                            .ok_or_else(|| {
4246                                AppError::NotFound("workflow run does not exist".into())
4247                            })?
4248                    }
4249                };
4250                if let Some((context, lineage, category)) = self.cell.source_failure(generation)
4251                    && context.checkpoint == loaded.record.checkpoint.base_sequence
4252                    && !loaded.record.checkpoint.is_stopping()
4253                    && !loaded.record.checkpoint.is_terminal()
4254                    && loaded
4255                        .record
4256                        .checkpoint
4257                        .position
4258                        .as_ref()
4259                        .is_none_or(|committed| committed.ordinal < context.action_ordinal)
4260                {
4261                    let position = ActionPosition {
4262                        node: context.node.clone(),
4263                        ordinal: context.action_ordinal,
4264                    };
4265                    let definition = self.app.definition_for(&loaded.record)?;
4266                    let (position, category, message) = match loaded
4267                        .record
4268                        .checkpoint
4269                        .preceding_failure(&definition, &position, &lineage)?
4270                    {
4271                        Some((position, category, message)) => {
4272                            (position, category, message.to_owned())
4273                        }
4274                        None => (position, category, "caller action execution failed".into()),
4275                    };
4276                    loaded = self
4277                        .commit_for_recovery(
4278                            loaded,
4279                            ControlTransition::Failed {
4280                                category,
4281                                message,
4282                                observation: Some(position),
4283                            },
4284                        )
4285                        .await
4286                        .map_err(RuntimeOperationFailure::into_error)?;
4287                }
4288                if !loaded.record.checkpoint.is_terminal()
4289                    && !loaded.record.checkpoint.is_stopping()
4290                {
4291                    let definition = self.app.definition_for(&loaded.record)?;
4292                    let failed = loaded
4293                        .record
4294                        .checkpoint
4295                        .selected_staged(&definition)
4296                        .filter_map(|activation| {
4297                            if let StagedOutcomeRecord::Failed { category, message } =
4298                                &activation.outcome
4299                            {
4300                                Some((activation, *category, message))
4301                            } else {
4302                                None
4303                            }
4304                        })
4305                        .min_by(|left, right| {
4306                            (&left.0.node, &left.0.lineage).cmp(&(&right.0.node, &right.0.lineage))
4307                        });
4308                    if let Some((activation, category, message)) = failed {
4309                        let message = message.clone();
4310                        let observation = loaded.record.checkpoint.proposed_position(
4311                            &definition,
4312                            &activation.node,
4313                            &activation.lineage,
4314                            &[],
4315                        )?;
4316                        loaded = self
4317                            .commit_for_recovery(
4318                                loaded,
4319                                ControlTransition::Failed {
4320                                    category,
4321                                    message,
4322                                    observation: Some(observation),
4323                                },
4324                            )
4325                            .await
4326                            .map_err(RuntimeOperationFailure::into_error)?;
4327                    }
4328                }
4329                // No lower owner remains in this case. Complete the accepted
4330                // rollback on this generation before making a retry admissible.
4331                if loaded.record.checkpoint.is_stopping()
4332                    && loaded.record.checkpoint.controls.is_empty()
4333                {
4334                    loaded = match Box::pin(self.settle_stopping(loaded)).await {
4335                        Some(loaded) => loaded,
4336                        None => {
4337                            // No terminal receipt owns the result. Observe
4338                            // current authority under the helper's finite pass;
4339                            // native mutations keep their separate ownership.
4340                            self.load_execution_observation(None, None)
4341                                .await
4342                                .map_err(|failure| {
4343                                    self.retain_execution_read_failure(&failure);
4344                                    failure.into_error()
4345                                })?
4346                                .ok_or_else(|| {
4347                                    AppError::NotFound("workflow run does not exist".into())
4348                                })?
4349                        }
4350                    };
4351                }
4352                if !loaded.record.checkpoint.staged.is_empty()
4353                    && !loaded.record.checkpoint.is_terminal()
4354                    && !loaded.record.checkpoint.is_stopping()
4355                {
4356                    let definition = self.app.definition_for(&loaded.record)?;
4357                    let discard_staged = loaded.record.checkpoint.rollback_stage_keys(&definition);
4358                    return self
4359                        .commit_for_recovery(loaded, ControlTransition::Pause { discard_staged })
4360                        .await
4361                        .map_err(RuntimeOperationFailure::into_error);
4362                }
4363                if loaded.record.checkpoint.is_terminal() {
4364                    // Registration has no active execution guards. Completed
4365                    // guards likewise permit the original capacity cleanup.
4366                    let deadline = if self.cell.lifecycle_generation().is_none() {
4367                        Box::pin(self.maintain_retention(Some(loaded))).await;
4368                        // Retention can advance the head revision. Its native
4369                        // receipt starts this required fresh capacity-read pass.
4370                        let deadline = LifecycleEvent::deadline(None);
4371                        loaded = self
4372                            .load_execution_observation(None, Some(deadline))
4373                            .await
4374                            .map_err(|failure| {
4375                                self.retain_execution_read_failure(&failure);
4376                                failure.into_error()
4377                            })?
4378                            .ok_or_else(|| {
4379                                AppError::NotFound("workflow run does not exist".into())
4380                            })?;
4381                        deadline
4382                    } else {
4383                        LifecycleEvent::deadline(None)
4384                    };
4385                    retained = tokio::time::timeout_at(deadline, async {
4386                        let mut delay = RECONCILIATION_INITIAL_DELAY;
4387                        loop {
4388                            match self.app.storage().run_is_active(&loaded.record).await {
4389                                Ok(active) => {
4390                                    // Marker authority cannot replace the App
4391                                    // generation fence of the native result.
4392                                    self.observe_generation(deadline).await?;
4393                                    return Ok(!active);
4394                                }
4395                                Err(failure)
4396                                    if failure.is_recoverable()
4397                                        && self.schedule_after_storage_failure(&failure) =>
4398                                {
4399                                    tokio::time::sleep(delay).await;
4400                                    delay = delay.saturating_mul(2).min(RECONCILIATION_MAX_DELAY);
4401                                }
4402                                Err(failure) => return Err(failure),
4403                            }
4404                        }
4405                    })
4406                    .await
4407                    .unwrap_or_else(|_| {
4408                        Err(StorageFailure::permanent(AppError::Timeout(
4409                            "workflow capacity observation deadline expired".into(),
4410                        )))
4411                    })
4412                    .map_err(|failure| {
4413                        self.retain_execution_read_failure(&failure);
4414                        failure.into_error()
4415                    })?;
4416                } else if loaded.record.checkpoint.is_stopping() {
4417                    // Failed may retain admitted controls in Failing. Returning
4418                    // that native receipt still fences this App generation.
4419                    self.observe_generation(LifecycleEvent::deadline(None))
4420                        .await
4421                        .map_err(|failure| {
4422                            self.retain_execution_read_failure(&failure);
4423                            failure.into_error()
4424                        })?;
4425                }
4426                Ok(loaded)
4427            }
4428            .await;
4429            result.map(|loaded| (loaded, retained))
4430        })
4431    }
4432}
4433
4434impl<
4435    CS: StateStoreBackend,
4436    A: AgentBackend,
4437    X: ActionBackend,
4438    C: ConditionBackend,
4439    T: StateTyping,
4440    H: LifecycleHooks,
4441    E: ChildExecutor,
4442> WorkflowRunRuntime<CS, A, X, C, T, H, E>
4443{
4444    pub(super) async fn begin_execution_observation(&self) -> Result<u64, WorkflowFailure> {
4445        let owner = self
4446            .cell
4447            .observation_owner()
4448            .map_err(|error| WorkflowFailure::Execution(error.to_string()))?;
4449        let (generation, opening) = self
4450            .cell
4451            .begin_lifecycle(owner)
4452            .map_err(|error| WorkflowFailure::Execution(error.to_string()))?;
4453        if opening {
4454            let deadline = self
4455                .cell
4456                .begin_action_cleanup(generation)
4457                .map_err(|error| WorkflowFailure::Execution(error.to_string()))?;
4458            LifecycleEvent::ExecuteStarted {
4459                run: self.id(),
4460                occurrence: self.occurrence(),
4461            }
4462            .deliver(
4463                self.app.hooks(),
4464                self.cell
4465                    .execution_mode()
4466                    .map_err(|error| WorkflowFailure::Execution(error.to_string()))?,
4467                deadline,
4468            )
4469            .await?;
4470            self.cell
4471                .opened_lifecycle(generation)
4472                .map_err(|error| WorkflowFailure::Execution(error.to_string()))?;
4473        }
4474        Ok(generation)
4475    }
4476
4477    async fn observe_activation_start(
4478        &self,
4479        loaded: &LoadedRun,
4480        activation: &FrontierRecord,
4481        context: &Arc<ActionContext>,
4482        inputs: &Map<String, Value>,
4483        deadline: tokio::time::Instant,
4484    ) -> Result<(), WorkflowFailure> {
4485        let generation = self.begin_execution_observation().await?;
4486        let definition = self
4487            .app
4488            .definition_for(&loaded.record)
4489            .map_err(|error| WorkflowFailure::Execution(error.to_string()))?;
4490        let Some(Step::Action {
4491            action, revision, ..
4492        }) = definition.node(&activation.node).map(|node| node.step())
4493        else {
4494            return self
4495                .observe_step_start(
4496                    generation,
4497                    &loaded.record,
4498                    context,
4499                    &activation.lineage,
4500                    inputs,
4501                    deadline,
4502                )
4503                .await
4504                .map(|_| ());
4505        };
4506        let binding = self
4507            .app
4508            .action(action, revision)
4509            .map_err(|error| WorkflowFailure::Execution(error.to_string()))?;
4510        let inputs = catch_unwind(AssertUnwindSafe(|| {
4511            binding
4512                .declaration()
4513                .resolve_inputs(inputs, self.cell.execution_mode()?)
4514                .and_then(ActionInputs::into_values)
4515        }))
4516        .map_err(|_| WorkflowFailure::Execution("action declaration panicked".into()))?
4517        .map_err(|error| WorkflowFailure::State(error.to_string()))?;
4518        let context = self
4519            .observe_step_start(
4520                generation,
4521                &loaded.record,
4522                context,
4523                &activation.lineage,
4524                &inputs,
4525                deadline,
4526            )
4527            .await?;
4528        self.observe_stream_start(generation, &context, deadline)
4529            .await
4530    }
4531
4532    pub(super) async fn observe_stream_start(
4533        &self,
4534        generation: u64,
4535        context: &ActionContext,
4536        deadline: tokio::time::Instant,
4537    ) -> Result<(), WorkflowFailure> {
4538        if self.cell.lifecycle_generation() != Some(generation)
4539            || self.cell.execution_was_dropped(generation)
4540        {
4541            return Ok(());
4542        }
4543        LifecycleEvent::StreamStarted {
4544            context,
4545            span: None,
4546        }
4547        .deliver(
4548            self.app.hooks(),
4549            context.mode,
4550            LifecycleEvent::deadline(Some(deadline)),
4551        )
4552        .await?;
4553        self.cell
4554            .open_action_stream(generation, context)
4555            .map_err(|error| WorkflowFailure::Execution(error.to_string()))?;
4556        Ok(())
4557    }
4558
4559    pub(super) async fn observe_stream_item(
4560        &self,
4561        generation: u64,
4562        context: &ActionContext,
4563        result: &Map<String, Value>,
4564        deadline: tokio::time::Instant,
4565    ) -> Result<(), WorkflowFailure> {
4566        if self.cell.lifecycle_generation() != Some(generation)
4567            || self.cell.execution_was_dropped(generation)
4568        {
4569            return Ok(());
4570        }
4571        let index = self
4572            .cell
4573            .action_stream_item(generation, context, false)
4574            .map_err(|error| WorkflowFailure::Execution(error.to_string()))?;
4575        LifecycleEvent::StreamItem {
4576            context,
4577            span: None,
4578            index,
4579            result,
4580        }
4581        .deliver(
4582            self.app.hooks(),
4583            context.mode,
4584            LifecycleEvent::deadline(Some(deadline)),
4585        )
4586        .await?;
4587        self.cell
4588            .action_stream_item(generation, context, true)
4589            .map_err(|error| WorkflowFailure::Execution(error.to_string()))?;
4590        Ok(())
4591    }
4592
4593    async fn observe_source_event(
4594        &self,
4595        context: &ActionContext,
4596        event: ActionEvent,
4597        deadline: tokio::time::Instant,
4598    ) -> Result<Option<Map<String, Value>>, WorkflowFailure> {
4599        let generation = self.cell.lifecycle_generation().ok_or_else(|| {
4600            WorkflowFailure::Execution("source observation generation closed".into())
4601        })?;
4602        if let ActionEvent::Partial { result } = event {
4603            self.observe_stream_item(generation, context, &result, deadline)
4604                .await?;
4605            return Ok(Some(result));
4606        }
4607        if self.cell.execution_was_dropped(generation) {
4608            return Ok(None);
4609        }
4610        event
4611            .validate_span()
4612            .map_err(|error| WorkflowFailure::Execution(error.to_string()))?;
4613        let deadline = LifecycleEvent::deadline(Some(deadline));
4614        match event {
4615            ActionEvent::SpanStarted {
4616                id,
4617                name,
4618                parent,
4619                dependencies,
4620            } => {
4621                let parent_index = self
4622                    .cell
4623                    .span_parent(generation, context, &id, parent.as_deref())
4624                    .map_err(|error| WorkflowFailure::Execution(error.to_string()))?;
4625                LifecycleEvent::SpanStarted {
4626                    context,
4627                    id: &id,
4628                    name: &name,
4629                    parent: parent.as_deref(),
4630                    dependencies: &dependencies,
4631                }
4632                .deliver(self.app.hooks(), context.mode, deadline)
4633                .await?;
4634                self.cell
4635                    .opened_span(generation, context, id, parent_index)
4636                    .map_err(|error| WorkflowFailure::Execution(error.to_string()))?;
4637            }
4638            ActionEvent::SpanAttribute {
4639                span,
4640                key,
4641                value,
4642                tags,
4643            } => {
4644                self.cell
4645                    .span_attribute(generation, context, span.as_deref())
4646                    .map_err(|error| WorkflowFailure::Execution(error.to_string()))?;
4647                LifecycleEvent::SpanAttribute {
4648                    context,
4649                    span: span.as_deref(),
4650                    key: &key,
4651                    value: &value,
4652                    tags: &tags,
4653                }
4654                .deliver(self.app.hooks(), context.mode, deadline)
4655                .await?;
4656            }
4657            ActionEvent::SpanFinished { id } => {
4658                let id = self
4659                    .cell
4660                    .take_span(generation, context, Some(&id))
4661                    .map_err(|error| WorkflowFailure::Execution(error.to_string()))?
4662                    .ok_or_else(|| {
4663                        WorkflowFailure::Execution(
4664                            "caller span finish target is unavailable".into(),
4665                        )
4666                    })?;
4667                LifecycleEvent::SpanFinished { context, id: &id }
4668                    .deliver(self.app.hooks(), context.mode, deadline)
4669                    .await?;
4670            }
4671            ActionEvent::Partial { .. } | ActionEvent::Completed { .. } => unreachable!(),
4672        }
4673        Ok(None)
4674    }
4675
4676    async fn finish_spans(
4677        &self,
4678        generation: u64,
4679        context: &ActionContext,
4680        async_deadline: Option<tokio::time::Instant>,
4681    ) -> Result<(), WorkflowFailure> {
4682        let mut failure = None;
4683        loop {
4684            let id = match self.cell.take_span(generation, context, None) {
4685                Ok(Some(id)) => id,
4686                Ok(None) => break,
4687                Err(error) => {
4688                    return Err(
4689                        failure.unwrap_or_else(|| WorkflowFailure::Execution(error.to_string()))
4690                    );
4691                }
4692            };
4693            let event = LifecycleEvent::SpanFinished { context, id: &id };
4694            let observed =
4695                match async_deadline.filter(|_| !self.cell.execution_was_dropped(generation)) {
4696                    Some(deadline) => {
4697                        event
4698                            .deliver(self.app.hooks(), context.mode, deadline)
4699                            .await
4700                    }
4701                    None => event.deliver_sync(self.app.hooks()),
4702                };
4703            if let Err(next) = observed
4704                && failure.is_none()
4705            {
4706                failure = Some(next);
4707            }
4708        }
4709        failure.map_or(Ok(()), Err)
4710    }
4711
4712    fn finish_spans_sync(
4713        &self,
4714        generation: u64,
4715        context: &ActionContext,
4716    ) -> Result<(), WorkflowFailure> {
4717        let mut failure = None;
4718        loop {
4719            let id = match self.cell.take_span(generation, context, None) {
4720                Ok(Some(id)) => id,
4721                Ok(None) => break,
4722                Err(error) => {
4723                    return Err(
4724                        failure.unwrap_or_else(|| WorkflowFailure::Execution(error.to_string()))
4725                    );
4726                }
4727            };
4728            if let Err(next) =
4729                (LifecycleEvent::SpanFinished { context, id: &id }).deliver_sync(self.app.hooks())
4730                && failure.is_none()
4731            {
4732                failure = Some(next);
4733            }
4734        }
4735        failure.map_or(Ok(()), Err)
4736    }
4737
4738    async fn finish_staged_spans(
4739        &self,
4740        loaded: &LoadedRun,
4741        decision: &BspDecision,
4742        context: Option<&ActionContext>,
4743    ) -> Result<(), AppError> {
4744        let (BspDecision::Stage(staged), Some(context)) = (decision, context) else {
4745            return Ok(());
4746        };
4747        let Some(generation) = self.cell.lifecycle_generation() else {
4748            return Ok(());
4749        };
4750        let Some(staged) = staged.iter().find(|staged| {
4751            self.cell
4752                .action_context(
4753                    loaded.record.checkpoint.base_sequence,
4754                    &staged.node,
4755                    &staged.lineage,
4756                )
4757                .is_some_and(|observed| std::ptr::eq(observed.as_ref(), context))
4758                && matches!(staged.outcome, StagedOutcomeRecord::Complete { .. })
4759        }) else {
4760            return Ok(());
4761        };
4762        let activation = loaded
4763            .record
4764            .checkpoint
4765            .frontier
4766            .iter()
4767            .find(|activation| {
4768                activation.node == staged.node && activation.lineage == staged.lineage
4769            })
4770            .ok_or_else(|| {
4771                AppError::Integration("completed caller activation is unavailable".into())
4772            })?;
4773        // The one caller Stage carries its original context across sibling
4774        // completion; no ordinal is resampled after another source settles.
4775        if let Err(failure) = self
4776            .finish_spans(generation, context, Some(LifecycleEvent::deadline(None)))
4777            .await
4778        {
4779            return Err(self
4780                .fail_hook(loaded, Some((activation, context)), failure, Vec::new())
4781                .await);
4782        }
4783        Ok(())
4784    }
4785
4786    pub(super) async fn fail_hook(
4787        &self,
4788        loaded: &LoadedRun,
4789        activation: Option<(&FrontierRecord, &ActionContext)>,
4790        failure: WorkflowFailure,
4791        staged: Vec<StagedActivationRecord>,
4792    ) -> AppError {
4793        let error = failure.application_error();
4794        let mut prior = None;
4795        let settled = async {
4796            let generation = self
4797                .cell
4798                .lifecycle_generation()
4799                .ok_or_else(|| AppError::Closed("lifecycle failure cleanup is closed".into()))?;
4800            let deadline = self.cell.begin_action_cleanup(generation)?;
4801            let mut loaded = LoadedRun {
4802                record: loaded.record.clone(),
4803                revision: loaded.revision,
4804            };
4805            if !staged.is_empty() {
4806                // Preserve the exact prefix before failure selection: its
4807                // Complete entries determine the earlier failure's ordinal.
4808                loaded = self
4809                    .commit_for_recovery(
4810                        loaded,
4811                        ControlTransition::Stage {
4812                            activations: staged,
4813                        },
4814                    )
4815                    .await
4816                    .map_err(RuntimeOperationFailure::into_error)?;
4817            }
4818            let definition = self.app.definition_for(&loaded.record)?;
4819            let retained = self
4820                .cell
4821                .lifecycle_generation()
4822                .and_then(|generation| self.cell.source_failure(generation));
4823            let source = retained.as_ref().filter(|(context, lineage, _)| {
4824                context.checkpoint == loaded.record.checkpoint.base_sequence
4825                    && activation.is_some_and(|(activation, _)| {
4826                        (&context.node, lineage) < (&activation.node, &activation.lineage)
4827                    })
4828            });
4829            // An earlier source can finish while this original callback is
4830            // held. Its sampled identity wins; a same-action destructor never
4831            // replaces the callback's own primary Hook failure.
4832            let (mut observation, lineage, mut category, mut message) = match source {
4833                Some((context, lineage, category)) => {
4834                    let message = "caller action execution failed".to_owned();
4835                    prior = Some(category.project(message.clone()));
4836                    (
4837                        Some(ActionPosition {
4838                            node: context.node.clone(),
4839                            ordinal: context.action_ordinal,
4840                        }),
4841                        Some(lineage.as_str()),
4842                        *category,
4843                        message,
4844                    )
4845                }
4846                None => (
4847                    activation.map(|(_, context)| ActionPosition {
4848                        node: context.node.clone(),
4849                        ordinal: context.action_ordinal,
4850                    }),
4851                    activation.map(|(activation, _)| activation.lineage.as_str()),
4852                    FailureCategory::from_failure(&failure),
4853                    failure.message().to_owned(),
4854                ),
4855            };
4856            if let Some((position, lineage)) = observation.as_ref().zip(lineage)
4857                && let Some((position, preceding, text)) = loaded
4858                    .record
4859                    .checkpoint
4860                    .preceding_failure(&definition, position, lineage)?
4861            {
4862                observation = Some(position);
4863                category = preceding;
4864                message = text.to_owned();
4865                prior = Some(category.project(message.clone()));
4866            }
4867            let mut loaded = self
4868                .commit_for_recovery(
4869                    loaded,
4870                    ControlTransition::Failed {
4871                        category,
4872                        message,
4873                        observation,
4874                    },
4875                )
4876                .await
4877                .map_err(RuntimeOperationFailure::into_error)?;
4878            if !loaded.record.checkpoint.is_terminal() {
4879                // Accepted controls can progress independently while their
4880                // original failure is published. Refresh only that authority;
4881                // the native Stage and terminal receipts remain exact values.
4882                loaded = self
4883                    .load_execution_observation(None, Some(deadline))
4884                    .await
4885                    .map_err(|failure| {
4886                        self.retain_execution_read_failure(&failure);
4887                        failure.into_error()
4888                    })?
4889                    .ok_or_else(|| AppError::NotFound("workflow run does not exist".into()))?;
4890                if loaded.record.checkpoint.is_stopping()
4891                    && loaded.record.checkpoint.controls.is_empty()
4892                {
4893                    loaded = match Box::pin(self.settle_stopping(loaded)).await {
4894                        Some(loaded) => loaded,
4895                        None => self
4896                            .load_execution_observation(None, Some(deadline))
4897                            .await
4898                            .map_err(|failure| {
4899                                self.retain_execution_read_failure(&failure);
4900                                failure.into_error()
4901                            })?
4902                            .ok_or_else(|| {
4903                                AppError::NotFound("workflow run does not exist".into())
4904                            })?,
4905                    };
4906                }
4907            }
4908            if loaded.record.checkpoint.is_terminal() {
4909                // The accepted failure receipt is complete authority. Preserve
4910                // generation fencing without requiring the same head again.
4911                self.observe_generation(deadline).await.map_err(|failure| {
4912                    self.retain_execution_read_failure(&failure);
4913                    failure.into_error()
4914                })?;
4915            }
4916            Ok(loaded)
4917        }
4918        .await;
4919        match settled {
4920            Ok(loaded) => {
4921                let record = &loaded.record;
4922                // A prior canonical action failure remains durable; this later
4923                // callback error is its execution-completion context. Ordinary
4924                // precommit Hook failure still projects directly from storage.
4925                let completion = prior.map(|prior| failure.with_cause(Some(prior)));
4926                let _ = self
4927                    .finish_execution_observation(record, completion, None, false)
4928                    .await;
4929                if record.checkpoint.is_terminal() && self.cell.child_parent().is_none() {
4930                    Box::pin(self.maintain_retention(Some(loaded))).await;
4931                }
4932            }
4933            Err(AppError::Closed(_)) => {}
4934            Err(_) => self.app.tasks_failed(FinalResult::Integration(
4935                "lifecycle failure cleanup authority failed".into(),
4936            )),
4937        }
4938        error
4939    }
4940
4941    async fn observe_step_start(
4942        &self,
4943        generation: u64,
4944        record: &RunRecord,
4945        context: &Arc<ActionContext>,
4946        lineage: &str,
4947        inputs: &Map<String, Value>,
4948        deadline: tokio::time::Instant,
4949    ) -> Result<Arc<ActionContext>, WorkflowFailure> {
4950        if let Some(context) = self
4951            .cell
4952            .action_context(context.checkpoint, &context.node, lineage)
4953        {
4954            return Ok(context);
4955        }
4956        let state = WorkflowState {
4957            values: record.checkpoint.state.clone(),
4958        };
4959        LifecycleEvent::StepStarted {
4960            context,
4961            state: &state,
4962            inputs,
4963        }
4964        .deliver(
4965            self.app.hooks(),
4966            context.mode,
4967            LifecycleEvent::deadline(Some(deadline)),
4968        )
4969        .await?;
4970        self.cell
4971            .observe_action(generation, Arc::clone(context), lineage)
4972            .map_err(|error| WorkflowFailure::Execution(error.to_string()))?;
4973        Ok(Arc::clone(context))
4974    }
4975}
4976
4977impl<
4978    CS: StateStoreBackend,
4979    A: AgentBackend,
4980    X: ActionBackend,
4981    C: ConditionBackend,
4982    T: StateTyping,
4983    H: LifecycleHooks,
4984    E: ChildExecutor,
4985> WorkflowRunRuntime<CS, A, X, C, T, H, E>
4986{
4987    async fn observe_step_finish(
4988        &self,
4989        generation: u64,
4990        context: &ActionContext,
4991        outcome: Option<Result<&ActionOutcome, &WorkflowFailure>>,
4992        record: &RunRecord,
4993        unresolved: bool,
4994        async_deadline: Option<tokio::time::Instant>,
4995    ) -> Result<(), WorkflowFailure> {
4996        let checkpoint = self
4997            .project_history(record)
4998            .map_err(|error| WorkflowFailure::Execution(error.to_string()))?;
4999        let spans = self.finish_spans(generation, context, async_deadline).await;
5000        let Some((context, items)) = self.cell.take_action_observation(generation, context) else {
5001            if !unresolved {
5002                self.cell.retire_closed_observation(generation, context);
5003            }
5004            return spans;
5005        };
5006        let event = LifecycleEvent::StepFinished {
5007            context: &context,
5008            outcome,
5009            checkpoint: &checkpoint,
5010        };
5011        let step = match async_deadline.filter(|_| !self.cell.execution_was_dropped(generation)) {
5012            Some(deadline) => {
5013                event
5014                    .deliver(self.app.hooks(), context.mode, deadline)
5015                    .await
5016            }
5017            None => event.deliver_sync(self.app.hooks()),
5018        };
5019        let result = spans.and(step);
5020        let result = if let Some(items) = items {
5021            let failure = result
5022                .as_ref()
5023                .err()
5024                .or_else(|| outcome.and_then(Result::err));
5025            let event = LifecycleEvent::StreamFinished {
5026                context: &context,
5027                span: None,
5028                items,
5029                failure,
5030            };
5031            // A failed step guard unwinds the still-open stream guard. Its
5032            // synchronous Drop callback must not start another async lane.
5033            let stream_result = match async_deadline
5034                .filter(|_| result.is_ok() && !self.cell.execution_was_dropped(generation))
5035            {
5036                Some(deadline) => {
5037                    event
5038                        .deliver(self.app.hooks(), context.mode, deadline)
5039                        .await
5040                }
5041                None => event.deliver_sync(self.app.hooks()),
5042            };
5043            result.and(stream_result)
5044        } else {
5045            result
5046        };
5047        self.cell
5048            .finish_action_observation(generation, &context, unresolved);
5049        result
5050    }
5051
5052    fn observe_step_finish_sync(
5053        &self,
5054        generation: u64,
5055        context: &ActionContext,
5056        record: &RunRecord,
5057        unresolved: bool,
5058    ) -> Result<(), WorkflowFailure> {
5059        let checkpoint = self
5060            .project_history(record)
5061            .map_err(|error| WorkflowFailure::Execution(error.to_string()))?;
5062        let spans = self.finish_spans_sync(generation, context);
5063        let Some((context, items)) = self.cell.take_action_observation(generation, context) else {
5064            if !unresolved {
5065                self.cell.retire_closed_observation(generation, context);
5066            }
5067            return spans;
5068        };
5069        let step = LifecycleEvent::StepFinished {
5070            context: &context,
5071            outcome: None,
5072            checkpoint: &checkpoint,
5073        }
5074        .deliver_sync(self.app.hooks());
5075        let result = spans.and(step);
5076        let result = if let Some(items) = items {
5077            let next = LifecycleEvent::StreamFinished {
5078                context: &context,
5079                span: None,
5080                items,
5081                failure: result.as_ref().err(),
5082            }
5083            .deliver_sync(self.app.hooks());
5084            result.and(next)
5085        } else {
5086            result
5087        };
5088        self.cell
5089            .finish_action_observation(generation, &context, unresolved);
5090        result
5091    }
5092
5093    fn finish_execution_observation_sync(&self, record: &RunRecord) -> Result<(), AppError> {
5094        let Some(generation) = self.cell.lifecycle_generation() else {
5095            return self
5096                .cell
5097                .lifecycle_observation(record.checkpoint.sequence, false)
5098                .map(|_| ());
5099        };
5100        let unfinished = record
5101            .checkpoint
5102            .controls
5103            .values()
5104            .filter_map(|control| {
5105                self.cell.action_context(
5106                    record.checkpoint.base_sequence,
5107                    &control.node,
5108                    &control.lineage,
5109                )
5110            })
5111            .collect::<Vec<_>>();
5112        if !self
5113            .cell
5114            .begin_lifecycle_finish(generation, record.checkpoint.sequence)
5115        {
5116            return self
5117                .cell
5118                .lifecycle_observation(record.checkpoint.sequence, false)
5119                .map(|_| ());
5120        }
5121        let mut failure = None;
5122        for context in self.cell.action_observations() {
5123            let unresolved = unfinished
5124                .iter()
5125                .any(|unfinished| Arc::ptr_eq(unfinished, &context));
5126            if let Err(next) =
5127                self.observe_step_finish_sync(generation, &context, record, unresolved)
5128                && failure.is_none()
5129            {
5130                failure = Some(next);
5131            }
5132        }
5133        match self.project(record) {
5134            Ok(snapshot) => {
5135                if let Err(next) = (LifecycleEvent::ExecuteFinished {
5136                    snapshot: &snapshot,
5137                    failure: failure.as_ref(),
5138                })
5139                .deliver_sync(self.app.hooks())
5140                    && failure.is_none()
5141                {
5142                    failure = Some(next);
5143                }
5144            }
5145            Err(error) => {
5146                if failure.is_none() {
5147                    failure = Some(WorkflowFailure::Execution(error.to_string()));
5148                }
5149            }
5150        }
5151        if record.checkpoint.is_terminal() || record.checkpoint.is_paused() {
5152            self.cell.clear_execution_options(Some(generation));
5153        }
5154        let error = failure.as_ref().map(WorkflowFailure::application_error);
5155        let completed =
5156            self.cell
5157                .complete_lifecycle(generation, record.checkpoint.sequence, failure);
5158        match error {
5159            Some(error) => Err(error),
5160            None => completed,
5161        }
5162    }
5163
5164    pub(super) async fn finish_execution_observation(
5165        &self,
5166        record: &RunRecord,
5167        mut failure: Option<WorkflowFailure>,
5168        deadline: Option<tokio::time::Instant>,
5169        synchronous: bool,
5170    ) -> Result<(), AppError> {
5171        let Some(generation) = self.cell.lifecycle_generation() else {
5172            return self
5173                .cell
5174                .lifecycle_observation(record.checkpoint.sequence, false)
5175                .map(|_| ());
5176        };
5177        let synchronous = synchronous || self.cell.execution_was_dropped(generation);
5178        let unfinished = record
5179            .checkpoint
5180            .controls
5181            .values()
5182            .filter_map(|control| {
5183                self.cell.action_context(
5184                    record.checkpoint.base_sequence,
5185                    &control.node,
5186                    &control.lineage,
5187                )
5188            })
5189            .collect::<Vec<_>>();
5190        if !self
5191            .cell
5192            .begin_lifecycle_finish(generation, record.checkpoint.sequence)
5193        {
5194            return self
5195                .cell
5196                .lifecycle_observation(record.checkpoint.sequence, false)
5197                .map(|_| ());
5198        }
5199        let deadline = match deadline {
5200            Some(deadline) => deadline,
5201            None => self.cell.begin_action_cleanup(generation)?,
5202        };
5203        let primary = match &record.checkpoint.phase {
5204            CheckpointPhaseRecord::Failing {
5205                category, message, ..
5206            }
5207            | CheckpointPhaseRecord::Failed {
5208                category, message, ..
5209            } => Some(category.project(message.clone())),
5210            _ => None,
5211        };
5212        for context in self.cell.action_observations() {
5213            let unresolved = unfinished
5214                .iter()
5215                .any(|unfinished| Arc::ptr_eq(unfinished, &context));
5216            if unresolved && !record.checkpoint.is_stopping() && !synchronous {
5217                continue;
5218            }
5219            let outcome = record.checkpoint.outcomes.iter().find(|outcome| {
5220                outcome.ordinal == context.action_ordinal && outcome.node == context.node
5221            });
5222            let failed = record
5223                .checkpoint
5224                .failed_observation()
5225                .is_some_and(|position| {
5226                    position.ordinal == context.action_ordinal && position.node == context.node
5227                });
5228            let observed = outcome
5229                .map(Ok)
5230                .or_else(|| failed.then(|| primary.as_ref().map(Err)).flatten());
5231            if let Err(next) = self
5232                .observe_step_finish(
5233                    generation,
5234                    &context,
5235                    observed,
5236                    record,
5237                    unresolved,
5238                    (!synchronous && failure.is_none()).then_some(deadline),
5239                )
5240                .await
5241                && failure.is_none()
5242            {
5243                failure = Some(next.with_cause(match &record.checkpoint.phase {
5244                    CheckpointPhaseRecord::Failing {
5245                        category, message, ..
5246                    }
5247                    | CheckpointPhaseRecord::Failed {
5248                        category, message, ..
5249                    } => Some(category.project(message.clone())),
5250                    _ => None,
5251                }));
5252            }
5253        }
5254        match self.project(record) {
5255            Ok(snapshot) => {
5256                let event = LifecycleEvent::ExecuteFinished {
5257                    snapshot: &snapshot,
5258                    failure: failure.as_ref().or(primary.as_ref()),
5259                };
5260                let observed = if synchronous || self.cell.execution_was_dropped(generation) {
5261                    event.deliver_sync(self.app.hooks())
5262                } else {
5263                    event
5264                        .deliver(self.app.hooks(), self.cell.execution_mode()?, deadline)
5265                        .await
5266                };
5267                if let Err(next) = observed
5268                    && failure.is_none()
5269                {
5270                    failure = Some(next.with_cause(primary));
5271                }
5272            }
5273            Err(error) => {
5274                if failure.is_none() {
5275                    failure = Some(WorkflowFailure::Execution(error.to_string()));
5276                }
5277            }
5278        }
5279        if record.parent.is_some()
5280            && record.checkpoint.is_terminal()
5281            && let Err(next) = self.observe_child_finished(record, deadline).await
5282            && failure.is_none()
5283        {
5284            failure = Some(next);
5285        }
5286        if record.checkpoint.is_terminal() || record.checkpoint.is_paused() {
5287            self.cell.clear_execution_options(Some(generation));
5288        }
5289        let error = failure.as_ref().map(WorkflowFailure::application_error);
5290        let completed =
5291            self.cell
5292                .complete_lifecycle(generation, record.checkpoint.sequence, failure);
5293        match error {
5294            Some(error) => Err(error),
5295            None => completed,
5296        }
5297    }
5298
5299    pub(super) async fn observe_barrier(&self, loaded: LoadedRun) -> Result<LoadedRun, AppError> {
5300        let Some(generation) = self.cell.lifecycle_generation() else {
5301            return Ok(loaded);
5302        };
5303        let record = &loaded.record;
5304        let observations = self.cell.action_observations();
5305        for outcome in &record.checkpoint.outcomes {
5306            let context = observations.iter().find(|context| {
5307                context.action_ordinal == outcome.ordinal && context.node == outcome.node
5308            });
5309            let Some(context) = context else {
5310                continue;
5311            };
5312            let deadline = self.cell.begin_action_cleanup(generation)?;
5313            if let Err(failure) = self
5314                .observe_step_finish(
5315                    generation,
5316                    context,
5317                    Some(Ok(outcome)),
5318                    record,
5319                    false,
5320                    Some(deadline),
5321                )
5322                .await
5323            {
5324                let error = failure.application_error();
5325                // The committed barrier owns this Pause intent even when the
5326                // callback spent its cleanup budget. Native revision-fenced
5327                // reconciliation preserves independent cancellation.
5328                let parked = self
5329                    .commit_for_recovery(
5330                        loaded,
5331                        ControlTransition::Pause {
5332                            discard_staged: Vec::new(),
5333                        },
5334                    )
5335                    .await
5336                    .map_err(RuntimeOperationFailure::into_error);
5337                match parked {
5338                    Ok(parked) => {
5339                        let _ = self
5340                            .finish_execution_observation(
5341                                &parked.record,
5342                                Some(failure),
5343                                Some(deadline),
5344                                false,
5345                            )
5346                            .await;
5347                        return Ok(parked);
5348                    }
5349                    Err(AppError::Closed(_)) => {}
5350                    Err(_) => self.app.tasks_failed(FinalResult::Integration(
5351                        "post-hook cleanup authority failed".into(),
5352                    )),
5353                }
5354                return Err(error);
5355            }
5356        }
5357        Ok(loaded)
5358    }
5359}