Skip to main content

app_kit/workflow/run/
session.rs

1use std::future::{Future, poll_fn};
2use std::panic::{AssertUnwindSafe, catch_unwind};
3use std::pin::Pin;
4use std::sync::Arc;
5use std::task::{Context, Poll};
6use std::time::Duration;
7
8use agents_kit::AgentBackend;
9use futures_util::{FutureExt, Stream};
10use serde_json::{Map, Value};
11use storage_kit::StateStoreBackend;
12use tokio::sync::{OwnedSemaphorePermit, mpsc, oneshot};
13
14use crate::AppError;
15use crate::app::{AppRuntime, DriverCommand, FinalResult, RunTaskKind};
16use crate::storage::StorageFailure;
17use crate::workflow::action::ActionInputs;
18use crate::workflow::definition::{JsonBoundary, Step};
19use crate::workflow::execution::{
20    BspDecision, ExecutionEngine, StagedActivationRecord, StagedOutcomeRecord,
21};
22use crate::workflow::run::{
23    ActionPosition, CheckpointPhaseRecord, ChildAdmissionReceipt, ControlRecord,
24    ControlRecordState, ControlTransition, FailureCategory, FrontierRecord, LoadedRun,
25    MAX_DRIVER_TRANSITIONS, RECONCILIATION_INITIAL_DELAY, RECONCILIATION_MAX_DELAY, RunAdmission,
26    RunRecord, RuntimeOperationFailure, ScheduleCause, WorkflowRunRuntime,
27};
28use crate::workflow::{
29    ActionBackend, ActionContext, ActionEvent, ActionMode, ActionOutcome, ActionRequest,
30    ChildBatch, ChildCompletion, ChildExecutor, ConditionBackend, ExecutionEvent, ExecutionOptions,
31    InlineExecutor, JsonState, LifecycleEvent, LifecycleHooks, NoActions, NoConditions, NoHooks,
32    RunSnapshot, RunStatus, StateTyping, WorkflowFailure, WorkflowRun, WorkflowState,
33};
34
35/// Exclusive lazy advancement whose caller source stays on the polling task.
36///
37/// Dropping an operation future preserves the source and any admitted write on
38/// this owner. Dropping the owner stops source polling and retains tracked write
39/// reconciliation before another execution can acquire the run.
40///
41/// Before native commit admission, owner drop discards uncommitted state and
42/// revokes later advancement. After admission, App retains native commit authority
43/// and reconciles the original receipt; observation loss cannot promise rollback.
44/// This does not undo caller Action I/O. Prefer explicit finish/abort while the
45/// runtime is driven; ordinary run and history observers never own cancellation.
46///
47/// # Example
48///
49/// ```
50/// # use agents_kit::Agent;
51/// # use app_kit::{App, AppConfig, Shutdown, WorkspaceConfig};
52/// # use app_kit::workflow::{CheckpointSelector, ExecutionEvent, ExecutionOptions, RunSelector,
53/// #     RunStatus, WorkflowDefinition, WorkflowState};
54/// # use storage_kit::StateStore;
55/// # use storage_kit::state::config::{LanceStateStoreConfig, StateStoreConfig};
56/// #
57/// # let runtime = tokio::runtime::Builder::new_current_thread().enable_all().build()?;
58/// # runtime.block_on(async {
59/// # let store = StateStore::connect(StateStoreConfig::Lance(LanceStateStoreConfig::memory())).await?;
60/// # let definition = WorkflowDefinition::parse(r#"{
61/// #     "schema":"app-kit.workflow/v2", "id":"counter", "revision":"r1",
62/// #     "state":{"count":0}, "required":["count"],
63/// #     "reducers":{"count":"reject_concurrent"}, "entry":"increment",
64/// #     "nodes":[{"id":"increment","step":{"kind":"state","mutations":[
65/// #         {"kind":"increment","target":"count","value":{"kind":"literal","value":1}}
66/// #     ]}}], "routes":{}
67/// # }"#)?;
68/// # let config = AppConfig::<_, Agent>::new(store, Vec::new(), vec![definition]);
69/// # let mut app = App::open(config).await?;
70/// # let mut workspace = app.create_workspace(WorkspaceConfig {
71/// #     name: Some("local counter".into()), root: std::env::current_dir()?,
72/// # }).await?;
73/// # let workflow = workspace.workflow("counter".parse()?).await?;
74/// # let mut run = workflow.prepare(WorkflowState::default()).await?;
75/// let mut execution = run.execution(ExecutionOptions::default()).await?;
76/// while let Some(event) = execution.next().await? {
77///     if let ExecutionEvent::Committed { outcomes, .. } = event {
78///         assert_eq!(outcomes[0].node, "increment");
79///     }
80/// }
81/// let snapshot = execution.finish().await?;
82/// assert_eq!(snapshot.state.values()["count"], 1);
83/// #
84/// # app.shutdown(Shutdown::CancelAndDrain { timeout: std::time::Duration::from_secs(5) }).await?;
85/// # Ok::<(), Box<dyn std::error::Error>>(())
86/// # })?;
87/// # Ok::<(), Box<dyn std::error::Error>>(())
88/// ```
89#[must_use = "execution owns lazy advancement; finish or abort it explicitly"]
90pub struct Execution<
91    'a,
92    CS: StateStoreBackend,
93    A: AgentBackend,
94    X: ActionBackend = NoActions,
95    C: ConditionBackend = NoConditions,
96    T: StateTyping = JsonState,
97    H: LifecycleHooks = NoHooks,
98    E: ChildExecutor = InlineExecutor,
99> {
100    runtime: WorkflowRunRuntime<CS, A, X, C, T, H, E>,
101    app: &'a AppRuntime<CS, A, X, C, T, H, E>,
102    lane: ExecutionLane,
103    admission: Option<OwnedSemaphorePermit>,
104    children: Vec<ChildExecution<'a, ChildFuture<Self>, E>>,
105    wake: Option<Pin<Box<tokio::sync::futures::OwnedNotified>>>,
106    timer: Option<Pin<Box<tokio::time::Sleep>>>,
107    generation: Option<u64>,
108    parent_deadline: Option<tokio::time::Instant>,
109    state: SessionState,
110    sources: Vec<CallerSource<X>>,
111    next_source: usize,
112    native_partials: Option<(mpsc::Sender<ExecutionEvent>, mpsc::Receiver<ExecutionEvent>)>,
113    prefetched: Option<(usize, Result<Option<ActionEvent>, FailureCategory>)>,
114    transitions: usize,
115    observed: Option<RunRecord>,
116}
117
118/// Blocking advancement of the same caller-owned execution source.
119///
120/// Calls require a thread outside any entered Tokio runtime context. The App's
121/// captured runtime and caller-provided lower runtimes must remain alive and
122/// driven elsewhere. Source polling and destruction remain on the caller thread;
123/// dropping this owner delegates nonblocking revocation to [`Execution`].
124///
125/// # Example
126///
127/// ```
128/// # use agents_kit::Agent;
129/// # use app_kit::{App, AppConfig, Shutdown, WorkspaceConfig};
130/// # use app_kit::workflow::{CheckpointSelector, ExecutionEvent, ExecutionOptions, RunSelector,
131/// #     RunStatus, WorkflowDefinition, WorkflowState};
132/// # use storage_kit::StateStore;
133/// # use storage_kit::state::config::{LanceStateStoreConfig, StateStoreConfig};
134/// # use app_kit::workflow::SyncExecution;
135/// let runtime = tokio::runtime::Builder::new_multi_thread()
136///     .worker_threads(2).enable_all().build()?;
137/// # let (mut app, mut run) = runtime.block_on(async {
138/// # let store = StateStore::connect(StateStoreConfig::Lance(LanceStateStoreConfig::memory())).await?;
139/// # let definition = WorkflowDefinition::parse(r#"{
140/// #     "schema":"app-kit.workflow/v2", "id":"counter", "revision":"r1",
141/// #     "state":{"count":0}, "required":["count"],
142/// #     "reducers":{"count":"reject_concurrent"}, "entry":"increment",
143/// #     "nodes":[{"id":"increment","step":{"kind":"state","mutations":[
144/// #         {"kind":"increment","target":"count","value":{"kind":"literal","value":1}}
145/// #     ]}}], "routes":{}
146/// # }"#)?;
147/// # let config = AppConfig::<_, Agent>::new(store, Vec::new(), vec![definition]);
148/// # let mut app = App::open(config).await?;
149/// # let mut workspace = app.create_workspace(WorkspaceConfig {
150/// #     name: Some("local counter".into()), root: std::env::current_dir()?,
151/// # }).await?;
152/// # let workflow = workspace.workflow("counter".parse()?).await?;
153/// # let mut run = workflow.prepare(WorkflowState::default()).await?;
154/// # Ok::<_, Box<dyn std::error::Error>>((app, run))
155/// # })?;
156/// // Runtime workers stay alive; this thread is outside any entered Tokio context.
157/// let mut execution = SyncExecution::open(&mut run, ExecutionOptions::default())?;
158/// while execution.try_next()?.is_some() {}
159/// assert_eq!(execution.finish()?.state.values()["count"], 1);
160/// # runtime.block_on(app.shutdown(Shutdown::CancelAndDrain { timeout: std::time::Duration::from_secs(5) }))?;
161/// # Ok::<(), Box<dyn std::error::Error>>(())
162/// ```
163#[must_use = "execution owns lazy advancement; finish or abort it explicitly"]
164pub struct SyncExecution<
165    'a,
166    CS: StateStoreBackend,
167    A: AgentBackend,
168    X: ActionBackend = NoActions,
169    C: ConditionBackend = NoConditions,
170    T: StateTyping = JsonState,
171    H: LifecycleHooks = NoHooks,
172    E: ChildExecutor = InlineExecutor,
173> {
174    execution: Execution<'a, CS, A, X, C, T, H, E>,
175    runtime: tokio::runtime::Handle,
176}
177
178impl<
179    'a,
180    CS: StateStoreBackend,
181    A: AgentBackend,
182    X: ActionBackend,
183    C: ConditionBackend,
184    T: StateTyping,
185    H: LifecycleHooks,
186    E: ChildExecutor,
187> SyncExecution<'a, CS, A, X, C, T, H, E>
188{
189    /// Acquires lazy synchronous advancement on a thread outside Tokio.
190    ///
191    /// The App's captured runtime must remain alive and driven elsewhere.
192    /// Acquisition validates options before callbacks or source construction.
193    /// The returned owner borrows the run exclusively through its last use.
194    ///
195    /// A live synchronous owner prevents overlapping access to the run:
196    ///
197    /// ```compile_fail
198    /// use agents_kit::AgentBackend;
199    /// use app_kit::{AppError, workflow::{ExecutionOptions, RunSnapshot, SyncExecution, WorkflowRun}};
200    /// use storage_kit::StateStoreBackend;
201    /// fn overlapping<CS: StateStoreBackend, A: AgentBackend>(
202    /// run: &mut WorkflowRun<CS, A>,
203    /// ) -> Result<RunSnapshot, AppError> {
204    /// let execution = SyncExecution::open(run, ExecutionOptions::default())?;
205    /// let _ = run.id();
206    /// execution.abort()
207    /// }
208    /// ```
209    ///
210    /// # Errors
211    ///
212    /// Returns `InvalidInput` inside an entered Tokio runtime context before blocking.
213    /// Otherwise propagates the corresponding asynchronous execution errors, including
214    /// stale/revoked authority, invalid options, durable failures and timeouts.
215    /// The captured runtime and lower runtimes must stay alive and driven elsewhere.
216    ///
217    /// # Example
218    ///
219    /// ```
220    /// # use agents_kit::Agent;
221    /// # use app_kit::{App, AppConfig, Shutdown, WorkspaceConfig};
222    /// # use app_kit::workflow::{CheckpointSelector, ExecutionEvent, ExecutionOptions, RunSelector,
223    /// #     RunStatus, WorkflowDefinition, WorkflowState};
224    /// # use storage_kit::StateStore;
225    /// # use storage_kit::state::config::{LanceStateStoreConfig, StateStoreConfig};
226    /// # use app_kit::workflow::SyncExecution;
227    /// let runtime = tokio::runtime::Builder::new_multi_thread()
228    ///     .worker_threads(2).enable_all().build()?;
229    /// # let (mut app, mut run) = runtime.block_on(async {
230    /// # let store = StateStore::connect(StateStoreConfig::Lance(LanceStateStoreConfig::memory())).await?;
231    /// # let definition = WorkflowDefinition::parse(r#"{
232    /// #     "schema":"app-kit.workflow/v2", "id":"counter", "revision":"r1",
233    /// #     "state":{"count":0}, "required":["count"],
234    /// #     "reducers":{"count":"reject_concurrent"}, "entry":"increment",
235    /// #     "nodes":[{"id":"increment","step":{"kind":"state","mutations":[
236    /// #         {"kind":"increment","target":"count","value":{"kind":"literal","value":1}}
237    /// #     ]}}], "routes":{}
238    /// # }"#)?;
239    /// # let config = AppConfig::<_, Agent>::new(store, Vec::new(), vec![definition]);
240    /// # let mut app = App::open(config).await?;
241    /// # let mut workspace = app.create_workspace(WorkspaceConfig {
242    /// #     name: Some("local counter".into()), root: std::env::current_dir()?,
243    /// # }).await?;
244    /// # let workflow = workspace.workflow("counter".parse()?).await?;
245    /// # let mut run = workflow.prepare(WorkflowState::default()).await?;
246    /// # Ok::<_, Box<dyn std::error::Error>>((app, run))
247    /// # })?;
248    /// // Runtime workers stay alive; this thread is outside any entered Tokio context.
249    /// let execution = SyncExecution::open(&mut run, ExecutionOptions::default())?;
250    /// assert_eq!(execution.finish()?.state.values()["count"], 1);
251    /// # runtime.block_on(app.shutdown(Shutdown::CancelAndDrain { timeout: std::time::Duration::from_secs(5) }))?;
252    /// # Ok::<(), Box<dyn std::error::Error>>(())
253    /// ```
254    pub fn open(
255        run: &'a mut WorkflowRun<CS, A, X, C, T, H, E>,
256        options: ExecutionOptions,
257    ) -> Result<Self, AppError> {
258        Self::ensure_blocking()?;
259        let runtime = run.runtime.app.execution_handle()?;
260        let execution = runtime.block_on(Execution::open(run, options, ActionMode::Synchronous))?;
261        Ok(Self { execution, runtime })
262    }
263
264    fn ensure_blocking() -> Result<(), AppError> {
265        if tokio::runtime::Handle::try_current().is_ok() {
266            return Err(AppError::InvalidInput(
267                "synchronous execution requires a thread outside an entered Tokio runtime".into(),
268            ));
269        }
270        Ok(())
271    }
272
273    /// Blocks until one partial result or committed boundary is available.
274    ///
275    /// An entered runtime context is rejected before advancing this owner.
276    ///
277    /// # Errors
278    ///
279    /// Returns `InvalidInput` inside an entered Tokio runtime context before blocking.
280    /// Otherwise propagates the corresponding asynchronous execution errors, including
281    /// stale/revoked authority, invalid options, durable failures and timeouts.
282    /// The captured runtime and lower runtimes must stay alive and driven elsewhere.
283    ///
284    /// # Example
285    ///
286    /// ```
287    /// # use agents_kit::Agent;
288    /// # use app_kit::{App, AppConfig, Shutdown, WorkspaceConfig};
289    /// # use app_kit::workflow::{CheckpointSelector, ExecutionEvent, ExecutionOptions, RunSelector,
290    /// #     RunStatus, WorkflowDefinition, WorkflowState};
291    /// # use storage_kit::StateStore;
292    /// # use storage_kit::state::config::{LanceStateStoreConfig, StateStoreConfig};
293    /// # use app_kit::workflow::SyncExecution;
294    /// let runtime = tokio::runtime::Builder::new_multi_thread()
295    ///     .worker_threads(2).enable_all().build()?;
296    /// # let (mut app, mut run) = runtime.block_on(async {
297    /// # let store = StateStore::connect(StateStoreConfig::Lance(LanceStateStoreConfig::memory())).await?;
298    /// # let definition = WorkflowDefinition::parse(r#"{
299    /// #     "schema":"app-kit.workflow/v2", "id":"counter", "revision":"r1",
300    /// #     "state":{"count":0}, "required":["count"],
301    /// #     "reducers":{"count":"reject_concurrent"}, "entry":"increment",
302    /// #     "nodes":[{"id":"increment","step":{"kind":"state","mutations":[
303    /// #         {"kind":"increment","target":"count","value":{"kind":"literal","value":1}}
304    /// #     ]}}], "routes":{}
305    /// # }"#)?;
306    /// # let config = AppConfig::<_, Agent>::new(store, Vec::new(), vec![definition]);
307    /// # let mut app = App::open(config).await?;
308    /// # let mut workspace = app.create_workspace(WorkspaceConfig {
309    /// #     name: Some("local counter".into()), root: std::env::current_dir()?,
310    /// # }).await?;
311    /// # let workflow = workspace.workflow("counter".parse()?).await?;
312    /// # let mut run = workflow.prepare(WorkflowState::default()).await?;
313    /// # Ok::<_, Box<dyn std::error::Error>>((app, run))
314    /// # })?;
315    /// // Runtime workers stay alive; this thread is outside any entered Tokio context.
316    /// let mut execution = SyncExecution::open(&mut run, ExecutionOptions::default())?;
317    /// assert!(matches!(execution.try_next()?, Some(ExecutionEvent::Committed { .. })));
318    /// execution.finish()?;
319    /// # runtime.block_on(app.shutdown(Shutdown::CancelAndDrain { timeout: std::time::Duration::from_secs(5) }))?;
320    /// # Ok::<(), Box<dyn std::error::Error>>(())
321    /// ```
322    pub fn try_next(&mut self) -> Result<Option<ExecutionEvent>, AppError> {
323        Self::ensure_blocking()?;
324        self.runtime.block_on(self.execution.next())
325    }
326
327    /// Drains partial results and blocks until one committed boundary.
328    ///
329    /// An entered runtime context is rejected before advancing this owner.
330    ///
331    /// # Errors
332    ///
333    /// Returns `InvalidInput` inside an entered Tokio runtime context before blocking.
334    /// Otherwise propagates the corresponding asynchronous execution errors, including
335    /// stale/revoked authority, invalid options, durable failures and timeouts.
336    /// The captured runtime and lower runtimes must stay alive and driven elsewhere.
337    ///
338    /// # Example
339    ///
340    /// ```
341    /// # use agents_kit::Agent;
342    /// # use app_kit::{App, AppConfig, Shutdown, WorkspaceConfig};
343    /// # use app_kit::workflow::{CheckpointSelector, ExecutionEvent, ExecutionOptions, RunSelector,
344    /// #     RunStatus, WorkflowDefinition, WorkflowState};
345    /// # use storage_kit::StateStore;
346    /// # use storage_kit::state::config::{LanceStateStoreConfig, StateStoreConfig};
347    /// # use app_kit::workflow::SyncExecution;
348    /// let runtime = tokio::runtime::Builder::new_multi_thread()
349    ///     .worker_threads(2).enable_all().build()?;
350    /// # let (mut app, mut run) = runtime.block_on(async {
351    /// # let store = StateStore::connect(StateStoreConfig::Lance(LanceStateStoreConfig::memory())).await?;
352    /// # let definition = WorkflowDefinition::parse(r#"{
353    /// #     "schema":"app-kit.workflow/v2", "id":"counter", "revision":"r1",
354    /// #     "state":{"count":0}, "required":["count"],
355    /// #     "reducers":{"count":"reject_concurrent"}, "entry":"increment",
356    /// #     "nodes":[{"id":"increment","step":{"kind":"state","mutations":[
357    /// #         {"kind":"increment","target":"count","value":{"kind":"literal","value":1}}
358    /// #     ]}}], "routes":{}
359    /// # }"#)?;
360    /// # let config = AppConfig::<_, Agent>::new(store, Vec::new(), vec![definition]);
361    /// # let mut app = App::open(config).await?;
362    /// # let mut workspace = app.create_workspace(WorkspaceConfig {
363    /// #     name: Some("local counter".into()), root: std::env::current_dir()?,
364    /// # }).await?;
365    /// # let workflow = workspace.workflow("counter".parse()?).await?;
366    /// # let mut run = workflow.prepare(WorkflowState::default()).await?;
367    /// # Ok::<_, Box<dyn std::error::Error>>((app, run))
368    /// # })?;
369    /// // Runtime workers stay alive; this thread is outside any entered Tokio context.
370    /// let mut execution = SyncExecution::open(&mut run, ExecutionOptions::default())?;
371    /// assert!(matches!(execution.step()?, Some(ExecutionEvent::Committed { .. })));
372    /// assert_eq!(execution.finish()?.state.values()["count"], 1);
373    /// # runtime.block_on(app.shutdown(Shutdown::CancelAndDrain { timeout: std::time::Duration::from_secs(5) }))?;
374    /// # Ok::<(), Box<dyn std::error::Error>>(())
375    /// ```
376    pub fn step(&mut self) -> Result<Option<ExecutionEvent>, AppError> {
377        Self::ensure_blocking()?;
378        self.runtime.block_on(self.execution.step())
379    }
380
381    /// Drains execution to a halt or exhaustion and finalizes its observation.
382    ///
383    /// Rejection inside an entered runtime drops this consumed owner; existing
384    /// execution cleanup retains any previously admitted publication.
385    ///
386    /// # Errors
387    ///
388    /// Returns `InvalidInput` inside an entered Tokio runtime context before blocking.
389    /// Otherwise propagates the corresponding asynchronous execution errors, including
390    /// stale/revoked authority, invalid options, durable failures and timeouts.
391    /// The captured runtime and lower runtimes must stay alive and driven elsewhere.
392    ///
393    /// # Example
394    ///
395    /// ```
396    /// # use agents_kit::Agent;
397    /// # use app_kit::{App, AppConfig, Shutdown, WorkspaceConfig};
398    /// # use app_kit::workflow::{CheckpointSelector, ExecutionEvent, ExecutionOptions, RunSelector,
399    /// #     RunStatus, WorkflowDefinition, WorkflowState};
400    /// # use storage_kit::StateStore;
401    /// # use storage_kit::state::config::{LanceStateStoreConfig, StateStoreConfig};
402    /// # use app_kit::workflow::SyncExecution;
403    /// let runtime = tokio::runtime::Builder::new_multi_thread()
404    ///     .worker_threads(2).enable_all().build()?;
405    /// # let (mut app, mut run) = runtime.block_on(async {
406    /// # let store = StateStore::connect(StateStoreConfig::Lance(LanceStateStoreConfig::memory())).await?;
407    /// # let definition = WorkflowDefinition::parse(r#"{
408    /// #     "schema":"app-kit.workflow/v2", "id":"counter", "revision":"r1",
409    /// #     "state":{"count":0}, "required":["count"],
410    /// #     "reducers":{"count":"reject_concurrent"}, "entry":"increment",
411    /// #     "nodes":[{"id":"increment","step":{"kind":"state","mutations":[
412    /// #         {"kind":"increment","target":"count","value":{"kind":"literal","value":1}}
413    /// #     ]}}], "routes":{}
414    /// # }"#)?;
415    /// # let config = AppConfig::<_, Agent>::new(store, Vec::new(), vec![definition]);
416    /// # let mut app = App::open(config).await?;
417    /// # let mut workspace = app.create_workspace(WorkspaceConfig {
418    /// #     name: Some("local counter".into()), root: std::env::current_dir()?,
419    /// # }).await?;
420    /// # let workflow = workspace.workflow("counter".parse()?).await?;
421    /// # let mut run = workflow.prepare(WorkflowState::default()).await?;
422    /// # Ok::<_, Box<dyn std::error::Error>>((app, run))
423    /// # })?;
424    /// // Runtime workers stay alive; this thread is outside any entered Tokio context.
425    /// let execution = SyncExecution::open(&mut run, ExecutionOptions::default())?;
426    /// let snapshot = execution.finish()?;
427    /// assert!(matches!(snapshot.status, RunStatus::Succeeded));
428    /// # runtime.block_on(app.shutdown(Shutdown::CancelAndDrain { timeout: std::time::Duration::from_secs(5) }))?;
429    /// # Ok::<(), Box<dyn std::error::Error>>(())
430    /// ```
431    pub fn finish(self) -> Result<RunSnapshot, AppError> {
432        Self::ensure_blocking()?;
433        self.runtime.block_on(self.execution.finish())
434    }
435
436    /// Finalizes the current uncommitted boundary without cancelling the run.
437    ///
438    /// Rejection inside an entered runtime drops this consumed owner; it does
439    /// not dispatch a new operation or discard accepted publication ownership.
440    ///
441    /// # Errors
442    ///
443    /// Returns `InvalidInput` inside an entered Tokio runtime context before blocking.
444    /// Otherwise propagates the corresponding asynchronous execution errors, including
445    /// stale/revoked authority, invalid options, durable failures and timeouts.
446    /// The captured runtime and lower runtimes must stay alive and driven elsewhere.
447    ///
448    /// # Example
449    ///
450    /// ```
451    /// # use agents_kit::Agent;
452    /// # use app_kit::{App, AppConfig, Shutdown, WorkspaceConfig};
453    /// # use app_kit::workflow::{CheckpointSelector, ExecutionEvent, ExecutionOptions, RunSelector,
454    /// #     RunStatus, WorkflowDefinition, WorkflowState};
455    /// # use storage_kit::StateStore;
456    /// # use storage_kit::state::config::{LanceStateStoreConfig, StateStoreConfig};
457    /// # use app_kit::workflow::SyncExecution;
458    /// let runtime = tokio::runtime::Builder::new_multi_thread()
459    ///     .worker_threads(2).enable_all().build()?;
460    /// # let (mut app, mut run) = runtime.block_on(async {
461    /// # let store = StateStore::connect(StateStoreConfig::Lance(LanceStateStoreConfig::memory())).await?;
462    /// # let definition = WorkflowDefinition::parse(r#"{
463    /// #     "schema":"app-kit.workflow/v2", "id":"counter", "revision":"r1",
464    /// #     "state":{"count":0}, "required":["count"],
465    /// #     "reducers":{"count":"reject_concurrent"}, "entry":"increment",
466    /// #     "nodes":[{"id":"increment","step":{"kind":"state","mutations":[
467    /// #         {"kind":"increment","target":"count","value":{"kind":"literal","value":1}}
468    /// #     ]}}], "routes":{}
469    /// # }"#)?;
470    /// # let config = AppConfig::<_, Agent>::new(store, Vec::new(), vec![definition]);
471    /// # let mut app = App::open(config).await?;
472    /// # let mut workspace = app.create_workspace(WorkspaceConfig {
473    /// #     name: Some("local counter".into()), root: std::env::current_dir()?,
474    /// # }).await?;
475    /// # let workflow = workspace.workflow("counter".parse()?).await?;
476    /// # let mut run = workflow.prepare(WorkflowState::default()).await?;
477    /// # Ok::<_, Box<dyn std::error::Error>>((app, run))
478    /// # })?;
479    /// // Runtime workers stay alive; this thread is outside any entered Tokio context.
480    /// let execution = SyncExecution::open(&mut run, ExecutionOptions::default())?;
481    /// let parked = execution.abort()?;
482    /// assert_eq!(parked.state.values()["count"], 0);
483    /// assert_eq!(SyncExecution::open(&mut run, ExecutionOptions::default())?.finish()?.state.values()["count"], 1);
484    /// # runtime.block_on(app.shutdown(Shutdown::CancelAndDrain { timeout: std::time::Duration::from_secs(5) }))?;
485    /// # Ok::<(), Box<dyn std::error::Error>>(())
486    /// ```
487    pub fn abort(self) -> Result<RunSnapshot, AppError> {
488        Self::ensure_blocking()?;
489        self.runtime.block_on(self.execution.abort())
490    }
491}
492
493struct CallerSource<X: ActionBackend> {
494    activation: FrontierRecord,
495    context: Arc<ActionContext>,
496    deadline: tokio::time::Instant,
497    state: CallerSourceState<X>,
498}
499
500enum CallerSourceState<X: ActionBackend> {
501    Polling(Pin<Box<X::Stream>>),
502    Failed(FailureCategory),
503    Ended,
504    Eof,
505}
506
507impl<X: ActionBackend> CallerSource<X> {
508    fn stop(&mut self) -> Result<(), FailureCategory> {
509        let previous = std::mem::replace(&mut self.state, CallerSourceState::Ended);
510        match previous {
511            CallerSourceState::Polling(source) => catch_unwind(AssertUnwindSafe(|| drop(source)))
512                .map_err(|_| FailureCategory::Execution),
513            previous => {
514                self.state = previous;
515                Ok(())
516            }
517        }
518    }
519
520    fn poll(
521        &mut self,
522        context: &mut Context<'_>,
523    ) -> Poll<Result<Option<ActionEvent>, FailureCategory>> {
524        let event = match &mut self.state {
525            CallerSourceState::Polling(source) => {
526                match catch_unwind(AssertUnwindSafe(|| source.as_mut().poll_next(context))) {
527                    Ok(Poll::Pending) => return Poll::Pending,
528                    Ok(Poll::Ready(Some(Ok(event)))) => Ok(Some(event)),
529                    Ok(Poll::Ready(Some(Err(_)))) => Err(FailureCategory::Action),
530                    Ok(Poll::Ready(None)) => Ok(None),
531                    Err(_) => Err(FailureCategory::Execution),
532                }
533            }
534            CallerSourceState::Failed(category) => {
535                let category = *category;
536                self.state = CallerSourceState::Ended;
537                return Poll::Ready(Err(category));
538            }
539            CallerSourceState::Ended | CallerSourceState::Eof => return Poll::Pending,
540        };
541        Poll::Ready(event)
542    }
543}
544
545#[derive(Clone, Copy, PartialEq)]
546enum ExecutionLane {
547    Caller,
548    Worker,
549}
550
551struct ChildExecution<'a, F, E>
552where
553    F: Future<Output = Result<ChildCompletion, AppError>> + 'a,
554    E: ChildExecutor,
555{
556    sequence: u64,
557    parent: Option<Arc<RunRecord>>,
558    context: Arc<ActionContext>,
559    lineage: String,
560    deadline: tokio::time::Instant,
561    state: ChildExecutionState<'a, F, E>,
562    inputs: Vec<Map<String, Value>>,
563}
564
565enum ChildExecutionState<'a, F, E>
566where
567    F: Future<Output = Result<ChildCompletion, AppError>> + 'a,
568    E: ChildExecutor,
569{
570    Awaiting,
571    Prepared(Vec<RunRecord>),
572    Local(Pin<Box<E::Execution<'a, F>>>),
573    Pipeline(ChildBatch<F>),
574    Worker(oneshot::Receiver<Result<SessionReceipt, RuntimeOperationFailure>>),
575    Receipt(Box<Result<SessionReceipt, RuntimeOperationFailure>>),
576    Complete(Result<Vec<ChildCompletion>, FailureCategory>),
577    Closed,
578}
579
580impl<
581    'a,
582    CS: StateStoreBackend,
583    A: AgentBackend,
584    X: ActionBackend,
585    C: ConditionBackend,
586    T: StateTyping,
587    H: LifecycleHooks,
588    E: ChildExecutor,
589> ChildExecution<'a, ChildFuture<Execution<'a, CS, A, X, C, T, H, E>>, E>
590{
591    fn start_local(
592        &mut self,
593        runtime: &WorkflowRunRuntime<CS, A, X, C, T, H, E>,
594        app: &'a AppRuntime<CS, A, X, C, T, H, E>,
595        lane: ExecutionLane,
596        children: Vec<RunRecord>,
597    ) -> Result<(), AppError> {
598        if tokio::time::Instant::now() >= self.deadline {
599            runtime.retain_source_failure(&self.context, &self.lineage, FailureCategory::Timeout);
600            self.state = ChildExecutionState::Complete(Err(FailureCategory::Timeout));
601            return Ok(());
602        }
603        let parent = self.parent.as_ref().ok_or_else(|| {
604            AppError::Integration("child group parent evidence is unavailable".into())
605        })?;
606        let definition = app.definition_for(parent)?;
607        let control = parent
608            .checkpoint
609            .control(self.sequence)
610            .ok_or_else(|| AppError::Conflict("child control is unavailable".into()))?;
611        let step = definition
612            .node(&control.node)
613            .ok_or_else(|| AppError::Conflict("child definition changed".into()))?
614            .step();
615        let (parallelism, _) = step
616            .child_plan()
617            .ok_or_else(|| AppError::Conflict("child definition changed".into()))?;
618        let slots = Arc::new(tokio::sync::Semaphore::new(parallelism));
619        let notification = runtime.notifier()?;
620        let mut tasks = Vec::with_capacity(children.len());
621        for record in children {
622            record.validate_child(parent, &definition)?;
623            let child = WorkflowRunRuntime::attach_record(Arc::clone(&runtime.app), &record)?;
624            if !record.checkpoint.is_terminal() {
625                child.cell.bind_child(runtime.key.clone())?;
626                child
627                    .cell
628                    .retain_child_observation(Arc::clone(&self.context))?;
629            }
630            tasks.push(ChildFuture::new(
631                Execution::owned(child, app, lane),
632                Arc::clone(parent),
633                self.sequence,
634                self.deadline,
635                Arc::clone(&slots),
636                Arc::clone(&notification),
637            ));
638        }
639        let batch = ChildBatch::new(tasks, self.context.mode)?;
640        self.state = if matches!(step, Step::Pipeline { .. }) {
641            ChildExecutionState::Pipeline(batch)
642        } else {
643            match catch_unwind(AssertUnwindSafe(|| app.child_executor().execute(batch))) {
644                Ok(future) => ChildExecutionState::Local(Box::pin(future)),
645                Err(_) => {
646                    runtime.retain_source_failure(
647                        &self.context,
648                        &self.lineage,
649                        FailureCategory::Execution,
650                    );
651                    ChildExecutionState::Complete(Err(FailureCategory::Execution))
652                }
653            }
654        };
655        Ok(())
656    }
657
658    fn stop(&mut self, runtime: &WorkflowRunRuntime<CS, A, X, C, T, H, E>) {
659        let previous = std::mem::replace(&mut self.state, ChildExecutionState::Closed);
660        if let ChildExecutionState::Complete(Err(category)) = &previous {
661            runtime.retain_source_failure(&self.context, &self.lineage, *category);
662        }
663        if catch_unwind(AssertUnwindSafe(|| drop(previous))).is_err() {
664            // An already retained source/E error remains primary in the
665            // original observation. This only supplies a destructor fallback.
666            runtime.retain_source_failure(&self.context, &self.lineage, FailureCategory::Execution);
667        }
668    }
669
670    fn poll_ready(
671        &mut self,
672        cx: &mut Context<'_>,
673        runtime: &WorkflowRunRuntime<CS, A, X, C, T, H, E>,
674    ) -> Poll<()> {
675        if matches!(
676            self.state,
677            ChildExecutionState::Complete(_)
678                | ChildExecutionState::Receipt(_)
679                | ChildExecutionState::Closed
680        ) {
681            return Poll::Ready(());
682        }
683        if matches!(
684            self.state,
685            ChildExecutionState::Awaiting | ChildExecutionState::Prepared(_)
686        ) {
687            return Poll::Pending;
688        }
689        if matches!(self.state, ChildExecutionState::Worker(_)) {
690            return self.poll_receipt(cx);
691        }
692        let result = if tokio::time::Instant::now() >= self.deadline {
693            Err(FailureCategory::Timeout)
694        } else {
695            match catch_unwind(AssertUnwindSafe(|| match &mut self.state {
696                ChildExecutionState::Local(future) => future.as_mut().poll(cx),
697                ChildExecutionState::Pipeline(batch) => Pin::new(batch).poll(cx),
698                _ => unreachable!(),
699            })) {
700                Ok(Poll::Pending) => return Poll::Pending,
701                Ok(Poll::Ready(result)) => result.map_err(|_| FailureCategory::Child),
702                Err(_) => Err(FailureCategory::Execution),
703            }
704        };
705        self.complete(runtime, result);
706        Poll::Ready(())
707    }
708
709    fn poll_receipt(&mut self, cx: &mut Context<'_>) -> Poll<()> {
710        let ChildExecutionState::Worker(receipt) = &mut self.state else {
711            unreachable!()
712        };
713        match Pin::new(receipt).poll(cx) {
714            Poll::Pending => Poll::Pending,
715            Poll::Ready(result) => {
716                self.state = ChildExecutionState::Receipt(Box::new(
717                    result
718                        .map_err(|_| {
719                            RuntimeOperationFailure::from(AppError::Closed(
720                                "child executor receipt closed".into(),
721                            ))
722                        })
723                        .and_then(|result| result),
724                ));
725                Poll::Ready(())
726            }
727        }
728    }
729
730    fn complete(
731        &mut self,
732        runtime: &WorkflowRunRuntime<CS, A, X, C, T, H, E>,
733        result: Result<Vec<ChildCompletion>, FailureCategory>,
734    ) {
735        if let Err(category) = result.as_ref() {
736            runtime.retain_source_failure(&self.context, &self.lineage, *category);
737        }
738        let previous = std::mem::replace(&mut self.state, ChildExecutionState::Complete(result));
739        if catch_unwind(AssertUnwindSafe(|| drop(previous))).is_err()
740            && matches!(&self.state, ChildExecutionState::Complete(Ok(_)))
741        {
742            runtime.retain_source_failure(&self.context, &self.lineage, FailureCategory::Execution);
743            self.state = ChildExecutionState::Complete(Err(FailureCategory::Execution));
744        }
745    }
746}
747
748struct ChildFuture<EX> {
749    execution: EX,
750    parent: Arc<RunRecord>,
751    sequence: u64,
752    slots: Arc<tokio::sync::Semaphore>,
753    permit: Option<tokio::sync::OwnedSemaphorePermit>,
754    notification: Arc<tokio::sync::Notify>,
755    wake: Pin<Box<tokio::sync::futures::OwnedNotified>>,
756    state: ChildFutureState,
757}
758
759enum ChildFutureState {
760    Admission,
761    Driving,
762    Reconcile(Option<oneshot::Receiver<Result<LoadedRun, AppError>>>),
763    Retry(tokio::time::Instant),
764    Finishing,
765    Waiting,
766    Finished,
767}
768
769impl<
770    'a,
771    CS: StateStoreBackend,
772    A: AgentBackend,
773    X: ActionBackend,
774    C: ConditionBackend,
775    T: StateTyping,
776    H: LifecycleHooks,
777    E: ChildExecutor,
778> ChildFuture<Execution<'a, CS, A, X, C, T, H, E>>
779{
780    fn new(
781        mut execution: Execution<'a, CS, A, X, C, T, H, E>,
782        parent: Arc<RunRecord>,
783        sequence: u64,
784        deadline: tokio::time::Instant,
785        slots: Arc<tokio::sync::Semaphore>,
786        notification: Arc<tokio::sync::Notify>,
787    ) -> Self {
788        let wake = Box::pin(Arc::clone(&notification).notified_owned());
789        execution.parent_deadline = Some(deadline);
790        Self {
791            execution,
792            parent,
793            sequence,
794            slots,
795            permit: None,
796            notification,
797            wake,
798            state: ChildFutureState::Admission,
799        }
800    }
801
802    fn poll_reconciliation(&mut self, cx: &mut Context<'_>) -> Poll<Result<(), AppError>> {
803        let ChildFutureState::Reconcile(receipt) = &mut self.state else {
804            unreachable!()
805        };
806        if receipt.is_none() {
807            // Finish an already admitted publication before sampling
808            // failure authority. Its older result must not restart
809            // caller work after the panic is reconciled.
810            match &mut self.execution.state {
811                SessionState::Publishing { receipt, .. } => {
812                    if Pin::new(receipt).poll(cx).is_pending() {
813                        return Poll::Pending;
814                    }
815                }
816                SessionState::ObservingItem { receipt, .. } => {
817                    if Pin::new(receipt).poll(cx).is_pending() {
818                        return Poll::Pending;
819                    }
820                }
821                _ => {}
822            }
823            self.execution.state = SessionState::Finished;
824            *receipt = Some(
825                self.execution
826                    .runtime
827                    .reconcile_child_panic(self.execution.parent_deadline),
828            );
829        }
830        let loaded = match Pin::new(receipt.as_mut().expect("reconciliation was admitted")).poll(cx)
831        {
832            Poll::Pending => return Poll::Pending,
833            Poll::Ready(Ok(result)) => result?,
834            Poll::Ready(Err(_)) => {
835                self.execution
836                    .runtime
837                    .app
838                    .tasks_failed(FinalResult::Integration(
839                        "child panic reconciliation receipt closed".into(),
840                    ));
841                return Poll::Ready(Err(AppError::Integration(
842                    "child panic reconciliation receipt closed".into(),
843                )));
844            }
845        };
846        self.execution.state = SessionState::Loaded(loaded);
847        self.state = ChildFutureState::Driving;
848        Poll::Ready(Ok(()))
849    }
850
851    fn poll_completion(
852        &mut self,
853        cx: &mut Context<'_>,
854    ) -> Poll<Result<Option<ChildCompletion>, AppError>> {
855        match self.execution.poll_publication(cx, false) {
856            Poll::Pending => Poll::Pending,
857            Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
858            Poll::Ready(Ok(Some(_))) => Poll::Ready(Ok(None)),
859            Poll::Ready(Ok(None)) => {
860                let SessionState::Loaded(loaded) =
861                    std::mem::replace(&mut self.execution.state, SessionState::Finished)
862                else {
863                    return Poll::Ready(Err(AppError::Integration(
864                        "child terminal observation is unavailable".into(),
865                    )));
866                };
867                if !loaded.record.checkpoint.is_terminal() {
868                    self.execution.state = SessionState::Loaded(loaded);
869                    self.state = ChildFutureState::Waiting;
870                    return Poll::Ready(Ok(None));
871                }
872                let definition = self.execution.runtime.app.definition_for(&self.parent)?;
873                let result = if matches!(
874                    loaded.record.checkpoint.phase,
875                    CheckpointPhaseRecord::Succeeded
876                ) {
877                    ChildCompletion::admitted(loaded.record, &self.parent, &definition)
878                } else {
879                    Err(AppError::Integration(
880                        "child workflow did not succeed".into(),
881                    ))
882                };
883                Poll::Ready(result.map(Some))
884            }
885        }
886    }
887}
888
889impl<
890    CS: StateStoreBackend,
891    A: AgentBackend,
892    X: ActionBackend,
893    C: ConditionBackend,
894    T: StateTyping,
895    H: LifecycleHooks,
896    E: ChildExecutor,
897> Future for ChildFuture<Execution<'_, CS, A, X, C, T, H, E>>
898{
899    type Output = Result<ChildCompletion, AppError>;
900
901    fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
902        let child = self.get_mut();
903        let observed = (|| -> Poll<Result<ChildCompletion, AppError>> {
904            loop {
905                match child.state {
906                    ChildFutureState::Admission => {
907                        child.wake.as_mut().enable();
908                        match Arc::clone(&child.slots).try_acquire_owned() {
909                            Ok(permit) => {
910                                child.permit = Some(permit);
911                                child.state = ChildFutureState::Driving;
912                            }
913                            Err(tokio::sync::TryAcquireError::NoPermits) => {
914                                if child.wake.as_mut().poll(cx).is_pending() {
915                                    return Poll::Pending;
916                                }
917                                child.wake =
918                                    Box::pin(Arc::clone(&child.notification).notified_owned());
919                            }
920                            Err(tokio::sync::TryAcquireError::Closed) => {
921                                return Poll::Ready(Err(AppError::Closed(
922                                    "child execution capacity is closed".into(),
923                                )));
924                            }
925                        }
926                    }
927                    ChildFutureState::Reconcile(_) => match child.poll_reconciliation(cx) {
928                        Poll::Pending => return Poll::Pending,
929                        Poll::Ready(result) => result?,
930                    },
931                    ChildFutureState::Retry(deadline) => {
932                        if !child.execution.poll_timer(cx, Some(deadline)) {
933                            return Poll::Pending;
934                        }
935                        child.execution.state = SessionState::Ready;
936                        child.state = ChildFutureState::Driving;
937                    }
938                    ChildFutureState::Driving => {
939                        let observed = catch_unwind(AssertUnwindSafe(|| {
940                            child.execution.poll_advance(cx, true)
941                        }));
942                        match observed {
943                            Err(_) => {
944                                // Unrelated wakes cannot re-admit this child before
945                                // its original failure publication is authoritative.
946                                child.state = ChildFutureState::Reconcile(None);
947                            }
948                            Ok(Poll::Pending) => return Poll::Pending,
949                            Ok(Poll::Ready(Err(error))) => {
950                                if let Some(delay) = child.execution.runtime.cell.complete_drive() {
951                                    child.state = ChildFutureState::Retry(
952                                        tokio::time::Instant::now() + delay,
953                                    );
954                                } else {
955                                    return Poll::Ready(Err(error));
956                                }
957                            }
958                            Ok(Poll::Ready(Ok(Some(_)))) => continue,
959                            Ok(Poll::Ready(Ok(None))) => {
960                                child.execution.publish(
961                                    SessionPublication::FinishChild {
962                                        parent: Arc::clone(&child.parent),
963                                        sequence: child.sequence,
964                                    },
965                                    false,
966                                )?;
967                                child.state = ChildFutureState::Finishing;
968                            }
969                        }
970                    }
971                    ChildFutureState::Finishing => match child.poll_completion(cx) {
972                        Poll::Pending => return Poll::Pending,
973                        Poll::Ready(Err(error)) => return Poll::Ready(Err(error)),
974                        Poll::Ready(Ok(Some(completion))) => return Poll::Ready(Ok(completion)),
975                        Poll::Ready(Ok(None)) => continue,
976                    },
977                    ChildFutureState::Waiting => {
978                        // A child's public Input can pause its lifecycle, but the
979                        // same admitted batch future remains owned by its parent.
980                        if !child.execution.poll_notification(cx)? {
981                            return Poll::Pending;
982                        }
983                        child.execution.state = SessionState::Ready;
984                        child.state = ChildFutureState::Driving;
985                    }
986                    ChildFutureState::Finished => {
987                        return Poll::Ready(Err(AppError::Conflict(
988                            "child future already completed".into(),
989                        )));
990                    }
991                }
992            }
993        })();
994        if observed.is_ready() {
995            child.release();
996            child.state = ChildFutureState::Finished;
997        }
998        observed
999    }
1000}
1001
1002impl<EX> ChildFuture<EX> {
1003    fn release(&mut self) {
1004        if self.permit.take().is_some() {
1005            self.notification.notify_waiters();
1006        }
1007    }
1008}
1009
1010impl<EX> Drop for ChildFuture<EX> {
1011    fn drop(&mut self) {
1012        self.release();
1013    }
1014}
1015
1016enum SessionState {
1017    Ready,
1018    Loaded(LoadedRun),
1019    ActivationReady {
1020        loaded: LoadedRun,
1021        activation: FrontierRecord,
1022        context: Arc<ActionContext>,
1023        remaining: std::vec::IntoIter<FrontierRecord>,
1024        staged: Vec<StagedActivationRecord>,
1025        deadline: tokio::time::Instant,
1026        deadline_millis: u64,
1027    },
1028    Sources {
1029        loaded: LoadedRun,
1030    },
1031    ObservingItem {
1032        loaded: LoadedRun,
1033        index: usize,
1034        receipt: oneshot::Receiver<Result<Option<Map<String, Value>>, AppError>>,
1035    },
1036    Publishing {
1037        receipt: oneshot::Receiver<Result<SessionReceipt, RuntimeOperationFailure>>,
1038        purpose: PublicationPurpose,
1039    },
1040    Eof,
1041    Finished,
1042}
1043
1044enum PublicationPurpose {
1045    Load,
1046    Reload,
1047    ObserveChildren,
1048    AdmitChildren,
1049    VerifyChildren,
1050    Start {
1051        activation: FrontierRecord,
1052        context: Arc<ActionContext>,
1053        remaining: std::vec::IntoIter<FrontierRecord>,
1054        deadline: tokio::time::Instant,
1055        deadline_millis: u64,
1056    },
1057    Continue {
1058        remaining: std::vec::IntoIter<FrontierRecord>,
1059        ordinary: bool,
1060    },
1061    Descendants {
1062        context: Arc<ActionContext>,
1063        lineage: String,
1064        inputs: Vec<Map<String, Value>>,
1065        remaining: std::vec::IntoIter<FrontierRecord>,
1066        ordinary: bool,
1067    },
1068    Commit {
1069        ordinary: bool,
1070    },
1071    Advance,
1072    EndSource,
1073}
1074
1075pub(crate) enum SessionReceipt {
1076    Checkpoint {
1077        loaded: LoadedRun,
1078        staged: Vec<StagedActivationRecord>,
1079    },
1080    Children {
1081        loaded: LoadedRun,
1082        sequence: u64,
1083        children: Vec<RunRecord>,
1084        verified: bool,
1085    },
1086}
1087
1088impl SessionReceipt {
1089    fn record(&self) -> &RunRecord {
1090        match self {
1091            Self::Checkpoint { loaded, .. } | Self::Children { loaded, .. } => &loaded.record,
1092        }
1093    }
1094
1095    fn with_loaded(self, loaded: LoadedRun) -> Self {
1096        match self {
1097            Self::Checkpoint { staged, .. } => Self::Checkpoint { loaded, staged },
1098            Self::Children {
1099                sequence,
1100                children,
1101                verified,
1102                ..
1103            } => Self::Children {
1104                loaded,
1105                sequence,
1106                children,
1107                verified,
1108            },
1109        }
1110    }
1111}
1112
1113enum SessionPublication {
1114    ObserveChildren {
1115        loaded: LoadedRun,
1116        sequence: u64,
1117    },
1118    VerifyChildren {
1119        parent: Arc<RunRecord>,
1120        sequence: u64,
1121        completions: Vec<ChildCompletion>,
1122    },
1123    ChildOutcome {
1124        control: ControlRecord,
1125        transition: ControlTransition,
1126    },
1127    FinishChild {
1128        parent: Arc<RunRecord>,
1129        sequence: u64,
1130    },
1131    SettleStopping {
1132        loaded: LoadedRun,
1133    },
1134    Load {
1135        terminal: Option<u64>,
1136    },
1137    Reload {
1138        receipt: SessionReceipt,
1139        deadline: Option<tokio::time::Instant>,
1140    },
1141    BeginActivation {
1142        loaded: LoadedRun,
1143        activation: FrontierRecord,
1144        context: Arc<ActionContext>,
1145        remaining: std::vec::IntoIter<FrontierRecord>,
1146        staged: Vec<StagedActivationRecord>,
1147        deadline: tokio::time::Instant,
1148        deadline_millis: u64,
1149    },
1150    Decision {
1151        loaded: LoadedRun,
1152        decision: BspDecision,
1153        activation_deadline: Option<u64>,
1154        context: Option<Arc<ActionContext>>,
1155        staged: Vec<StagedActivationRecord>,
1156        remaining: std::vec::IntoIter<FrontierRecord>,
1157    },
1158    Transition {
1159        loaded: LoadedRun,
1160        transition: ControlTransition,
1161    },
1162    Retry {
1163        loaded: LoadedRun,
1164    },
1165    EndSource {
1166        loaded: LoadedRun,
1167    },
1168    Release,
1169}
1170
1171impl<
1172    'a,
1173    CS: StateStoreBackend,
1174    A: AgentBackend,
1175    X: ActionBackend,
1176    C: ConditionBackend,
1177    T: StateTyping,
1178    H: LifecycleHooks,
1179    E: ChildExecutor,
1180> Execution<'a, CS, A, X, C, T, H, E>
1181{
1182    pub(crate) async fn open(
1183        run: &'a mut WorkflowRun<CS, A, X, C, T, H, E>,
1184        mut options: ExecutionOptions,
1185        mode: ActionMode,
1186    ) -> Result<Self, AppError> {
1187        let admission = run.runtime.app.reserve_caller()?;
1188        if !options.state.values().is_empty() {
1189            return Err(AppError::InvalidInput(
1190                "existing execution cannot replace initial state".into(),
1191            ));
1192        }
1193        JsonBoundary::validate_inputs(&options.inputs)?;
1194        run.runtime.app.ensure_open().await?;
1195        let loaded = run.runtime.load().await?;
1196        let definition = run.runtime.app.definition_for(&loaded.record)?;
1197        let graph = run.runtime.app.graph(&definition);
1198        options.resolve_selectors(&graph)?;
1199        let pending_controls = !loaded.record.checkpoint.controls.is_empty();
1200        let discard_staged = loaded.record.checkpoint.rollback_stage_keys(&definition);
1201        let generation = run.runtime.cell.claim_execution(mode)?;
1202        run.runtime.cell.configure_execution(generation, options)?;
1203        let mut execution = Self {
1204            runtime: run.runtime.clone(),
1205            app: &run.runtime.app,
1206            lane: ExecutionLane::Caller,
1207            admission: Some(admission),
1208            children: Vec::new(),
1209            wake: None,
1210            timer: None,
1211            generation: Some(generation),
1212            parent_deadline: None,
1213            state: SessionState::Ready,
1214            sources: Vec::new(),
1215            next_source: 0,
1216            native_partials: None,
1217            prefetched: None,
1218            transitions: 0,
1219            observed: Some(loaded.record.clone()),
1220        };
1221        if loaded.record.checkpoint.can_retry_failed() {
1222            execution.publish(SessionPublication::Retry { loaded }, false)?;
1223            poll_fn(|cx| execution.poll_publication(cx, false)).await?;
1224        } else if !loaded.record.checkpoint.is_terminal()
1225            && (loaded.record.checkpoint.admission != RunAdmission::Prepared
1226                || !discard_staged.is_empty())
1227        {
1228            execution.publish_transition(loaded, ControlTransition::Pause { discard_staged })?;
1229            poll_fn(|cx| execution.poll_publication(cx, false)).await?;
1230        }
1231        // Acquisition is bounded; a returned lazy execution owns no caller slot.
1232        drop(execution.admission.take());
1233        execution.runtime.cell.admit_lifecycle(generation)?;
1234        if pending_controls {
1235            execution.runtime.schedule(ScheduleCause::Reconcile);
1236        }
1237        Ok(execution)
1238    }
1239
1240    pub(super) fn background(run: &'a mut WorkflowRun<CS, A, X, C, T, H, E>) -> Self {
1241        Self::owned(run.runtime.clone(), &run.runtime.app, ExecutionLane::Worker)
1242    }
1243
1244    fn owned(
1245        runtime: WorkflowRunRuntime<CS, A, X, C, T, H, E>,
1246        app: &'a AppRuntime<CS, A, X, C, T, H, E>,
1247        lane: ExecutionLane,
1248    ) -> Self {
1249        Self {
1250            runtime,
1251            app,
1252            lane,
1253            admission: None,
1254            children: Vec::new(),
1255            wake: None,
1256            timer: None,
1257            generation: None,
1258            parent_deadline: None,
1259            state: SessionState::Ready,
1260            sources: Vec::new(),
1261            next_source: 0,
1262            native_partials: None,
1263            prefetched: None,
1264            transitions: 0,
1265            observed: None,
1266        }
1267    }
1268
1269    /// Advances lazily to one partial result or committed boundary.
1270    ///
1271    /// Partials are transient; only Committed events publish state/outcomes. Streaming
1272    /// iteration checks halt-after-prior before halt-before-next. A retained halt is
1273    /// not reclassified by a later step/finish call. EOF without Completed commits
1274    /// no action and ends this observation without automatically retrying it.
1275    ///
1276    /// # Errors
1277    ///
1278    /// Returns `Closed` or `Conflict` when execution authority is revoked or stale,
1279    /// `Integration` for unavailable durable/lower evidence, and `Timeout` for expired
1280    /// observation. Invalid execution input may return `InvalidInput`; accepted action
1281    /// failures are retained in the returned run state. Cleanup preserves primary
1282    /// failures and does not detach an admitted native commit.
1283    ///
1284    /// # Example
1285    ///
1286    /// ```
1287    /// # use agents_kit::Agent;
1288    /// # use app_kit::{App, AppConfig, Shutdown, WorkspaceConfig};
1289    /// # use app_kit::workflow::{CheckpointSelector, ExecutionEvent, ExecutionOptions, RunSelector,
1290    /// #     RunStatus, WorkflowDefinition, WorkflowState};
1291    /// # use storage_kit::StateStore;
1292    /// # use storage_kit::state::config::{LanceStateStoreConfig, StateStoreConfig};
1293    /// #
1294    /// # let runtime = tokio::runtime::Builder::new_current_thread().enable_all().build()?;
1295    /// # runtime.block_on(async {
1296    /// # let store = StateStore::connect(StateStoreConfig::Lance(LanceStateStoreConfig::memory())).await?;
1297    /// # let definition = WorkflowDefinition::parse(r#"{
1298    /// #     "schema":"app-kit.workflow/v2", "id":"counter", "revision":"r1",
1299    /// #     "state":{"count":0}, "required":["count"],
1300    /// #     "reducers":{"count":"reject_concurrent"}, "entry":"increment",
1301    /// #     "nodes":[{"id":"increment","step":{"kind":"state","mutations":[
1302    /// #         {"kind":"increment","target":"count","value":{"kind":"literal","value":1}}
1303    /// #     ]}}], "routes":{}
1304    /// # }"#)?;
1305    /// # let config = AppConfig::<_, Agent>::new(store, Vec::new(), vec![definition]);
1306    /// # let mut app = App::open(config).await?;
1307    /// # let mut workspace = app.create_workspace(WorkspaceConfig {
1308    /// #     name: Some("local counter".into()), root: std::env::current_dir()?,
1309    /// # }).await?;
1310    /// # let workflow = workspace.workflow("counter".parse()?).await?;
1311    /// # let mut run = workflow.prepare(WorkflowState::default()).await?;
1312    /// let mut execution = run.execution(ExecutionOptions::default()).await?;
1313    /// let event = execution.next().await?.expect("the state action commits");
1314    /// assert!(matches!(event, ExecutionEvent::Committed { .. }));
1315    /// execution.finish().await?;
1316    /// #
1317    /// # app.shutdown(Shutdown::CancelAndDrain { timeout: std::time::Duration::from_secs(5) }).await?;
1318    /// # Ok::<(), Box<dyn std::error::Error>>(())
1319    /// # })?;
1320    /// # Ok::<(), Box<dyn std::error::Error>>(())
1321    /// ```
1322    pub async fn next(&mut self) -> Result<Option<ExecutionEvent>, AppError> {
1323        let event = poll_fn(|cx| self.poll_advance(cx, false)).await;
1324        if (matches!(&event, Ok(Some(ExecutionEvent::Committed { .. })))
1325            || matches!(&event, Ok(None)) && matches!(&self.state, SessionState::Eof)
1326            || event.is_err() && self.runtime.cell.lifecycle_generation().is_none())
1327            && let Some(generation) = self.generation
1328        {
1329            self.runtime.cell.returned_boundary(generation);
1330        }
1331        event
1332    }
1333
1334    fn poll_advance(
1335        &mut self,
1336        cx: &mut Context<'_>,
1337        ordinary: bool,
1338    ) -> Poll<Result<Option<ExecutionEvent>, AppError>> {
1339        loop {
1340            // Sources polls once after starting newly prepared groups. Keep
1341            // its pre-start authority check, but do not double-poll pending
1342            // descendants at every nested level.
1343            if !matches!(&self.state, SessionState::Sources { .. })
1344                || self
1345                    .children
1346                    .iter()
1347                    .any(|group| matches!(group.state, ChildExecutionState::Prepared(_)))
1348            {
1349                self.poll_child_groups(cx)?;
1350            }
1351            match &mut self.state {
1352                SessionState::ObservingItem { .. } => match self.poll_item(cx) {
1353                    Poll::Pending => return Poll::Pending,
1354                    Poll::Ready(Err(error)) => return Poll::Ready(Err(error)),
1355                    Poll::Ready(Ok(Some(event))) if !ordinary => {
1356                        return Poll::Ready(Ok(Some(event)));
1357                    }
1358                    Poll::Ready(Ok(_)) => continue,
1359                },
1360                SessionState::Finished => return Poll::Ready(Ok(None)),
1361                SessionState::Eof => self.state = SessionState::Ready,
1362                SessionState::ActivationReady { .. } => self.activate_ready(ordinary)?,
1363                SessionState::Publishing { .. } => {
1364                    match self.poll_publication(cx, false) {
1365                        Poll::Pending => return Poll::Pending,
1366                        Poll::Ready(Err(error)) => return Poll::Ready(Err(error)),
1367                        Poll::Ready(Ok(Some(event))) => {
1368                            if ordinary && matches!(event, ExecutionEvent::Partial { .. }) {
1369                                continue;
1370                            }
1371                            return Poll::Ready(Ok(Some(event)));
1372                        }
1373                        Poll::Ready(Ok(None)) => {}
1374                    }
1375                    if matches!(&self.state, SessionState::Eof) {
1376                        return Poll::Ready(Ok(None));
1377                    }
1378                }
1379                SessionState::Ready => {
1380                    self.ensure_owned()?;
1381                    self.arm_notification()?;
1382                    let terminal = self
1383                        .observed
1384                        .as_ref()
1385                        .filter(|record| {
1386                            self.generation.is_none() && record.checkpoint.is_terminal()
1387                        })
1388                        .map(|record| record.checkpoint.sequence);
1389                    self.publish(SessionPublication::Load { terminal }, ordinary)?;
1390                }
1391                SessionState::Loaded(_) => {
1392                    if self.advance_loaded(ordinary)? {
1393                        return Poll::Ready(Ok(None));
1394                    }
1395                }
1396                SessionState::Sources { .. } => {
1397                    self.start_child_groups(cx)?;
1398                    if !matches!(self.state, SessionState::Sources { .. }) {
1399                        continue;
1400                    }
1401                    self.poll_child_groups(cx)?;
1402                    if self.publish_child_completion(ordinary, false)? {
1403                        continue;
1404                    }
1405                    if self.prefetched.is_none()
1406                        && !self.sources.is_empty()
1407                        && self.sources.iter().all(|source| {
1408                            matches!(
1409                                source.state,
1410                                CallerSourceState::Ended | CallerSourceState::Eof
1411                            )
1412                        })
1413                    {
1414                        let SessionState::Sources { loaded, .. } = &self.state else {
1415                            unreachable!()
1416                        };
1417                        if !self.retains_external_continuation(&loaded.record) {
1418                            self.finish_wave(ordinary)?;
1419                            continue;
1420                        }
1421                    }
1422                    if self.prefetched.is_none() {
1423                        match self.poll_sources_observation(cx) {
1424                            Poll::Pending => return Poll::Pending,
1425                            Poll::Ready(Err(error)) => return Poll::Ready(Err(error)),
1426                            Poll::Ready(Ok(Some(event))) if !ordinary => {
1427                                return Poll::Ready(Ok(Some(event)));
1428                            }
1429                            Poll::Ready(Ok(Some(_))) => continue,
1430                            Poll::Ready(Ok(None)) => {}
1431                        }
1432                        if !matches!(&self.state, SessionState::Sources { .. }) {
1433                            continue;
1434                        }
1435                    }
1436                    if let Some((index, event)) = self.prefetched.take() {
1437                        self.consume_source_event(index, event, ordinary)?;
1438                    } else {
1439                        return Poll::Ready(Ok(None));
1440                    }
1441                }
1442            }
1443        }
1444    }
1445
1446    fn advance_loaded(&mut self, ordinary: bool) -> Result<bool, AppError> {
1447        let SessionState::Loaded(loaded) =
1448            std::mem::replace(&mut self.state, SessionState::Finished)
1449        else {
1450            unreachable!()
1451        };
1452        self.ensure_owned()?;
1453        self.runtime.supervise_controls(&loaded.record);
1454        if loaded.record.checkpoint.is_stopping() {
1455            self.stop_children();
1456            self.stop_sources();
1457            self.publish(SessionPublication::SettleStopping { loaded }, ordinary)?;
1458            return Ok(false);
1459        }
1460        if loaded.record.checkpoint.is_terminal() {
1461            self.stop_children();
1462            self.stop_sources();
1463            self.observed = Some(loaded.record);
1464            return Ok(true);
1465        }
1466        self.observed = Some(loaded.record.clone());
1467        if self.children.iter().any(|group| {
1468            matches!(
1469                group.state,
1470                ChildExecutionState::Worker(_) | ChildExecutionState::Receipt(_)
1471            ) && group
1472                .parent
1473                .as_ref()
1474                .is_some_and(|parent| !loaded.record.owns_child_activation(parent, group.sequence))
1475        }) {
1476            // A worker may have staged its result before its final
1477            // receipt arrives. Keep that original receipt ahead of
1478            // parent evaluation, halting and barrier publication.
1479            self.state = SessionState::Sources { loaded };
1480            return Ok(false);
1481        }
1482        if self.generation.is_none()
1483            && self.parent_deadline.is_none()
1484            && loaded.record.checkpoint.is_paused()
1485        {
1486            // Background cleanup may finish already admitted children, but
1487            // their receipt cannot authorize a later prepared parent barrier.
1488            self.state = SessionState::Sources { loaded };
1489            return Ok(true);
1490        }
1491        if self.runtime.cell.execution_boundary()? == Some(false) && self.halt_after(&loaded)? {
1492            return Ok(false);
1493        }
1494        if self.transitions >= MAX_DRIVER_TRANSITIONS {
1495            self.publish_decision(
1496                loaded,
1497                BspDecision::Failed {
1498                    category: FailureCategory::Limit,
1499                    message: "workflow transition bound exhausted".into(),
1500                    observation: None,
1501                },
1502                ordinary,
1503            )?;
1504            return Ok(false);
1505        }
1506        let mut decision = match self.runtime.decide_checkpoint(&loaded) {
1507            Ok(decision) => decision,
1508            Err(failure) => {
1509                if self.generation.is_none() && self.runtime.schedule_operation_failure(&failure) {
1510                    self.observed = None;
1511                }
1512                return Err(failure.into_error());
1513            }
1514        };
1515        if let BspDecision::Activate(selected) = &mut decision {
1516            selected.retain(|activation| {
1517                !self
1518                    .sources
1519                    .iter()
1520                    .any(|source| source.activation == *activation)
1521            });
1522            if selected.is_empty() {
1523                decision = BspDecision::AwaitControls;
1524            }
1525        }
1526        let boundary = self.runtime.cell.execution_boundary()?;
1527        if boundary.is_some()
1528            && let Some(node) =
1529                decision.pending_node(&self.runtime.cell.execution_options()?.halt_before)
1530        {
1531            let node = node.to_owned();
1532            self.publish_transition(loaded, ControlTransition::Halt { node, result: None })?;
1533            return Ok(false);
1534        }
1535        if boundary == Some(true)
1536            && !matches!(&decision, BspDecision::Routed { .. })
1537            && self.halt_after(&loaded)?
1538        {
1539            return Ok(false);
1540        }
1541        if !matches!(&decision, BspDecision::Routed { .. }) {
1542            self.runtime.cell.clear_execution_boundary()?;
1543        }
1544        match decision {
1545            BspDecision::AwaitControls => {
1546                let pending = loaded
1547                    .record
1548                    .checkpoint
1549                    .controls
1550                    .values()
1551                    .find(|control| {
1552                        matches!(control.state, ControlRecordState::Child { .. })
1553                            && !self.children.iter().any(|group| {
1554                                group.sequence == control.sequence
1555                                    && !matches!(group.state, ChildExecutionState::Awaiting)
1556                            })
1557                    })
1558                    .map(|control| control.sequence);
1559                if let Some(sequence) = pending {
1560                    self.publish(
1561                        SessionPublication::ObserveChildren { loaded, sequence },
1562                        ordinary,
1563                    )?;
1564                } else {
1565                    self.state = SessionState::Sources { loaded };
1566                }
1567            }
1568            BspDecision::Activate(selected) => {
1569                self.transitions += 1;
1570                let mut remaining = selected.into_iter();
1571                let activation = remaining.next().ok_or_else(|| {
1572                    AppError::Integration("native selected prefix is empty".into())
1573                })?;
1574                self.begin_activation(loaded, activation, remaining, Vec::new(), ordinary)?;
1575            }
1576            decision => {
1577                self.transitions += 1;
1578                self.publish_decision(loaded, decision, ordinary)?;
1579            }
1580        }
1581        Ok(false)
1582    }
1583
1584    /// Drains transient partials through one committed boundary.
1585    ///
1586    /// Consumes partials until a committed boundary or halt. Ordinary iteration
1587    /// checks halt-before-next before halt-after-prior. Only a committed action
1588    /// advances its ordinal; control checkpoints and partials do not.
1589    ///
1590    /// # Errors
1591    ///
1592    /// Returns `Closed` or `Conflict` when execution authority is revoked or stale,
1593    /// `Integration` for unavailable durable/lower evidence, and `Timeout` for expired
1594    /// observation. Invalid execution input may return `InvalidInput`; accepted action
1595    /// failures are retained in the returned run state. Cleanup preserves primary
1596    /// failures and does not detach an admitted native commit.
1597    ///
1598    /// # Example
1599    ///
1600    /// ```
1601    /// # use agents_kit::Agent;
1602    /// # use app_kit::{App, AppConfig, Shutdown, WorkspaceConfig};
1603    /// # use app_kit::workflow::{CheckpointSelector, ExecutionEvent, ExecutionOptions, RunSelector,
1604    /// #     RunStatus, WorkflowDefinition, WorkflowState};
1605    /// # use storage_kit::StateStore;
1606    /// # use storage_kit::state::config::{LanceStateStoreConfig, StateStoreConfig};
1607    /// #
1608    /// # let runtime = tokio::runtime::Builder::new_current_thread().enable_all().build()?;
1609    /// # runtime.block_on(async {
1610    /// # let store = StateStore::connect(StateStoreConfig::Lance(LanceStateStoreConfig::memory())).await?;
1611    /// # let definition = WorkflowDefinition::parse(r#"{
1612    /// #     "schema":"app-kit.workflow/v2", "id":"counter", "revision":"r1",
1613    /// #     "state":{"count":0}, "required":["count"],
1614    /// #     "reducers":{"count":"reject_concurrent"}, "entry":"increment",
1615    /// #     "nodes":[{"id":"increment","step":{"kind":"state","mutations":[
1616    /// #         {"kind":"increment","target":"count","value":{"kind":"literal","value":1}}
1617    /// #     ]}}], "routes":{}
1618    /// # }"#)?;
1619    /// # let config = AppConfig::<_, Agent>::new(store, Vec::new(), vec![definition]);
1620    /// # let mut app = App::open(config).await?;
1621    /// # let mut workspace = app.create_workspace(WorkspaceConfig {
1622    /// #     name: Some("local counter".into()), root: std::env::current_dir()?,
1623    /// # }).await?;
1624    /// # let workflow = workspace.workflow("counter".parse()?).await?;
1625    /// # let mut run = workflow.prepare(WorkflowState::default()).await?;
1626    /// let mut execution = run.execution(ExecutionOptions::default()).await?;
1627    /// let event = execution.step().await?.expect("the state action commits");
1628    /// assert!(matches!(event, ExecutionEvent::Committed { .. }));
1629    /// assert_eq!(execution.finish().await?.state.values()["count"], 1);
1630    /// #
1631    /// # app.shutdown(Shutdown::CancelAndDrain { timeout: std::time::Duration::from_secs(5) }).await?;
1632    /// # Ok::<(), Box<dyn std::error::Error>>(())
1633    /// # })?;
1634    /// # Ok::<(), Box<dyn std::error::Error>>(())
1635    /// ```
1636    pub async fn step(&mut self) -> Result<Option<ExecutionEvent>, AppError> {
1637        loop {
1638            match poll_fn(|cx| self.poll_advance(cx, true)).await {
1639                Ok(Some(ExecutionEvent::Partial { .. })) => {}
1640                event => {
1641                    if (matches!(&event, Ok(Some(ExecutionEvent::Committed { .. })))
1642                        || matches!(&event, Ok(None)) && matches!(&self.state, SessionState::Eof)
1643                        || event.is_err() && self.runtime.cell.lifecycle_generation().is_none())
1644                        && let Some(generation) = self.generation
1645                    {
1646                        self.runtime.cell.returned_boundary(generation);
1647                    }
1648                    return event;
1649                }
1650            }
1651        }
1652    }
1653
1654    /// Drains this execution to exhaustion, a halt or an actionable control.
1655    ///
1656    /// Consumes this owner and drains advancement until halt, actionable parking, EOF
1657    /// or exhaustion, then finalizes observation. A returned snapshot need not be
1658    /// Succeeded: inspect status and interactions. Accepted child/native cleanup
1659    /// retains its original authority and deadlines.
1660    ///
1661    /// # Errors
1662    ///
1663    /// Returns `Closed` or `Conflict` when execution authority is revoked or stale,
1664    /// `Integration` for unavailable durable/lower evidence, and `Timeout` for expired
1665    /// observation. Invalid execution input may return `InvalidInput`; accepted action
1666    /// failures are retained in the returned run state. Cleanup preserves primary
1667    /// failures and does not detach an admitted native commit.
1668    ///
1669    /// # Example
1670    ///
1671    /// ```
1672    /// # use agents_kit::Agent;
1673    /// # use app_kit::{App, AppConfig, Shutdown, WorkspaceConfig};
1674    /// # use app_kit::workflow::{CheckpointSelector, ExecutionEvent, ExecutionOptions, RunSelector,
1675    /// #     RunStatus, WorkflowDefinition, WorkflowState};
1676    /// # use storage_kit::StateStore;
1677    /// # use storage_kit::state::config::{LanceStateStoreConfig, StateStoreConfig};
1678    /// #
1679    /// # let runtime = tokio::runtime::Builder::new_current_thread().enable_all().build()?;
1680    /// # runtime.block_on(async {
1681    /// # let store = StateStore::connect(StateStoreConfig::Lance(LanceStateStoreConfig::memory())).await?;
1682    /// # let definition = WorkflowDefinition::parse(r#"{
1683    /// #     "schema":"app-kit.workflow/v2", "id":"counter", "revision":"r1",
1684    /// #     "state":{"count":0}, "required":["count"],
1685    /// #     "reducers":{"count":"reject_concurrent"}, "entry":"increment",
1686    /// #     "nodes":[{"id":"increment","step":{"kind":"state","mutations":[
1687    /// #         {"kind":"increment","target":"count","value":{"kind":"literal","value":1}}
1688    /// #     ]}}], "routes":{}
1689    /// # }"#)?;
1690    /// # let config = AppConfig::<_, Agent>::new(store, Vec::new(), vec![definition]);
1691    /// # let mut app = App::open(config).await?;
1692    /// # let mut workspace = app.create_workspace(WorkspaceConfig {
1693    /// #     name: Some("local counter".into()), root: std::env::current_dir()?,
1694    /// # }).await?;
1695    /// # let workflow = workspace.workflow("counter".parse()?).await?;
1696    /// # let mut run = workflow.prepare(WorkflowState::default()).await?;
1697    /// let terminal = run.execution(ExecutionOptions::default()).await?.finish().await?;
1698    /// assert!(matches!(terminal.status, RunStatus::Succeeded));
1699    /// assert_eq!(terminal.state.values()["count"], 1);
1700    /// #
1701    /// # app.shutdown(Shutdown::CancelAndDrain { timeout: std::time::Duration::from_secs(5) }).await?;
1702    /// # Ok::<(), Box<dyn std::error::Error>>(())
1703    /// # })?;
1704    /// # Ok::<(), Box<dyn std::error::Error>>(())
1705    /// ```
1706    pub async fn finish(mut self) -> Result<RunSnapshot, AppError> {
1707        if !matches!(&self.state, SessionState::Eof) {
1708            while poll_fn(|cx| self.poll_advance(cx, true)).await?.is_some() {}
1709        }
1710        self.close().await
1711    }
1712
1713    /// Discards uncommitted caller work after joining admitted publication.
1714    ///
1715    /// Consumes this owner and finalizes its uncommitted boundary without requesting
1716    /// durable run cancellation. An already admitted native publication retains its
1717    /// commit authority; abort is not a guarantee that external work was rolled back.
1718    ///
1719    /// # Errors
1720    ///
1721    /// Returns `Closed` or `Conflict` when execution authority is revoked or stale,
1722    /// `Integration` for unavailable durable/lower evidence, and `Timeout` for expired
1723    /// observation. Invalid execution input may return `InvalidInput`; accepted action
1724    /// failures are retained in the returned run state. Cleanup preserves primary
1725    /// failures and does not detach an admitted native commit.
1726    ///
1727    /// # Example
1728    ///
1729    /// ```
1730    /// # use agents_kit::Agent;
1731    /// # use app_kit::{App, AppConfig, Shutdown, WorkspaceConfig};
1732    /// # use app_kit::workflow::{CheckpointSelector, ExecutionEvent, ExecutionOptions, RunSelector,
1733    /// #     RunStatus, WorkflowDefinition, WorkflowState};
1734    /// # use storage_kit::StateStore;
1735    /// # use storage_kit::state::config::{LanceStateStoreConfig, StateStoreConfig};
1736    /// #
1737    /// # let runtime = tokio::runtime::Builder::new_current_thread().enable_all().build()?;
1738    /// # runtime.block_on(async {
1739    /// # let store = StateStore::connect(StateStoreConfig::Lance(LanceStateStoreConfig::memory())).await?;
1740    /// # let definition = WorkflowDefinition::parse(r#"{
1741    /// #     "schema":"app-kit.workflow/v2", "id":"counter", "revision":"r1",
1742    /// #     "state":{"count":0}, "required":["count"],
1743    /// #     "reducers":{"count":"reject_concurrent"}, "entry":"increment",
1744    /// #     "nodes":[{"id":"increment","step":{"kind":"state","mutations":[
1745    /// #         {"kind":"increment","target":"count","value":{"kind":"literal","value":1}}
1746    /// #     ]}}], "routes":{}
1747    /// # }"#)?;
1748    /// # let config = AppConfig::<_, Agent>::new(store, Vec::new(), vec![definition]);
1749    /// # let mut app = App::open(config).await?;
1750    /// # let mut workspace = app.create_workspace(WorkspaceConfig {
1751    /// #     name: Some("local counter".into()), root: std::env::current_dir()?,
1752    /// # }).await?;
1753    /// # let workflow = workspace.workflow("counter".parse()?).await?;
1754    /// # let mut run = workflow.prepare(WorkflowState::default()).await?;
1755    /// let execution = run.execution(ExecutionOptions::default()).await?;
1756    /// let parked = execution.abort().await?;
1757    /// assert_eq!(parked.state.values()["count"], 0);
1758    /// assert!(parked.last_action_ordinal.is_none());
1759    /// let completed = run.execution(ExecutionOptions::default()).await?.finish().await?;
1760    /// assert_eq!(completed.state.values()["count"], 1);
1761    /// #
1762    /// # app.shutdown(Shutdown::CancelAndDrain { timeout: std::time::Duration::from_secs(5) }).await?;
1763    /// # Ok::<(), Box<dyn std::error::Error>>(())
1764    /// # })?;
1765    /// # Ok::<(), Box<dyn std::error::Error>>(())
1766    /// ```
1767    pub async fn abort(mut self) -> Result<RunSnapshot, AppError> {
1768        self.close().await
1769    }
1770
1771    async fn close(&mut self) -> Result<RunSnapshot, AppError> {
1772        self.stop_sources();
1773        self.children.retain(|group| {
1774            !matches!(
1775                group.state,
1776                ChildExecutionState::Awaiting | ChildExecutionState::Prepared(_)
1777            )
1778        });
1779        poll_fn(|cx| -> Poll<Result<(), AppError>> {
1780            loop {
1781                self.poll_child_groups(cx)?;
1782                let deadline = self.source_deadline(None);
1783                if self.poll_timer(cx, deadline) {
1784                    self.poll_child_groups(cx)?;
1785                }
1786                if matches!(self.state, SessionState::Publishing { .. }) {
1787                    match self.poll_publication(cx, true) {
1788                        Poll::Pending => return Poll::Pending,
1789                        Poll::Ready(result) => {
1790                            result?;
1791                            continue;
1792                        }
1793                    }
1794                }
1795                if let SessionState::ObservingItem { receipt, .. } = &mut self.state {
1796                    let observed = match Pin::new(receipt).poll(cx) {
1797                        Poll::Pending => return Poll::Pending,
1798                        Poll::Ready(result) => result.map_err(|_| {
1799                            AppError::Closed("stream observation receipt closed".into())
1800                        })?,
1801                    };
1802                    self.finish_item(observed)?;
1803                    continue;
1804                }
1805                if self.children.is_empty() {
1806                    return Poll::Ready(Ok(()));
1807                }
1808                if let SessionState::Loaded(_) = &self.state {
1809                    let SessionState::Loaded(loaded) =
1810                        std::mem::replace(&mut self.state, SessionState::Finished)
1811                    else {
1812                        unreachable!()
1813                    };
1814                    self.state = SessionState::Sources { loaded };
1815                }
1816                if matches!(self.state, SessionState::Sources { .. }) {
1817                    if self.publish_child_completion(false, true)? {
1818                        continue;
1819                    }
1820                    // Observe cancellation under the original receipt/deadline;
1821                    // no new group or source can be constructed during close.
1822                    if self.poll_notification(cx)? {
1823                        let SessionState::Sources { loaded } =
1824                            std::mem::replace(&mut self.state, SessionState::Finished)
1825                        else {
1826                            unreachable!()
1827                        };
1828                        self.publish(
1829                            SessionPublication::Reload {
1830                                receipt: SessionReceipt::Checkpoint {
1831                                    loaded,
1832                                    staged: Vec::new(),
1833                                },
1834                                deadline,
1835                            },
1836                            false,
1837                        )?;
1838                        continue;
1839                    }
1840                }
1841                return Poll::Pending;
1842            }
1843        })
1844        .await?;
1845        if !self.sources.is_empty() {
1846            if !matches!(&self.state, SessionState::Sources { .. }) {
1847                let deadline = LifecycleEvent::deadline(self.parent_deadline);
1848                let loaded = if matches!(&self.state, SessionState::Loaded(_)) {
1849                    // Closing an already-admitted publication kept its exact
1850                    // receipt. Preserve the generation fence without decoding
1851                    // that same checkpoint again.
1852                    self.runtime
1853                        .observe_generation(deadline)
1854                        .await
1855                        .map_err(|failure| {
1856                            self.runtime.retain_execution_read_failure(&failure);
1857                            failure.into_error()
1858                        })?;
1859                    let SessionState::Loaded(loaded) =
1860                        std::mem::replace(&mut self.state, SessionState::Finished)
1861                    else {
1862                        unreachable!()
1863                    };
1864                    loaded
1865                } else {
1866                    self.runtime
1867                        .load_execution_observation(None, Some(deadline))
1868                        .await
1869                        .map_err(|failure| {
1870                            self.runtime.retain_execution_read_failure(&failure);
1871                            failure.into_error()
1872                        })?
1873                        .ok_or_else(|| AppError::NotFound("workflow run does not exist".into()))?
1874                };
1875                self.state = SessionState::Sources { loaded };
1876            }
1877            let buffered_failure = self
1878                .prefetched
1879                .take()
1880                .and_then(|(index, event)| event.err().map(|category| (index, category)));
1881            for index in 0..self.sources.len() {
1882                let source = &mut self.sources[index];
1883                let failure = buffered_failure
1884                    .filter(|(buffered, _)| *buffered == index)
1885                    .map(|(_, category)| category)
1886                    .or(match source.state {
1887                        CallerSourceState::Failed(category) => Some(category),
1888                        _ => None,
1889                    });
1890                source.state = CallerSourceState::Ended;
1891                if let Some(category) = failure {
1892                    self.stage_source_failure(index, category, false)?;
1893                    poll_fn(|cx| self.poll_publication(cx, true)).await?;
1894                    self.runtime
1895                        .observe_generation(LifecycleEvent::deadline(self.parent_deadline))
1896                        .await
1897                        .map_err(|failure| {
1898                            self.runtime.retain_execution_read_failure(&failure);
1899                            failure.into_error()
1900                        })?;
1901                    let SessionState::Loaded(loaded) =
1902                        std::mem::replace(&mut self.state, SessionState::Finished)
1903                    else {
1904                        unreachable!("closing publication retains its original checkpoint receipt")
1905                    };
1906                    self.state = SessionState::Sources { loaded };
1907                }
1908            }
1909            let SessionState::Sources { loaded, .. } = &self.state else {
1910                unreachable!()
1911            };
1912            if loaded
1913                .record
1914                .checkpoint
1915                .staged
1916                .iter()
1917                .any(|activation| matches!(activation.outcome, StagedOutcomeRecord::Failed { .. }))
1918            {
1919                self.finish_wave(false)?;
1920                poll_fn(|cx| self.poll_publication(cx, true)).await?;
1921            }
1922        }
1923        if let SessionState::Sources { loaded, .. } =
1924            std::mem::replace(&mut self.state, SessionState::Finished)
1925        {
1926            self.observed = Some(loaded.record);
1927        }
1928        self.sources.clear();
1929        let foreground = self.generation.is_some();
1930        if foreground {
1931            self.publish(SessionPublication::Release, false)?;
1932            poll_fn(|cx| self.poll_publication(cx, true)).await?;
1933            self.state = SessionState::Finished;
1934            self.generation = None;
1935        }
1936        let snapshot = if foreground {
1937            self.runtime.snapshot().await?
1938        } else {
1939            self.runtime.app.ensure_snapshot_read()?;
1940            let terminal = self
1941                .observed
1942                .as_ref()
1943                .filter(|record| record.checkpoint.is_terminal())
1944                .map(|record| record.checkpoint.sequence);
1945            let loaded = self
1946                .runtime
1947                .load_execution_observation(terminal, self.parent_deadline)
1948                .await
1949                .map_err(|failure| {
1950                    self.runtime.retain_execution_read_failure(&failure);
1951                    failure.into_error()
1952                })?
1953                .ok_or_else(|| AppError::NotFound("workflow run does not exist".into()))?;
1954            let mut deadline = LifecycleEvent::deadline(self.parent_deadline);
1955            if loaded.record.checkpoint.is_terminal()
1956                && let Some(generation) = self.runtime.cell.lifecycle_generation()
1957            {
1958                deadline = self
1959                    .runtime
1960                    .cell
1961                    .begin_action_cleanup(generation)?
1962                    .min(deadline);
1963            }
1964            if !self.retains_execution_lifecycle(&loaded.record)
1965                && (loaded.record.checkpoint.controls.is_empty()
1966                    || loaded.record.checkpoint.is_public_outcome())
1967            {
1968                self.runtime
1969                    .finish_execution_observation(
1970                        &loaded.record,
1971                        None,
1972                        loaded.record.checkpoint.is_terminal().then_some(deadline),
1973                        false,
1974                    )
1975                    .await?;
1976            }
1977            if loaded.record.checkpoint.is_terminal() {
1978                // Recheck the generation after awaited hooks, while retaining
1979                // this terminal receipt instead of reading its run head again.
1980                self.runtime
1981                    .observe_generation(deadline)
1982                    .await
1983                    .map_err(|failure| {
1984                        self.runtime.retain_execution_read_failure(&failure);
1985                        failure.into_error()
1986                    })?;
1987                let snapshot = self.runtime.project(&loaded.record)?;
1988                self.runtime.app.ensure_snapshot_read()?;
1989                snapshot
1990            } else {
1991                self.runtime.snapshot().await?
1992            }
1993        };
1994        if !foreground
1995            && matches!(
1996                snapshot.status,
1997                RunStatus::Succeeded
1998                    | RunStatus::Cancelled
1999                    | RunStatus::Failed(_)
2000                    | RunStatus::Halted { .. }
2001            )
2002        {
2003            self.runtime.cell.clear_execution_options(None);
2004        }
2005        if foreground && matches!(snapshot.status, crate::workflow::RunStatus::Running) {
2006            self.runtime.wait().await
2007        } else {
2008            Ok(snapshot)
2009        }
2010    }
2011
2012    fn stop_sources(&mut self) {
2013        for source in &mut self.sources {
2014            let failure = match source.state {
2015                CallerSourceState::Failed(category) => Some(category),
2016                _ => None,
2017            };
2018            let destroyed = source.stop().err();
2019            if let Some(category) = failure.or(destroyed) {
2020                self.runtime.retain_source_failure(
2021                    &source.context,
2022                    &source.activation.lineage,
2023                    category,
2024                );
2025                source.state = CallerSourceState::Failed(category);
2026            }
2027        }
2028    }
2029
2030    fn stop_children(&mut self) {
2031        for group in &mut self.children {
2032            group.stop(&self.runtime);
2033        }
2034        self.children.clear();
2035    }
2036
2037    fn plan_child_group(
2038        &mut self,
2039        loaded: LoadedRun,
2040        sequence: u64,
2041        children: Vec<RunRecord>,
2042        verified: bool,
2043    ) -> Result<(), AppError> {
2044        if loaded.record.checkpoint.is_stopping() || loaded.record.checkpoint.is_terminal() {
2045            self.state = SessionState::Loaded(loaded);
2046            return Ok(());
2047        }
2048        if let Some(group) = self
2049            .children
2050            .iter_mut()
2051            .find(|group| group.sequence == sequence)
2052            && let Some(ControlRecord {
2053                state: ControlRecordState::Child { admitted, .. },
2054                ..
2055            }) = loaded.record.checkpoint.control(sequence)
2056        {
2057            // Only confirmed native admission transfers these maps to the
2058            // child's concrete execution options. Retry retains the original.
2059            for values in group.inputs.iter_mut().take(admitted.len()) {
2060                values.clear();
2061            }
2062        }
2063        let decision = catch_unwind(AssertUnwindSafe(|| {
2064            self.runtime
2065                .plan_children(&loaded, sequence, children, verified)
2066        }))
2067        .map_err(|_| AppError::Integration("child planning callback panicked".into()))??;
2068        match decision {
2069            BspDecision::ExecuteChildren { sequence, children } => {
2070                let control = loaded
2071                    .record
2072                    .checkpoint
2073                    .control(sequence)
2074                    .ok_or_else(|| AppError::Conflict("child control is unavailable".into()))?;
2075                let context = self.runtime.child_context(&loaded.record, sequence)?;
2076                let definition = self.runtime.app.definition_for(&loaded.record)?;
2077                let step = definition
2078                    .node(&control.node)
2079                    .ok_or_else(|| AppError::Integration("child definition is unavailable".into()))?
2080                    .step();
2081                step.validate_mode(context.mode)?;
2082                let deadline = tokio::time::Instant::now()
2083                    .checked_add(WorkflowRunRuntime::<CS, A, X, C, T, H, E>::remaining(
2084                        control.deadline_millis,
2085                    )?)
2086                    .ok_or_else(|| {
2087                        AppError::Timeout("child deadline is outside the clock range".into())
2088                    })?;
2089                if let Some(group) = self
2090                    .children
2091                    .iter_mut()
2092                    .find(|group| group.sequence == sequence)
2093                {
2094                    if !matches!(group.state, ChildExecutionState::Awaiting) {
2095                        return Err(AppError::Conflict(
2096                            "child group already has a polling owner".into(),
2097                        ));
2098                    }
2099                    group.state = ChildExecutionState::Prepared(children);
2100                } else {
2101                    self.children.push(ChildExecution {
2102                        sequence,
2103                        parent: None,
2104                        context,
2105                        lineage: control.lineage.clone(),
2106                        deadline,
2107                        state: ChildExecutionState::Prepared(children),
2108                        inputs: Vec::new(),
2109                    });
2110                }
2111                self.state = SessionState::Loaded(loaded);
2112                Ok(())
2113            }
2114            BspDecision::AwaitControls => {
2115                self.state = SessionState::Loaded(loaded);
2116                Ok(())
2117            }
2118            decision => {
2119                let transition = match decision {
2120                    BspDecision::Stage(activations) => ControlTransition::Stage { activations },
2121                    BspDecision::Failed {
2122                        category,
2123                        message,
2124                        observation,
2125                    } => ControlTransition::Failed {
2126                        category,
2127                        message,
2128                        observation,
2129                    },
2130                    decision => return self.publish_decision(loaded, decision, false),
2131                };
2132                let control = loaded
2133                    .record
2134                    .checkpoint
2135                    .control(sequence)
2136                    .cloned()
2137                    .ok_or_else(|| {
2138                        AppError::Conflict("completed child control is unavailable".into())
2139                    })?;
2140                self.publish(
2141                    SessionPublication::ChildOutcome {
2142                        control,
2143                        transition,
2144                    },
2145                    false,
2146                )
2147            }
2148        }
2149    }
2150
2151    fn start_child_groups(&mut self, cx: &mut Context<'_>) -> Result<(), AppError> {
2152        if !self
2153            .children
2154            .iter()
2155            .any(|group| matches!(group.state, ChildExecutionState::Prepared(_)))
2156        {
2157            return Ok(());
2158        }
2159        if self.poll_notification(cx)? {
2160            let deadline = self.source_deadline(None);
2161            let SessionState::Sources { loaded } =
2162                std::mem::replace(&mut self.state, SessionState::Finished)
2163            else {
2164                unreachable!()
2165            };
2166            self.publish(
2167                SessionPublication::Reload {
2168                    receipt: SessionReceipt::Checkpoint {
2169                        loaded,
2170                        staged: Vec::new(),
2171                    },
2172                    deadline,
2173                },
2174                false,
2175            )?;
2176            return Ok(());
2177        }
2178        let SessionState::Sources { loaded, .. } = &self.state else {
2179            return Ok(());
2180        };
2181        let parent = Arc::new(loaded.record.clone());
2182        for index in 0..self.children.len() {
2183            if !matches!(self.children[index].state, ChildExecutionState::Prepared(_)) {
2184                continue;
2185            }
2186            let group = &mut self.children[index];
2187            let ChildExecutionState::Prepared(children) =
2188                std::mem::replace(&mut group.state, ChildExecutionState::Closed)
2189            else {
2190                unreachable!()
2191            };
2192            group.parent = Some(Arc::clone(&parent));
2193            if group.context.mode == ActionMode::Asynchronous && self.lane == ExecutionLane::Caller
2194            {
2195                let (sender, receipt) = oneshot::channel();
2196                if !self.runtime.app.spawn_driver(
2197                    self.runtime.key.clone(),
2198                    Duration::ZERO,
2199                    DriverCommand::Child {
2200                        sequence: group.sequence,
2201                        generation: self.generation,
2202                        parent: Arc::clone(&parent),
2203                        children,
2204                        receipt: sender,
2205                    },
2206                ) {
2207                    return Err(AppError::Busy(
2208                        "child group already has an accepted owner".into(),
2209                    ));
2210                }
2211                group.state = ChildExecutionState::Worker(receipt);
2212            } else {
2213                group.start_local(&self.runtime, self.app, self.lane, children)?;
2214            }
2215        }
2216        Ok(())
2217    }
2218
2219    fn poll_child_groups(&mut self, cx: &mut Context<'_>) -> Result<(), AppError> {
2220        let stopped = match &self.state {
2221            SessionState::Loaded(loaded)
2222            | SessionState::Sources { loaded }
2223            | SessionState::ObservingItem { loaded, .. } => {
2224                loaded.record.checkpoint.is_stopping() || loaded.record.checkpoint.is_terminal()
2225            }
2226            _ => false,
2227        };
2228        if stopped {
2229            self.stop_children();
2230            return Ok(());
2231        }
2232        if let SessionState::Loaded(loaded)
2233        | SessionState::Sources { loaded }
2234        | SessionState::ObservingItem { loaded, .. } = &self.state
2235        {
2236            self.children.retain(|group| {
2237                !matches!(group.state, ChildExecutionState::Awaiting)
2238                    || loaded.record.checkpoint.control(group.sequence).is_some()
2239            });
2240            if self.children.iter().any(|group| {
2241                !matches!(
2242                    group.state,
2243                    ChildExecutionState::Awaiting
2244                        | ChildExecutionState::Worker(_)
2245                        | ChildExecutionState::Receipt(_)
2246                ) && group.parent.as_ref().is_some_and(|parent| {
2247                    !loaded.record.owns_child_activation(parent, group.sequence)
2248                })
2249            }) {
2250                return Err(AppError::Conflict(
2251                    "child parent reservation changed".into(),
2252                ));
2253            }
2254        }
2255        let observing_parent = matches!(
2256            self.state,
2257            SessionState::Publishing {
2258                purpose: PublicationPurpose::Reload,
2259                ..
2260            }
2261        );
2262        for group in &mut self.children {
2263            if observing_parent
2264                && tokio::time::Instant::now() >= group.deadline
2265                && matches!(
2266                    group.state,
2267                    ChildExecutionState::Local(_) | ChildExecutionState::Pipeline(_)
2268                )
2269            {
2270                // The outstanding authority read owns deadline disposition.
2271                // Keep the original executor until that receipt or fatal cleanup.
2272                continue;
2273            }
2274            let _ = group.poll_ready(cx, &self.runtime);
2275        }
2276        Ok(())
2277    }
2278
2279    fn publish_child_completion(
2280        &mut self,
2281        ordinary: bool,
2282        closing: bool,
2283    ) -> Result<bool, AppError> {
2284        let Some(index) = self.children.iter().position(|group| {
2285            matches!(
2286                group.state,
2287                ChildExecutionState::Complete(_) | ChildExecutionState::Receipt(_)
2288            )
2289        }) else {
2290            return Ok(false);
2291        };
2292        let group = &mut self.children[index];
2293        let sequence = group.sequence;
2294        let parent = group.parent.take();
2295        let context = Arc::clone(&group.context);
2296        let lineage = group.lineage.clone();
2297        let completed = std::mem::replace(&mut group.state, ChildExecutionState::Awaiting);
2298        if closing {
2299            self.children.remove(index);
2300        }
2301        if let ChildExecutionState::Receipt(result) = completed {
2302            match (*result).map_err(RuntimeOperationFailure::into_error)? {
2303                SessionReceipt::Children {
2304                    loaded,
2305                    sequence,
2306                    children,
2307                    verified: true,
2308                } => {
2309                    if closing {
2310                        self.state = SessionState::Loaded(loaded);
2311                    } else {
2312                        self.plan_child_group(loaded, sequence, children, true)?;
2313                    }
2314                }
2315                SessionReceipt::Checkpoint { loaded, .. } => {
2316                    self.state = SessionState::Loaded(loaded)
2317                }
2318                SessionReceipt::Children {
2319                    verified: false, ..
2320                } => {
2321                    return Err(AppError::Integration(
2322                        "child worker returned an unverified execution result".into(),
2323                    ));
2324                }
2325            }
2326            return Ok(true);
2327        }
2328        let ChildExecutionState::Complete(result) = completed else {
2329            unreachable!()
2330        };
2331        let SessionState::Sources { loaded } =
2332            std::mem::replace(&mut self.state, SessionState::Finished)
2333        else {
2334            unreachable!()
2335        };
2336        match result {
2337            Ok(completions) => self.publish(
2338                SessionPublication::VerifyChildren {
2339                    parent: parent.ok_or_else(|| {
2340                        AppError::Integration("child group parent evidence is unavailable".into())
2341                    })?,
2342                    sequence,
2343                    completions,
2344                },
2345                ordinary,
2346            )?,
2347            Err(category) => {
2348                let activation = loaded
2349                    .record
2350                    .checkpoint
2351                    .frontier
2352                    .iter()
2353                    .find(|activation| {
2354                        activation.node == context.node && activation.lineage == lineage
2355                    })
2356                    .cloned()
2357                    .ok_or_else(|| AppError::Conflict("child activation changed".into()))?;
2358                self.publish_failure(loaded, &activation, &context, category)?;
2359            }
2360        }
2361        Ok(true)
2362    }
2363
2364    fn retains_external_continuation(&self, record: &RunRecord) -> bool {
2365        !self.children.is_empty()
2366            || self.runtime.app.has_child_work(&self.runtime.key)
2367            || record.checkpoint.controls.values().any(|control| {
2368                matches!(&control.state, ControlRecordState::Attempt(_))
2369                    && self
2370                        .runtime
2371                        .app
2372                        .has_control_work(&self.runtime.key, control.sequence)
2373            })
2374    }
2375
2376    fn retains_execution_lifecycle(&self, record: &RunRecord) -> bool {
2377        self.retains_execution_work(record)
2378            || record
2379                .checkpoint
2380                .controls
2381                .values()
2382                .any(|control| matches!(control.state, ControlRecordState::Inputs { .. }))
2383    }
2384
2385    fn retains_execution_work(&self, record: &RunRecord) -> bool {
2386        self.sources.iter().any(|source| {
2387            matches!(
2388                source.state,
2389                CallerSourceState::Polling(_) | CallerSourceState::Failed(_)
2390            )
2391        }) || self.prefetched.is_some()
2392            || self.retains_external_continuation(record)
2393            || record.checkpoint.controls.values().any(|control| {
2394                matches!(control.state, ControlRecordState::Sleep)
2395                    && self
2396                        .runtime
2397                        .app
2398                        .has_control_work(&self.runtime.key, control.sequence)
2399            })
2400    }
2401
2402    fn ensure_owned(&self) -> Result<(), AppError> {
2403        self.runtime.app.ensure_runtime_owned()?;
2404        if self
2405            .generation
2406            .is_some_and(|generation| !self.runtime.cell.execution_is_live(generation))
2407        {
2408            return Err(AppError::Closed(
2409                "run execution generation is closed".into(),
2410            ));
2411        }
2412        Ok(())
2413    }
2414
2415    fn source_deadline(&self, excluded: Option<usize>) -> Option<tokio::time::Instant> {
2416        self.sources
2417            .iter()
2418            .enumerate()
2419            .filter_map(|(index, source)| {
2420                (Some(index) != excluded && matches!(source.state, CallerSourceState::Polling(_)))
2421                    .then_some(source.deadline)
2422            })
2423            .chain(self.children.iter().filter_map(|group| {
2424                (!matches!(
2425                    group.state,
2426                    ChildExecutionState::Complete(_) | ChildExecutionState::Receipt(_)
2427                ))
2428                .then_some(group.deadline)
2429            }))
2430            .min()
2431    }
2432
2433    fn expire_sources(&mut self, excluded: Option<usize>) {
2434        let now = tokio::time::Instant::now();
2435        for (index, source) in self.sources.iter_mut().enumerate() {
2436            if Some(index) == excluded
2437                || now < source.deadline
2438                || !matches!(source.state, CallerSourceState::Polling(_))
2439            {
2440                continue;
2441            }
2442            self.runtime.retain_source_failure(
2443                &source.context,
2444                &source.activation.lineage,
2445                FailureCategory::Timeout,
2446            );
2447            let _ = source.stop();
2448            if self
2449                .prefetched
2450                .as_ref()
2451                .is_some_and(|(buffered, _)| *buffered == index)
2452            {
2453                self.prefetched = Some((index, Err(FailureCategory::Timeout)));
2454            } else {
2455                source.state = CallerSourceState::Failed(FailureCategory::Timeout);
2456            }
2457        }
2458    }
2459
2460    fn poll_sources(&mut self, context: &mut Context<'_>, excluded: Option<usize>) {
2461        self.expire_sources(excluded);
2462        if self.prefetched.is_some() {
2463            return;
2464        }
2465        let count = self.sources.len();
2466        for offset in 0..count {
2467            let index = (self.next_source + offset) % count;
2468            if Some(index) == excluded {
2469                continue;
2470            }
2471            if let Poll::Ready(mut event) = self.sources[index].poll(context) {
2472                let source = &mut self.sources[index];
2473                if let Err(category) = &event {
2474                    // Preserve sampled failure before invoking the destructor:
2475                    // the original sibling callback may settle during Drop.
2476                    self.runtime.retain_source_failure(
2477                        &source.context,
2478                        &source.activation.lineage,
2479                        *category,
2480                    );
2481                }
2482                if matches!(
2483                    &event,
2484                    Ok(Some(ActionEvent::Completed { .. })) | Ok(None) | Err(_)
2485                ) {
2486                    let destroyed = source.stop();
2487                    if event.is_ok()
2488                        && let Err(category) = destroyed
2489                    {
2490                        self.runtime.retain_source_failure(
2491                            &source.context,
2492                            &source.activation.lineage,
2493                            category,
2494                        );
2495                        event = Err(category);
2496                    }
2497                    if matches!(&event, Ok(None)) {
2498                        source.state = CallerSourceState::Eof;
2499                    }
2500                }
2501                self.next_source = (index + 1) % count;
2502                self.prefetched = Some((index, event));
2503                return;
2504            }
2505        }
2506    }
2507
2508    fn arm_notification(&mut self) -> Result<(), AppError> {
2509        if self.wake.is_none() {
2510            self.wake = Some(Box::pin(self.runtime.notifier()?.notified_owned()));
2511        }
2512        let wake = self.wake.as_mut().unwrap();
2513        wake.as_mut().enable();
2514        // Retain registration before admitting reads; polling installs the
2515        // current task waker without recreating this notification future.
2516        Ok(())
2517    }
2518
2519    fn poll_notification(&mut self, cx: &mut Context<'_>) -> Result<bool, AppError> {
2520        self.arm_notification()?;
2521        let ready = self.wake.as_mut().unwrap().as_mut().poll(cx).is_ready();
2522        if ready {
2523            self.wake = None;
2524        }
2525        Ok(ready)
2526    }
2527
2528    fn poll_timer(&mut self, cx: &mut Context<'_>, deadline: Option<tokio::time::Instant>) -> bool {
2529        let Some(deadline) = deadline else {
2530            self.timer = None;
2531            return false;
2532        };
2533        let timer = self
2534            .timer
2535            .get_or_insert_with(|| Box::pin(tokio::time::sleep_until(deadline)));
2536        if timer.deadline() != deadline {
2537            timer.as_mut().reset(deadline);
2538        }
2539        timer.as_mut().poll(cx).is_ready()
2540    }
2541
2542    fn poll_native_partial(&mut self, cx: &mut Context<'_>) -> Option<ExecutionEvent> {
2543        let Some((_, receiver)) = &mut self.native_partials else {
2544            return None;
2545        };
2546        match receiver.poll_recv(cx) {
2547            Poll::Ready(Some(event)) => Some(event),
2548            Poll::Ready(None) => {
2549                self.native_partials = None;
2550                None
2551            }
2552            Poll::Pending => None,
2553        }
2554    }
2555
2556    fn poll_sources_observation(
2557        &mut self,
2558        cx: &mut Context<'_>,
2559    ) -> Poll<Result<Option<ExecutionEvent>, AppError>> {
2560        self.ensure_owned()?;
2561        self.expire_sources(None);
2562        let deadline = self.source_deadline(None);
2563        if self.poll_timer(cx, deadline) {
2564            self.expire_sources(None);
2565        }
2566        let pending_failure = self
2567            .sources
2568            .iter()
2569            .any(|source| matches!(source.state, CallerSourceState::Failed(_)));
2570        if !pending_failure && self.poll_notification(cx)? {
2571            let SessionState::Sources { loaded } =
2572                std::mem::replace(&mut self.state, SessionState::Finished)
2573            else {
2574                unreachable!()
2575            };
2576            self.publish(
2577                SessionPublication::Reload {
2578                    receipt: SessionReceipt::Checkpoint {
2579                        loaded,
2580                        staged: Vec::new(),
2581                    },
2582                    deadline,
2583                },
2584                false,
2585            )?;
2586            return Poll::Ready(Ok(None));
2587        }
2588        self.poll_sources(cx, None);
2589        if let Some(event) = self.poll_native_partial(cx) {
2590            return Poll::Ready(Ok(Some(event)));
2591        }
2592        if self.prefetched.is_some() {
2593            return Poll::Ready(Ok(None));
2594        }
2595        let SessionState::Sources { loaded, .. } = &self.state else {
2596            unreachable!()
2597        };
2598        let known_eof = self
2599            .sources
2600            .iter()
2601            .any(|source| matches!(source.state, CallerSourceState::Eof));
2602        if (self.generation.is_some() || !self.retains_execution_work(&loaded.record))
2603            && !known_eof
2604            && !self.children.iter().any(|group| {
2605                matches!(
2606                    group.state,
2607                    ChildExecutionState::Complete(_) | ChildExecutionState::Receipt(_)
2608                )
2609            })
2610            && loaded.record.checkpoint.is_actionable()
2611        {
2612            return Poll::Ready(Ok(None));
2613        }
2614        Poll::Pending
2615    }
2616
2617    fn poll_item(
2618        &mut self,
2619        cx: &mut Context<'_>,
2620    ) -> Poll<Result<Option<ExecutionEvent>, AppError>> {
2621        let SessionState::ObservingItem { index, .. } = &self.state else {
2622            unreachable!()
2623        };
2624        let index = *index;
2625        self.expire_sources(Some(index));
2626        let deadline = self.source_deadline(Some(index));
2627        if self.poll_timer(cx, deadline) {
2628            self.expire_sources(Some(index));
2629        }
2630        let SessionState::ObservingItem { receipt, .. } = &mut self.state else {
2631            unreachable!()
2632        };
2633        if let Poll::Ready(result) = Pin::new(receipt).poll(cx) {
2634            let result = result
2635                .map_err(|_| AppError::Closed("stream observation receipt closed".into()))
2636                .and_then(|result| result);
2637            return Poll::Ready(self.finish_item(result).map(|result| {
2638                result.map(|result| ExecutionEvent::Partial {
2639                    node: self.sources[index].activation.node.clone(),
2640                    result,
2641                })
2642            }));
2643        }
2644        self.poll_sources(cx, Some(index));
2645        if let Some(event) = self.poll_native_partial(cx) {
2646            return Poll::Ready(Ok(Some(event)));
2647        }
2648        Poll::Pending
2649    }
2650
2651    fn finish_item(
2652        &mut self,
2653        result: Result<Option<Map<String, Value>>, AppError>,
2654    ) -> Result<Option<Map<String, Value>>, AppError> {
2655        let SessionState::ObservingItem {
2656            loaded, receipt, ..
2657        } = std::mem::replace(&mut self.state, SessionState::Finished)
2658        else {
2659            unreachable!()
2660        };
2661        drop(receipt);
2662        if result.is_err() {
2663            self.observed = Some(loaded.record);
2664            for source in &mut self.sources {
2665                let _ = source.stop();
2666            }
2667        } else {
2668            self.state = SessionState::Sources { loaded };
2669        }
2670        result
2671    }
2672
2673    fn consume_source_event(
2674        &mut self,
2675        index: usize,
2676        event: Result<Option<ActionEvent>, FailureCategory>,
2677        ordinary: bool,
2678    ) -> Result<(), AppError> {
2679        match event {
2680            Ok(Some(ActionEvent::Partial { result })) => {
2681                if JsonBoundary::validate_size(&result).is_err() {
2682                    let _ = self.sources[index].stop();
2683                    return self.stage_source_failure(index, FailureCategory::State, ordinary);
2684                }
2685                self.publish_item(index, ActionEvent::Partial { result })
2686            }
2687            Ok(Some(
2688                event @ (ActionEvent::SpanStarted { .. }
2689                | ActionEvent::SpanAttribute { .. }
2690                | ActionEvent::SpanFinished { .. }),
2691            )) => self.publish_item(index, event),
2692            Ok(Some(ActionEvent::Completed { result, mut writes })) => {
2693                let SessionState::Sources { loaded, .. } = &self.state else {
2694                    unreachable!()
2695                };
2696                let source = &self.sources[index];
2697                let staged = catch_unwind(AssertUnwindSafe(|| {
2698                    let definition = self.runtime.app.definition_for(&loaded.record)?;
2699                    if let Some(Step::Action {
2700                        action,
2701                        revision,
2702                        project_result: true,
2703                    }) = definition
2704                        .node(&source.activation.node)
2705                        .map(|node| node.step())
2706                    {
2707                        writes.insert(
2708                            0,
2709                            self.runtime
2710                                .app
2711                                .action(action, revision)?
2712                                .declaration()
2713                                .result_projection(&result),
2714                        );
2715                    }
2716                    StagedActivationRecord::action(
2717                        &source.activation,
2718                        &loaded.record.checkpoint.state,
2719                        &writes,
2720                        result,
2721                        self.runtime.app.typing(),
2722                    )
2723                }));
2724                match staged {
2725                    Ok(Ok(staged)) => self.stage_source(index, staged, ordinary),
2726                    Ok(Err(_)) => {
2727                        self.stage_source_failure(index, FailureCategory::State, ordinary)
2728                    }
2729                    Err(_) => {
2730                        self.stage_source_failure(index, FailureCategory::Execution, ordinary)
2731                    }
2732                }
2733            }
2734            Ok(None) => Ok(()),
2735            Err(category) => self.stage_source_failure(index, category, ordinary),
2736        }
2737    }
2738
2739    fn stage_source_failure(
2740        &mut self,
2741        index: usize,
2742        category: FailureCategory,
2743        ordinary: bool,
2744    ) -> Result<(), AppError> {
2745        let source = &self.sources[index];
2746        self.runtime
2747            .retain_source_failure(&source.context, &source.activation.lineage, category);
2748        let staged = StagedActivationRecord::failed(
2749            source.activation.node.clone(),
2750            source.activation.lineage.clone(),
2751            category,
2752            "caller action execution failed".into(),
2753        );
2754        self.stage_source(index, staged, ordinary)
2755    }
2756
2757    fn stage_source(
2758        &mut self,
2759        index: usize,
2760        staged: StagedActivationRecord,
2761        ordinary: bool,
2762    ) -> Result<(), AppError> {
2763        let SessionState::Sources { loaded } =
2764            std::mem::replace(&mut self.state, SessionState::Finished)
2765        else {
2766            unreachable!()
2767        };
2768        self.observed = Some(loaded.record.clone());
2769        self.publish(
2770            SessionPublication::Decision {
2771                loaded,
2772                decision: BspDecision::Stage(vec![staged]),
2773                activation_deadline: None,
2774                context: Some(Arc::clone(&self.sources[index].context)),
2775                staged: Vec::new(),
2776                remaining: Vec::new().into_iter(),
2777            },
2778            ordinary,
2779        )
2780    }
2781
2782    fn finish_wave(&mut self, ordinary: bool) -> Result<(), AppError> {
2783        let SessionState::Sources { loaded } =
2784            std::mem::replace(&mut self.state, SessionState::Finished)
2785        else {
2786            unreachable!()
2787        };
2788        let definition = self.runtime.app.definition_for(&loaded.record)?;
2789        let failed = loaded
2790            .record
2791            .checkpoint
2792            .selected_staged(&definition)
2793            .filter_map(|activation| {
2794                if let StagedOutcomeRecord::Failed { category, message } = &activation.outcome {
2795                    Some((activation, *category, message))
2796                } else {
2797                    None
2798                }
2799            })
2800            .min_by(|left, right| {
2801                (&left.0.node, &left.0.lineage).cmp(&(&right.0.node, &right.0.lineage))
2802            });
2803        if let Some((activation, category, message)) = failed {
2804            let message = message.clone();
2805            let observation = match self.sources.iter().find(|source| {
2806                source.activation.node == activation.node
2807                    && source.activation.lineage == activation.lineage
2808            }) {
2809                Some(source) => ActionPosition {
2810                    node: source.context.node.clone(),
2811                    ordinal: source.context.action_ordinal,
2812                },
2813                None => loaded.record.checkpoint.proposed_position(
2814                    &definition,
2815                    &activation.node,
2816                    &activation.lineage,
2817                    &[],
2818                )?,
2819            };
2820            self.sources.clear();
2821            self.publish_decision(
2822                loaded,
2823                BspDecision::Failed {
2824                    category,
2825                    message,
2826                    observation: Some(observation),
2827                },
2828                ordinary,
2829            )
2830        } else if self
2831            .sources
2832            .iter()
2833            .any(|source| matches!(source.state, CallerSourceState::Eof))
2834        {
2835            self.sources.clear();
2836            self.publish(SessionPublication::EndSource { loaded }, ordinary)
2837        } else {
2838            self.sources.clear();
2839            self.observed = Some(loaded.record);
2840            self.state = SessionState::Ready;
2841            Ok(())
2842        }
2843    }
2844
2845    fn begin_activation(
2846        &mut self,
2847        loaded: LoadedRun,
2848        activation: FrontierRecord,
2849        remaining: std::vec::IntoIter<FrontierRecord>,
2850        staged: Vec<StagedActivationRecord>,
2851        ordinary: bool,
2852    ) -> Result<(), AppError> {
2853        self.ensure_owned()?;
2854        let definition = self.runtime.app.definition_for(&loaded.record)?;
2855        if let Some(node) = definition.node(&activation.node) {
2856            node.step()
2857                .validate_mode(self.runtime.cell.execution_mode()?)?;
2858        }
2859        let (started, deadline_millis) = WorkflowRunRuntime::<CS, A, X, C, T, H, E>::node_deadline(
2860            &definition,
2861            &activation.node,
2862        )?;
2863        let deadline = tokio::time::Instant::now()
2864            .checked_add(Duration::from_millis(deadline_millis - started))
2865            .ok_or_else(|| {
2866                AppError::Timeout("activation deadline is outside the clock range".into())
2867            })?;
2868        let context = match self.runtime.cell.action_context(
2869            loaded.record.checkpoint.base_sequence,
2870            &activation.node,
2871            &activation.lineage,
2872        ) {
2873            Some(context) => context,
2874            None => Arc::new(loaded.record.action_context(
2875                &definition,
2876                &activation,
2877                &staged,
2878                self.runtime.cell.execution_mode()?,
2879            )?),
2880        };
2881        if loaded.record.checkpoint.loses_join_first(
2882            &definition,
2883            &activation.node,
2884            &activation.lineage,
2885        ) {
2886            self.state = SessionState::ActivationReady {
2887                loaded,
2888                activation,
2889                context,
2890                remaining,
2891                staged,
2892                deadline,
2893                deadline_millis,
2894            };
2895            return Ok(());
2896        }
2897        if let Some(Step::Action {
2898            action, revision, ..
2899        }) = definition.node(&activation.node).map(|node| node.step())
2900        {
2901            let binding = self.runtime.app.action(action, revision)?;
2902            let validated = catch_unwind(AssertUnwindSafe(|| {
2903                binding.declaration().validate_reads(&WorkflowState {
2904                    values: loaded.record.checkpoint.state.clone(),
2905                })?;
2906                binding.declaration().resolve_inputs(
2907                    &self.runtime.cell.execution_options()?.inputs,
2908                    self.runtime.cell.execution_mode()?,
2909                )
2910            }));
2911            match validated {
2912                Ok(Err(AppError::InvalidInput(message))) if self.generation.is_none() => {
2913                    return self.publish_decision(
2914                        loaded,
2915                        BspDecision::Failed {
2916                            category: FailureCategory::Execution,
2917                            message,
2918                            observation: None,
2919                        },
2920                        ordinary,
2921                    );
2922                }
2923                Ok(Ok(ActionInputs::Required(required))) => {
2924                    return self.publish(
2925                        SessionPublication::Decision {
2926                            loaded,
2927                            decision: BspDecision::Inputs {
2928                                node: activation.node,
2929                                lineage: activation.lineage,
2930                                required,
2931                            },
2932                            activation_deadline: Some(deadline_millis),
2933                            context: None,
2934                            staged,
2935                            remaining,
2936                        },
2937                        ordinary,
2938                    );
2939                }
2940                Ok(result) => {
2941                    result?;
2942                }
2943                Err(_) => {
2944                    return self.publish_failure(
2945                        loaded,
2946                        &activation,
2947                        &context,
2948                        FailureCategory::Execution,
2949                    );
2950                }
2951            }
2952        }
2953        self.publish(
2954            SessionPublication::BeginActivation {
2955                loaded,
2956                activation,
2957                context,
2958                remaining,
2959                staged,
2960                deadline,
2961                deadline_millis,
2962            },
2963            ordinary,
2964        )
2965    }
2966
2967    fn activate_ready(&mut self, ordinary: bool) -> Result<(), AppError> {
2968        self.ensure_owned()?;
2969        let SessionState::ActivationReady {
2970            loaded,
2971            activation,
2972            context,
2973            mut remaining,
2974            mut staged,
2975            deadline,
2976            deadline_millis,
2977        } = std::mem::replace(&mut self.state, SessionState::Finished)
2978        else {
2979            unreachable!()
2980        };
2981        let definition = self.runtime.app.definition_for(&loaded.record)?;
2982        let skipped = loaded.record.checkpoint.loses_join_first(
2983            &definition,
2984            &activation.node,
2985            &activation.lineage,
2986        );
2987        if !skipped
2988            && matches!(
2989                definition.node(&activation.node).map(|node| node.step()),
2990                Some(Step::Action { .. })
2991            )
2992        {
2993            self.construct_source(&loaded, activation, context, deadline)?;
2994            return match remaining.next() {
2995                Some(activation) => {
2996                    self.begin_activation(loaded, activation, remaining, staged, ordinary)
2997                }
2998                None if staged.is_empty() => {
2999                    self.observed = None;
3000                    self.state = SessionState::Sources { loaded };
3001                    Ok(())
3002                }
3003                None => self.publish_decision(loaded, BspDecision::Stage(staged), ordinary),
3004            };
3005        }
3006        if !skipped && tokio::time::Instant::now() >= deadline {
3007            return self.publish_failure(loaded, &activation, &context, FailureCategory::Timeout);
3008        }
3009        let engine = ExecutionEngine::new(
3010            &definition,
3011            self.runtime.app.typing(),
3012            self.runtime.app.conditions(),
3013        );
3014        let decision = match catch_unwind(AssertUnwindSafe(|| {
3015            let decision = engine.activate(
3016                &loaded.record.checkpoint,
3017                &activation,
3018                loaded
3019                    .record
3020                    .parent
3021                    .as_ref()
3022                    .map_or(0, |parent| parent.depth),
3023                |workflow, revision| {
3024                    let revision = match revision {
3025                        Some(revision) => revision,
3026                        None => loaded.record.child_revision(&workflow)?.to_owned(),
3027                    };
3028                    self.runtime.app.definition_revision(&workflow, &revision)?;
3029                    Ok((workflow, revision))
3030                },
3031            )?;
3032            if let BspDecision::Descendants { inputs, .. } = &decision {
3033                let step = definition
3034                    .node(&activation.node)
3035                    .ok_or_else(|| {
3036                        AppError::Integration("child declaration is unavailable".into())
3037                    })?
3038                    .step();
3039                for task in inputs {
3040                    let child = self
3041                        .runtime
3042                        .app
3043                        .definition_revision(&task.workflow, &task.revision)?;
3044                    step.validate_child_binding(&definition, &child, &task.input)?;
3045                }
3046            }
3047            Ok(decision)
3048        })) {
3049            Ok(Ok(decision)) => decision,
3050            Ok(Err(error)) => BspDecision::Failed {
3051                category: FailureCategory::Evaluation,
3052                message: match error {
3053                    AppError::InvalidInput(message) => message,
3054                    _ => "workflow evaluation failed".into(),
3055                },
3056                observation: Some(loaded.record.checkpoint.proposed_position(
3057                    &definition,
3058                    &activation.node,
3059                    &activation.lineage,
3060                    &staged,
3061                )?),
3062            },
3063            Err(_) => BspDecision::Failed {
3064                category: FailureCategory::Execution,
3065                message: "workflow callback panicked".into(),
3066                observation: Some(loaded.record.checkpoint.proposed_position(
3067                    &definition,
3068                    &activation.node,
3069                    &activation.lineage,
3070                    &staged,
3071                )?),
3072            },
3073        };
3074        match decision {
3075            BspDecision::Stage(mut activations) => {
3076                staged.append(&mut activations);
3077                match remaining.next() {
3078                    Some(activation) => {
3079                        self.begin_activation(loaded, activation, remaining, staged, ordinary)
3080                    }
3081                    None => self.publish_decision(loaded, BspDecision::Stage(staged), ordinary),
3082                }
3083            }
3084            decision => self.publish(
3085                SessionPublication::Decision {
3086                    loaded,
3087                    decision,
3088                    activation_deadline: Some(deadline_millis),
3089                    context: Some(context),
3090                    staged,
3091                    remaining,
3092                },
3093                ordinary,
3094            ),
3095        }
3096    }
3097
3098    fn construct_source(
3099        &mut self,
3100        loaded: &LoadedRun,
3101        activation: FrontierRecord,
3102        context: Arc<ActionContext>,
3103        deadline: tokio::time::Instant,
3104    ) -> Result<(), AppError> {
3105        if tokio::time::Instant::now() >= deadline {
3106            self.runtime.retain_source_failure(
3107                &context,
3108                &activation.lineage,
3109                FailureCategory::Timeout,
3110            );
3111            self.sources.push(CallerSource {
3112                activation,
3113                context,
3114                deadline,
3115                state: CallerSourceState::Failed(FailureCategory::Timeout),
3116            });
3117            return Ok(());
3118        }
3119        let definition = self.runtime.app.definition_for(&loaded.record)?;
3120        let Some(Step::Action {
3121            action, revision, ..
3122        }) = definition.node(&activation.node).map(|node| node.step())
3123        else {
3124            return Err(AppError::Integration(
3125                "action decision has no exact declaration".into(),
3126            ));
3127        };
3128        let binding = self.runtime.app.action(action, revision)?;
3129        let state = WorkflowState {
3130            values: loaded.record.checkpoint.state.clone(),
3131        };
3132        let constructed = catch_unwind(AssertUnwindSafe(|| {
3133            binding.declaration().validate_reads(&state)?;
3134            let inputs = binding
3135                .declaration()
3136                .resolve_inputs(
3137                    &self.runtime.cell.execution_options()?.inputs,
3138                    self.runtime.cell.execution_mode()?,
3139                )?
3140                .into_values()?;
3141            Ok::<_, AppError>(binding.start(ActionRequest {
3142                state,
3143                inputs,
3144                context: context.as_ref().clone(),
3145            }))
3146        }));
3147        let state = match constructed {
3148            Ok(Ok(source)) => CallerSourceState::Polling(Box::pin(source)),
3149            Ok(Err(error)) => return Err(error),
3150            Err(_) => CallerSourceState::Failed(FailureCategory::Execution),
3151        };
3152        if let CallerSourceState::Failed(category) = &state {
3153            self.runtime
3154                .retain_source_failure(&context, &activation.lineage, *category);
3155        }
3156        self.sources.push(CallerSource {
3157            activation,
3158            context,
3159            deadline,
3160            state,
3161        });
3162        self.expire_sources(None);
3163        Ok(())
3164    }
3165
3166    fn halt_after(&mut self, loaded: &LoadedRun) -> Result<bool, AppError> {
3167        let outcomes = &loaded.record.checkpoint.outcomes;
3168        let Some(last) = outcomes.last() else {
3169            return Ok(false);
3170        };
3171        let options = self.runtime.cell.execution_options()?;
3172        let selected = outcomes
3173            .iter()
3174            .find(|outcome| options.halt_after.iter().any(|node| node == &outcome.node));
3175        let outcome = match selected {
3176            Some(outcome) => outcome,
3177            None if self.runtime.cell.halt_after(&last.node)? => last,
3178            None => return Ok(false),
3179        };
3180        self.publish_transition(
3181            LoadedRun {
3182                record: loaded.record.clone(),
3183                revision: loaded.revision,
3184            },
3185            ControlTransition::Halt {
3186                node: outcome.node.clone(),
3187                result: Some(outcome.result.clone()),
3188            },
3189        )?;
3190        Ok(true)
3191    }
3192
3193    fn publish_item(&mut self, index: usize, event: ActionEvent) -> Result<(), AppError> {
3194        let SessionState::Sources { loaded } = &self.state else {
3195            return Err(AppError::Integration(
3196                "stream item has no source owner".into(),
3197            ));
3198        };
3199        let source = &self.sources[index];
3200        let context = Arc::clone(&source.context);
3201        let deadline = source.deadline;
3202        let generation = self.generation;
3203        if let Some(generation) = generation {
3204            self.runtime.cell.begin_publication(generation)?;
3205        }
3206        let runtime = self.runtime.clone();
3207        let loaded = LoadedRun {
3208            record: loaded.record.clone(),
3209            revision: loaded.revision,
3210        };
3211        let activation = source.activation.clone();
3212        let (sender, receipt) = oneshot::channel();
3213        let kind = generation.map_or(RunTaskKind::Operation, |generation| {
3214            RunTaskKind::Foreground { generation }
3215        });
3216        if !self
3217            .runtime
3218            .app
3219            .spawn_run(self.runtime.key.clone(), kind, None, async move {
3220                // The admitted observer owns its failure even if the caller
3221                // drops the source or never polls this receipt again.
3222                let mut result = match runtime
3223                    .observe_source_event(&context, event, deadline)
3224                    .await
3225                {
3226                    Ok(result) => Ok(result),
3227                    Err(failure) => Err(runtime
3228                        .fail_hook(&loaded, Some((&activation, &context)), failure, Vec::new())
3229                        .await),
3230                };
3231                if let Some(generation) = generation
3232                    && runtime.cell.end_publication(generation)
3233                    && let Err(error) = runtime.finish_execution(generation, None).await
3234                    && result.is_ok()
3235                {
3236                    result = Err(error);
3237                }
3238                let _ = sender.send(result);
3239            })
3240        {
3241            if let Some(generation) = generation {
3242                self.runtime.cell.end_publication(generation);
3243            }
3244            return Err(AppError::Closed(
3245                "stream observation admission closed".into(),
3246            ));
3247        }
3248        let SessionState::Sources { loaded } =
3249            std::mem::replace(&mut self.state, SessionState::Finished)
3250        else {
3251            unreachable!()
3252        };
3253        self.state = SessionState::ObservingItem {
3254            loaded,
3255            index,
3256            receipt,
3257        };
3258        Ok(())
3259    }
3260
3261    fn publish_failure(
3262        &mut self,
3263        loaded: LoadedRun,
3264        activation: &FrontierRecord,
3265        context: &ActionContext,
3266        category: FailureCategory,
3267    ) -> Result<(), AppError> {
3268        let definition = self.runtime.app.definition_for(&loaded.record)?;
3269        let position = ActionPosition {
3270            node: context.node.clone(),
3271            ordinal: context.action_ordinal,
3272        };
3273        let (position, category, message) = match loaded.record.checkpoint.preceding_failure(
3274            &definition,
3275            &position,
3276            &activation.lineage,
3277        )? {
3278            Some((position, category, message)) => (position, category, message.to_owned()),
3279            None => (position, category, "caller action execution failed".into()),
3280        };
3281        self.publish_decision(
3282            loaded,
3283            BspDecision::Failed {
3284                category,
3285                message,
3286                observation: Some(position),
3287            },
3288            false,
3289        )
3290    }
3291
3292    fn publish_decision(
3293        &mut self,
3294        loaded: LoadedRun,
3295        decision: BspDecision,
3296        ordinary: bool,
3297    ) -> Result<(), AppError> {
3298        self.publish(
3299            SessionPublication::Decision {
3300                loaded,
3301                decision,
3302                activation_deadline: None,
3303                context: None,
3304                staged: Vec::new(),
3305                remaining: Vec::new().into_iter(),
3306            },
3307            ordinary,
3308        )
3309    }
3310
3311    fn publish_transition(
3312        &mut self,
3313        loaded: LoadedRun,
3314        transition: ControlTransition,
3315    ) -> Result<(), AppError> {
3316        self.publish(SessionPublication::Transition { loaded, transition }, false)
3317    }
3318
3319    fn publish(
3320        &mut self,
3321        mut publication: SessionPublication,
3322        ordinary: bool,
3323    ) -> Result<(), AppError> {
3324        let mut staged_prefix = Vec::new();
3325        let mut admitted_inputs = Vec::new();
3326        if let SessionPublication::Decision {
3327            loaded,
3328            decision: BspDecision::AdmitChildren { sequence, children },
3329            ..
3330        } = &publication
3331            && let Some(group) = self
3332                .children
3333                .iter()
3334                .find(|group| group.sequence == *sequence)
3335        {
3336            let original = loaded
3337                .record
3338                .checkpoint
3339                .control(*sequence)
3340                .and_then(|control| match &control.state {
3341                    ControlRecordState::Child { admitted, .. } => Some(admitted.len()),
3342                    _ => None,
3343                })
3344                .ok_or_else(|| {
3345                    AppError::Integration("child admission evidence is unavailable".into())
3346                })?;
3347            if !group.inputs.is_empty() {
3348                for index in original..children.len() {
3349                    let values = group.inputs.get(index).ok_or_else(|| {
3350                        AppError::Integration("child runtime input membership changed".into())
3351                    })?;
3352                    // The original group retains this bounded map until its
3353                    // native admitted prefix is confirmed by the receipt.
3354                    admitted_inputs.push(values.clone());
3355                }
3356            }
3357        }
3358        let purpose = match &mut publication {
3359            SessionPublication::Decision {
3360                decision:
3361                    BspDecision::Descendants {
3362                        lineage,
3363                        runtime_inputs,
3364                        ..
3365                    },
3366                context,
3367                remaining,
3368                ..
3369            } => PublicationPurpose::Descendants {
3370                context: Arc::clone(context.as_ref().ok_or_else(|| {
3371                    AppError::Integration(
3372                        "child runtime inputs have no original action context".into(),
3373                    )
3374                })?),
3375                lineage: lineage.clone(),
3376                inputs: std::mem::take(runtime_inputs),
3377                remaining: std::mem::take(remaining),
3378                ordinary,
3379            },
3380            SessionPublication::Load { .. } => PublicationPurpose::Load,
3381            SessionPublication::Reload { .. } => PublicationPurpose::Reload,
3382            SessionPublication::ObserveChildren { .. } => PublicationPurpose::ObserveChildren,
3383            SessionPublication::VerifyChildren { .. } => PublicationPurpose::VerifyChildren,
3384            SessionPublication::FinishChild { .. } | SessionPublication::SettleStopping { .. } => {
3385                PublicationPurpose::Load
3386            }
3387            SessionPublication::Decision {
3388                decision: BspDecision::AdmitChildren { .. },
3389                ..
3390            } => PublicationPurpose::AdmitChildren,
3391            SessionPublication::BeginActivation {
3392                activation,
3393                context,
3394                remaining,
3395                staged,
3396                deadline,
3397                deadline_millis,
3398                ..
3399            } => {
3400                // The same receipt returns this prefix after successful hooks.
3401                // A rejected hook first preserves its native failure evidence.
3402                staged_prefix = std::mem::take(staged);
3403                PublicationPurpose::Start {
3404                    activation: activation.clone(),
3405                    context: Arc::clone(context),
3406                    remaining: std::mem::take(remaining),
3407                    deadline: *deadline,
3408                    deadline_millis: *deadline_millis,
3409                }
3410            }
3411            SessionPublication::Decision { remaining, .. } if !remaining.as_slice().is_empty() => {
3412                PublicationPurpose::Continue {
3413                    remaining: std::mem::take(remaining),
3414                    ordinary,
3415                }
3416            }
3417            SessionPublication::Decision {
3418                decision: BspDecision::Barrier { .. },
3419                ..
3420            } => PublicationPurpose::Commit { ordinary },
3421            SessionPublication::EndSource { .. } => PublicationPurpose::EndSource,
3422            _ => PublicationPurpose::Advance,
3423        };
3424        if let SessionPublication::Decision {
3425            decision: BspDecision::Agent(request),
3426            ..
3427        } = &mut publication
3428            && request.is_streaming()
3429        {
3430            let (sender, _) = self.native_partials.get_or_insert_with(|| mpsc::channel(1));
3431            request.observe_partials(sender.clone());
3432        }
3433        let generation = self.generation;
3434        if let Some(generation) = generation {
3435            self.runtime.cell.begin_publication(generation)?;
3436            if matches!(&publication, SessionPublication::Release) {
3437                self.runtime.cell.revoke_execution(generation);
3438            }
3439        }
3440        let child_observation = match &publication {
3441            SessionPublication::Reload { receipt, .. } => receipt
3442                .record()
3443                .checkpoint
3444                .controls
3445                .values()
3446                .filter(|control| {
3447                    matches!(control.state, ControlRecordState::Child { .. })
3448                        && !self.children.iter().any(|group| {
3449                            group.sequence == control.sequence
3450                                && matches!(
3451                                    group.state,
3452                                    ChildExecutionState::Worker(_)
3453                                        | ChildExecutionState::Receipt(_)
3454                                )
3455                        })
3456                })
3457                .min_by_key(|control| control.deadline_millis)
3458                .map(|control| control.sequence),
3459            _ => None,
3460        };
3461        let options = self.runtime.cell.execution_options()?;
3462        let parent_deadline = self.parent_deadline;
3463        let reconcile_read = generation.is_none()
3464            && self.runtime.cell.child_parent().is_some()
3465            && matches!(
3466                &publication,
3467                SessionPublication::Load { .. }
3468                    | SessionPublication::Reload { .. }
3469                    | SessionPublication::ObserveChildren { .. }
3470                    | SessionPublication::VerifyChildren { .. }
3471            );
3472        let execution_read = matches!(
3473            &publication,
3474            SessionPublication::Load { .. }
3475                | SessionPublication::Reload { .. }
3476                | SessionPublication::ObserveChildren { .. }
3477                | SessionPublication::FinishChild { .. }
3478                | SessionPublication::BeginActivation { .. }
3479        );
3480        let runtime = self.runtime.clone();
3481        let (sender, receipt) = oneshot::channel();
3482        let kind = generation.map_or(RunTaskKind::Operation, |generation| {
3483            RunTaskKind::Foreground { generation }
3484        });
3485        if !self
3486            .runtime
3487            .app
3488            .spawn_run(self.runtime.key.clone(), kind, self.admission.take(), async move {
3489                let inputs = &options.inputs;
3490                let observed = AssertUnwindSafe(async {
3491                    match publication {
3492                    SessionPublication::ObserveChildren { loaded, sequence } => async {
3493                        let (loaded, children) = runtime.observe_children(loaded, sequence).await?;
3494                        Ok(SessionReceipt::Children { loaded, sequence, children, verified: false })
3495                    }.await,
3496                    SessionPublication::VerifyChildren { parent, sequence, completions } => {
3497                        match runtime.verify_child_completions(&parent, sequence, completions).await {
3498                            Ok((loaded, children)) => {
3499                                let control = parent.checkpoint.control(sequence).ok_or_else(|| {
3500                                    AppError::Conflict("verified child control is unavailable".into())
3501                                })?;
3502                                let complete = matches!(&control.state,
3503                                    ControlRecordState::Child { inputs, .. } if children.len() == inputs.len());
3504                                if loaded.record.checkpoint.is_stopping() || loaded.record.checkpoint.is_terminal() || !complete {
3505                                    runtime.schedule_child_retention(control);
3506                                    Ok(SessionReceipt::Children { loaded, sequence, children, verified: true })
3507                                } else {
3508                                    // Keep final collection and Stage in this accepted
3509                                    // publication even when its local waiter is dropped.
3510                                    // A partial Pipeline remains caller-owned above.
3511                                    let transition = runtime.plan_child_completion(&loaded, control, children)?;
3512                                    runtime.publish_child_outcome(control, transition).await
3513                                }
3514                            },
3515                            Err(RuntimeOperationFailure::Application(error)) => runtime
3516                                .finish_child_admission(sequence, ChildAdmissionReceipt::Rejected(error))
3517                                .await
3518                                .map(|(loaded, children)| SessionReceipt::Children {
3519                                    loaded, sequence, children, verified: false,
3520                                }),
3521                            Err(failure) => Err(failure),
3522                        }
3523                    },
3524                    SessionPublication::Decision { loaded, decision: BspDecision::AdmitChildren { sequence, children }, .. } => {
3525                        let receipt = runtime.register_children(loaded, sequence, children, admitted_inputs).await;
3526                        runtime.finish_child_admission(sequence, receipt).await.map(|(loaded, children)| {
3527                            SessionReceipt::Children { loaded, sequence, children, verified: false }
3528                        })
3529                    },
3530                    SessionPublication::ChildOutcome { control, transition } => {
3531                        runtime.publish_child_outcome(&control, transition).await
3532                    },
3533                    SessionPublication::FinishChild { parent, sequence } => async {
3534                        let loaded = runtime.load_execution_observation(None, parent_deadline).await?
3535                            .ok_or_else(|| AppError::NotFound("admitted child is unavailable".into()))?;
3536                        let definition = runtime.app.definition_for(&parent)?;
3537                        loaded.record.validate_child(&parent, &definition)?;
3538                        if loaded.record.parent.as_ref().is_none_or(|relation| relation.control_sequence != sequence) {
3539                            return Err(AppError::Conflict("child completion group changed".into()).into());
3540                        }
3541                        if loaded.record.checkpoint.is_stopping()
3542                            || loaded.record.checkpoint.is_terminal()
3543                            || !loaded.record.checkpoint.controls.values().any(|control| {
3544                                matches!(control.state, ControlRecordState::Inputs { .. })
3545                            })
3546                        {
3547                            runtime.finish_execution_observation(&loaded.record, None, None, false).await?;
3548                            runtime.cell.lifecycle_observation(loaded.record.checkpoint.sequence, false)?;
3549                        }
3550                        Ok(SessionReceipt::Checkpoint { loaded, staged: Vec::new() })
3551                    }.await,
3552                    SessionPublication::SettleStopping { loaded } => async {
3553                        if let Some(loaded) = Box::pin(runtime.settle_stopping(loaded)).await {
3554                            let observed = runtime.finish_execution_observation(&loaded.record, None, None, false).await;
3555                            if runtime.cell.child_parent().is_none() {
3556                                runtime.maintain_retention(None).await;
3557                            }
3558                            observed?;
3559                        }
3560                        // Retention may have advanced the head revision after
3561                        // the terminal checkpoint. Observe that current authority
3562                        // in one finite pass after the native receipt returns.
3563                        let loaded = runtime.load_execution_observation(None, parent_deadline).await
3564                            .inspect_err(|failure| runtime.retain_execution_read_failure(failure))?
3565                            .ok_or_else(|| AppError::NotFound("workflow run does not exist".into()))?;
3566                        Ok(SessionReceipt::Checkpoint { loaded, staged: Vec::new() })
3567                    }.await,
3568                    SessionPublication::Load { terminal } => async {
3569                        let loaded = runtime.load_execution_observation(terminal, parent_deadline).await?
3570                            .ok_or_else(|| AppError::NotFound("workflow run does not exist".into()))?;
3571                        if !loaded.record.checkpoint.is_stopping()
3572                            && !loaded.record.checkpoint.is_terminal()
3573                            && loaded.record.checkpoint.controls.values().any(|control| {
3574                                matches!(control.state, ControlRecordState::Inputs { .. })
3575                            })
3576                            && let Err(failure) = runtime.begin_execution_observation().await
3577                        {
3578                            return Err(runtime.fail_hook(&loaded, None, failure, Vec::new()).await.into());
3579                        }
3580                        Ok(SessionReceipt::Checkpoint { loaded, staged: Vec::new() })
3581                    }.await,
3582                    SessionPublication::Reload { receipt, deadline } => {
3583                        let observed = tokio::select! {
3584                            biased;
3585                            _ = async { match deadline {
3586                                Some(deadline) => tokio::time::sleep_until(deadline).await,
3587                                None => std::future::pending().await,
3588                            }}, if child_observation.is_none() && parent_deadline.is_none() => Ok(None),
3589                            observed = async {
3590                                // Child authority retains its own sampled deadline;
3591                                // it cannot fall back to a stale receipt on timeout.
3592                                // The original worker retains its own exact child
3593                                // watchdog and final receipt after staging its result.
3594                                let observed = match child_observation {
3595                                    Some(sequence) => runtime.observe_child_parent(receipt.record(), sequence, false, None).await?,
3596                                    None => runtime.load_execution_observation(None, deadline.into_iter().chain(parent_deadline).min()).await?
3597                                        .ok_or_else(|| AppError::NotFound("workflow run does not exist".into()))?,
3598                                };
3599                                Ok::<_, RuntimeOperationFailure>(Some(observed))
3600                            } => observed,
3601                        };
3602                        observed.map(|observed| match observed {
3603                            Some(loaded) => receipt.with_loaded(loaded),
3604                            None => receipt,
3605                        })
3606                    },
3607                    SessionPublication::BeginActivation { loaded, activation, context, deadline, .. } => async {
3608                        if let Err(failure) = runtime.observe_activation_start(&loaded, &activation, &context, inputs, deadline).await {
3609                            return Err(runtime.fail_hook(&loaded, Some((&activation, &context)), failure, std::mem::take(&mut staged_prefix)).await.into());
3610                        }
3611                        let current = runtime.load_execution_observation(
3612                            None, parent_deadline.into_iter().chain(Some(deadline)).min(),
3613                        ).await?.ok_or_else(|| AppError::NotFound("workflow run does not exist".into()))?;
3614                        if !(current.record.checkpoint.is_terminal()
3615                            || current.record.checkpoint.is_stopping()
3616                            || current.record.same_run(&loaded.record)
3617                                && current.record.checkpoint.base_sequence == loaded.record.checkpoint.base_sequence
3618                                && current.record.checkpoint.frontier.iter().any(|candidate| candidate == &activation)
3619                                && current.record.checkpoint.control_for(&activation.node, &activation.lineage).is_none()
3620                                && !current.record.checkpoint.staged.iter().any(|staged| staged.node == activation.node && staged.lineage == activation.lineage))
3621                        {
3622                            return Err(AppError::Conflict("action pre-hook authority changed".into()).into());
3623                        }
3624                        Ok(SessionReceipt::Checkpoint { loaded: current, staged: std::mem::take(&mut staged_prefix) })
3625                    }.await,
3626                    publication => async {
3627                    match publication {
3628                        SessionPublication::Load { .. } | SessionPublication::Reload { .. }
3629                        | SessionPublication::ObserveChildren { .. } | SessionPublication::VerifyChildren { .. }
3630                        | SessionPublication::ChildOutcome { .. } | SessionPublication::FinishChild { .. } | SessionPublication::SettleStopping { .. }
3631                        | SessionPublication::BeginActivation { .. } => unreachable!(),
3632                        SessionPublication::Decision { loaded, decision, activation_deadline, context, staged, .. } => {
3633                            if matches!(decision, BspDecision::Inputs { .. })
3634                                && let Err(failure) = runtime.begin_execution_observation().await
3635                            {
3636                                return Err(runtime.fail_hook(&loaded, None, failure, staged).await);
3637                            }
3638                            runtime.finish_staged_spans(&loaded, &decision, context.as_deref()).await?;
3639                            runtime.apply_decision(loaded, decision, activation_deadline, staged).await
3640                        }
3641                        SessionPublication::Transition { loaded, transition } => {
3642                            let loaded = runtime.commit_for_recovery(loaded, transition).await
3643                                .map_err(RuntimeOperationFailure::into_error)?;
3644                            // The native checkpoint remains this publication's
3645                            // exact result. Only its App generation needs a
3646                            // finite post-receipt observation before returning.
3647                            runtime.observe_generation(LifecycleEvent::deadline(parent_deadline)).await
3648                                .map_err(|failure| {
3649                                    runtime.retain_execution_read_failure(&failure);
3650                                    failure.into_error()
3651                                })?;
3652                            Ok(loaded)
3653                        }
3654                        SessionPublication::Retry { loaded } => {
3655                            let loaded = runtime.readmit_failed(loaded).await?;
3656                            runtime.observe_generation(LifecycleEvent::deadline(parent_deadline)).await
3657                                .map_err(|failure| {
3658                                    runtime.retain_execution_read_failure(&failure);
3659                                    failure.into_error()
3660                                })?;
3661                            Ok(loaded)
3662                        }
3663                        SessionPublication::EndSource { loaded } => {
3664                            let loaded = runtime.end_source(loaded, generation).await?;
3665                            runtime.observe_generation(LifecycleEvent::deadline(parent_deadline)).await
3666                                .map_err(|failure| {
3667                                    runtime.retain_execution_read_failure(&failure);
3668                                    failure.into_error()
3669                                })?;
3670                            Ok(loaded)
3671                        }
3672                        SessionPublication::Release => {
3673                            if let Some(generation) = generation { runtime.finish_execution(generation, None).await?; }
3674                            runtime.load_execution_observation(None, parent_deadline).await
3675                                .map_err(|failure| {
3676                                    runtime.retain_execution_read_failure(&failure);
3677                                    failure.into_error()
3678                                })?
3679                                .ok_or_else(|| AppError::NotFound("workflow run does not exist".into()))
3680                        }
3681                    }
3682                    }.await.map(|loaded| SessionReceipt::Checkpoint { loaded, staged: std::mem::take(&mut staged_prefix) })
3683                        .map_err(RuntimeOperationFailure::from),
3684                    }
3685                }).catch_unwind().await;
3686                let mut result: Result<SessionReceipt, RuntimeOperationFailure> = match observed {
3687                    Ok(result) => result,
3688                    Err(payload) if reconcile_read => {
3689                        drop(payload);
3690                        match runtime.reconcile_child_panic(parent_deadline).await {
3691                            Ok(result) => result.map(|loaded| SessionReceipt::Checkpoint {
3692                                loaded, staged: Vec::new(),
3693                            }).map_err(RuntimeOperationFailure::from),
3694                            Err(_) => {
3695                                runtime.app.tasks_failed(FinalResult::Integration(
3696                                    "child read panic reconciliation receipt closed".into(),
3697                                ));
3698                                Err(AppError::Integration("child read panic reconciliation receipt closed".into()).into())
3699                            }
3700                        }
3701                    }
3702                    Err(payload) => std::panic::resume_unwind(payload),
3703                };
3704                if execution_read
3705                    && let Err(RuntimeOperationFailure::Storage(failure)) = &result
3706                {
3707                    runtime.retain_execution_read_failure(failure);
3708                }
3709                if let Some(generation) = generation
3710                    && runtime.cell.end_publication(generation)
3711                {
3712                    let loaded = match &result {
3713                        Ok(SessionReceipt::Checkpoint { loaded, .. })
3714                            if (loaded.record.checkpoint.is_paused() || loaded.record.checkpoint.is_stopping())
3715                                && loaded.record.checkpoint.controls.is_empty()
3716                                && runtime.cell.lifecycle_generation().is_none()
3717                                && !runtime.app.has_child_work(&runtime.key) =>
3718                        {
3719                            // The publication receipt remains available to a
3720                            // consuming close waiter while cleanup borrows its
3721                            // already-settled native authority.
3722                            Some(LoadedRun { record: loaded.record.clone(), revision: loaded.revision })
3723                        }
3724                        _ => None,
3725                    };
3726                    if let Err(error) = runtime.finish_execution(generation, loaded).await
3727                        && result.is_ok()
3728                    {
3729                        result = Err(error.into());
3730                    }
3731                }
3732                let _ = sender.send(result);
3733            })
3734        {
3735            if let Some(generation) = generation {
3736                self.runtime.cell.end_publication(generation);
3737            }
3738            return Err(AppError::Closed(
3739                "run publication admission is closed".into(),
3740            ));
3741        }
3742        self.state = SessionState::Publishing { receipt, purpose };
3743        Ok(())
3744    }
3745
3746    fn poll_publication(
3747        &mut self,
3748        cx: &mut Context<'_>,
3749        closing: bool,
3750    ) -> Poll<Result<Option<ExecutionEvent>, AppError>> {
3751        if !matches!(&self.state, SessionState::Publishing { .. }) {
3752            return Poll::Ready(Ok(None));
3753        }
3754        self.expire_sources(None);
3755        let deadline = self.source_deadline(None);
3756        if self.poll_timer(cx, deadline) {
3757            self.expire_sources(None);
3758        }
3759        let commit = matches!(
3760            &self.state,
3761            SessionState::Publishing {
3762                purpose: PublicationPurpose::Commit { .. },
3763                ..
3764            }
3765        );
3766        if commit && let Some(event) = self.poll_native_partial(cx) {
3767            return Poll::Ready(Ok(Some(event)));
3768        }
3769        let SessionState::Publishing { receipt, .. } = &mut self.state else {
3770            unreachable!()
3771        };
3772        let observed = match Pin::new(receipt).poll(cx) {
3773            Poll::Ready(observed) => observed
3774                .map_err(|_| {
3775                    RuntimeOperationFailure::from(AppError::Closed(
3776                        "run publication observation closed".into(),
3777                    ))
3778                })
3779                .and_then(|observed| observed),
3780            Poll::Pending => {
3781                if let Some(event) = self.poll_native_partial(cx) {
3782                    return Poll::Ready(Ok(Some(event)));
3783                }
3784                return Poll::Pending;
3785            }
3786        };
3787        let SessionState::Publishing { purpose, .. } =
3788            std::mem::replace(&mut self.state, SessionState::Finished)
3789        else {
3790            unreachable!()
3791        };
3792        let observed = match observed {
3793            Ok(observed) => observed,
3794            Err(failure) => {
3795                if self.generation.is_none() {
3796                    if matches!(
3797                        purpose,
3798                        PublicationPurpose::Load
3799                            | PublicationPurpose::Reload
3800                            | PublicationPurpose::ObserveChildren
3801                            | PublicationPurpose::VerifyChildren
3802                    ) {
3803                        if self.runtime.schedule_operation_failure(&failure) {
3804                            self.observed = None;
3805                        }
3806                    } else if matches!(purpose, PublicationPurpose::AdmitChildren) {
3807                        // The original admission result already ran its exact
3808                        // retry/failure policy. Do not spend its budget twice.
3809                        self.observed = None;
3810                    }
3811                }
3812                return Poll::Ready(Err(failure.into_error()));
3813            }
3814        };
3815        let (loaded, staged) = match observed {
3816            SessionReceipt::Checkpoint { loaded, staged } => (loaded, staged),
3817            SessionReceipt::Children {
3818                loaded,
3819                sequence,
3820                children,
3821                verified,
3822            } => {
3823                if closing {
3824                    self.state = SessionState::Loaded(loaded);
3825                } else if matches!(purpose, PublicationPurpose::AdmitChildren)
3826                    && self.poll_notification(cx)?
3827                {
3828                    let deadline = loaded
3829                        .record
3830                        .checkpoint
3831                        .control(sequence)
3832                        .map(|control| {
3833                            WorkflowRunRuntime::<CS, A, X, C, T, H, E>::remaining(
3834                                control.deadline_millis,
3835                            )
3836                        })
3837                        .transpose()?
3838                        .and_then(|remaining| tokio::time::Instant::now().checked_add(remaining));
3839                    self.publish(
3840                        SessionPublication::Reload {
3841                            receipt: SessionReceipt::Children {
3842                                loaded,
3843                                sequence,
3844                                children,
3845                                verified,
3846                            },
3847                            deadline,
3848                        },
3849                        false,
3850                    )?;
3851                } else {
3852                    self.plan_child_group(loaded, sequence, children, verified)?;
3853                }
3854                return Poll::Ready(Ok(None));
3855            }
3856        };
3857        if matches!(purpose, PublicationPurpose::Commit { .. })
3858            && let Err(error) = self
3859                .runtime
3860                .cell
3861                .lifecycle_observation(loaded.record.checkpoint.sequence, false)
3862        {
3863            // A failed post-hook may already have paused this exact native
3864            // barrier. Keep its receipt for owning cleanup before projecting
3865            // the original retained lifecycle failure to the caller.
3866            self.state = SessionState::Loaded(loaded);
3867            return Poll::Ready(Err(error));
3868        }
3869        if closing
3870            || matches!(
3871                purpose,
3872                PublicationPurpose::Load | PublicationPurpose::Reload
3873            )
3874        {
3875            self.state = SessionState::Loaded(loaded);
3876            return Poll::Ready(Ok(None));
3877        }
3878        self.state = SessionState::Ready;
3879        if let PublicationPurpose::Start {
3880            activation,
3881            context,
3882            remaining,
3883            deadline,
3884            deadline_millis,
3885        } = purpose
3886        {
3887            self.state = if loaded.record.checkpoint.is_terminal()
3888                || loaded.record.checkpoint.is_stopping()
3889            {
3890                self.observed = Some(loaded.record);
3891                SessionState::Finished
3892            } else {
3893                SessionState::ActivationReady {
3894                    loaded,
3895                    activation,
3896                    context,
3897                    remaining,
3898                    staged,
3899                    deadline,
3900                    deadline_millis,
3901                }
3902            };
3903            return Poll::Ready(Ok(None));
3904        }
3905        let purpose = match purpose {
3906            PublicationPurpose::Descendants {
3907                context,
3908                lineage,
3909                inputs,
3910                remaining,
3911                ordinary,
3912            } => {
3913                if !loaded.record.checkpoint.is_terminal()
3914                    && !loaded.record.checkpoint.is_stopping()
3915                {
3916                    let control = loaded
3917                        .record
3918                        .checkpoint
3919                        .control_for(&context.node, &lineage)
3920                        .ok_or_else(|| {
3921                            AppError::Integration(
3922                                "original child input control is unavailable".into(),
3923                            )
3924                        })?;
3925                    let deadline = tokio::time::Instant::now()
3926                        .checked_add(WorkflowRunRuntime::<CS, A, X, C, T, H, E>::remaining(
3927                            control.deadline_millis,
3928                        )?)
3929                        .ok_or_else(|| {
3930                            AppError::Timeout("child deadline is outside the clock range".into())
3931                        })?;
3932                    self.children.push(ChildExecution {
3933                        sequence: control.sequence,
3934                        parent: None,
3935                        context,
3936                        lineage,
3937                        deadline,
3938                        state: ChildExecutionState::Awaiting,
3939                        inputs,
3940                    });
3941                }
3942                if remaining.as_slice().is_empty() {
3943                    PublicationPurpose::Advance
3944                } else {
3945                    PublicationPurpose::Continue {
3946                        remaining,
3947                        ordinary,
3948                    }
3949                }
3950            }
3951            purpose => purpose,
3952        };
3953        if let PublicationPurpose::Continue {
3954            mut remaining,
3955            ordinary,
3956        } = purpose
3957        {
3958            if loaded.record.checkpoint.is_terminal() || loaded.record.checkpoint.is_stopping() {
3959                self.observed = Some(loaded.record);
3960                self.state = SessionState::Finished;
3961            } else {
3962                let activation = remaining.next().ok_or_else(|| {
3963                    AppError::Integration("native activation continuation is empty".into())
3964                })?;
3965                self.begin_activation(loaded, activation, remaining, staged, ordinary)?;
3966            }
3967            return Poll::Ready(Ok(None));
3968        }
3969        if let RunAdmission::Halted { node, result } = &loaded.record.checkpoint.admission {
3970            let event = ExecutionEvent::Halted {
3971                node: node.clone(),
3972                result: result.clone(),
3973            };
3974            self.observed = Some(loaded.record);
3975            self.state = SessionState::Finished;
3976            return Poll::Ready(Ok(Some(event)));
3977        }
3978        if let PublicationPurpose::Commit { ordinary } = purpose {
3979            self.observed = Some(loaded.record.clone());
3980            self.runtime
3981                .cell
3982                .record_execution_boundary(loaded.record.checkpoint.outcomes.len(), ordinary)?;
3983            return Poll::Ready(Ok(Some(ExecutionEvent::Committed {
3984                checkpoint: loaded.record.checkpoint.sequence,
3985                outcomes: loaded.record.checkpoint.outcomes,
3986            })));
3987        }
3988        if matches!(purpose, PublicationPurpose::EndSource) {
3989            self.observed = Some(loaded.record);
3990            self.state = SessionState::Eof;
3991        } else {
3992            self.observed = None;
3993            self.state = SessionState::Loaded(loaded);
3994        }
3995        Poll::Ready(Ok(None))
3996    }
3997}
3998
3999impl<
4000    CS: StateStoreBackend,
4001    A: AgentBackend,
4002    X: ActionBackend,
4003    C: ConditionBackend,
4004    T: StateTyping,
4005    H: LifecycleHooks,
4006    E: ChildExecutor,
4007> Drop for Execution<'_, CS, A, X, C, T, H, E>
4008{
4009    fn drop(&mut self) {
4010        if let Some(generation) = self.generation {
4011            self.runtime.cell.clear_execution_options(Some(generation));
4012        }
4013        let pending_observation = matches!(
4014            &self.state,
4015            SessionState::Publishing { .. } | SessionState::ObservingItem { .. }
4016        );
4017        self.stop_sources();
4018        self.stop_children();
4019        if self.generation.is_none()
4020            && !pending_observation
4021            && matches!(&self.state, SessionState::Sources { .. })
4022        {
4023            let failed = self
4024                .sources
4025                .iter()
4026                .enumerate()
4027                .filter_map(|(index, source)| {
4028                    let category = self
4029                        .prefetched
4030                        .as_ref()
4031                        .filter(|(buffered, _)| *buffered == index)
4032                        .and_then(|(_, event)| event.as_ref().err().copied())
4033                        .or(match source.state {
4034                            CallerSourceState::Failed(category) => Some(category),
4035                            _ => None,
4036                        });
4037                    category.map(|category| (index, category))
4038                })
4039                .min_by(|left, right| {
4040                    let left = &self.sources[left.0].activation;
4041                    let right = &self.sources[right.0].activation;
4042                    (&left.node, &left.lineage).cmp(&(&right.node, &right.lineage))
4043                });
4044            if let Some((index, category)) = failed {
4045                let _ = self.stage_source_failure(index, category, false);
4046            }
4047        }
4048        let source_failed = self
4049            .runtime
4050            .cell
4051            .lifecycle_generation()
4052            .is_some_and(|generation| self.runtime.cell.source_failure(generation).is_some());
4053        let record = match &self.state {
4054            SessionState::Loaded(loaded)
4055            | SessionState::Sources { loaded }
4056            | SessionState::ObservingItem { loaded, .. }
4057            | SessionState::ActivationReady { loaded, .. } => Some(&loaded.record),
4058            _ => self.observed.as_ref(),
4059        };
4060        let child_pending = self.runtime.app.has_child_work(&self.runtime.key);
4061        let local_only = !child_pending
4062            && !pending_observation
4063            && !matches!(&self.state, SessionState::Publishing { .. })
4064            && !source_failed
4065            && !self.sources.is_empty()
4066            && record.is_some_and(|record| {
4067                record.checkpoint.staged.is_empty() && record.checkpoint.controls.is_empty()
4068            });
4069        if !child_pending
4070            && !pending_observation
4071            && !matches!(&self.state, SessionState::Publishing { .. })
4072            && !source_failed
4073            && let Some(record) = record
4074            && !self.retains_execution_lifecycle(record)
4075            && (self.generation.is_some()
4076                || record.checkpoint.controls.is_empty()
4077                || record.checkpoint.is_public_outcome())
4078            && !record
4079                .checkpoint
4080                .staged
4081                .iter()
4082                .any(|activation| matches!(activation.outcome, StagedOutcomeRecord::Failed { .. }))
4083        {
4084            let _ = self.runtime.finish_execution_observation_sync(record);
4085        }
4086        if let Some(generation) = self.generation.take()
4087            && self.runtime.cell.drop_execution(generation)
4088        {
4089            if local_only {
4090                if self.runtime.cell.release_execution(generation) {
4091                    self.runtime.schedule(ScheduleCause::Reconcile);
4092                }
4093                return;
4094            }
4095            let loaded = if !child_pending
4096                && !pending_observation
4097                && !source_failed
4098                && self.runtime.cell.lifecycle_generation().is_none()
4099                && matches!(&self.state, SessionState::Loaded(loaded)
4100                    if (loaded.record.checkpoint.is_paused() || loaded.record.checkpoint.is_stopping())
4101                        && loaded.record.checkpoint.controls.is_empty())
4102            {
4103                let SessionState::Loaded(loaded) =
4104                    std::mem::replace(&mut self.state, SessionState::Finished)
4105                else {
4106                    unreachable!()
4107                };
4108                Some(loaded)
4109            } else {
4110                None
4111            };
4112            let runtime = self.runtime.clone();
4113            self.runtime.app.spawn_run(
4114                self.runtime.key.clone(),
4115                RunTaskKind::Foreground { generation },
4116                None,
4117                async move {
4118                    let _ = runtime.finish_execution(generation, loaded).await;
4119                },
4120            );
4121        }
4122    }
4123}
4124
4125impl<
4126    CS: StateStoreBackend,
4127    A: AgentBackend,
4128    X: ActionBackend,
4129    C: ConditionBackend,
4130    T: StateTyping,
4131    H: LifecycleHooks,
4132    E: ChildExecutor,
4133> WorkflowRunRuntime<CS, A, X, C, T, H, E>
4134{
4135    pub(crate) async fn execute_child_group(
4136        &self,
4137        sequence: u64,
4138        generation: Option<u64>,
4139        parent: Arc<RunRecord>,
4140        children: Vec<RunRecord>,
4141    ) -> Result<SessionReceipt, RuntimeOperationFailure> {
4142        if generation.is_some_and(|generation| !self.cell.execution_is_live(generation)) {
4143            return Err(
4144                AppError::Closed("child executor admission generation changed".into()).into(),
4145            );
4146        }
4147        let mut outcome_published = false;
4148        let result = async {
4149            let control = parent.checkpoint.control(sequence).ok_or_else(|| AppError::Conflict("child control is unavailable".into()))?;
4150            let remaining = Self::remaining(control.deadline_millis)?;
4151            if remaining.is_zero() {
4152                // Queue time belongs to the original activation. Publish its
4153                // fenced Timeout without constructing E or starting a read
4154                // against an already exhausted observation budget.
4155                outcome_published = true;
4156                let loaded = self.commit_scheduled_transition(control, ControlTransition::Failed {
4157                    category: FailureCategory::Timeout, message: "child workflow deadline expired".into(), observation: None,
4158                }).await.ok_or_else(|| AppError::Closed(
4159                    "child timeout publication requires reconciliation".into(),
4160                ))?;
4161                return Ok(SessionReceipt::Checkpoint { loaded, staged: Vec::new() });
4162            }
4163            let deadline = tokio::time::Instant::now().checked_add(remaining)
4164                .ok_or_else(|| AppError::Timeout("child deadline is outside the clock range".into()))?;
4165            // Retain the original ordered child records. Only parent authority
4166            // needs a fresh observation after waiting in the worker queue.
4167            let current = self.load_execution_observation(None, Some(deadline)).await?
4168                .ok_or_else(|| AppError::NotFound("child parent is unavailable".into()))?;
4169            if current.record.checkpoint.is_stopping() || current.record.checkpoint.is_terminal() {
4170                return Ok(SessionReceipt::Checkpoint { loaded: current, staged: Vec::new() });
4171            }
4172            if !current.record.owns_child_activation(&parent, sequence) {
4173                return Err(AppError::Conflict("child parent reservation changed".into()).into());
4174            }
4175            drop(current);
4176            let context = self.child_context(&parent, sequence)?;
4177            let mut group = ChildExecution {
4178                sequence, parent: Some(Arc::clone(&parent)), context,
4179                lineage: control.lineage.clone(), deadline,
4180                state: ChildExecutionState::Prepared(Vec::new()), inputs: Vec::new(),
4181            };
4182            group.start_local(self, &self.app, ExecutionLane::Worker, children)?;
4183            let context = Arc::clone(&group.context);
4184            let stopped = {
4185                let mut executor = Box::pin(poll_fn(|cx| group.poll_ready(cx, self)));
4186                let mut stopping = Box::pin(self.observe_child_parent(&parent, sequence, true, None));
4187                tokio::select! {
4188                    biased;
4189                    result = &mut stopping => {
4190                        let loaded = result?;
4191                        let loaded = if !loaded.record.checkpoint.is_stopping() {
4192                            self.retain_source_failure(&context, &control.lineage, FailureCategory::Timeout);
4193                            outcome_published = true;
4194                            let mut failure = Box::pin(self.commit_scheduled_transition(control, ControlTransition::Failed {
4195                                category: FailureCategory::Timeout, message: "child workflow deadline expired".into(), observation: None,
4196                            }));
4197                            let committed = tokio::select! {
4198                                _ = &mut executor => failure.await,
4199                                result = &mut failure => result,
4200                            };
4201                            committed.ok_or_else(|| AppError::Closed(
4202                                "child timeout publication requires reconciliation".into(),
4203                            ))?
4204                        } else { loaded };
4205                        Some(loaded)
4206                    }
4207                    _ = &mut executor => None,
4208                }
4209            };
4210            if let Some(loaded) = stopped {
4211                // The original parent observer or native timeout publication
4212                // already supplied authority. E destruction does not require
4213                // another head read before returning that exact receipt.
4214                group.stop(self);
4215                return Ok(SessionReceipt::Checkpoint { loaded, staged: Vec::new() });
4216            }
4217            let ChildExecutionState::Complete(completed) = group.state else {
4218                return Err(AppError::Integration("child worker completed without its local executor".into()).into());
4219            };
4220            let completions = match completed {
4221                Ok(completions) => completions,
4222                Err(category) => {
4223                    outcome_published = true;
4224                    let loaded = self.commit_scheduled_transition(control, ControlTransition::Failed {
4225                        category, message: "child execution failed".into(), observation: None,
4226                    }).await.ok_or_else(|| AppError::Closed(
4227                        "child failure publication requires reconciliation".into(),
4228                    ))?;
4229                    return Ok(SessionReceipt::Checkpoint { loaded, staged: Vec::new() });
4230                }
4231            };
4232            let (loaded, children) = self.verify_child_completions(&parent, sequence, completions).await?;
4233            if loaded.record.checkpoint.is_stopping() || loaded.record.checkpoint.is_terminal() {
4234                self.schedule_child_retention(control);
4235                return Ok(SessionReceipt::Checkpoint { loaded, staged: Vec::new() });
4236            }
4237            let transition = self.plan_child_completion(&loaded, control, children)?;
4238            // The original worker publishes only its fenced stage before
4239            // sending. Dropping either a pending or queued caller receipt
4240            // cannot lose completed children or permit another E invocation.
4241            outcome_published = true;
4242            self.publish_child_outcome(control, transition).await
4243        }.await;
4244        match result {
4245            Ok(receipt) => Ok(receipt),
4246            Err(RuntimeOperationFailure::Storage(failure)) => {
4247                self.retain_execution_read_failure(&failure);
4248                self.schedule_after_storage_failure(&failure);
4249                Err(failure.into())
4250            }
4251            Err(RuntimeOperationFailure::Application(error)) => {
4252                if outcome_published {
4253                    return Err(error.into());
4254                }
4255                let failure = self
4256                    .finish_child_admission(sequence, ChildAdmissionReceipt::Rejected(error))
4257                    .await;
4258                let loaded = self
4259                    .load_execution_observation(None, None)
4260                    .await
4261                    .inspect_err(|failure| self.retain_execution_read_failure(failure))?
4262                    .ok_or_else(|| AppError::NotFound("child parent is unavailable".into()))?;
4263                if loaded.record.checkpoint.is_stopping() || loaded.record.checkpoint.is_terminal()
4264                {
4265                    Ok(SessionReceipt::Checkpoint {
4266                        loaded,
4267                        staged: Vec::new(),
4268                    })
4269                } else {
4270                    match failure {
4271                        Err(failure) => Err(failure),
4272                        Ok(_) => Err(AppError::Integration(
4273                            "child failure policy returned admission success".into(),
4274                        )
4275                        .into()),
4276                    }
4277                }
4278            }
4279        }
4280    }
4281
4282    fn schedule_child_retention(&self, control: &ControlRecord) {
4283        let ControlRecordState::Child { admitted, .. } = &control.state else {
4284            return;
4285        };
4286        for member in admitted {
4287            match Self::attach(Arc::clone(&self.app), member.key.clone()) {
4288                Ok(child) => {
4289                    child.schedule(ScheduleCause::Reconcile);
4290                }
4291                Err(error) => self.app.tasks_failed(FinalResult::from(&error)),
4292            }
4293        }
4294    }
4295
4296    fn plan_child_completion(
4297        &self,
4298        loaded: &LoadedRun,
4299        control: &ControlRecord,
4300        children: Vec<RunRecord>,
4301    ) -> Result<ControlTransition, AppError> {
4302        let planned = catch_unwind(AssertUnwindSafe(|| {
4303            self.plan_children(loaded, control.sequence, children, true)
4304        }))
4305        .map_err(|_| AppError::Integration("child planning callback panicked".into()))
4306        .and_then(|result| result)
4307        .and_then(|decision| match decision {
4308            BspDecision::Stage(activations) => Ok(ControlTransition::Stage { activations }),
4309            BspDecision::Failed {
4310                category,
4311                message,
4312                observation,
4313            } => Ok(ControlTransition::Failed {
4314                category,
4315                message,
4316                observation,
4317            }),
4318            _ => Err(AppError::Integration(
4319                "completed child group has no final outcome".into(),
4320            )),
4321        });
4322        if !matches!(&planned, Ok(ControlTransition::Stage { activations })
4323            if activations.iter().all(|activation| matches!(activation.outcome, StagedOutcomeRecord::Complete { .. })))
4324        {
4325            // Failed children, output validation, expired deadlines and planning
4326            // errors keep their existing independent terminal cleanup.
4327            self.schedule_child_retention(control);
4328        }
4329        planned
4330    }
4331
4332    async fn publish_child_outcome(
4333        &self,
4334        control: &ControlRecord,
4335        transition: ControlTransition,
4336    ) -> Result<SessionReceipt, RuntimeOperationFailure> {
4337        // Retain the verified result under its original Child control while
4338        // unrelated sibling resolutions advance the parent revision. Both
4339        // local and worker execution keep this one publication receipt.
4340        let completed = matches!(&transition, ControlTransition::Stage { activations }
4341            if activations.iter().all(|activation| matches!(activation.outcome, StagedOutcomeRecord::Complete { .. })));
4342        if let Some(loaded) = self.commit_scheduled_transition(control, transition).await {
4343            // The native commit owner already retained Applied authority or
4344            // reconciled Unknown. Return that exact receipt and revision.
4345            return Ok(SessionReceipt::Checkpoint {
4346                loaded,
4347                staged: Vec::new(),
4348            });
4349        }
4350        if completed {
4351            // No applied Stage receipt owns the batch. The original exact
4352            // terminal children retain independent cleanup, including a stop
4353            // that won before this publication or unresolved native authority.
4354            self.schedule_child_retention(control);
4355        }
4356        let loaded = match tokio::time::timeout(
4357            Self::remaining(control.deadline_millis)?,
4358            self.load_for_recovery(),
4359        )
4360        .await
4361        {
4362            Ok(result) => result
4363                .map_err(RuntimeOperationFailure::from)
4364                .and_then(|loaded| {
4365                    loaded.ok_or_else(|| {
4366                        AppError::NotFound("child parent is unavailable".into()).into()
4367                    })
4368                }),
4369            Err(_) => Err(AppError::Timeout("child outcome observation timed out".into()).into()),
4370        };
4371        if !loaded.as_ref().is_ok_and(|loaded| {
4372            loaded.record.checkpoint.is_stopping() || loaded.record.checkpoint.is_terminal()
4373        }) {
4374            self.app.tasks_failed(FinalResult::Integration(
4375                "child outcome publication authority is unresolved".into(),
4376            ));
4377            return Err(loaded.err().unwrap_or_else(|| {
4378                AppError::Closed("child outcome publication requires reconciliation".into()).into()
4379            }));
4380        }
4381        Ok(SessionReceipt::Checkpoint {
4382            loaded: loaded?,
4383            staged: Vec::new(),
4384        })
4385    }
4386
4387    fn retain_source_failure(
4388        &self,
4389        context: &ActionContext,
4390        lineage: &str,
4391        category: FailureCategory,
4392    ) {
4393        let Some(generation) = self.cell.lifecycle_generation() else {
4394            return;
4395        };
4396        match self
4397            .cell
4398            .observe_source_failure(generation, context, lineage, category)
4399        {
4400            Ok(()) | Err(AppError::Closed(_)) => {
4401                // Completed guards have already projected authoritative truth.
4402                // A later source destructor cannot reopen that observation.
4403            }
4404            Err(_) => self.app.tasks_failed(FinalResult::Integration(
4405                "source failure observation authority failed".into(),
4406            )),
4407        }
4408    }
4409
4410    async fn end_source(
4411        &self,
4412        loaded: LoadedRun,
4413        execution: Option<u64>,
4414    ) -> Result<LoadedRun, AppError> {
4415        let generation = self
4416            .cell
4417            .lifecycle_generation()
4418            .ok_or_else(|| AppError::Closed("source cleanup observation is unavailable".into()))?;
4419        let deadline = self.cell.begin_action_cleanup(generation)?;
4420        // EOF rolls back the complete uncommitted wave. Close its original
4421        // guards before discarding any evidence a failing hook may need.
4422        for context in self.cell.action_observations() {
4423            if context.checkpoint != loaded.record.checkpoint.base_sequence {
4424                continue;
4425            }
4426            let mut observed_activation = None;
4427            for activation in &loaded.record.checkpoint.frontier {
4428                if self
4429                    .cell
4430                    .action_context(
4431                        loaded.record.checkpoint.base_sequence,
4432                        &activation.node,
4433                        &activation.lineage,
4434                    )
4435                    .is_some_and(|observed| Arc::ptr_eq(&observed, &context))
4436                {
4437                    observed_activation = Some(activation);
4438                    break;
4439                }
4440            }
4441            let activation = observed_activation.ok_or_else(|| {
4442                AppError::Integration("source cleanup activation is unavailable".into())
4443            })?;
4444            if let Err(failure) = self
4445                .observe_step_finish(
4446                    generation,
4447                    &context,
4448                    None,
4449                    &loaded.record,
4450                    loaded
4451                        .record
4452                        .checkpoint
4453                        .control_for(&activation.node, &activation.lineage)
4454                        .is_some(),
4455                    Some(deadline),
4456                )
4457                .await
4458            {
4459                return Err(self
4460                    .fail_hook(&loaded, Some((activation, &context)), failure, Vec::new())
4461                    .await);
4462            }
4463        }
4464        let loaded = self
4465            .load_execution_observation(None, Some(deadline))
4466            .await
4467            .map_err(|failure| {
4468                self.retain_execution_read_failure(&failure);
4469                failure.into_error()
4470            })?
4471            .ok_or_else(|| AppError::NotFound("workflow run does not exist".into()))?;
4472        let loaded = if let Some(generation) = execution {
4473            // Keep this Execution's lease and options: its caller must first
4474            // observe the retained None before choosing retry or finalization.
4475            self.settle_execution(generation, Some(loaded)).await?.0
4476        } else if loaded.record.checkpoint.is_stopping() || loaded.record.checkpoint.is_terminal() {
4477            loaded
4478        } else {
4479            let definition = self.app.definition_for(&loaded.record)?;
4480            let discard_staged = loaded.record.checkpoint.rollback_stage_keys(&definition);
4481            // The original EndSource operation owns the native pause receipt.
4482            self.commit_for_recovery(loaded, ControlTransition::Pause { discard_staged })
4483                .await
4484                .map_err(RuntimeOperationFailure::into_error)?
4485        };
4486        Ok(loaded)
4487    }
4488
4489    pub(super) async fn finish_execution(
4490        &self,
4491        generation: u64,
4492        loaded: Option<LoadedRun>,
4493    ) -> Result<RunRecord, AppError> {
4494        if !self.cell.owns_execution(generation) {
4495            return Err(AppError::Closed(
4496                "run execution generation no longer owns cleanup".into(),
4497            ));
4498        }
4499        let mut changes = self.app.runtime_changes();
4500        while self.app.has_child_work(&self.key) {
4501            changes
4502                .changed()
4503                .await
4504                .map_err(|_| AppError::Closed("child receipt observation closed".into()))?;
4505        }
4506        let settled = self.settle_execution(generation, loaded).await;
4507        if let Err(error) = &settled
4508            && self.cell.source_failure(generation).is_some()
4509        {
4510            if !matches!(error, AppError::Closed(_)) {
4511                self.app.tasks_failed(FinalResult::Integration(
4512                    "source failure cleanup authority failed".into(),
4513                ));
4514            }
4515            return settled.map(|(loaded, _)| loaded.record);
4516        }
4517        let retained = settled.as_ref().is_ok_and(|(_, retained)| *retained);
4518        let mut result = settled.map(|(loaded, _)| loaded.record);
4519        let cleanup_pending = result.as_ref().is_ok_and(|record| {
4520            record.checkpoint.is_stopping()
4521                || !record.checkpoint.controls.is_empty()
4522                || record.checkpoint.is_terminal() && !retained
4523        });
4524        if let Ok(record) = &result {
4525            let observed = self
4526                .finish_execution_observation(record, None, None, false)
4527                .await;
4528            if record.checkpoint.is_terminal() && self.cell.child_parent().is_none() {
4529                self.maintain_retention(None).await;
4530            }
4531            if let Err(error) = observed {
4532                result = Err(error);
4533            }
4534        }
4535        self.cell.end_publication(generation);
4536        self.cell.clear_execution_options(Some(generation));
4537        let scheduled = self.cell.release_execution(generation);
4538        if !retained && (cleanup_pending || scheduled) {
4539            self.schedule(ScheduleCause::Reconcile);
4540        }
4541        result
4542    }
4543
4544    pub(super) fn settle_execution(
4545        &self,
4546        generation: u64,
4547        loaded: Option<LoadedRun>,
4548    ) -> Pin<Box<impl Future<Output = Result<(LoadedRun, bool), AppError>> + '_>> {
4549        Box::pin(async move {
4550            if !self.cell.owns_execution(generation) {
4551                return Err(AppError::Closed(
4552                    "run execution generation no longer owns cleanup".into(),
4553                ));
4554            }
4555            let mut retained = false;
4556            let result = async {
4557                let mut loaded = match loaded {
4558                    Some(loaded) => loaded,
4559                    None => {
4560                        let deadline = self.cell.begin_action_cleanup(generation)?;
4561                        // Pure settlement reads and unfinished guards retain
4562                        // the same originally sampled cleanup budget.
4563                        self.load_execution_observation(None, Some(deadline))
4564                            .await
4565                            .map_err(|failure| {
4566                                self.retain_execution_read_failure(&failure);
4567                                failure.into_error()
4568                            })?
4569                            .ok_or_else(|| {
4570                                AppError::NotFound("workflow run does not exist".into())
4571                            })?
4572                    }
4573                };
4574                if let Some((context, lineage, category)) = self.cell.source_failure(generation)
4575                    && context.checkpoint == loaded.record.checkpoint.base_sequence
4576                    && !loaded.record.checkpoint.is_stopping()
4577                    && !loaded.record.checkpoint.is_terminal()
4578                    && loaded
4579                        .record
4580                        .checkpoint
4581                        .position
4582                        .as_ref()
4583                        .is_none_or(|committed| committed.ordinal < context.action_ordinal)
4584                {
4585                    let position = ActionPosition {
4586                        node: context.node.clone(),
4587                        ordinal: context.action_ordinal,
4588                    };
4589                    let definition = self.app.definition_for(&loaded.record)?;
4590                    let (position, category, message) = match loaded
4591                        .record
4592                        .checkpoint
4593                        .preceding_failure(&definition, &position, &lineage)?
4594                    {
4595                        Some((position, category, message)) => {
4596                            (position, category, message.to_owned())
4597                        }
4598                        None => (position, category, "caller action execution failed".into()),
4599                    };
4600                    loaded = self
4601                        .commit_for_recovery(
4602                            loaded,
4603                            ControlTransition::Failed {
4604                                category,
4605                                message,
4606                                observation: Some(position),
4607                            },
4608                        )
4609                        .await
4610                        .map_err(RuntimeOperationFailure::into_error)?;
4611                }
4612                if !loaded.record.checkpoint.is_terminal()
4613                    && !loaded.record.checkpoint.is_stopping()
4614                {
4615                    let definition = self.app.definition_for(&loaded.record)?;
4616                    let failed = loaded
4617                        .record
4618                        .checkpoint
4619                        .selected_staged(&definition)
4620                        .filter_map(|activation| {
4621                            if let StagedOutcomeRecord::Failed { category, message } =
4622                                &activation.outcome
4623                            {
4624                                Some((activation, *category, message))
4625                            } else {
4626                                None
4627                            }
4628                        })
4629                        .min_by(|left, right| {
4630                            (&left.0.node, &left.0.lineage).cmp(&(&right.0.node, &right.0.lineage))
4631                        });
4632                    if let Some((activation, category, message)) = failed {
4633                        let message = message.clone();
4634                        let observation = loaded.record.checkpoint.proposed_position(
4635                            &definition,
4636                            &activation.node,
4637                            &activation.lineage,
4638                            &[],
4639                        )?;
4640                        loaded = self
4641                            .commit_for_recovery(
4642                                loaded,
4643                                ControlTransition::Failed {
4644                                    category,
4645                                    message,
4646                                    observation: Some(observation),
4647                                },
4648                            )
4649                            .await
4650                            .map_err(RuntimeOperationFailure::into_error)?;
4651                    }
4652                }
4653                // No lower owner remains in this case. Complete the accepted
4654                // rollback on this generation before making a retry admissible.
4655                if loaded.record.checkpoint.is_stopping()
4656                    && loaded.record.checkpoint.controls.is_empty()
4657                {
4658                    loaded = match Box::pin(self.settle_stopping(loaded)).await {
4659                        Some(loaded) => loaded,
4660                        None => {
4661                            // No terminal receipt owns the result. Observe
4662                            // current authority under the helper's finite pass;
4663                            // native mutations keep their separate ownership.
4664                            self.load_execution_observation(None, None)
4665                                .await
4666                                .map_err(|failure| {
4667                                    self.retain_execution_read_failure(&failure);
4668                                    failure.into_error()
4669                                })?
4670                                .ok_or_else(|| {
4671                                    AppError::NotFound("workflow run does not exist".into())
4672                                })?
4673                        }
4674                    };
4675                }
4676                if !loaded.record.checkpoint.staged.is_empty()
4677                    && !loaded.record.checkpoint.is_terminal()
4678                    && !loaded.record.checkpoint.is_stopping()
4679                {
4680                    let definition = self.app.definition_for(&loaded.record)?;
4681                    let discard_staged = loaded.record.checkpoint.rollback_stage_keys(&definition);
4682                    return self
4683                        .commit_for_recovery(loaded, ControlTransition::Pause { discard_staged })
4684                        .await
4685                        .map_err(RuntimeOperationFailure::into_error);
4686                }
4687                if loaded.record.checkpoint.is_terminal() {
4688                    // Registration has no active execution guards. Completed
4689                    // guards likewise permit the original capacity cleanup.
4690                    let deadline = if self.cell.lifecycle_generation().is_none() {
4691                        Box::pin(self.maintain_retention(Some(loaded))).await;
4692                        // Retention can advance the head revision. Its native
4693                        // receipt starts this required fresh capacity-read pass.
4694                        let deadline = LifecycleEvent::deadline(None);
4695                        loaded = self
4696                            .load_execution_observation(None, Some(deadline))
4697                            .await
4698                            .map_err(|failure| {
4699                                self.retain_execution_read_failure(&failure);
4700                                failure.into_error()
4701                            })?
4702                            .ok_or_else(|| {
4703                                AppError::NotFound("workflow run does not exist".into())
4704                            })?;
4705                        deadline
4706                    } else {
4707                        LifecycleEvent::deadline(None)
4708                    };
4709                    retained = tokio::time::timeout_at(deadline, async {
4710                        let mut delay = RECONCILIATION_INITIAL_DELAY;
4711                        loop {
4712                            match self.app.storage().run_is_active(&loaded.record).await {
4713                                Ok(active) => {
4714                                    // Marker authority cannot replace the App
4715                                    // generation fence of the native result.
4716                                    self.observe_generation(deadline).await?;
4717                                    return Ok(!active);
4718                                }
4719                                Err(failure)
4720                                    if failure.is_recoverable()
4721                                        && self.schedule_after_storage_failure(&failure) =>
4722                                {
4723                                    tokio::time::sleep(delay).await;
4724                                    delay = delay.saturating_mul(2).min(RECONCILIATION_MAX_DELAY);
4725                                }
4726                                Err(failure) => return Err(failure),
4727                            }
4728                        }
4729                    })
4730                    .await
4731                    .unwrap_or_else(|_| {
4732                        Err(StorageFailure::permanent(AppError::Timeout(
4733                            "workflow capacity observation deadline expired".into(),
4734                        )))
4735                    })
4736                    .map_err(|failure| {
4737                        self.retain_execution_read_failure(&failure);
4738                        failure.into_error()
4739                    })?;
4740                } else if loaded.record.checkpoint.is_stopping() {
4741                    // Failed may retain admitted controls in Failing. Returning
4742                    // that native receipt still fences this App generation.
4743                    self.observe_generation(LifecycleEvent::deadline(None))
4744                        .await
4745                        .map_err(|failure| {
4746                            self.retain_execution_read_failure(&failure);
4747                            failure.into_error()
4748                        })?;
4749                }
4750                Ok(loaded)
4751            }
4752            .await;
4753            result.map(|loaded| (loaded, retained))
4754        })
4755    }
4756}
4757
4758impl<
4759    CS: StateStoreBackend,
4760    A: AgentBackend,
4761    X: ActionBackend,
4762    C: ConditionBackend,
4763    T: StateTyping,
4764    H: LifecycleHooks,
4765    E: ChildExecutor,
4766> WorkflowRunRuntime<CS, A, X, C, T, H, E>
4767{
4768    pub(super) async fn begin_execution_observation(&self) -> Result<u64, WorkflowFailure> {
4769        let owner = self
4770            .cell
4771            .observation_owner()
4772            .map_err(|error| WorkflowFailure::Execution(error.to_string()))?;
4773        let (generation, opening) = self
4774            .cell
4775            .begin_lifecycle(owner)
4776            .map_err(|error| WorkflowFailure::Execution(error.to_string()))?;
4777        if opening {
4778            let deadline = self
4779                .cell
4780                .begin_action_cleanup(generation)
4781                .map_err(|error| WorkflowFailure::Execution(error.to_string()))?;
4782            LifecycleEvent::ExecuteStarted {
4783                run: self.id(),
4784                occurrence: self.occurrence(),
4785            }
4786            .deliver(
4787                self.app.hooks(),
4788                self.cell
4789                    .execution_mode()
4790                    .map_err(|error| WorkflowFailure::Execution(error.to_string()))?,
4791                deadline,
4792            )
4793            .await?;
4794            self.cell
4795                .opened_lifecycle(generation)
4796                .map_err(|error| WorkflowFailure::Execution(error.to_string()))?;
4797        }
4798        Ok(generation)
4799    }
4800
4801    async fn observe_activation_start(
4802        &self,
4803        loaded: &LoadedRun,
4804        activation: &FrontierRecord,
4805        context: &Arc<ActionContext>,
4806        inputs: &Map<String, Value>,
4807        deadline: tokio::time::Instant,
4808    ) -> Result<(), WorkflowFailure> {
4809        let generation = self.begin_execution_observation().await?;
4810        let definition = self
4811            .app
4812            .definition_for(&loaded.record)
4813            .map_err(|error| WorkflowFailure::Execution(error.to_string()))?;
4814        let Some(Step::Action {
4815            action, revision, ..
4816        }) = definition.node(&activation.node).map(|node| node.step())
4817        else {
4818            return self
4819                .observe_step_start(
4820                    generation,
4821                    &loaded.record,
4822                    context,
4823                    &activation.lineage,
4824                    inputs,
4825                    deadline,
4826                )
4827                .await
4828                .map(|_| ());
4829        };
4830        let binding = self
4831            .app
4832            .action(action, revision)
4833            .map_err(|error| WorkflowFailure::Execution(error.to_string()))?;
4834        let inputs = catch_unwind(AssertUnwindSafe(|| {
4835            binding
4836                .declaration()
4837                .resolve_inputs(inputs, self.cell.execution_mode()?)
4838                .and_then(ActionInputs::into_values)
4839        }))
4840        .map_err(|_| WorkflowFailure::Execution("action declaration panicked".into()))?
4841        .map_err(|error| WorkflowFailure::State(error.to_string()))?;
4842        let context = self
4843            .observe_step_start(
4844                generation,
4845                &loaded.record,
4846                context,
4847                &activation.lineage,
4848                &inputs,
4849                deadline,
4850            )
4851            .await?;
4852        self.observe_stream_start(generation, &context, deadline)
4853            .await
4854    }
4855
4856    pub(super) async fn observe_stream_start(
4857        &self,
4858        generation: u64,
4859        context: &ActionContext,
4860        deadline: tokio::time::Instant,
4861    ) -> Result<(), WorkflowFailure> {
4862        if self.cell.lifecycle_generation() != Some(generation)
4863            || self.cell.execution_was_dropped(generation)
4864        {
4865            return Ok(());
4866        }
4867        LifecycleEvent::StreamStarted {
4868            context,
4869            span: None,
4870        }
4871        .deliver(
4872            self.app.hooks(),
4873            context.mode,
4874            LifecycleEvent::deadline(Some(deadline)),
4875        )
4876        .await?;
4877        self.cell
4878            .open_action_stream(generation, context)
4879            .map_err(|error| WorkflowFailure::Execution(error.to_string()))?;
4880        Ok(())
4881    }
4882
4883    pub(super) async fn observe_stream_item(
4884        &self,
4885        generation: u64,
4886        context: &ActionContext,
4887        result: &Map<String, Value>,
4888        deadline: tokio::time::Instant,
4889    ) -> Result<(), WorkflowFailure> {
4890        if self.cell.lifecycle_generation() != Some(generation)
4891            || self.cell.execution_was_dropped(generation)
4892        {
4893            return Ok(());
4894        }
4895        let index = self
4896            .cell
4897            .action_stream_item(generation, context, false)
4898            .map_err(|error| WorkflowFailure::Execution(error.to_string()))?;
4899        LifecycleEvent::StreamItem {
4900            context,
4901            span: None,
4902            index,
4903            result,
4904        }
4905        .deliver(
4906            self.app.hooks(),
4907            context.mode,
4908            LifecycleEvent::deadline(Some(deadline)),
4909        )
4910        .await?;
4911        self.cell
4912            .action_stream_item(generation, context, true)
4913            .map_err(|error| WorkflowFailure::Execution(error.to_string()))?;
4914        Ok(())
4915    }
4916
4917    async fn observe_source_event(
4918        &self,
4919        context: &ActionContext,
4920        event: ActionEvent,
4921        deadline: tokio::time::Instant,
4922    ) -> Result<Option<Map<String, Value>>, WorkflowFailure> {
4923        let generation = self.cell.lifecycle_generation().ok_or_else(|| {
4924            WorkflowFailure::Execution("source observation generation closed".into())
4925        })?;
4926        if let ActionEvent::Partial { result } = event {
4927            self.observe_stream_item(generation, context, &result, deadline)
4928                .await?;
4929            return Ok(Some(result));
4930        }
4931        if self.cell.execution_was_dropped(generation) {
4932            return Ok(None);
4933        }
4934        event
4935            .validate_span()
4936            .map_err(|error| WorkflowFailure::Execution(error.to_string()))?;
4937        let deadline = LifecycleEvent::deadline(Some(deadline));
4938        match event {
4939            ActionEvent::SpanStarted {
4940                id,
4941                name,
4942                parent,
4943                dependencies,
4944            } => {
4945                let parent_index = self
4946                    .cell
4947                    .span_parent(generation, context, &id, parent.as_deref())
4948                    .map_err(|error| WorkflowFailure::Execution(error.to_string()))?;
4949                LifecycleEvent::SpanStarted {
4950                    context,
4951                    id: &id,
4952                    name: &name,
4953                    parent: parent.as_deref(),
4954                    dependencies: &dependencies,
4955                }
4956                .deliver(self.app.hooks(), context.mode, deadline)
4957                .await?;
4958                self.cell
4959                    .opened_span(generation, context, id, parent_index)
4960                    .map_err(|error| WorkflowFailure::Execution(error.to_string()))?;
4961            }
4962            ActionEvent::SpanAttribute {
4963                span,
4964                key,
4965                value,
4966                tags,
4967            } => {
4968                self.cell
4969                    .span_attribute(generation, context, span.as_deref())
4970                    .map_err(|error| WorkflowFailure::Execution(error.to_string()))?;
4971                LifecycleEvent::SpanAttribute {
4972                    context,
4973                    span: span.as_deref(),
4974                    key: &key,
4975                    value: &value,
4976                    tags: &tags,
4977                }
4978                .deliver(self.app.hooks(), context.mode, deadline)
4979                .await?;
4980            }
4981            ActionEvent::SpanFinished { id } => {
4982                let id = self
4983                    .cell
4984                    .take_span(generation, context, Some(&id))
4985                    .map_err(|error| WorkflowFailure::Execution(error.to_string()))?
4986                    .ok_or_else(|| {
4987                        WorkflowFailure::Execution(
4988                            "caller span finish target is unavailable".into(),
4989                        )
4990                    })?;
4991                LifecycleEvent::SpanFinished { context, id: &id }
4992                    .deliver(self.app.hooks(), context.mode, deadline)
4993                    .await?;
4994            }
4995            ActionEvent::Partial { .. } | ActionEvent::Completed { .. } => unreachable!(),
4996        }
4997        Ok(None)
4998    }
4999
5000    async fn finish_spans(
5001        &self,
5002        generation: u64,
5003        context: &ActionContext,
5004        async_deadline: Option<tokio::time::Instant>,
5005    ) -> Result<(), WorkflowFailure> {
5006        let mut failure = None;
5007        loop {
5008            let id = match self.cell.take_span(generation, context, None) {
5009                Ok(Some(id)) => id,
5010                Ok(None) => break,
5011                Err(error) => {
5012                    return Err(
5013                        failure.unwrap_or_else(|| WorkflowFailure::Execution(error.to_string()))
5014                    );
5015                }
5016            };
5017            let event = LifecycleEvent::SpanFinished { context, id: &id };
5018            let observed =
5019                match async_deadline.filter(|_| !self.cell.execution_was_dropped(generation)) {
5020                    Some(deadline) => {
5021                        event
5022                            .deliver(self.app.hooks(), context.mode, deadline)
5023                            .await
5024                    }
5025                    None => event.deliver_sync(self.app.hooks()),
5026                };
5027            if let Err(next) = observed
5028                && failure.is_none()
5029            {
5030                failure = Some(next);
5031            }
5032        }
5033        failure.map_or(Ok(()), Err)
5034    }
5035
5036    fn finish_spans_sync(
5037        &self,
5038        generation: u64,
5039        context: &ActionContext,
5040    ) -> Result<(), WorkflowFailure> {
5041        let mut failure = None;
5042        loop {
5043            let id = match self.cell.take_span(generation, context, None) {
5044                Ok(Some(id)) => id,
5045                Ok(None) => break,
5046                Err(error) => {
5047                    return Err(
5048                        failure.unwrap_or_else(|| WorkflowFailure::Execution(error.to_string()))
5049                    );
5050                }
5051            };
5052            if let Err(next) =
5053                (LifecycleEvent::SpanFinished { context, id: &id }).deliver_sync(self.app.hooks())
5054                && failure.is_none()
5055            {
5056                failure = Some(next);
5057            }
5058        }
5059        failure.map_or(Ok(()), Err)
5060    }
5061
5062    async fn finish_staged_spans(
5063        &self,
5064        loaded: &LoadedRun,
5065        decision: &BspDecision,
5066        context: Option<&ActionContext>,
5067    ) -> Result<(), AppError> {
5068        let (BspDecision::Stage(staged), Some(context)) = (decision, context) else {
5069            return Ok(());
5070        };
5071        let Some(generation) = self.cell.lifecycle_generation() else {
5072            return Ok(());
5073        };
5074        let Some(staged) = staged.iter().find(|staged| {
5075            self.cell
5076                .action_context(
5077                    loaded.record.checkpoint.base_sequence,
5078                    &staged.node,
5079                    &staged.lineage,
5080                )
5081                .is_some_and(|observed| std::ptr::eq(observed.as_ref(), context))
5082                && matches!(staged.outcome, StagedOutcomeRecord::Complete { .. })
5083        }) else {
5084            return Ok(());
5085        };
5086        let activation = loaded
5087            .record
5088            .checkpoint
5089            .frontier
5090            .iter()
5091            .find(|activation| {
5092                activation.node == staged.node && activation.lineage == staged.lineage
5093            })
5094            .ok_or_else(|| {
5095                AppError::Integration("completed caller activation is unavailable".into())
5096            })?;
5097        // The one caller Stage carries its original context across sibling
5098        // completion; no ordinal is resampled after another source settles.
5099        if let Err(failure) = self
5100            .finish_spans(generation, context, Some(LifecycleEvent::deadline(None)))
5101            .await
5102        {
5103            return Err(self
5104                .fail_hook(loaded, Some((activation, context)), failure, Vec::new())
5105                .await);
5106        }
5107        Ok(())
5108    }
5109
5110    pub(super) async fn fail_hook(
5111        &self,
5112        loaded: &LoadedRun,
5113        activation: Option<(&FrontierRecord, &ActionContext)>,
5114        failure: WorkflowFailure,
5115        staged: Vec<StagedActivationRecord>,
5116    ) -> AppError {
5117        let error = failure.application_error();
5118        let mut prior = None;
5119        let settled = async {
5120            let generation = self
5121                .cell
5122                .lifecycle_generation()
5123                .ok_or_else(|| AppError::Closed("lifecycle failure cleanup is closed".into()))?;
5124            let deadline = self.cell.begin_action_cleanup(generation)?;
5125            let mut loaded = LoadedRun {
5126                record: loaded.record.clone(),
5127                revision: loaded.revision,
5128            };
5129            if !staged.is_empty() {
5130                // Preserve the exact prefix before failure selection: its
5131                // Complete entries determine the earlier failure's ordinal.
5132                loaded = self
5133                    .commit_for_recovery(
5134                        loaded,
5135                        ControlTransition::Stage {
5136                            activations: staged,
5137                        },
5138                    )
5139                    .await
5140                    .map_err(RuntimeOperationFailure::into_error)?;
5141            }
5142            let definition = self.app.definition_for(&loaded.record)?;
5143            let retained = self
5144                .cell
5145                .lifecycle_generation()
5146                .and_then(|generation| self.cell.source_failure(generation));
5147            let source = retained.as_ref().filter(|(context, lineage, _)| {
5148                context.checkpoint == loaded.record.checkpoint.base_sequence
5149                    && activation.is_some_and(|(activation, _)| {
5150                        (&context.node, lineage) < (&activation.node, &activation.lineage)
5151                    })
5152            });
5153            // An earlier source can finish while this original callback is
5154            // held. Its sampled identity wins; a same-action destructor never
5155            // replaces the callback's own primary Hook failure.
5156            let (mut observation, lineage, mut category, mut message) = match source {
5157                Some((context, lineage, category)) => {
5158                    let message = "caller action execution failed".to_owned();
5159                    prior = Some(category.project(message.clone()));
5160                    (
5161                        Some(ActionPosition {
5162                            node: context.node.clone(),
5163                            ordinal: context.action_ordinal,
5164                        }),
5165                        Some(lineage.as_str()),
5166                        *category,
5167                        message,
5168                    )
5169                }
5170                None => (
5171                    activation.map(|(_, context)| ActionPosition {
5172                        node: context.node.clone(),
5173                        ordinal: context.action_ordinal,
5174                    }),
5175                    activation.map(|(activation, _)| activation.lineage.as_str()),
5176                    FailureCategory::from_failure(&failure),
5177                    failure.message().to_owned(),
5178                ),
5179            };
5180            if let Some((position, lineage)) = observation.as_ref().zip(lineage)
5181                && let Some((position, preceding, text)) = loaded
5182                    .record
5183                    .checkpoint
5184                    .preceding_failure(&definition, position, lineage)?
5185            {
5186                observation = Some(position);
5187                category = preceding;
5188                message = text.to_owned();
5189                prior = Some(category.project(message.clone()));
5190            }
5191            let mut loaded = self
5192                .commit_for_recovery(
5193                    loaded,
5194                    ControlTransition::Failed {
5195                        category,
5196                        message,
5197                        observation,
5198                    },
5199                )
5200                .await
5201                .map_err(RuntimeOperationFailure::into_error)?;
5202            if !loaded.record.checkpoint.is_terminal() {
5203                // Accepted controls can progress independently while their
5204                // original failure is published. Refresh only that authority;
5205                // the native Stage and terminal receipts remain exact values.
5206                loaded = self
5207                    .load_execution_observation(None, Some(deadline))
5208                    .await
5209                    .map_err(|failure| {
5210                        self.retain_execution_read_failure(&failure);
5211                        failure.into_error()
5212                    })?
5213                    .ok_or_else(|| AppError::NotFound("workflow run does not exist".into()))?;
5214                if loaded.record.checkpoint.is_stopping()
5215                    && loaded.record.checkpoint.controls.is_empty()
5216                {
5217                    loaded = match Box::pin(self.settle_stopping(loaded)).await {
5218                        Some(loaded) => loaded,
5219                        None => self
5220                            .load_execution_observation(None, Some(deadline))
5221                            .await
5222                            .map_err(|failure| {
5223                                self.retain_execution_read_failure(&failure);
5224                                failure.into_error()
5225                            })?
5226                            .ok_or_else(|| {
5227                                AppError::NotFound("workflow run does not exist".into())
5228                            })?,
5229                    };
5230                }
5231            }
5232            if loaded.record.checkpoint.is_terminal() {
5233                // The accepted failure receipt is complete authority. Preserve
5234                // generation fencing without requiring the same head again.
5235                self.observe_generation(deadline).await.map_err(|failure| {
5236                    self.retain_execution_read_failure(&failure);
5237                    failure.into_error()
5238                })?;
5239            }
5240            Ok(loaded)
5241        }
5242        .await;
5243        match settled {
5244            Ok(loaded) => {
5245                let record = &loaded.record;
5246                // A prior canonical action failure remains durable; this later
5247                // callback error is its execution-completion context. Ordinary
5248                // precommit Hook failure still projects directly from storage.
5249                let completion = prior.map(|prior| failure.with_cause(Some(prior)));
5250                let _ = self
5251                    .finish_execution_observation(record, completion, None, false)
5252                    .await;
5253                if record.checkpoint.is_terminal() && self.cell.child_parent().is_none() {
5254                    Box::pin(self.maintain_retention(Some(loaded))).await;
5255                }
5256            }
5257            Err(AppError::Closed(_)) => {}
5258            Err(_) => self.app.tasks_failed(FinalResult::Integration(
5259                "lifecycle failure cleanup authority failed".into(),
5260            )),
5261        }
5262        error
5263    }
5264
5265    async fn observe_step_start(
5266        &self,
5267        generation: u64,
5268        record: &RunRecord,
5269        context: &Arc<ActionContext>,
5270        lineage: &str,
5271        inputs: &Map<String, Value>,
5272        deadline: tokio::time::Instant,
5273    ) -> Result<Arc<ActionContext>, WorkflowFailure> {
5274        if let Some(context) = self
5275            .cell
5276            .action_context(context.checkpoint, &context.node, lineage)
5277        {
5278            return Ok(context);
5279        }
5280        let state = WorkflowState {
5281            values: record.checkpoint.state.clone(),
5282        };
5283        LifecycleEvent::StepStarted {
5284            context,
5285            state: &state,
5286            inputs,
5287        }
5288        .deliver(
5289            self.app.hooks(),
5290            context.mode,
5291            LifecycleEvent::deadline(Some(deadline)),
5292        )
5293        .await?;
5294        self.cell
5295            .observe_action(generation, Arc::clone(context), lineage)
5296            .map_err(|error| WorkflowFailure::Execution(error.to_string()))?;
5297        Ok(Arc::clone(context))
5298    }
5299}
5300
5301impl<
5302    CS: StateStoreBackend,
5303    A: AgentBackend,
5304    X: ActionBackend,
5305    C: ConditionBackend,
5306    T: StateTyping,
5307    H: LifecycleHooks,
5308    E: ChildExecutor,
5309> WorkflowRunRuntime<CS, A, X, C, T, H, E>
5310{
5311    async fn observe_step_finish(
5312        &self,
5313        generation: u64,
5314        context: &ActionContext,
5315        outcome: Option<Result<&ActionOutcome, &WorkflowFailure>>,
5316        record: &RunRecord,
5317        unresolved: bool,
5318        async_deadline: Option<tokio::time::Instant>,
5319    ) -> Result<(), WorkflowFailure> {
5320        let checkpoint = self
5321            .project_history(record)
5322            .map_err(|error| WorkflowFailure::Execution(error.to_string()))?;
5323        let spans = self.finish_spans(generation, context, async_deadline).await;
5324        let Some((context, items)) = self.cell.take_action_observation(generation, context) else {
5325            if !unresolved {
5326                self.cell.retire_closed_observation(generation, context);
5327            }
5328            return spans;
5329        };
5330        let event = LifecycleEvent::StepFinished {
5331            context: &context,
5332            outcome,
5333            checkpoint: &checkpoint,
5334        };
5335        let step = match async_deadline.filter(|_| !self.cell.execution_was_dropped(generation)) {
5336            Some(deadline) => {
5337                event
5338                    .deliver(self.app.hooks(), context.mode, deadline)
5339                    .await
5340            }
5341            None => event.deliver_sync(self.app.hooks()),
5342        };
5343        let result = spans.and(step);
5344        let result = if let Some(items) = items {
5345            let failure = result
5346                .as_ref()
5347                .err()
5348                .or_else(|| outcome.and_then(Result::err));
5349            let event = LifecycleEvent::StreamFinished {
5350                context: &context,
5351                span: None,
5352                items,
5353                failure,
5354            };
5355            // A failed step guard unwinds the still-open stream guard. Its
5356            // synchronous Drop callback must not start another async lane.
5357            let stream_result = match async_deadline
5358                .filter(|_| result.is_ok() && !self.cell.execution_was_dropped(generation))
5359            {
5360                Some(deadline) => {
5361                    event
5362                        .deliver(self.app.hooks(), context.mode, deadline)
5363                        .await
5364                }
5365                None => event.deliver_sync(self.app.hooks()),
5366            };
5367            result.and(stream_result)
5368        } else {
5369            result
5370        };
5371        self.cell
5372            .finish_action_observation(generation, &context, unresolved);
5373        result
5374    }
5375
5376    fn observe_step_finish_sync(
5377        &self,
5378        generation: u64,
5379        context: &ActionContext,
5380        record: &RunRecord,
5381        unresolved: bool,
5382    ) -> Result<(), WorkflowFailure> {
5383        let checkpoint = self
5384            .project_history(record)
5385            .map_err(|error| WorkflowFailure::Execution(error.to_string()))?;
5386        let spans = self.finish_spans_sync(generation, context);
5387        let Some((context, items)) = self.cell.take_action_observation(generation, context) else {
5388            if !unresolved {
5389                self.cell.retire_closed_observation(generation, context);
5390            }
5391            return spans;
5392        };
5393        let step = LifecycleEvent::StepFinished {
5394            context: &context,
5395            outcome: None,
5396            checkpoint: &checkpoint,
5397        }
5398        .deliver_sync(self.app.hooks());
5399        let result = spans.and(step);
5400        let result = if let Some(items) = items {
5401            let next = LifecycleEvent::StreamFinished {
5402                context: &context,
5403                span: None,
5404                items,
5405                failure: result.as_ref().err(),
5406            }
5407            .deliver_sync(self.app.hooks());
5408            result.and(next)
5409        } else {
5410            result
5411        };
5412        self.cell
5413            .finish_action_observation(generation, &context, unresolved);
5414        result
5415    }
5416
5417    fn finish_execution_observation_sync(&self, record: &RunRecord) -> Result<(), AppError> {
5418        let Some(generation) = self.cell.lifecycle_generation() else {
5419            return self
5420                .cell
5421                .lifecycle_observation(record.checkpoint.sequence, false)
5422                .map(|_| ());
5423        };
5424        let unfinished = record
5425            .checkpoint
5426            .controls
5427            .values()
5428            .filter_map(|control| {
5429                self.cell.action_context(
5430                    record.checkpoint.base_sequence,
5431                    &control.node,
5432                    &control.lineage,
5433                )
5434            })
5435            .collect::<Vec<_>>();
5436        if !self
5437            .cell
5438            .begin_lifecycle_finish(generation, record.checkpoint.sequence)
5439        {
5440            return self
5441                .cell
5442                .lifecycle_observation(record.checkpoint.sequence, false)
5443                .map(|_| ());
5444        }
5445        let mut failure = None;
5446        for context in self.cell.action_observations() {
5447            let unresolved = unfinished
5448                .iter()
5449                .any(|unfinished| Arc::ptr_eq(unfinished, &context));
5450            if let Err(next) =
5451                self.observe_step_finish_sync(generation, &context, record, unresolved)
5452                && failure.is_none()
5453            {
5454                failure = Some(next);
5455            }
5456        }
5457        match self.project(record) {
5458            Ok(snapshot) => {
5459                if let Err(next) = (LifecycleEvent::ExecuteFinished {
5460                    snapshot: &snapshot,
5461                    failure: failure.as_ref(),
5462                })
5463                .deliver_sync(self.app.hooks())
5464                    && failure.is_none()
5465                {
5466                    failure = Some(next);
5467                }
5468            }
5469            Err(error) => {
5470                if failure.is_none() {
5471                    failure = Some(WorkflowFailure::Execution(error.to_string()));
5472                }
5473            }
5474        }
5475        if record.checkpoint.is_terminal() || record.checkpoint.is_paused() {
5476            self.cell.clear_execution_options(Some(generation));
5477        }
5478        let error = failure.as_ref().map(WorkflowFailure::application_error);
5479        let completed =
5480            self.cell
5481                .complete_lifecycle(generation, record.checkpoint.sequence, failure);
5482        match error {
5483            Some(error) => Err(error),
5484            None => completed,
5485        }
5486    }
5487
5488    pub(super) async fn finish_execution_observation(
5489        &self,
5490        record: &RunRecord,
5491        mut failure: Option<WorkflowFailure>,
5492        deadline: Option<tokio::time::Instant>,
5493        synchronous: bool,
5494    ) -> Result<(), AppError> {
5495        let Some(generation) = self.cell.lifecycle_generation() else {
5496            return self
5497                .cell
5498                .lifecycle_observation(record.checkpoint.sequence, false)
5499                .map(|_| ());
5500        };
5501        let synchronous = synchronous || self.cell.execution_was_dropped(generation);
5502        let unfinished = record
5503            .checkpoint
5504            .controls
5505            .values()
5506            .filter_map(|control| {
5507                self.cell.action_context(
5508                    record.checkpoint.base_sequence,
5509                    &control.node,
5510                    &control.lineage,
5511                )
5512            })
5513            .collect::<Vec<_>>();
5514        if !self
5515            .cell
5516            .begin_lifecycle_finish(generation, record.checkpoint.sequence)
5517        {
5518            return self
5519                .cell
5520                .lifecycle_observation(record.checkpoint.sequence, false)
5521                .map(|_| ());
5522        }
5523        let deadline = match deadline {
5524            Some(deadline) => deadline,
5525            None => self.cell.begin_action_cleanup(generation)?,
5526        };
5527        let primary = match &record.checkpoint.phase {
5528            CheckpointPhaseRecord::Failing {
5529                category, message, ..
5530            }
5531            | CheckpointPhaseRecord::Failed {
5532                category, message, ..
5533            } => Some(category.project(message.clone())),
5534            _ => None,
5535        };
5536        for context in self.cell.action_observations() {
5537            let unresolved = unfinished
5538                .iter()
5539                .any(|unfinished| Arc::ptr_eq(unfinished, &context));
5540            if unresolved && !record.checkpoint.is_stopping() && !synchronous {
5541                continue;
5542            }
5543            let outcome = record.checkpoint.outcomes.iter().find(|outcome| {
5544                outcome.ordinal == context.action_ordinal && outcome.node == context.node
5545            });
5546            let failed = record
5547                .checkpoint
5548                .failed_observation()
5549                .is_some_and(|position| {
5550                    position.ordinal == context.action_ordinal && position.node == context.node
5551                });
5552            let observed = outcome
5553                .map(Ok)
5554                .or_else(|| failed.then(|| primary.as_ref().map(Err)).flatten());
5555            if let Err(next) = self
5556                .observe_step_finish(
5557                    generation,
5558                    &context,
5559                    observed,
5560                    record,
5561                    unresolved,
5562                    (!synchronous && failure.is_none()).then_some(deadline),
5563                )
5564                .await
5565                && failure.is_none()
5566            {
5567                failure = Some(next.with_cause(match &record.checkpoint.phase {
5568                    CheckpointPhaseRecord::Failing {
5569                        category, message, ..
5570                    }
5571                    | CheckpointPhaseRecord::Failed {
5572                        category, message, ..
5573                    } => Some(category.project(message.clone())),
5574                    _ => None,
5575                }));
5576            }
5577        }
5578        match self.project(record) {
5579            Ok(snapshot) => {
5580                let event = LifecycleEvent::ExecuteFinished {
5581                    snapshot: &snapshot,
5582                    failure: failure.as_ref().or(primary.as_ref()),
5583                };
5584                let observed = if synchronous || self.cell.execution_was_dropped(generation) {
5585                    event.deliver_sync(self.app.hooks())
5586                } else {
5587                    event
5588                        .deliver(self.app.hooks(), self.cell.execution_mode()?, deadline)
5589                        .await
5590                };
5591                if let Err(next) = observed
5592                    && failure.is_none()
5593                {
5594                    failure = Some(next.with_cause(primary));
5595                }
5596            }
5597            Err(error) => {
5598                if failure.is_none() {
5599                    failure = Some(WorkflowFailure::Execution(error.to_string()));
5600                }
5601            }
5602        }
5603        if record.parent.is_some()
5604            && record.checkpoint.is_terminal()
5605            && let Err(next) = self.observe_child_finished(record, deadline).await
5606            && failure.is_none()
5607        {
5608            failure = Some(next);
5609        }
5610        if record.checkpoint.is_terminal() || record.checkpoint.is_paused() {
5611            self.cell.clear_execution_options(Some(generation));
5612        }
5613        let error = failure.as_ref().map(WorkflowFailure::application_error);
5614        let completed =
5615            self.cell
5616                .complete_lifecycle(generation, record.checkpoint.sequence, failure);
5617        match error {
5618            Some(error) => Err(error),
5619            None => completed,
5620        }
5621    }
5622
5623    pub(super) async fn observe_barrier(&self, loaded: LoadedRun) -> Result<LoadedRun, AppError> {
5624        let Some(generation) = self.cell.lifecycle_generation() else {
5625            return Ok(loaded);
5626        };
5627        let record = &loaded.record;
5628        let observations = self.cell.action_observations();
5629        for outcome in &record.checkpoint.outcomes {
5630            let context = observations.iter().find(|context| {
5631                context.action_ordinal == outcome.ordinal && context.node == outcome.node
5632            });
5633            let Some(context) = context else {
5634                continue;
5635            };
5636            let deadline = self.cell.begin_action_cleanup(generation)?;
5637            if let Err(failure) = self
5638                .observe_step_finish(
5639                    generation,
5640                    context,
5641                    Some(Ok(outcome)),
5642                    record,
5643                    false,
5644                    Some(deadline),
5645                )
5646                .await
5647            {
5648                let error = failure.application_error();
5649                // The committed barrier owns this Pause intent even when the
5650                // callback spent its cleanup budget. Native revision-fenced
5651                // reconciliation preserves independent cancellation.
5652                let parked = self
5653                    .commit_for_recovery(
5654                        loaded,
5655                        ControlTransition::Pause {
5656                            discard_staged: Vec::new(),
5657                        },
5658                    )
5659                    .await
5660                    .map_err(RuntimeOperationFailure::into_error);
5661                match parked {
5662                    Ok(parked) => {
5663                        let _ = self
5664                            .finish_execution_observation(
5665                                &parked.record,
5666                                Some(failure),
5667                                Some(deadline),
5668                                false,
5669                            )
5670                            .await;
5671                        return Ok(parked);
5672                    }
5673                    Err(AppError::Closed(_)) => {}
5674                    Err(_) => self.app.tasks_failed(FinalResult::Integration(
5675                        "post-hook cleanup authority failed".into(),
5676                    )),
5677                }
5678                return Err(error);
5679            }
5680        }
5681        Ok(loaded)
5682    }
5683}