Skip to main content

bamboo_engine/external_agents/
local_task.rs

1//! Native Child execution in the Host process. Identity, input, history and
2//! terminal publication still belong to the canonical Actor/Session stores.
3//! There is no broker, Worker IPC, or remote fallback in this backend.
4use super::runtime::NativeToolCeilingSource;
5use crate::execution::spawn::{ExternalChildRunner, SessionInboxRuntimeBinding, SpawnJob};
6use crate::runtime::{Agent, ExecuteRequestBuilder};
7use async_trait::async_trait;
8use bamboo_agent_core::tools::ToolExecutor;
9use bamboo_agent_core::{AgentError, AgentEvent, Session};
10use bamboo_domain::{
11    ActorActivationClaim, ActorActivationFinish, ActorDirectoryPort, ChildActorRuntimeWrite,
12    RuntimeSessionPersistence, SessionKind, Storage,
13};
14use bamboo_llm::{Config, ProviderModelRouter};
15use futures::FutureExt;
16use std::{
17    collections::HashMap,
18    path::PathBuf,
19    sync::{
20        atomic::{AtomicBool, Ordering},
21        Arc, Mutex,
22    },
23    time::Duration,
24};
25use tokio::sync::{mpsc, Semaphore};
26use tokio_util::sync::CancellationToken;
27
28#[derive(Clone)]
29pub struct LocalTaskRuntime {
30    pub agent: Arc<Agent>,
31    /// The Host's complete permission-checking base chain, without the Root role fence.
32    pub tools: Arc<dyn ToolExecutor>,
33    pub provider_router: Option<Arc<ProviderModelRouter>>,
34    pub app_data_dir: Option<PathBuf>,
35    pub sessions_cache: crate::SessionCache,
36}
37
38mod local_skill_binding;
39pub use local_skill_binding::{
40    LocalChildSkillAuthority, LocalChildSkillBinding, LocalChildSkillFactory,
41};
42
43#[derive(Default)]
44struct Bindings {
45    skills: Option<Arc<dyn LocalChildSkillFactory>>,
46    runtime: Option<LocalTaskRuntime>,
47    inbox: Option<SessionInboxRuntimeBinding>,
48    store: Option<Arc<bamboo_storage::SessionStoreV2>>,
49    tickets: Option<Arc<bamboo_tickets::TicketService>>,
50    subagent: Option<Arc<dyn bamboo_agent_core::tools::Tool>>,
51    observer: Option<Arc<dyn super::actor_event_stream::ActorEventObserver>>,
52}
53
54/// Constructed only after the actual Agent task was joined and its owned
55/// resources drained. A request to cancel, a PID, and a durable status cannot
56/// manufacture this value.
57#[derive(Clone, Debug)]
58pub struct ConfirmedTaskStop {
59    session_id: String,
60    child_birth: chrono::DateTime<chrono::Utc>,
61    run_id: String,
62    instance_id: uuid::Uuid,
63}
64impl ConfirmedTaskStop {
65    pub(crate) fn session_id(&self) -> &str {
66        &self.session_id
67    }
68    pub(crate) fn child_birth(&self) -> chrono::DateTime<chrono::Utc> {
69        self.child_birth
70    }
71    pub(crate) fn run_id(&self) -> &str {
72        &self.run_id
73    }
74    pub(crate) fn instance_id(&self) -> uuid::Uuid {
75        self.instance_id
76    }
77}
78
79struct ExecutionOutput {
80    session: Session,
81    result: crate::runtime::runner::Result<()>,
82    resources: std::io::Result<bamboo_tools::DrainedLocalTaskResources>,
83}
84#[derive(Clone, Copy, PartialEq, Eq)]
85enum LaunchState {
86    Queued,
87    Preparing,
88    Running,
89    Stopping,
90    Stopped,
91}
92
93struct Execution {
94    owner: ChildActorRuntimeWrite,
95    persistence: Arc<dyn RuntimeSessionPersistence>,
96    cancel: CancellationToken,
97    resources: bamboo_tools::LocalTaskResources,
98    task: tokio::sync::Mutex<Option<tokio::task::JoinHandle<ExecutionOutput>>>,
99    stop: Mutex<Option<ConfirmedTaskStop>>,
100    finalized: AtomicBool,
101    launch: Mutex<LaunchState>,
102    admission: Mutex<Option<tokio::sync::OwnedSemaphorePermit>>,
103    slot: Mutex<Option<tokio::sync::OwnedSemaphorePermit>>,
104    launch_handoff: tokio::sync::watch::Sender<bool>,
105    heartbeat_stop: CancellationToken,
106    heartbeat: tokio::sync::Mutex<Option<tokio::task::JoinHandle<()>>>,
107}
108
109impl Execution {
110    async fn persist_stop_unconfirmed(&self) -> Result<Session, String> {
111        // The same birth/fence-bound writer owns this narrow observation. A
112        // task panic must never checkpoint its partially mutated transcript.
113        let mut current = self
114            .persistence
115            .load_runtime_control_plane(&self.owner.fence.actor_id)
116            .await
117            .map_err(|e| e.to_string())?
118            .ok_or("Child missing while recording unconfirmed stop")?;
119        let same_observation = current
120            .metadata
121            .get("runtime.local_task.state")
122            .map(String::as_str)
123            == Some("stop_unconfirmed")
124            && current.metadata.get("runtime.local_task.run_id") == Some(&self.owner.fence.run_id)
125            && current.metadata.get("runtime.local_task.child_birth")
126                == Some(&self.owner.created_at.to_rfc3339())
127            && current
128                .metadata
129                .get("runtime.local_task.reason")
130                .map(String::as_str)
131                == Some("execution_stop_unconfirmed")
132            && current
133                .metadata
134                .get("runtime.local_task.observed_at")
135                .is_some_and(|value| chrono::DateTime::parse_from_rfc3339(value).is_ok());
136        if !same_observation {
137            record_local_task_observation(
138                &mut current,
139                &self.owner.fence.run_id,
140                self.owner.created_at,
141                "stop_unconfirmed",
142            );
143            current.metadata.insert(
144                "runtime.local_task.reason".into(),
145                "execution_stop_unconfirmed".into(),
146            );
147        }
148        current.metadata.remove("runtime.local_task.instance");
149        self.persistence
150            .save_runtime_control_plane(&mut current)
151            .await
152            .map_err(|e| e.to_string())?;
153        let readback = self
154            .persistence
155            .load_runtime_control_plane(&self.owner.fence.actor_id)
156            .await
157            .map_err(|e| e.to_string())?
158            .ok_or("Child missing after unconfirmed-stop checkpoint")?;
159        if LOCAL_TASK_OBSERVATION_KEYS
160            .iter()
161            .any(|key| current.metadata.get(*key) != readback.metadata.get(*key))
162        {
163            return Err("unconfirmed-stop checkpoint readback differs".into());
164        }
165        Ok(readback)
166    }
167
168    async fn wait_for_handoff(&self) {
169        let mut handoff = self.launch_handoff.subscribe();
170        while !*handoff.borrow_and_update() {
171            // This Execution owns the sender, so cancellation of any other
172            // waiter cannot close it or consume another waiter's wakeup.
173            handoff
174                .changed()
175                .await
176                .expect("execution retains handoff sender");
177        }
178    }
179}
180
181pub struct LocalTaskChildRunner {
182    bindings: Mutex<Bindings>,
183    executions: Mutex<HashMap<String, Arc<Execution>>>,
184    coordinators: Mutex<Vec<CoordinatorHandle>>,
185    coordinator_errors: Mutex<Vec<String>>,
186    concurrency: Arc<Semaphore>,
187    admission: Arc<Semaphore>,
188    preparations: Arc<Semaphore>,
189    closed: AtomicBool,
190    ceiling: Option<Arc<dyn NativeToolCeilingSource>>,
191    permission: Option<Arc<bamboo_tools::permission::PermissionConfig>>,
192    remote_roles: Vec<String>,
193    approval_registry: Option<super::approval_registry::SharedApprovalRegistry>,
194    approval_reviewer: Option<Arc<dyn super::actor_adapter::ChildApprovalReviewer>>,
195}
196
197type CoordinatorHandle = tokio::task::JoinHandle<Result<(), String>>;
198
199/// Shutdown may be cancelled while a publication tail still owns a writer.
200/// Keep the real join handle available to the next shutdown attempt.
201struct RetainedCoordinatorJoin<'a> {
202    queue: &'a Mutex<Vec<CoordinatorHandle>>,
203    handle: Option<CoordinatorHandle>,
204}
205impl Drop for RetainedCoordinatorJoin<'_> {
206    fn drop(&mut self) {
207        if let Some(handle) = self.handle.take() {
208            lock(self.queue).push(handle);
209        }
210    }
211}
212
213fn failure(error: impl std::fmt::Display) -> AgentError {
214    AgentError::LLM(format!("local task backend: {error}"))
215}
216const LOCAL_TASK_OBSERVATION_KEYS: [&str; 6] = [
217    "runtime.local_task.state",
218    "runtime.local_task.run_id",
219    "runtime.local_task.child_birth",
220    "runtime.local_task.observed_at",
221    "runtime.local_task.reason",
222    "runtime.local_task.instance",
223];
224
225fn record_local_task_observation(
226    session: &mut Session,
227    run_id: &str,
228    child_birth: chrono::DateTime<chrono::Utc>,
229    state: &str,
230) {
231    for key in LOCAL_TASK_OBSERVATION_KEYS {
232        session.metadata.remove(key);
233    }
234    session
235        .metadata
236        .insert("runtime.local_task.state".into(), state.into());
237    session
238        .metadata
239        .insert("runtime.local_task.run_id".into(), run_id.into());
240    session.metadata.insert(
241        "runtime.local_task.child_birth".into(),
242        child_birth.to_rfc3339(),
243    );
244    session.metadata.insert(
245        "runtime.local_task.observed_at".into(),
246        chrono::Utc::now().to_rfc3339(),
247    );
248}
249
250fn copy_local_task_observation(from: &Session, to: &mut Session) {
251    for key in LOCAL_TASK_OBSERVATION_KEYS {
252        match from.metadata.get(key) {
253            Some(value) => {
254                to.metadata.insert(key.into(), value.clone());
255            }
256            None => {
257                to.metadata.remove(key);
258            }
259        }
260    }
261}
262
263fn record_stop_metadata(session: &mut Session, proof: &ConfirmedTaskStop) {
264    record_local_task_observation(session, &proof.run_id, proof.child_birth, "stopped");
265    session.metadata.insert(
266        "runtime.local_task.instance".into(),
267        proof.instance_id.to_string(),
268    );
269    session
270        .metadata
271        .insert("runtime.local_task.run_id".into(), proof.run_id.clone());
272    if session
273        .metadata
274        .get(bamboo_subagent::proto::WORKFLOW_USAGE_REQUESTED_KEY)
275        .is_some_and(|v| v == "true")
276    {
277        session.metadata.insert(
278            bamboo_subagent::proto::WORKFLOW_TERMINAL_OBSERVATION_KEY.into(),
279            proof.run_id.clone(),
280        );
281    }
282}
283
284fn reconcile_question_handoff(session: &mut Session, previous_request: Option<&str>) {
285    let issued_here = session
286        .metadata
287        .get(bamboo_domain::PARENT_QUESTION_REQUEST_KEY)
288        .is_some_and(|request| Some(request.as_str()) != previous_request);
289    if issued_here {
290        // Preserve this run's handoff even if a fast parent answer already
291        // cleared pending_question before the Agent returned.
292        session.metadata.insert(
293            "runtime.actor_parent_question_handoff".into(),
294            "true".into(),
295        );
296    } else {
297        session
298            .metadata
299            .remove("runtime.actor_parent_question_handoff");
300    }
301}
302
303fn lock<T>(mutex: &Mutex<T>) -> std::sync::MutexGuard<'_, T> {
304    mutex.lock().unwrap_or_else(|error| error.into_inner())
305}
306
307impl LocalTaskChildRunner {
308    fn retain_shutdown_error(&self, error: String) {
309        let mut errors = lock(&self.coordinator_errors);
310        if !errors.contains(&error) {
311            errors.push(error);
312        }
313    }
314
315    pub fn new(
316        config: &Config,
317        ceiling: Option<Arc<dyn NativeToolCeilingSource>>,
318        permission: Option<Arc<bamboo_tools::permission::PermissionConfig>>,
319    ) -> Self {
320        Self {
321            bindings: Mutex::new(Bindings::default()),
322            executions: Mutex::new(HashMap::new()),
323            coordinators: Mutex::new(Vec::new()),
324            coordinator_errors: Mutex::new(Vec::new()),
325            concurrency: Arc::new(Semaphore::new(
326                config
327                    .subagents()
328                    .max_concurrent
329                    .unwrap_or(super::actor_adapter::DEFAULT_MAX_CONCURRENT_ACTORS)
330                    .max(1),
331            )),
332            admission: Arc::new(Semaphore::new(
333                config
334                    .subagents()
335                    .max_concurrent
336                    .unwrap_or(super::actor_adapter::DEFAULT_MAX_CONCURRENT_ACTORS)
337                    .max(1)
338                    .saturating_add(128),
339            )),
340            preparations: Arc::new(Semaphore::new(128)),
341            closed: AtomicBool::new(false),
342            ceiling,
343            permission,
344            approval_registry: None,
345            approval_reviewer: None,
346            remote_roles: config
347                .subagents()
348                .remote_placements
349                .iter()
350                .map(|p| p.role.clone())
351                .chain(
352                    config
353                        .subagents()
354                        .schedulable_placements
355                        .iter()
356                        .map(|p| p.role.clone()),
357                )
358                .collect(),
359        }
360    }
361    pub fn with_approval(
362        mut self,
363        registry: Option<super::approval_registry::SharedApprovalRegistry>,
364        reviewer: Option<Arc<dyn super::actor_adapter::ChildApprovalReviewer>>,
365    ) -> Self {
366        self.approval_registry = registry;
367        self.approval_reviewer = reviewer;
368        self
369    }
370    async fn stop_unstarted(
371        &self,
372        execution: &Execution,
373        session: &mut Session,
374        run_id: &str,
375    ) -> Result<(), String> {
376        {
377            let mut state = lock(&execution.launch);
378            match *state {
379                LaunchState::Queued | LaunchState::Preparing => *state = LaunchState::Stopping,
380                LaunchState::Stopped => return Ok(()),
381                _ => return Err("local execution launch or stop is already in progress".into()),
382            }
383        }
384        let drained = execution
385            .resources
386            .close_and_drain()
387            .await
388            .map_err(|e| e.to_string())?;
389        let identity = drained.identity();
390        if identity.session_id != session.id
391            || identity.run_id != run_id
392            || identity.child_birth != session.created_at
393        {
394            return Err("local resource identity changed during rollback".into());
395        }
396        let proof = ConfirmedTaskStop {
397            session_id: session.id.clone(),
398            child_birth: session.created_at,
399            run_id: run_id.into(),
400            instance_id: identity.owner_id,
401        };
402        record_stop_metadata(session, &proof);
403        *lock(&execution.stop) = Some(proof);
404        *lock(&execution.launch) = LaunchState::Stopped;
405        Ok(())
406    }
407    fn execution(&self, session: &Session, run_id: &str) -> Result<Arc<Execution>, String> {
408        let execution = lock(&self.executions)
409            .get(&session.id)
410            .cloned()
411            .ok_or("local execution ownership is missing")?;
412        if execution.owner.created_at != session.created_at
413            || execution.owner.fence.run_id != run_id
414        {
415            return Err("local execution birth or generation changed".into());
416        }
417        Ok(execution)
418    }
419    fn runtime(&self) -> Result<LocalTaskRuntime, String> {
420        lock(&self.bindings)
421            .runtime
422            .clone()
423            .ok_or("Host local task runtime is unavailable".into())
424    }
425}
426
427#[async_trait]
428impl ExternalChildRunner for LocalTaskChildRunner {
429    fn supports_native_ticket_budget(&self) -> bool {
430        true
431    }
432
433    fn set_local_child_skill_factory(&self, factory: Option<Arc<dyn LocalChildSkillFactory>>) {
434        lock(&self.bindings).skills = factory;
435    }
436    fn set_local_task_runtime(&self, runtime: Option<LocalTaskRuntime>) {
437        lock(&self.bindings).runtime = runtime;
438    }
439    fn set_session_inbox_runtime(&self, inbox: Option<SessionInboxRuntimeBinding>) {
440        lock(&self.bindings).inbox = inbox;
441    }
442    fn set_actor_directory_store(&self, store: Option<Arc<bamboo_storage::SessionStoreV2>>) {
443        lock(&self.bindings).store = store;
444    }
445    fn set_ticket_service(&self, service: Option<Arc<bamboo_tickets::TicketService>>) {
446        let mut bindings = lock(&self.bindings);
447        // Serialize this check with the successful drain's cache release. A
448        // late scope publisher must not reattach a writer to a closed runner.
449        if service.is_some() && self.closed.load(Ordering::Acquire) {
450            return;
451        }
452        bindings.tickets = service;
453    }
454    fn set_canonical_subagent_tool(&self, tool: Option<Arc<dyn bamboo_agent_core::tools::Tool>>) {
455        lock(&self.bindings).subagent = tool;
456    }
457    fn set_actor_event_observer(
458        &self,
459        observer: Option<Arc<dyn super::actor_event_stream::ActorEventObserver>>,
460    ) {
461        lock(&self.bindings).observer = observer;
462    }
463
464    async fn should_handle(&self, session: &Session) -> bool {
465        session.kind == SessionKind::Child
466            && session
467                .metadata
468                .get("external.agent_id")
469                .is_none_or(|id| id == super::config::LOCAL_ACTOR_AGENT_ID)
470            && !self
471                .remote_roles
472                .iter()
473                .any(|role| session.metadata.get("subagent_type") == Some(role))
474    }
475    async fn validate_required_child_context_route(&self, session: &Session) -> Result<(), String> {
476        // Admission probes use a synthetic Child before its durable birth.
477        // Check only its configured route here; execution separately proves
478        // the canonical Child, lineage and activation ownership.
479        if session
480            .metadata
481            .get("external.agent_id")
482            .is_some_and(|id| id != super::config::LOCAL_ACTOR_AGENT_ID)
483            || self
484                .remote_roles
485                .iter()
486                .any(|role| session.metadata.get("subagent_type") == Some(role))
487            || self.ceiling.is_none()
488        {
489            return Err(
490                "required_child_context_unsupported: local native ceiling unavailable".into(),
491            );
492        }
493        self.runtime()?;
494        let bindings = lock(&self.bindings);
495        if bindings.store.is_none() || bindings.inbox.is_none() {
496            return Err("canonical local Child storage is unavailable".into());
497        }
498        Ok(())
499    }
500
501    async fn prepare_child_execution(
502        &self,
503        session: &Session,
504        run_id: &str,
505        cancel: CancellationToken,
506    ) -> Result<Option<Arc<dyn RuntimeSessionPersistence>>, String> {
507        let _preparation = self
508            .preparations
509            .clone()
510            .try_acquire_owned()
511            .map_err(|_| "local Child preparation capacity exhausted")?;
512        if self.closed.load(Ordering::Acquire) {
513            return Err("local task backend is shutting down".into());
514        }
515        // Keep scheduler backpressure: an accepted setup waits for bounded
516        // capacity instead of detaching an unbounded pending coordinator.
517        let admission = tokio::select! {
518            biased;
519            _ = cancel.cancelled() => return Err("local Child cancelled before admission".into()),
520            permit = self.admission.clone().acquire_owned() => permit.map_err(|_| "local task backend is shutting down")?,
521        };
522        let runtime = self.runtime()?;
523        let (store, inbox) = {
524            let bindings = lock(&self.bindings);
525            (
526                bindings
527                    .store
528                    .clone()
529                    .ok_or("canonical local Actor store unavailable")?,
530                bindings
531                    .inbox
532                    .clone()
533                    .ok_or("canonical local Inbox unavailable")?,
534            )
535        };
536        if !inbox.router.owns_run(&session.id, run_id).await {
537            return Err("local Child activation is not owned by this Host run".into());
538        }
539        if lock(&self.executions).contains_key(&session.id) {
540            return Err("previous local execution has not been finalized".into());
541        }
542        let durable = store
543            .load_session(&session.id)
544            .await
545            .map_err(|e| e.to_string())?
546            .ok_or("canonical Child is missing")?;
547        if durable.created_at != session.created_at
548            || durable.parent_created_at != session.parent_created_at
549            || durable.parent_session_id != session.parent_session_id
550            || durable.root_session_id != session.root_session_id
551            || durable.project_id_meta() != session.project_id_meta()
552        {
553            return Err("canonical Child birth or lineage changed".into());
554        }
555        if durable
556            .metadata
557            .get("runtime.local_task.state")
558            .is_some_and(|state| state != "stopped")
559        {
560            return Err("previous local task has no confirmed stop; recovery is blocked".into());
561        }
562        store
563            .ensure_actor(&session.id)
564            .await
565            .map_err(|e| e.to_string())?;
566        store
567            .bind_control_plane(
568                &session.id,
569                bamboo_domain::ActorControlPlaneBinding {
570                    policy_revision: self.permission.as_ref().map(|p| p.policy_revision()),
571                    placement_intent: Some(bamboo_domain::ActorPlacementIntent::Local),
572                },
573            )
574            .await
575            .map_err(|e| e.to_string())?;
576        let locked = inbox
577            .parent_question_lock
578            .as_ref()
579            .ok_or("canonical local write coordinator unavailable")?;
580        let now = chrono::Utc::now();
581        let claimed = store
582            .claim_activation(&ActorActivationClaim {
583                actor_id: session.id.clone(),
584                run_id: run_id.into(),
585                lease_owner: format!("host-task-{}", uuid::Uuid::new_v4()),
586                lease_expires_at: now + chrono::Duration::seconds(30),
587                inbox_generation: session
588                    .session_inbox_admission()
589                    .map_or(0, |state| state.last_admitted_sequence),
590                placement_ref: None,
591                now,
592            })
593            .await
594            .map_err(|e| e.to_string())?;
595        let setup = async {
596            let owner = ChildActorRuntimeWrite::from_session(claimed.fence(), session)
597                .map_err(|e| e.to_string())?;
598            let cache = runtime.sessions_cache.clone();
599            let persistence: Arc<dyn RuntimeSessionPersistence> = Arc::new(
600                locked
601                    .bind_child_actor_writer(
602                        owner.clone(),
603                        store.clone(),
604                        Arc::new(move |saved| {
605                            cache.insert(
606                                saved.id.clone(),
607                                Arc::new(crate::SessionSnapshot::new(saved.clone())),
608                            );
609                        }),
610                    )
611                    .map_err(|e| e.to_string())?,
612            );
613            store
614                .start_activation(&owner.fence, now)
615                .await
616                .map_err(|e| e.to_string())?;
617            Ok::<_, String>((owner, persistence))
618        }
619        .await;
620        let (owner, persistence) = match setup {
621            Ok(bound) => bound,
622            Err(error) => {
623                return match store
624                    .finish_activation(
625                        &claimed.fence(),
626                        chrono::Utc::now(),
627                        ActorActivationFinish::Failed,
628                    )
629                    .await
630                {
631                    Ok(_) => Err(error),
632                    Err(rollback) => Err(format!(
633                        "{error}; exact claim rollback unconfirmed: {rollback}"
634                    )),
635                };
636            }
637        };
638        let heartbeat_stop = CancellationToken::new();
639        let heartbeat = {
640            let store = store.clone();
641            let fence = owner.fence.clone();
642            let done = heartbeat_stop.clone();
643            let cancel = cancel.clone();
644            tokio::spawn(async move {
645                let mut interval = tokio::time::interval(Duration::from_secs(5));
646                interval.tick().await;
647                loop {
648                    tokio::select! { _ = done.cancelled() => break, _ = interval.tick() => {} }
649                    let now = chrono::Utc::now();
650                    if store
651                        .renew_activation(&fence, now, now + chrono::Duration::seconds(30))
652                        .await
653                        .is_err()
654                    {
655                        cancel.cancel();
656                        break;
657                    }
658                }
659            })
660        };
661        let execution = Arc::new(Execution {
662            owner,
663            persistence: persistence.clone(),
664            cancel,
665            resources: bamboo_tools::LocalTaskResources::new(
666                &session.id,
667                run_id,
668                session.created_at,
669            ),
670            task: tokio::sync::Mutex::new(None),
671            stop: Mutex::new(None),
672            finalized: AtomicBool::new(false),
673            launch: Mutex::new(LaunchState::Queued),
674            admission: Mutex::new(Some(admission)),
675            slot: Mutex::new(None),
676            launch_handoff: tokio::sync::watch::channel(false).0,
677            heartbeat_stop,
678            heartbeat: tokio::sync::Mutex::new(Some(heartbeat)),
679        });
680        lock(&self.executions).insert(session.id.clone(), execution);
681        Ok(Some(persistence))
682    }
683
684    async fn execute_external_child(
685        &self,
686        session: &mut Session,
687        job: &SpawnJob,
688        event_tx: mpsc::Sender<AgentEvent>,
689        cancel_token: CancellationToken,
690    ) -> crate::runtime::runner::Result<()> {
691        let runtime = self.runtime().map_err(failure)?;
692        let (binding, tickets) = {
693            let b = lock(&self.bindings);
694            (
695                b.inbox
696                    .clone()
697                    .ok_or_else(|| failure("Inbox unavailable"))?,
698                b.tickets.clone(),
699            )
700        };
701        let run_id = binding
702            .router
703            .current_run_id(&session.id)
704            .await
705            .ok_or_else(|| failure("activation missing"))?;
706        let execution = self.execution(session, &run_id).map_err(failure)?;
707        {
708            let mut state = lock(&execution.launch);
709            if *state != LaunchState::Queued {
710                return Err(failure("duplicate or closed local task launch"));
711            }
712            *state = LaunchState::Preparing;
713        }
714        let result = async {
715        let slot = tokio::select! { biased; _ = cancel_token.cancelled() => return Err(AgentError::Cancelled), slot = self.concurrency.clone().acquire_owned() => slot.map_err(failure)? };
716        *lock(&execution.slot) = Some(slot);
717        crate::session_app::child_session::named_profile::validate_named_profile(session)
718            .map_err(failure)?;
719        if let (Some(policy), Some(workspace)) = (&self.permission, &session.workspace) {
720            policy.register_session_workspace(session.id.clone(), workspace);
721        }
722        let strict = skill_host::strict_child(session)?;
723        let names = if strict { match &self.ceiling {
724            Some(source) => Some(source.observe(session).await.map_err(failure)?),
725            None => return Err(failure("strict native tool ceiling unavailable")),
726        }} else { None };
727        let mut ticket_plan = None;
728        if session.metadata.contains_key(crate::ticket_worker_plan::TICKET_LOCAL_PLAN_KEY) {
729            let service = tickets.clone().ok_or_else(||failure("Ticket authority unavailable"))?;
730            crate::ticket_runtime::admit_ticket_run(&service, &binding, session, &run_id).await.map_err(failure)?;
731            let assignment = session.metadata[crate::ticket_worker_plan::TICKET_LOCAL_PLAN_KEY].clone();
732            let plan = Arc::new(crate::ticket_worker_plan::TicketWorkerPlan::from_runtime_receipt(service, &assignment).map_err(failure)?);
733            session.agent_runtime_state.get_or_insert_with(Default::default).run_id = run_id.clone();
734            plan.bind_session(session).map_err(failure)?;
735            ticket_plan = Some(plan);
736        }
737        let mut tools: Arc<dyn ToolExecutor> = runtime.tools.clone();
738        let store = lock(&self.bindings).store.clone().ok_or_else(||failure("local store unavailable"))?;
739        let observer = store.supervisor_observer_activation(session).await.map_err(failure)?;
740        let factory = if strict || observer { None } else { lock(&self.bindings).skills.clone() };
741        let mut child_skill_host = None;
742        let mut child_skill_authority = None;
743        if let Some(factory) = factory {
744            let authority = LocalChildSkillAuthority::bind(store.clone(), &execution);
745            let binding = factory.bind(authority.clone(), session, tools).await?;
746            child_skill_authority = Some(authority);
747            tools = binding.tools;
748            child_skill_host = Some(binding.host);
749        }
750        let names = names.unwrap_or_else(||tools.list_tools().into_iter().map(|s|s.function.name).collect());
751        let local_tools = tool_policy::LocalChildTools::new(tools, names.clone(), session, store.clone(), execution.owner.clone(), job.disabled_tools.as_deref())?.with_canonical_subagent(lock(&self.bindings).subagent.clone());
752        tools = Arc::new(if observer { local_tools.restrict_to_observer() } else { local_tools });
753        if session.metadata.contains_key(crate::ticket_worker_plan::TICKET_LOCAL_PLAN_KEY) {
754            tools = Arc::new(crate::ticket_worker_plan::files::LocalFileExecutor::new(
755                tools, tickets.clone().ok_or_else(||failure("Ticket authority unavailable"))?, session.clone(), run_id.clone(), &names
756            ).map_err(failure)?);
757        }
758        if let Some(authority) = child_skill_authority { authority.finalize_tools(tools.as_ref()); }
759        let (local_event_tx, local_event_rx) = mpsc::channel(256);
760        let mut request = ExecuteRequestBuilder::new("", local_event_tx.clone(), cancel_token.clone())
761            .tools(tools)
762            .model(job.model.clone());
763        if strict || observer { request = request.selected_skill_ids(Vec::new()); }
764        else {
765            if let Some(ids) = session.metadata.get("selected_skill_ids").and_then(|raw|bamboo_skills::selection::parse_selected_skill_ids_metadata(raw)) { request = request.selected_skill_ids(ids); }
766            if let Some(mode) = session.metadata.get("skill_mode").or_else(||session.metadata.get("mode")).map(|v|v.trim()).filter(|v|!v.is_empty()) { request = request.selected_skill_mode(mode); }
767        }
768        if let Some(data_dir) = runtime.app_data_dir.clone() {
769            request = request.app_data_dir(data_dir);
770        }
771        if let Some(disabled) = &job.disabled_tools {
772            request = request.disabled_tools(disabled.iter().cloned().collect());
773        }
774        if observer {
775            request = request.run_budget(bamboo_config::RunBudgetConfig {
776                max_rounds: Some(6),
777                max_tool_calls: Some(8),
778                ..Default::default()
779            });
780        }
781        if let Some(reference) = crate::session_app::provider_model::session_effective_model_ref(session) {
782            if let Some(router) = &runtime.provider_router {
783                request = request.model(reference.model.clone())
784                    .provider_override(router.route(&reference).map_err(failure)?)
785                    .provider_name(reference.provider.clone());
786                if let Some(kind) = router.provider_type_for(&reference) {
787                    request = request.provider_type(kind);
788                }
789            }
790        }
791        if let Some(effort) = session.reasoning_effort {
792            request = request.reasoning_effort(effort);
793        }
794        if let Some(plan) = ticket_plan {
795            // Admission above binds this exact canonical Child/Run/dispatch.
796            // Carry its Host ceiling into the request even if queued execution
797            // observes a later relaxation of the global configuration.
798            if let Some(cap) = crate::ticket_runtime::read_dispatch(session).map_err(failure)?.and_then(|binding| binding.max_rounds) {
799                if cap == 0 { return Err(failure("zero Host round ceiling")); }
800                request = request.run_budget(bamboo_config::RunBudgetConfig { max_rounds: Some(cap), ..Default::default() });
801            }
802            request = request.ticket_worker_plan(plan);
803        }
804        if let Some(budget) = crate::ticket_token_budget::NativeWorkTokenBudget::capture(
805            tickets.clone(), store.clone(), session
806        ).await.map_err(failure)? {
807            request = request.ticket_token_budget(budget);
808        }
809        let mut owned_session = session.clone();
810        // The question fact survives its answer for replay/deduplication. The
811        // handoff is only for the run that issued that fact, not every later
812        // run that happens to retain it in canonical history.
813        let parent_question_at_start = owned_session.metadata.get(bamboo_domain::PARENT_QUESTION_REQUEST_KEY).cloned();
814        owned_session.metadata.remove("runtime.actor_parent_question_handoff");
815        owned_session.metadata.remove(bamboo_subagent::proto::WORKFLOW_TERMINAL_OBSERVATION_KEY);
816        owned_session.metadata.remove(bamboo_subagent::proto::WORKFLOW_USAGE_OBSERVATION_KEY);
817        record_local_task_observation(&mut owned_session, &run_id, execution.owner.created_at, "running");
818        execution
819            .persistence
820            .checkpoint_runtime_session(&mut owned_session)
821            .await
822            .map_err(failure)?;
823        let mut agent = runtime
824            .agent
825            .with_execution_persistence(execution.persistence.clone());
826        if strict {
827            agent = agent.with_skill_execution_host(skill_host::StrictChildSkills::bind(store, &execution));
828        } else if let Some(host) = child_skill_host {
829            agent = agent.with_skill_execution_host(host);
830        }
831        let resources = execution.resources.clone();
832        let task_cancel = cancel_token.clone();
833        let mut handle = execution.task.lock().await;
834        {
835            let mut state = lock(&execution.launch);
836            if *state != LaunchState::Preparing { return Err(failure("local launch gate is closed")); }
837            if cancel_token.is_cancelled() || self.closed.load(Ordering::Acquire) { return Err(AgentError::Cancelled); }
838            *state = LaunchState::Running;
839        }
840        let approval_proxy: Option<Arc<dyn bamboo_tools::ApprovalProxy>> = match (&self.permission, &self.approval_reviewer) {
841            (Some(policy), Some(reviewer)) => Some(Arc::new(approval::LocalApprovalProxy::new(approval::LocalApprovalContext {
842                store: lock(&self.bindings).store.clone().expect("prepared store"), owner: execution.owner.clone(), router: binding.router.clone(), cancel: cancel_token.clone(), policy: policy.clone(), reviewer: reviewer.clone(), registry: self.approval_registry.clone(),
843            }))),
844            _ => None,
845        };
846        let (event_store, event_observer) = { let bindings=lock(&self.bindings); (bindings.store.clone().expect("prepared store"), bindings.observer.clone()) };
847        let event_owner = execution.owner.clone();
848        let event_cancel = cancel_token.clone();
849        let task = tokio::spawn(async move {
850            let forward = tokio::spawn(events::forward(event_store, event_owner, event_observer, local_event_rx, event_tx, event_cancel));
851            let result = std::panic::AssertUnwindSafe(
852                resources.scope(bamboo_tools::with_approval_proxy(approval_proxy, async {
853                    if let Some(audit) = bamboo_domain::PermissionAuditSnapshot::from_metadata(&owned_session.metadata) {
854                        local_event_tx.send(AgentEvent::PermissionPostureActivated { session_id: owned_session.id.clone(), policy_revision: audit.policy_revision, requested_mode: audit.resolution.requested.as_str().to_owned(), effective_mode: audit.resolution.effective.as_str().to_owned(), executor_mapping: audit.executor_mapping }).await.map_err(failure)?;
855                    }
856                    drop(local_event_tx);
857                    // This owned execution has now taken over the admitted
858                    // resume. Mirror the normal Root execution seam: retained
859                    // answer/retry handoff markers are not future pending work.
860                    // Do not consume them while queued or during preparation.
861                    crate::session_app::execute::consume_pending_clarification_resume(&mut owned_session);
862                    agent.execute(&mut owned_session, request.build()).await
863                })),
864            )
865            .catch_unwind()
866            .await;
867            // This is a private execution token. Finalization cancels remaining
868            // safe auxiliary requests even when the main loop returned normally.
869            task_cancel.cancel();
870            if result.is_err() { resources.record_unconfirmed("Agent task panicked; physical effects remain unconfirmed"); }
871            let mut drained = resources.close_and_drain().await;
872            let mut result = match result {
873                Ok(result) => result,
874                Err(_) => {
875                    // The loop itself may have unwound a Tokio filesystem or
876                    // custom-tool future outside registered builtin resources.
877                    // Joining the task proves exit, not those physical effects.
878                    drained = Err(std::io::Error::other("Agent task panicked; physical effects remain unconfirmed"));
879                    Err(failure("Agent task panicked"))
880                },
881            };
882            match forward.await {
883                Ok(Ok(())) => {},
884                Ok(Err(error)) => result = Err(failure(format!("local event publication failed: {error}"))),
885                Err(error) => { drained = Err(std::io::Error::other(format!("local event owner panicked: {error}"))); result = Err(failure(error)); },
886            }
887            reconcile_question_handoff(&mut owned_session, parent_question_at_start.as_deref());
888            ExecutionOutput {
889                session: owned_session,
890                result,
891                resources: drained,
892            }
893        });
894        *handle = Some(task);
895        let joined = handle.as_mut().expect("owned task").await;
896        handle.take();
897        let mut output = joined.map_err(|error| failure(format!("Agent task join failed: {error}")))?;
898        *session = output.session;
899        let drained = output
900            .resources
901            .map_err(|error| failure(format!("resource stop unconfirmed: {error}")))?;
902        let identity = drained.identity();
903        if identity.session_id != session.id
904            || identity.run_id != run_id
905            || identity.child_birth != session.created_at
906        {
907            return Err(failure("resource owner changed"));
908        }
909        let proof = ConfirmedTaskStop {
910            session_id: session.id.clone(),
911            child_birth: session.created_at,
912            run_id: run_id.clone(),
913            instance_id: identity.owner_id,
914        };
915        record_stop_metadata(session, &proof);
916        // A Host-limited Ticket which exhausted its rounds did not complete
917        // its contract. Preserve the real stop proof, then use the existing
918        // failure delivery path instead of attempting a success Submission
919        // from a summary turn which may contain only more tool calls.
920        if output.result.is_ok()
921            && session.metadata.get("runtime.completion_reason").map(String::as_str)
922                == Some("max_rounds_reached")
923            && crate::ticket_runtime::read_dispatch(session).map_err(failure)?
924                .is_some_and(|dispatch| dispatch.max_rounds.is_some())
925        {
926            output.result = Err(AgentError::Budget("Ticket run round budget exhausted".into()));
927        }
928        if session.metadata.get(bamboo_subagent::proto::WORKFLOW_USAGE_REQUESTED_KEY).is_some_and(|v|v == "true") {
929            if let Some(runtime) = session.agent_runtime_state.as_ref().filter(|runtime|runtime.run_id == run_id) {
930                let usage = bamboo_subagent::proto::WorkflowAgentUsage { kind: bamboo_subagent::proto::WorkflowAgentUsage::TYPE.into(), activation_run_id: run_id.clone(), child_session_id: session.id.clone(), child_created_at: session.created_at, prompt_tokens: runtime.round.total_prompt_tokens, completion_tokens: runtime.round.total_completion_tokens };
931                session.metadata.insert(bamboo_subagent::proto::WORKFLOW_USAGE_OBSERVATION_KEY.into(), serde_json::to_string(&usage).map_err(failure)?);
932            }
933        }
934        *lock(&execution.stop) = Some(proof.clone());
935        *lock(&execution.launch) = LaunchState::Stopped;
936        let stopped_tickets = lock(&self.bindings).tickets.clone();
937        if let Some(tickets) = stopped_tickets {
938            if session
939                .metadata
940                .contains_key(crate::ticket_worker_plan::TICKET_LOCAL_PLAN_KEY)
941            {
942                crate::ticket_runtime::checkpoint_owned_task_stop(
943                    &tickets,
944                    &binding,
945                    session,
946                    &run_id,
947                    &proof,
948                    output.result.is_ok(),
949                )
950                .await
951                .map_err(failure)?;
952            }
953        }
954        output.result
955        }.await;
956        // No-work stop is proved by permanently closing the launch gate and
957        // draining the resource domain, never by inventing a PID or a task join.
958        let result = async {
959            let unstarted = *lock(&execution.launch) == LaunchState::Preparing;
960            if unstarted {
961                self.stop_unstarted(&execution, session, &run_id)
962                    .await
963                    .map_err(failure)?;
964            }
965            if unstarted
966                && crate::ticket_runtime::read_dispatch(session)
967                    .ok()
968                    .flatten()
969                    .is_some_and(|d| d.receipt.is_some())
970            {
971                let proof = lock(&execution.stop)
972                    .clone()
973                    .ok_or_else(|| failure("local stop proof missing"))?;
974                if let Some(tickets) = tickets {
975                    crate::ticket_runtime::checkpoint_owned_task_stop(
976                        &tickets, &binding, session, &run_id, &proof, false,
977                    )
978                    .await
979                    .map_err(failure)?;
980                }
981            }
982            result
983        }
984        .await;
985        match result {
986            Err(error) if lock(&execution.stop).is_none() => {
987                match execution.persist_stop_unconfirmed().await {
988                    Ok(observed) => {
989                        copy_local_task_observation(&observed, session);
990                        Err(error)
991                    }
992                    Err(checkpoint) => Err(failure(format!(
993                        "{error}; unconfirmed-stop observation was not durably confirmed: {checkpoint}"
994                    ))),
995                }
996            }
997            result => result,
998        }
999    }
1000
1001    async fn abandon_child_execution(&self, session: &Session, run_id: &str) -> Result<(), String> {
1002        let execution = self.execution(session, run_id)?;
1003        execution.cancel.cancel();
1004        execution.launch_handoff.send_replace(true);
1005        let mut stopped = session.clone();
1006        if let Err(error) = self.stop_unstarted(&execution, &mut stopped, run_id).await {
1007            return match execution.persist_stop_unconfirmed().await {
1008                Ok(_) => Err(error),
1009                Err(checkpoint) => Err(format!(
1010                    "{error}; unconfirmed-stop observation was not durably confirmed: {checkpoint}"
1011                )),
1012            };
1013        }
1014        stopped.set_last_run_status("error");
1015        stopped.set_last_run_error("initial local Child checkpoint failed");
1016        execution
1017            .persistence
1018            .checkpoint_runtime_session(&mut stopped)
1019            .await
1020            .map_err(|e| e.to_string())?;
1021        self.finish_child_execution(&stopped, run_id, true).await
1022    }
1023
1024    fn quarantine_child_execution(&self, session: &Session, run_id: &str) {
1025        if let Ok(execution) = self.execution(session, run_id) {
1026            execution.cancel.cancel();
1027            let mut state = lock(&execution.launch);
1028            if matches!(*state, LaunchState::Queued | LaunchState::Preparing) {
1029                *state = LaunchState::Stopping;
1030            }
1031            execution.launch_handoff.send_replace(true);
1032        }
1033    }
1034
1035    fn track_child_task(
1036        &self,
1037        session_id: &str,
1038        run_id: &str,
1039        handle: tokio::task::JoinHandle<()>,
1040    ) -> Option<tokio::task::JoinHandle<()>> {
1041        let execution = lock(&self.executions).get(session_id).cloned();
1042        if let Some(execution) = execution.filter(|e| e.owner.fence.run_id == run_id) {
1043            // Admission remains occupied through the SDK publication tail.
1044            // The retained supervisor owns and joins the actual coordinator;
1045            // completed permits are returned without waiting for a later spawn.
1046            let permit = lock(&execution.admission).take();
1047            let retained_execution = execution.clone();
1048            let supervisor = tokio::spawn(async move {
1049                let mut result = handle.await.map_err(|error| error.to_string());
1050                if !retained_execution.finalized.load(Ordering::Acquire) {
1051                    // A coordinator exiting does not make an unconfirmed or
1052                    // unpublished execution eligible for replacement.
1053                    if let Some(permit) = permit {
1054                        *lock(&retained_execution.admission) = Some(permit);
1055                    }
1056                    if lock(&retained_execution.stop).is_none() {
1057                        retained_execution.cancel.cancel();
1058                        // The coordinator may have unwound while its separate
1059                        // Agent task still owned a canonical writer. Cancellation
1060                        // alone does not stop those writes: join that task and
1061                        // all registered resources before recording the final
1062                        // unknown observation. This recovery does not mint a
1063                        // normal execution/publication stop receipt.
1064                        let mut orphaned_task = retained_execution.task.lock().await;
1065                        if let Some(task) = orphaned_task.take() {
1066                            if let Err(error) = task.await {
1067                                retained_execution.resources.record_unconfirmed(format!(
1068                                    "orphaned Agent task join failed: {error}"
1069                                ));
1070                            }
1071                        }
1072                        drop(orphaned_task);
1073                        let _ = retained_execution.resources.close_and_drain().await;
1074                        if let Err(error) = retained_execution.persist_stop_unconfirmed().await {
1075                            result = Err(format!(
1076                                "{}unconfirmed-stop observation was not durably confirmed: {error}",
1077                                result
1078                                    .err()
1079                                    .map(|error| format!("{error}; "))
1080                                    .unwrap_or_default()
1081                            ));
1082                        }
1083                    }
1084                }
1085                result
1086            });
1087            let mut handles = lock(&self.coordinators);
1088            let mut pending = Vec::with_capacity(handles.len() + 1);
1089            for mut handle in handles.drain(..) {
1090                if handle.is_finished() {
1091                    match (&mut handle).now_or_never() {
1092                        Some(Err(error)) => lock(&self.coordinator_errors).push(error.to_string()),
1093                        Some(Ok(Err(error))) => lock(&self.coordinator_errors).push(error),
1094                        Some(Ok(Ok(()))) => {}
1095                        None => pending.push(handle),
1096                    }
1097                } else {
1098                    pending.push(handle);
1099                }
1100            }
1101            pending.push(supervisor);
1102            *handles = pending;
1103            execution.launch_handoff.send_replace(true);
1104            None
1105        } else {
1106            Some(handle)
1107        }
1108    }
1109
1110    async fn prepare_durable_child_delivery(
1111        &self,
1112        session: &Session,
1113        run_id: &str,
1114    ) -> Result<bool, String> {
1115        if !self.should_handle(session).await {
1116            return Ok(false);
1117        }
1118        let execution = self.execution(session, run_id)?;
1119        if lock(&execution.stop).is_none() {
1120            return Err("local execution stop is unconfirmed; terminal publication blocked".into());
1121        }
1122        Ok(true)
1123    }
1124    async fn commit_durable_child_delivery(
1125        &self,
1126        session: &Session,
1127        run_id: &str,
1128    ) -> Result<(), String> {
1129        if !self.should_handle(session).await {
1130            return Ok(());
1131        }
1132        let execution = self.execution(session, run_id)?;
1133        if lock(&execution.stop).is_none() {
1134            return Err("local task stop receipt missing".into());
1135        }
1136        let mut final_session = session.clone();
1137        crate::execution::ChildCompletionSource::publish_after_durable_delivery(
1138            &mut final_session,
1139            run_id,
1140        )?;
1141        execution
1142            .persistence
1143            .checkpoint_runtime_session(&mut final_session)
1144            .await
1145            .map_err(|e| e.to_string())?;
1146        let store = lock(&self.bindings)
1147            .store
1148            .clone()
1149            .ok_or("local store unavailable")?;
1150        let readback = store
1151            .load_session(&session.id)
1152            .await
1153            .map_err(|e| e.to_string())?
1154            .ok_or("Child missing after terminal checkpoint")?;
1155        if readback.created_at != session.created_at
1156            || readback
1157                .metadata
1158                .get("runtime.local_task.state")
1159                .map(String::as_str)
1160                != Some("stopped")
1161            || readback.metadata.get("runtime.local_task.instance")
1162                != final_session.metadata.get("runtime.local_task.instance")
1163            || readback.metadata.get("runtime.local_task.run_id") != Some(&run_id.to_string())
1164            || crate::execution::ChildCompletionSource::from_committed_session(&readback)
1165                != crate::execution::ChildCompletionSource::from_committed_session(&final_session)
1166        {
1167            return Err("local terminal checkpoint readback differs".into());
1168        }
1169        let tickets = lock(&self.bindings).tickets.clone();
1170        if let Some(tickets) = tickets.filter(|_| {
1171            crate::ticket_runtime::read_dispatch(&readback)
1172                .ok()
1173                .flatten()
1174                .is_some_and(|d| d.receipt.is_some())
1175        }) {
1176            crate::ticket_runtime::checkpoint_ticket_result(
1177                &tickets,
1178                store.as_ref(),
1179                &readback,
1180                run_id,
1181            )
1182            .await
1183            .map_err(|e| e.to_string())?;
1184        }
1185        Ok(())
1186    }
1187    async fn finish_child_execution(
1188        &self,
1189        session: &Session,
1190        run_id: &str,
1191        history_committed: bool,
1192    ) -> Result<(), String> {
1193        let execution = self.execution(session, run_id)?;
1194        // SDK registers the real coordinator immediately after spawning it.
1195        // Keep this record visible until its handle has another strong owner.
1196        execution.wait_for_handoff().await;
1197        if !history_committed || lock(&execution.stop).is_none() {
1198            return Err("local Child finalization remains unconfirmed".into());
1199        }
1200        execution.heartbeat_stop.cancel();
1201        if let Some(handle) = execution.heartbeat.lock().await.take() {
1202            handle.await.map_err(|e| e.to_string())?;
1203        }
1204        let store = lock(&self.bindings)
1205            .store
1206            .clone()
1207            .ok_or("local store unavailable")?;
1208        let outcome = match session.last_run_status().as_deref() {
1209            Some("completed" | "suspended") => ActorActivationFinish::Succeeded,
1210            Some("cancelled") => ActorActivationFinish::Cancelled,
1211            _ => ActorActivationFinish::Failed,
1212        };
1213        store
1214            .finish_activation(&execution.owner.fence, chrono::Utc::now(), outcome)
1215            .await
1216            .map_err(|e| e.to_string())?;
1217        lock(&execution.slot).take();
1218        lock(&self.executions).remove(&session.id);
1219        execution.finalized.store(true, Ordering::Release);
1220        Ok(())
1221    }
1222    async fn shutdown(&self) -> Result<(), String> {
1223        self.closed.store(true, Ordering::Release);
1224        self.admission.close();
1225        // Preparations take a permit before observing closed. Holding all of
1226        // them here closes the snapshot race with an already admitted claim
1227        // and serializes shutdown callers through all joins and cache release.
1228        let _preparations = self
1229            .preparations
1230            .acquire_many(128)
1231            .await
1232            .map_err(|e| e.to_string())?;
1233        let executions: Vec<_> = lock(&self.executions).values().cloned().collect();
1234        for execution in &executions {
1235            execution.cancel.cancel();
1236        }
1237        for execution in &executions {
1238            execution.wait_for_handoff().await;
1239        }
1240        loop {
1241            let next = lock(&self.coordinators).pop();
1242            let Some(coordinator) = next else { break };
1243            let mut retained = RetainedCoordinatorJoin {
1244                queue: &self.coordinators,
1245                handle: Some(coordinator),
1246            };
1247            let result = retained
1248                .handle
1249                .as_mut()
1250                .expect("retained coordinator")
1251                .await;
1252            retained.handle.take();
1253            // Record each observed error before the next await, so cancelling
1254            // this caller cannot discard a completed coordinator's failure.
1255            match result {
1256                Ok(Ok(())) => {}
1257                Ok(Err(error)) => self.retain_shutdown_error(error),
1258                Err(error) => self.retain_shutdown_error(error.to_string()),
1259            }
1260        }
1261        for execution in executions {
1262            execution.wait_for_handoff().await;
1263            execution.heartbeat_stop.cancel();
1264            if let Some(heartbeat) = execution.heartbeat.lock().await.take() {
1265                if let Err(error) = heartbeat.await {
1266                    self.retain_shutdown_error(error.to_string());
1267                }
1268            }
1269            if let Some(task) = execution.task.lock().await.take() {
1270                if let Err(error) = task.await {
1271                    self.retain_shutdown_error(error.to_string());
1272                }
1273            }
1274            if let Err(error) = execution.resources.close_and_drain().await {
1275                self.retain_shutdown_error(error.to_string());
1276            }
1277            if lock(&execution.stop).is_none() {
1278                self.retain_shutdown_error(format!(
1279                    "{} local stop remains unconfirmed",
1280                    execution.owner.fence.actor_id
1281                ));
1282            }
1283        }
1284        // A setup accepted before shutdown may harvest an older coordinator
1285        // while handing off. Read this queue only after all handoffs/joins, so
1286        // its retained error cannot be lost between the two snapshots.
1287        // A consumed failed join cannot be re-confirmed by a later empty
1288        // queue. Keep observed drain failures on this closed runtime, including
1289        // across cancellation; repeated shutdown is not permission to forget.
1290        let errors = lock(&self.coordinator_errors).clone();
1291        if errors.is_empty() {
1292            // Only a confirmed physical drain, including publication-tail
1293            // joins, can relinquish this cached canonical writer. Unknown stop
1294            // or a failed drain keeps the original service and its real lock.
1295            let retired_ticket_service = lock(&self.bindings).tickets.take();
1296            drop(retired_ticket_service);
1297            Ok(())
1298        } else {
1299            Err(errors.join("; "))
1300        }
1301    }
1302}
1303
1304mod approval;
1305mod events;
1306mod skill_host;
1307mod tool_policy;
1308
1309#[cfg(test)]
1310mod tests;
1311
1312#[cfg(test)]
1313mod question_handoff_tests {
1314    use super::*;
1315
1316    #[test]
1317    fn answered_question_fact_does_not_suspend_successor_run() {
1318        let mut session = Session::new("child", "model");
1319        session.metadata.insert(
1320            bamboo_domain::PARENT_QUESTION_REQUEST_KEY.into(),
1321            "durable-question".into(),
1322        );
1323        session.metadata.insert(
1324            "runtime.actor_parent_question_handoff".into(),
1325            "true".into(),
1326        );
1327        reconcile_question_handoff(&mut session, Some("durable-question"));
1328        assert!(!session
1329            .metadata
1330            .contains_key("runtime.actor_parent_question_handoff"));
1331    }
1332
1333    #[test]
1334    fn newly_issued_question_keeps_handoff_after_fast_answer() {
1335        let mut session = Session::new("child", "model");
1336        session.metadata.insert(
1337            bamboo_domain::PARENT_QUESTION_REQUEST_KEY.into(),
1338            "durable-question".into(),
1339        );
1340        assert!(session.pending_question.is_none());
1341        reconcile_question_handoff(&mut session, None);
1342        assert_eq!(
1343            session
1344                .metadata
1345                .get("runtime.actor_parent_question_handoff")
1346                .map(String::as_str),
1347            Some("true")
1348        );
1349    }
1350}