Skip to main content

bamboo_engine/external_agents/
actor_adapter.rs

1//! Actor external child runner.
2//!
3//! Runs a child session as an independent **actor**: a separate OS process with its own
4//! isolated context, speaking the `bamboo-subagent` WebSocket protocol. This is the
5//! engine-side adapter on the `wants_external` seam: it spawns the worker binary, waits for
6//! it to self-register into the Tier-1 file fabric, connects, sends the assignment, and
7//! forwards the child's `AgentEvent`s back onto the parent's `event_tx`.
8//!
9//! The built-in **local actor** instance of this runner is the default runtime for
10//! every sub-agent (the in-process runtime was removed). The expert `externalAgents`
11//! tables can additionally route specific roles to other actor/a2a agents.
12
13use std::collections::HashMap;
14use std::path::PathBuf;
15use std::sync::Arc;
16use std::time::Duration;
17
18use async_trait::async_trait;
19use bamboo_agent_core::{AgentError, AgentEvent, Role, Session};
20use bamboo_domain::poison::PoisonRecover;
21use tokio::sync::mpsc;
22use tokio_util::sync::CancellationToken;
23
24use bamboo_subagent::fleet::{spawn_worker_on_bus, SpawnedChild};
25use bamboo_subagent::proto::{
26    AgentRecord, ChildFrame, ParentFrame, PermissionPolicyContext, RunSpec, TerminalStatus,
27};
28use bamboo_subagent::provision::{
29    ChildIdentity, ExecutorSpec, ModelRefSpec, Placement, ProvisionSpec, ScopedCredential,
30};
31use bamboo_subagent::transport::{client_config_trusting_cert, ChildClient};
32
33use crate::runtime::execution::{ExternalChildRunner, SpawnJob};
34
35/// Default cap on simultaneously running actor processes.
36pub const DEFAULT_MAX_CONCURRENT_ACTORS: usize = 8;
37
38/// Max nesting depth for direct nested execution (Phase 6). A worker whose
39/// session `spawn_depth` is below this gets its own spawn stack + the real
40/// SubAgent tool; at/over it, neither (and the tool itself refuses). Mirrors
41/// `bamboo_server_tools::DEFAULT_MAX_SPAWN_DEPTH` (kept in sync; engine can't
42/// depend on server-tools). Root orchestrator = 0 ⇒ 4 levels of sub-agents.
43pub const MAX_SPAWN_DEPTH: u32 = 4;
44
45/// Default cap on idle pooled (warm, reusable) workers kept per fingerprint.
46const DEFAULT_MAX_IDLE_PER_KEY: usize = 4;
47
48/// How long a pooled worker waits for its next assignment before reclaiming
49/// itself (must comfortably exceed the gap between sibling spawns).
50const POOLED_IDLE_TIMEOUT_SECS: u64 = 300;
51
52/// Deadline for a local worker's FIRST frame after a Run is dispatched. A warm
53/// worker answers in seconds; a cold spawn within tens. Total silence past this
54/// means the worker is dead (e.g. a pooled worker that exited right after its
55/// liveness check) and its Run is queued with nobody to serve it — trip it so the
56/// runner respawns once instead of hanging forever. Generous, to never false-trip
57/// a slow-but-healthy cold start.
58const WORKER_FIRST_FRAME_TIMEOUT: Duration = Duration::from_secs(60);
59
60/// A warm worker on the mailbox bus, parked for reuse between runs. It stays
61/// dialed-in + subscribed to `mailbox_id`; the next interchangeable child
62/// delivers its `Run` there instead of spawning a fresh process. Dropping it
63/// kills a local kill-on-drop subprocess; a remote / schedulable handle is
64/// process-less (`kill()` is a no-op — it self-manages via its idle timeout).
65struct PooledWorker {
66    worker: SpawnedChild,
67    /// The bus mailbox this worker subscribes to (where its `Run`s are delivered).
68    mailbox_id: String,
69}
70
71/// A role pinned to a remote resident worker (remote-actor-plan §3.4 / P1.5,
72/// #193), resolved at runner-build time from `SubagentsConfig.remote_placements`:
73/// the env-named bearer is already READ into `token` here (the raw token never
74/// rides the config), and `ca_cert_file` is the path to a PEM pinning a
75/// self-signed worker cert (`None` ⇒ default webpki roots / plaintext `ws://`).
76#[derive(Debug, Clone)]
77pub struct ResolvedRemotePlacement {
78    pub endpoint: String,
79    pub token: Option<String>,
80    pub ca_cert_file: Option<PathBuf>,
81    /// Display name for the machine this role runs on — the matching cluster
82    /// node's `label`/host, surfaced on the UI placement badge. `None` ⇒ derive
83    /// from the endpoint host.
84    pub host_label: Option<String>,
85}
86
87/// A role routed to a SCHEDULED worker (remote-actor-plan §3.4 / P2b, #181),
88/// resolved at runner-build time from `SubagentsConfig.schedulable_placements`.
89/// Names the logical `pool` (= the bus role) whose LIVE connected workers are the
90/// scheduling candidates — the runner picks one via the bus presence query
91/// (`BrokerClient::list_connected`). Phase 3 retired the old HTTP registry, so a
92/// pool is now just a role on the bus.
93#[derive(Debug, Clone)]
94pub struct ResolvedSchedulablePlacement {
95    pub pool: String,
96    /// Display name for the machine this pool's workers run on — the matching
97    /// cluster node's `label`/host, surfaced on the UI placement badge. `None` ⇒
98    /// fall back to the pool name.
99    pub host_label: Option<String>,
100}
101
102/// How `execute_external_child` should obtain its worker connection, decided
103/// once from `spec.placement`. Splits the divergent acquire/connect + retire
104/// logic three ways while the shared middle (Run dispatch, live registration,
105/// drive, close) stays identical. `Local` is the unchanged pre-#193 path;
106/// `Remote` is the unchanged #194 path; `Schedulable` (#181, P2b) is new.
107enum PlacementKind {
108    Local,
109    Remote,
110    Schedulable,
111}
112
113/// Spawns and drives a child session as an independent actor: a `bamboo-subagent` worker process.
114pub struct ActorChildRunner {
115    approval_registry: Option<super::approval_registry::SharedApprovalRegistry>,
116    permission_config: Option<Arc<bamboo_tools::permission::PermissionConfig>>,
117    agent_id: String,
118    worker_bin: PathBuf,
119    worker_args: Vec<String>,
120    fabric_dir: PathBuf,
121    executor: ExecutorSpec,
122    /// Per-provider credentials snapshotted from the parent config at build
123    /// time; the spec carries only the ONE the child's provider needs.
124    credentials: Vec<ScopedCredential>,
125    /// Parent's default provider (used when the child has no explicit one).
126    default_provider: String,
127    /// The mailbox bus to run local children over (the unified transport). Local
128    /// sub-agents require it; `None` only when no broker could be embedded.
129    bus: Option<bamboo_subagent::BusEndpoint>,
130    /// Backpressure: bounds the number of concurrently *running* actors; further
131    /// runs wait for a slot instead of exploding the process table. (Idle pooled
132    /// workers do not hold a slot.)
133    concurrency: std::sync::Arc<tokio::sync::Semaphore>,
134    /// Warm-worker pool keyed by a reuse fingerprint
135    /// (role/provider/model/workspace/disabled-tools/baked-caps). A finished run
136    /// parks its bus worker here so the next interchangeable child reuses it
137    /// (delivers its `Run` to the same mailbox) instead of spawning a fresh
138    /// process — collapsing N sibling sub-agents onto a few warm workers.
139    pool: Arc<tokio::sync::Mutex<HashMap<String, Vec<PooledWorker>>>>,
140    max_idle_per_key: usize,
141    /// Host-side decision for a child's gated-tool approval request (Phase 2).
142    /// `None` ⇒ fail-closed DENY (the safe default). A wired decider (policy or
143    /// human-routing bridge) returns approve/deny over the actor WS.
144    approval_decider: Option<Arc<dyn ChildApprovalDecider>>,
145    /// Off-loop parent-agent reviewer for forced-ask requests. The root server
146    /// wires a session-aware reviewer; nested workers wire their owning model
147    /// reviewer directly into the per-run runner.
148    approval_reviewer: Option<Arc<dyn ChildApprovalReviewer>>,
149    /// Per-run escalation host bridge for non-bypass child-approval routing (#68;
150    /// Phase 6, Part B). The owning worker's `run()` installs its OWN host bridge
151    /// here via `set_escalation_bridge`; `execute_external_child` CAPTURES it at
152    /// grandchild-spawn time and hands the owned value to `drive()`, which uses it
153    /// to RE-PROXY a child's approval request UP to the parent run — chaining up
154    /// every level until a bypass level (model-review) or the top orchestrator
155    /// (human) decides, then relaying the reply back down. Was a process-global
156    /// slot; now per-runner so a fire-and-forget grandchild that OUTLIVES the run
157    /// that spawned it keeps that run's bridge for its whole lifetime instead of
158    /// reading a stale/overwritten global at approval time (→ fail-closed deny).
159    escalation_bridge: Arc<std::sync::Mutex<Option<bamboo_subagent::executor::HostBridge>>>,
160    /// Roles pinned to a REMOTE resident worker (#193), keyed by sub-agent role
161    /// (the child's `subagent_type`). A role present here routes through the
162    /// dedicated remote branch in `execute_external_child` (Bearer-authenticated
163    /// `wss://` connect, no spawn, no pool, no kill) instead of the local
164    /// subprocess + warm-pool path. Empty (the default) = all-local behavior.
165    remote_placements: HashMap<String, ResolvedRemotePlacement>,
166    /// Roles routed to a REGISTRY-SCHEDULED worker (#181, P2b), keyed by sub-agent
167    /// role. A role present here (AND not already in `remote_placements`, which
168    /// wins) routes through the dedicated SCHEDULABLE branch in
169    /// `execute_external_child`: query the registry for live workers in the pool,
170    /// pick one (round-robin), connect over `wss://` — no spawn, no pool, no kill,
171    /// and NO local-subprocess fallback (no live worker ⇒ a clear error). Empty
172    /// (the default) = all-local behavior.
173    schedulable_placements: HashMap<String, ResolvedSchedulablePlacement>,
174    /// Per-pool round-robin cursor for schedulable scheduling (#181, P2b). Bumped
175    /// once per pick so successive sibling spawns SPREAD across a pool's live
176    /// workers instead of all landing on the first candidate. Best-effort spread,
177    /// not a load balancer — the registry's live set can change between picks.
178    schedule_cursor: Arc<std::sync::Mutex<HashMap<String, usize>>>,
179}
180
181/// Decides how the host answers a child worker's gated-tool approval request
182/// (Phase 2: child → parent approval delegation). Async so an implementation
183/// can consult a policy or defer to a human. With no decider wired the host
184/// replies with a fail-closed DENY.
185///
186/// NOTE: `decide` is awaited inside the per-child frame pump, so an
187/// implementation must resolve promptly (e.g. a policy lookup). A human-in-the-
188/// loop decision that may block indefinitely should instead be delivered
189/// out-of-band as a `ParentFrame::ApprovalReply` via the live steering channel
190/// (`super::live`), which `drive()` already forwards to the worker without
191/// stalling the pump.
192#[async_trait]
193pub trait ChildApprovalDecider: Send + Sync {
194    /// Decide whether `child_session_id` may perform the gated action described
195    /// by `request` (`{tool_name, permission, resource}`).
196    async fn decide(&self, child_session_id: &str, request: &serde_json::Value) -> bool;
197}
198
199/// Resolve a child approval request to approve/deny. Fail-closed (DENY) when no
200/// decider is wired — the single, testable seam for the host-side decision.
201async fn decide_child_approval(
202    decider: Option<&Arc<dyn ChildApprovalDecider>>,
203    child_session_id: &str,
204    request: &serde_json::Value,
205) -> bool {
206    match decider {
207        Some(decider) => decider.decide(child_session_id, request).await,
208        None => false,
209    }
210}
211
212/// How long the host waits for a human approval decision before failing the
213/// child's gated tool closed (DENY). Bounds an unanswered request so it can't
214/// hang the worker indefinitely.
215const CHILD_APPROVAL_TIMEOUT: Duration = Duration::from_secs(300);
216
217/// Extract `(tool_name, permission, resource)` from a worker's approval request
218/// body (`{tool_name, permission, resource}`); missing fields default to empty.
219fn approval_request_fields(body: &serde_json::Value) -> (String, String, String) {
220    let field = |k: &str| {
221        body.get(k)
222            .and_then(|v| v.as_str())
223            .unwrap_or("")
224            .to_string()
225    };
226    (field("tool_name"), field("permission"), field("resource"))
227}
228
229/// Off-loop reviewer for a child's gated-tool approval request (Phase 6, Part B).
230///
231/// Installed (process-global) by a BYPASSED self-orchestrating worker so its
232/// children's forced-ask (dangerous) gated actions — which still raise
233/// `ConfirmationRequired` even under bypass — get an LLM reasonableness check
234/// rather than a blind pass. `review` is an LLM call: `drive()` invokes it in a
235/// SPAWNED task (NEVER in the frame pump) and delivers the verdict async via the
236/// live channel, so the agent loop is never blocked.
237#[async_trait]
238pub trait ChildApprovalReviewer: Send + Sync {
239    /// Judge whether the gated action `request` (`{tool_name, permission,
240    /// resource}`) is reasonable for `child_session_id`'s task. `true` = approve.
241    async fn review(
242        &self,
243        parent_session_id: &str,
244        child_session_id: &str,
245        request: &serde_json::Value,
246    ) -> bool;
247}
248
249fn child_approval_reviewer_slot() -> &'static std::sync::OnceLock<Arc<dyn ChildApprovalReviewer>> {
250    static SLOT: std::sync::OnceLock<Arc<dyn ChildApprovalReviewer>> = std::sync::OnceLock::new();
251    &SLOT
252}
253
254/// Install the process-global child-approval reviewer (idempotent; first wins).
255pub fn set_child_approval_reviewer(reviewer: Arc<dyn ChildApprovalReviewer>) {
256    let _ = child_approval_reviewer_slot().set(reviewer);
257}
258
259/// The process-global child-approval reviewer, if installed.
260pub fn child_approval_reviewer() -> Option<Arc<dyn ChildApprovalReviewer>> {
261    child_approval_reviewer_slot().get().cloned()
262}
263
264impl ActorChildRunner {
265    #[allow(clippy::too_many_arguments)]
266    pub fn new(
267        agent_id: String,
268        worker_bin: PathBuf,
269        worker_args: Vec<String>,
270        fabric_dir: PathBuf,
271        executor: ExecutorSpec,
272        credentials: Vec<ScopedCredential>,
273        default_provider: String,
274        max_concurrent: usize,
275    ) -> Self {
276        Self {
277            approval_registry: None,
278            permission_config: None,
279            agent_id,
280            worker_bin,
281            worker_args,
282            fabric_dir,
283            executor,
284            credentials,
285            default_provider,
286            bus: None,
287            concurrency: std::sync::Arc::new(tokio::sync::Semaphore::new(max_concurrent.max(1))),
288            pool: Arc::new(tokio::sync::Mutex::new(HashMap::new())),
289            max_idle_per_key: DEFAULT_MAX_IDLE_PER_KEY,
290            approval_decider: None,
291            approval_reviewer: None,
292            escalation_bridge: Arc::new(std::sync::Mutex::new(None)),
293            remote_placements: HashMap::new(),
294            schedulable_placements: HashMap::new(),
295            schedule_cursor: Arc::new(std::sync::Mutex::new(HashMap::new())),
296        }
297    }
298
299    pub fn with_approval_registry(
300        mut self,
301        registry: super::approval_registry::SharedApprovalRegistry,
302    ) -> Self {
303        self.approval_registry = Some(registry);
304        self
305    }
306
307    pub fn with_permission_config(
308        mut self,
309        config: Arc<bamboo_tools::permission::PermissionConfig>,
310    ) -> Self {
311        self.permission_config = Some(config);
312        self
313    }
314
315    /// Run children over the mailbox bus (the unified actor+mailbox transport).
316    /// When set, local children dial this bus and are driven by mailbox id; when
317    /// unset they use the legacy direct-WS path. The server passes its in-process
318    /// broker here (`subagents.broker`); tests without a broker leave it unset.
319    pub fn with_bus(mut self, bus: Option<bamboo_subagent::BusEndpoint>) -> Self {
320        self.bus = bus.filter(|b| !b.endpoint.trim().is_empty());
321        self
322    }
323
324    /// Wire the host-side decider for child gated-tool approval requests
325    /// (Phase 2). Without this the host fail-closed DENYs every request.
326    pub fn with_approval_decider(mut self, decider: Arc<dyn ChildApprovalDecider>) -> Self {
327        self.approval_decider = Some(decider);
328        self
329    }
330
331    pub fn with_approval_reviewer(mut self, reviewer: Arc<dyn ChildApprovalReviewer>) -> Self {
332        self.approval_reviewer = Some(reviewer);
333        self
334    }
335
336    /// Pin specific sub-agent roles to remote resident workers (#193). The map
337    /// is keyed by role (`subagent_type`); a child whose role is present connects
338    /// over `wss://` to the resolved endpoint instead of spawning a local
339    /// subprocess. Default (empty) keeps every role on the local path — exactly
340    /// today's behavior.
341    pub fn with_remote_placements(
342        mut self,
343        placements: HashMap<String, ResolvedRemotePlacement>,
344    ) -> Self {
345        self.remote_placements = placements;
346        self
347    }
348
349    /// Route specific sub-agent roles to a registry-SCHEDULED worker (#181, P2b).
350    /// The map is keyed by role (`subagent_type`); a child whose role is present
351    /// (and NOT already pinned by `remote_placements`, which takes precedence) is
352    /// run on a live worker discovered from the registry instead of a local
353    /// subprocess. Default (empty) keeps every role on the local path.
354    pub fn with_schedulable_placements(
355        mut self,
356        placements: HashMap<String, ResolvedSchedulablePlacement>,
357    ) -> Self {
358        self.schedulable_placements = placements;
359        self
360    }
361
362    /// Reuse fingerprint: two children are interchangeable on one warm worker iff
363    /// they share role, provider, model, workspace, disabled-tool set, AND every
364    /// capability the worker BAKES at provision time (`BambooRuntimeExecutor`
365    /// stamps these once and reuses them across runs): nesting depth, nested-spawn
366    /// stack, bypass mode, permission enforcement, and the depth cap. Omitting any
367    /// of these lets the pool hand a run a worker baked for a DIFFERENT posture —
368    /// e.g. a depth-1 worker (with its own spawn stack) reused for a depth-4
369    /// child would re-stamp `spawn_depth=1` and pass the depth-cap check, breaking
370    /// the recursion bound; or a bypass worker reused for a non-bypass child. So
371    /// these MUST split the pool bucket. Everything else (assignment, history) is
372    /// shipped per-run in the `RunSpec` and does not affect the fingerprint.
373    /// Reuse fingerprint (role/provider/model/workspace/disabled-tools/baked
374    /// caps): two children with the same fingerprint are interchangeable on one
375    /// warm worker, so they share a pool bucket. Any axis the worker bakes ONCE
376    /// at provision time MUST be in here, else a worker baked for one posture
377    /// gets reused for another (see the `fingerprint_*` tests).
378    fn fingerprint(spec: &ProvisionSpec) -> String {
379        let role = spec.identity.role.as_str();
380        let (provider, model) = spec
381            .model
382            .as_ref()
383            .map(|m| (m.provider.as_str(), m.model.as_str()))
384            .unwrap_or(("", ""));
385        let workspace = spec.workspace.as_deref().unwrap_or("");
386        let mut tools = spec.disabled_tools.clone().unwrap_or_default();
387        tools.sort();
388        let caps = &spec.capabilities;
389        format!(
390            "{role}\u{1}{provider}\u{1}{model}\u{1}{workspace}\u{1}{}\u{1}d={}\u{1}ns={}\u{1}by={}\u{1}ep={}\u{1}md={}\u{1}nha={}\u{1}gro={}",
391            tools.join(","),
392            spec.identity.depth,
393            caps.nested_spawn,
394            caps.bypass,
395            caps.enforce_permissions,
396            caps.max_spawn_depth.unwrap_or(0),
397            // #73 review (P1): a worker bakes `no_human_review` ONCE from this flag
398            // at build() and never re-reads it per run, so the pool MUST NOT hand a
399            // worker baked for one approval posture to a run of the opposite one —
400            // else a scheduled-root worker reused for an interactive child would
401            // silently model-review instead of asking the human (and vice-versa,
402            // reintroducing the 300s-deny). Split the bucket on it.
403            caps.no_human_approver,
404            // #71: the read-only Bash checker is baked once at build() from this
405            // flag, so a guardian-reviewer worker must NOT be reused for an
406            // ordinary child (which expects unrestricted Bash), and vice-versa.
407            caps.guardian_read_only,
408        )
409    }
410
411    /// Check out a warm bus worker for `key`, reusing a live parked one if any,
412    /// else spawning a fresh one that dials the bus. The returned worker is OWNED
413    /// by the caller for the run's duration (checkout removes it from the pool, so
414    /// a concurrent sibling gets a different worker or spawns its own — one run per
415    /// worker at a time, matching the pre-bus pool semantics).
416    async fn acquire_bus_worker(
417        &self,
418        key: &str,
419        spec: &ProvisionSpec,
420    ) -> crate::runtime::runner::Result<PooledWorker> {
421        // Drain the bucket, skipping (and reaping) any worker whose process exited
422        // while parked. A live one is handed straight out for reuse.
423        loop {
424            let candidate = {
425                let mut pool = self.pool.lock().await;
426                pool.get_mut(key).and_then(|bucket| bucket.pop())
427            };
428            let Some(mut candidate) = candidate else {
429                break;
430            };
431            if candidate.worker.is_alive() {
432                return Ok(candidate);
433            }
434            candidate.worker.kill().await;
435        }
436
437        let spawned = spawn_worker_on_bus(&self.worker_bin, &self.worker_args, spec)
438            .await
439            .map_err(|e| AgentError::LLM(format!("actor spawn (bus) failed: {e}")))?;
440        let mailbox_id = spawned.record.agent_id.clone();
441        Ok(PooledWorker {
442            worker: spawned,
443            mailbox_id,
444        })
445    }
446
447    /// Park a warm bus worker for reuse after a clean run; if its bucket is full
448    /// (or it died), kill it instead. The worker stays dialed-in + subscribed
449    /// while parked, so a reusing child just delivers a new `Run` to its mailbox.
450    async fn release_bus_worker(&self, key: &str, mut worker: PooledWorker) {
451        if !worker.worker.is_alive() {
452            worker.worker.kill().await;
453            return;
454        }
455        let mut pool = self.pool.lock().await;
456        let bucket = pool.entry(key.to_string()).or_default();
457        if bucket.len() >= self.max_idle_per_key {
458            drop(pool);
459            worker.worker.kill().await;
460            return;
461        }
462        bucket.push(worker);
463    }
464
465    /// Assemble the parent-resolved provisioning document for this child.
466    fn build_spec(&self, session: &Session, job: &SpawnJob) -> ProvisionSpec {
467        let mut spec = ProvisionSpec::new(
468            ChildIdentity {
469                child_id: job.child_session_id.clone(),
470                parent_id: Some(job.parent_session_id.clone()),
471                project_key: None,
472                role: session
473                    .metadata
474                    .get("subagent_type")
475                    .cloned()
476                    .unwrap_or_else(|| "worker".to_string()),
477                // The child session already carries the correct depth
478                // (create_child_action's new_child_of did parent.spawn_depth+1);
479                // stamp it so the worker can re-establish it on its run session
480                // and enforce the max-depth cap across the actor boundary.
481                depth: session.spawn_depth,
482            },
483            self.executor.clone(),
484            self.fabric_dir.to_string_lossy().into_owned(),
485        );
486        spec.workspace = session.workspace.clone();
487        // Unified transport: when a bus is configured, the child dials it (no
488        // listen socket / file discovery) and the parent drives it by mailbox id.
489        spec.bus = self.bus.clone();
490        // Final model: the session's pinned model_ref (create.model / routing already applied),
491        // falling back to the job's bare model on the parent's default provider.
492        spec.model = session
493            .model_ref
494            .as_ref()
495            .map(|r| ModelRefSpec {
496                provider: r.provider.clone(),
497                model: r.model.clone(),
498            })
499            .or_else(|| {
500                let m = job.model.trim();
501                (!m.is_empty()).then(|| ModelRefSpec {
502                    provider: self.default_provider.clone(),
503                    model: m.to_string(),
504                })
505            });
506        spec.disabled_tools = job.disabled_tools.clone();
507        // Least-privilege secrets: only the credential for the child's provider.
508        let provider = spec
509            .model
510            .as_ref()
511            .map(|m| m.provider.as_str())
512            .filter(|p| !p.trim().is_empty())
513            .unwrap_or(&self.default_provider);
514        if let Some(cred) = self.credentials.iter().find(|c| c.provider == provider) {
515            spec.secrets.provider_credentials.push(cred.clone());
516        } else {
517            tracing::warn!(
518                "actor child {}: no credential found for provider '{}'",
519                job.child_session_id,
520                provider
521            );
522        }
523        // Phase 6 (direct nested execution): a worker BELOW the depth cap may
524        // orchestrate its OWN children — on startup it builds its own spawn
525        // stack and runs the real SubAgent tool (no host proxy). The cap (the
526        // SubAgent tool refuses to spawn at/over `max_spawn_depth`) bounds the
527        // recursion. Driven purely by the child's depth, so it auto-propagates
528        // down the tree without any extra config threading.
529        spec.capabilities.nested_spawn = session.spawn_depth < MAX_SPAWN_DEPTH;
530        spec.capabilities.max_spawn_depth = Some(MAX_SPAWN_DEPTH);
531        // #69: activate child-approval review. Sub-agents enforce permissions so
532        // their DANGEROUS actions (the worker uses a HIGH threshold) reach the
533        // parent for review — escalated to the human, or model-reviewed off-loop
534        // when the parent is in bypass. The worker installs no checker without
535        // this, so the whole review chain would otherwise stay dormant.
536        spec.capabilities.enforce_permissions = true;
537        // Propagate "bypass permissions" so a self-orchestrating worker knows it
538        // is a bypassed parent and installs the off-loop model-reviewer for its
539        // children's forced-ask actions (Phase 6, Part B). The child session
540        // already carries the inherited flag (create_child_action seeds it).
541        spec.capabilities.bypass = session
542            .agent_runtime_state
543            .as_ref()
544            .is_some_and(|s| s.bypass_permissions);
545        // #73: propagate "no interactive human approver" (headless / scheduled /
546        // deployed root, inherited by the child session). When set, the worker's
547        // per-run approval proxy model-reviews a gated action locally instead of
548        // escalating to a human who will never answer (which would 300s-deny).
549        spec.capabilities.no_human_approver = session
550            .agent_runtime_state
551            .as_ref()
552            .is_some_and(|s| s.no_human_approver);
553        // #71: mark a READ-ONLY Guardian reviewer so the worker installs the
554        // read-only Bash allowlist checker. The reviewer is spawned by
555        // `spawn_guardian_review` with `subagent_type == "guardian"` (the SAME
556        // marker the completion coordinator branches on to parse the verdict) AND
557        // the `guardian_read_only_disabled_tools` denylist. Keyed off that role
558        // marker (already read above to set `identity.role`), so it rides the same
559        // session-metadata path the denylist/subagent_type use — no new wire seam.
560        // Without this the worker keeps an UNRESTRICTED Bash, so the reviewer could
561        // still `rm -rf` / `git push` / `curl | sh`, defeating "read-only".
562        spec.capabilities.guardian_read_only =
563            session.metadata.get("subagent_type").map(String::as_str) == Some("guardian");
564        // #193: route this role to a REMOTE resident worker when one is pinned.
565        // `spec.identity.role` was just computed from `subagent_type` above; a
566        // match flips the placement to Remote and rides the worker's bearer on the
567        // scoped secrets envelope (TLS handshake / Authorization header only — the
568        // token is never logged). No match leaves the default `Placement::Local`,
569        // so the local path is byte-for-byte unchanged for every non-pinned role.
570        if let Some(placement) = self.remote_placements.get(spec.identity.role.as_str()) {
571            spec.placement = Placement::Remote {
572                endpoint: placement.endpoint.clone(),
573            };
574            spec.secrets.worker_auth_token = placement.token.clone();
575        } else if let Some(placement) = self.schedulable_placements.get(spec.identity.role.as_str())
576        {
577            // #181 (P2b): route this role to a SCHEDULED worker — ONLY when it is
578            // NOT already pinned to a fixed remote endpoint (the `else if` makes
579            // remote_placements take precedence for a role in both). The concrete
580            // worker is picked at run time in `execute_external_child` from the bus
581            // (a live connected worker of the pool role). No per-placement bearer
582            // now — the bus connection uses the bus token. No match in either map
583            // leaves the default `Placement::Local`.
584            spec.placement = Placement::Schedulable {
585                pool: placement.pool.clone(),
586            };
587        }
588        spec
589    }
590
591    /// The `metadata["placement"]` JSON to stamp on a child from its resolved
592    /// placement, preferring the matching cluster node's `host_label` (its
593    /// operator label/host) over the raw endpoint/pool. `None` for a Local child
594    /// (the DTO defaults it to the backend's own host). Split out of
595    /// `execute_external_child` so the role→placement→host resolution is unit-testable.
596    fn placement_stamp_for(&self, spec: &ProvisionSpec) -> Option<String> {
597        let host_label = match &spec.placement {
598            Placement::Remote { .. } => self
599                .remote_placements
600                .get(spec.identity.role.as_str())
601                .and_then(|p| p.host_label.as_deref()),
602            Placement::Schedulable { .. } => self
603                .schedulable_placements
604                .get(spec.identity.role.as_str())
605                .and_then(|p| p.host_label.as_deref()),
606            Placement::Local => None,
607        };
608        placement_metadata(&spec.placement, host_label)
609    }
610
611    /// Pick a live worker for a SCHEDULABLE role from the BUS (#181, Phase 3):
612    /// ask the broker which actors are connected serving the pool role (presence
613    /// is connection-truth — no HTTP registry, no leases, no connect-fail
614    /// failover), then round-robin one per resolve for spread. Returns the chosen
615    /// worker's mailbox id. An empty pool ⇒ a terminal `AgentError` — NEVER a
616    /// local-subprocess fallback (that would silently defeat the placement).
617    async fn resolve_schedulable_worker(
618        &self,
619        role: &str,
620    ) -> std::result::Result<String, AgentError> {
621        let pool = self
622            .schedulable_placements
623            .get(role)
624            .ok_or_else(|| {
625                AgentError::LLM(format!(
626                    "schedulable placement for role '{role}' vanished before scheduling"
627                ))
628            })?
629            .pool
630            .clone();
631        let bus = self.bus.as_ref().ok_or_else(|| {
632            AgentError::LLM(format!(
633                "schedulable role '{role}': no mailbox bus configured (subagents.broker)"
634            ))
635        })?;
636
637        // Ask the BUS who is connected serving the pool role — presence is
638        // connection-truth (no HTTP registry, no leases, no stale-record failover).
639        let mut q = bamboo_broker::BrokerClient::connect(
640            &bus.endpoint,
641            bamboo_subagent::AgentRef {
642                session_id: format!("sched-q-{role}"),
643                role: None,
644            },
645            &bus.token,
646        )
647        .await
648        .map_err(|e| {
649            AgentError::LLM(format!(
650                "schedulable role '{role}': bus connect failed: {e}"
651            ))
652        })?;
653        let candidates = q.list_connected(&pool).await.map_err(|e| {
654            AgentError::LLM(format!(
655                "schedulable role '{role}': bus presence query failed: {e}"
656            ))
657        })?;
658
659        if candidates.is_empty() {
660            return Err(AgentError::LLM(format!(
661                "schedulable role '{role}': no live worker in pool '{pool}' on the bus \
662                 (NOT spawning a local subprocess — a schedulable role has no local fallback)"
663            )));
664        }
665
666        // Round-robin: advance a per-pool cursor once per resolve so successive
667        // sibling spawns spread across the connected pool workers. No failover
668        // needed — a listed worker is connected NOW (the bus only lists live
669        // subscribers), so there is no stale-but-leased candidate to skip.
670        let idx = {
671            let mut cursors = self.schedule_cursor.lock().recover_poison();
672            let cursor = cursors.entry(pool.clone()).or_insert(0);
673            let i = *cursor % candidates.len();
674            *cursor = cursor.wrapping_add(1);
675            i
676        };
677        Ok(candidates[idx].clone())
678    }
679}
680
681#[async_trait]
682impl ExternalChildRunner for ActorChildRunner {
683    async fn should_handle(&self, session: &Session) -> bool {
684        session.metadata.get("runtime.kind") == Some(&"external".to_string())
685            && session.metadata.get("external.protocol") == Some(&"actor".to_string())
686            && session.metadata.get("external.agent_id") == Some(&self.agent_id)
687    }
688
689    fn set_escalation_bridge(&self, bridge: Option<bamboo_subagent::executor::HostBridge>) {
690        *self.escalation_bridge.lock().recover_poison() = bridge;
691    }
692
693    async fn execute_external_child(
694        &self,
695        session: &mut Session,
696        job: &SpawnJob,
697        event_tx: mpsc::Sender<AgentEvent>,
698        cancel_token: CancellationToken,
699    ) -> crate::runtime::runner::Result<()> {
700        // #68 CORRECTNESS CRUX: capture the per-run escalation bridge HERE, at the
701        // moment this grandchild is spawned — while the parent run's bridge is
702        // still in our slot — into an owned local handed to `drive()` for this
703        // grandchild's whole lifetime. A fire-and-forget grandchild that OUTLIVES
704        // the run that spawned it must NOT re-read `self.escalation_bridge` at
705        // approval time: by then `run()` may have cleared/overwritten it (a worker
706        // serves runs sequentially), and re-proxying through a closed bridge
707        // fail-closed denies. Capturing at spawn pins the right bridge per run.
708        let escalation = self.escalation_bridge.lock().recover_poison().clone();
709        let assignment = extract_assignment(session);
710        let mut spec = self.build_spec(session, job);
711        // Mark the worker reusable + give it an idle timeout so it self-reaps if
712        // orphaned. Warm bus workers are pooled per fingerprint and reused.
713        spec.reusable = true;
714        if spec.limits.idle_timeout_secs.is_none() {
715            spec.limits.idle_timeout_secs = Some(POOLED_IDLE_TIMEOUT_SECS);
716        }
717        let pool_key = Self::fingerprint(&spec);
718        // Rehydration: the child session in the parent's store is the actor's
719        // durable state. Ship the full conversation so a reactivation
720        // (send_message / update / rerun) carries its history. A reused worker is
721        // stateless between runs, so this is also what isolates each child's
722        // context on a shared process.
723        let messages: Vec<serde_json::Value> = session
724            .messages
725            .iter()
726            .filter_map(|m| serde_json::to_value(m).ok())
727            .collect();
728        // Policy is captured per activation (not only when a worker is
729        // provisioned), so reused local workers and resident remote/broker
730        // workers observe the latest durable revision and bypass flag at the
731        // next run boundary. Session grants are intentionally not inherited.
732        let permission_policy = self.permission_config.as_ref().and_then(|config| {
733            serde_json::to_value(config.to_serializable())
734                .ok()
735                .map(|policy| PermissionPolicyContext {
736                    revision: config.policy_revision(),
737                    bypass_permissions: session
738                        .agent_runtime_state
739                        .as_ref()
740                        .is_some_and(|state| state.bypass_permissions),
741                    session_id: session.id.clone(),
742                    workspace_path: session.workspace.clone(),
743                    inherit_session_grants: false,
744                    policy,
745                })
746        });
747
748        // Backpressure: hold a concurrency slot for the lifetime of the *run*
749        // (cancellation still proceeds — the cancel branch in drive() runs while
750        // we hold the permit). Released when this fn returns, i.e. once the worker
751        // is parked back into the pool, so idle workers don't pin slots.
752        let _slot = self
753            .concurrency
754            .acquire()
755            .await
756            .map_err(|_| AgentError::LLM("actor concurrency limiter closed".to_string()))?;
757
758        // Split LOCAL (spawn + warm-pool) from the two process-less remote paths
759        // ONLY at the divergent spots — acquire/connect here and the park/retire at
760        // the end. Everything between (Run dispatch, live-actor registration,
761        // drive, the close) is identical for all three. `kind` is the single guard.
762        //   - Local       (#0):  byte-for-byte the pre-#193 reuse-or-spawn path.
763        //   - Remote       (#194): connect to a FIXED resident endpoint, no spawn.
764        //   - Schedulable  (#181): resolve a live worker from the registry, connect.
765        let kind = match spec.placement {
766            Placement::Remote { .. } => PlacementKind::Remote,
767            Placement::Schedulable { .. } => PlacementKind::Schedulable,
768            Placement::Local => PlacementKind::Local,
769        };
770        let remote = !matches!(kind, PlacementKind::Local);
771
772        // Stamp WHICH machine this child runs on onto its session metadata, so the
773        // UI can show it (mirrored into the session index → SessionSummary.placement).
774        // Only remote/scheduled placements need a stamp — a Local child falls through
775        // to the DTO default (this backend's own host). Persisted by the caller with
776        // the rest of the child session after we return.
777        if let Some(placement_meta) = self.placement_stamp_for(&spec) {
778            session
779                .metadata
780                .insert("placement".to_string(), placement_meta);
781        }
782
783        // Retry-once loop: a pooled local worker can die between its liveness
784        // check and handling the Run (a tiny TOCTOU window) — its Run then sits
785        // queued with no server. The first-frame watchdog in `drive` surfaces that
786        // as `WorkerUnresponsive`; we reap the dead worker and re-acquire ONCE
787        // (which spawns fresh / reuses the next live one). Remote/schedulable have
788        // no spawn fallback, so they never retry.
789        let mut attempt = 0u8;
790        let (result, actor) = loop {
791            let (actor, mut client) = match kind {
792                PlacementKind::Remote => {
793                    // REMOTE branch: connect to a resident worker. No spawn, no pool
794                    // touch, no drain. We do not own the worker, so a connect failure
795                    // has NO respawn fallback — it is a clear, terminal error.
796                    let placement = self
797                        .remote_placements
798                        .get(spec.identity.role.as_str())
799                        .ok_or_else(|| {
800                            AgentError::LLM(format!(
801                                "remote placement for role '{}' vanished before connect",
802                                spec.identity.role
803                            ))
804                        })?;
805                    let endpoint = placement.endpoint.clone();
806                    // Build the TLS trust: a pinned CA pins a self-signed worker cert;
807                    // otherwise default webpki roots (or plaintext for `ws://`).
808                    let trust_cfg = match placement.ca_cert_file.as_deref() {
809                        Some(path) => Some(client_config_trusting_cert(path).map_err(|e| {
810                            AgentError::LLM(format!(
811                                "remote worker CA cert '{}': {e}",
812                                path.display()
813                            ))
814                        })?),
815                        None => None,
816                    };
817                    let client = ChildClient::connect_with_auth_tls(
818                        &endpoint,
819                        placement.token.as_deref(),
820                        trust_cfg,
821                    )
822                    .await
823                    .map_err(|e| {
824                        AgentError::LLM(format!("remote actor connect to '{endpoint}' failed: {e}"))
825                    })?;
826                    // Process-less handle so live-actor registration (in-band steering)
827                    // works exactly as for a local worker; `kill()` is a no-op.
828                    let record = AgentRecord {
829                        agent_id: job.child_session_id.clone(),
830                        role: spec.identity.role.clone(),
831                        labels: Vec::new(),
832                        endpoint: endpoint.clone(),
833                        pid: 0,
834                        version: String::new(),
835                        started_at: chrono::Utc::now(),
836                        lease_expires_at: chrono::Utc::now(),
837                    };
838                    let _ = endpoint;
839                    let actor = PooledWorker {
840                        worker: SpawnedChild::remote(record),
841                        mailbox_id: job.child_session_id.clone(),
842                    };
843                    let client: Box<dyn bamboo_subagent::ChildLink> = Box::new(client);
844                    (actor, client)
845                }
846                PlacementKind::Schedulable => {
847                    // SCHEDULABLE branch (#181): pick a LIVE worker of the pool role
848                    // from the BUS (presence = connection-truth; no HTTP registry, no
849                    // leases, no failover) and drive it by mailbox id. The pool worker
850                    // stays connected and is reused next time. No spawn, no kill, NO
851                    // local fallback — an empty pool is a terminal error (raised in
852                    // resolve_schedulable_worker).
853                    let bus = self.bus.as_ref().ok_or_else(|| {
854                        AgentError::LLM(
855                            "schedulable sub-agents require a mailbox bus (subagents.broker)"
856                                .to_string(),
857                        )
858                    })?;
859                    let mailbox_id = self
860                        .resolve_schedulable_worker(spec.identity.role.as_str())
861                        .await?;
862                    let parent = bamboo_subagent::AgentRef {
863                        session_id: format!("p-{}", job.child_session_id),
864                        role: None,
865                    };
866                    let link = bamboo_broker::BrokerChildLink::connect(
867                        &bus.endpoint,
868                        parent,
869                        &bus.token,
870                        mailbox_id.clone(),
871                    )
872                    .await
873                    .map_err(|e| {
874                        AgentError::LLM(format!(
875                            "schedulable link connect to '{mailbox_id}' failed: {e}"
876                        ))
877                    })?;
878                    // Process-less handle — a bus-resident pool worker is never ours to
879                    // kill (remote ⇒ dropped, not pooled, after the run).
880                    let actor = PooledWorker {
881                        worker: SpawnedChild::remote(AgentRecord {
882                            agent_id: mailbox_id.clone(),
883                            role: spec.identity.role.clone(),
884                            labels: Vec::new(),
885                            endpoint: bus.endpoint.clone(),
886                            pid: 0,
887                            version: String::new(),
888                            started_at: chrono::Utc::now(),
889                            lease_expires_at: chrono::Utc::now(),
890                        }),
891                        mailbox_id,
892                    };
893                    let client: Box<dyn bamboo_subagent::ChildLink> = Box::new(link);
894                    (actor, client)
895                }
896                PlacementKind::Local => {
897                    // LOCAL = the mailbox bus (the unified transport): check out a warm
898                    // pooled worker (reuse a live parked one, else spawn fresh) and
899                    // drive it by mailbox id — no listen socket, no file discovery, no
900                    // respawn-on-connect-miss (the broker queues the Run until the
901                    // worker handles it). The legacy direct-WS path was retired; the bus
902                    // is required.
903                    let bus = self.bus.as_ref().ok_or_else(|| {
904                        AgentError::LLM(
905                            "local sub-agents require a mailbox bus (subagents.broker); none is \
906                         configured and the bus could not be embedded"
907                                .to_string(),
908                        )
909                    })?;
910                    let actor = self.acquire_bus_worker(&pool_key, &spec).await?;
911                    let parent = bamboo_subagent::AgentRef {
912                        session_id: format!("p-{}", job.child_session_id),
913                        role: None,
914                    };
915                    let link = bamboo_broker::BrokerChildLink::connect(
916                        &bus.endpoint,
917                        parent,
918                        &bus.token,
919                        actor.mailbox_id.clone(),
920                    )
921                    .await
922                    .map_err(|e| {
923                        AgentError::LLM(format!("broker child link connect failed: {e}"))
924                    })?;
925                    let client: Box<dyn bamboo_subagent::ChildLink> = Box::new(link);
926                    (actor, client)
927                }
928            };
929
930            if let Err(e) = client
931                .send(ParentFrame::Run(RunSpec {
932                    // Cloned (not moved) so a retry can re-dispatch to a fresh worker.
933                    assignment: assignment.clone(),
934                    reasoning_effort: None,
935                    permission_policy: permission_policy.clone(),
936                    messages: messages.clone(),
937                }))
938                .await
939            {
940                if !remote {
941                    actor.worker.kill().await;
942                }
943                return Err(AgentError::LLM(format!("actor run dispatch failed: {e}")));
944            }
945
946            // Register as a live actor so send_message (running, no interrupt) can
947            // steer this child in-band over the existing WS connection. The guard
948            // unregisters on every exit path.
949            let (live_tx, mut live_rx) = mpsc::unbounded_channel::<ParentFrame>();
950            let live_guard = super::live::register(
951                &job.child_session_id,
952                live_tx,
953                attempt as u32,
954                self.approval_registry.clone(),
955            );
956
957            let result = drive(
958                &mut *client,
959                &job.parent_session_id,
960                &job.child_session_id,
961                attempt as u32,
962                self.approval_registry.as_ref(),
963                self.approval_decider.as_ref(),
964                self.approval_reviewer.as_ref(),
965                escalation.clone(),
966                &event_tx,
967                &cancel_token,
968                &mut live_rx,
969                // First-frame watchdog for EVERY placement: a wedged-but-connected
970                // worker (subscribed ≠ serving — e.g. stuck on a prior LLM call) emits
971                // no first frame; without a deadline drive() blocks forever. Bounding it
972                // turns the "running-but-unresponsive" hang into a recoverable
973                // WorkerUnresponsive (reap+respawn local / re-pick schedulable / error
974                // on a fixed remote endpoint).
975                Some(WORKER_FIRST_FRAME_TIMEOUT),
976            )
977            .await;
978            // Unregister IMMEDIATELY: after drive returns nobody consumes live_rx,
979            // so a send_message landing in the close/park window below must see
980            // "not live" and take the durable-queue fallback instead of vanishing.
981            // (Even if one slipped in earlier, send_message also appends it to the
982            // durable transcript, so the next activation still rehydrates it.)
983            drop(live_guard);
984            // Close the parent link (dropping it closes our broker connection; the
985            // worker stays dialed-in + subscribed, ready for its next Run).
986            drop(client);
987
988            // No first frame ⇒ the worker is wedged. Recover ONCE before giving up:
989            //   - Local: reap the dead pooled worker + respawn.
990            //   - Schedulable: not ours to kill — drop it and re-select a live pool
991            //     member (a wedged worker must not fail the run when the pool has others).
992            //   - Remote: a FIXED endpoint has no alternative — fall through to a bounded
993            //     WorkerUnresponsive error (far better than the previous infinite hang).
994            if attempt == 0 && matches!(result, Err(AgentError::WorkerUnresponsive(_))) {
995                match kind {
996                    PlacementKind::Local => {
997                        tracing::warn!(
998                        "actor child {} got no first frame; reaping the worker and respawning once",
999                        job.child_session_id
1000                    );
1001                        actor.worker.kill().await;
1002                        attempt += 1;
1003                        continue;
1004                    }
1005                    PlacementKind::Schedulable => {
1006                        tracing::warn!(
1007                        "scheduled actor child {} got no first frame; re-selecting a pool worker",
1008                        job.child_session_id
1009                    );
1010                        drop(actor);
1011                        attempt += 1;
1012                        continue;
1013                    }
1014                    PlacementKind::Remote => {}
1015                }
1016            }
1017            break (result, actor);
1018        };
1019
1020        // Park the warm worker for reuse on a clean run, or kill it on
1021        // error/cancel (a wedged worker must not be reused). Remote / schedulable
1022        // workers are registry-managed — never ours to pool/kill, just drop.
1023        if remote {
1024            drop(actor);
1025        } else {
1026            match &result {
1027                Ok(_) => self.release_bus_worker(&pool_key, actor).await,
1028                Err(_) => actor.worker.kill().await,
1029            }
1030        }
1031
1032        // Write-back: persist the actor's final reply onto the child session so
1033        // the transcript survives and the NEXT activation sees it as history.
1034        // (run_child_spawn saves the session right after we return.)
1035        match result {
1036            Ok(Some(text)) => {
1037                if !text.is_empty() {
1038                    session.add_message(bamboo_agent_core::Message::assistant(text, None));
1039                }
1040                Ok(())
1041            }
1042            Ok(None) => Ok(()),
1043            Err(e) => Err(e),
1044        }
1045    }
1046}
1047
1048/// The `{kind,host}` placement descriptor stamped onto a child session's metadata
1049/// under `"placement"` — read back by the storage index → `SessionSummary.placement`
1050/// → the UI's machine badge. `None` for `Local` (those fall through to the DTO's
1051/// default of this backend's own host). The value is a JSON string matching
1052/// `bamboo_storage::SessionPlacement { kind, host }`.
1053fn placement_metadata(placement: &Placement, host_label: Option<&str>) -> Option<String> {
1054    // Prefer the cluster node's own label/host (its metadata) when the placement
1055    // maps to a node; else fall back to the raw endpoint host / pool name.
1056    let value = match placement {
1057        Placement::Local => return None,
1058        Placement::Remote { endpoint } => serde_json::json!({
1059            "kind": "remote",
1060            "host": host_label.map(str::to_string).unwrap_or_else(|| host_of_endpoint(endpoint)),
1061        }),
1062        Placement::Schedulable { pool } => serde_json::json!({
1063            "kind": "remote",
1064            "host": host_label.unwrap_or(pool),
1065        }),
1066    };
1067    serde_json::to_string(&value).ok()
1068}
1069
1070/// Extract the host from a `ws[s]://host:port[/path]` bus endpoint, for display.
1071fn host_of_endpoint(endpoint: &str) -> String {
1072    endpoint
1073        .trim()
1074        .trim_start_matches("wss://")
1075        .trim_start_matches("ws://")
1076        .split(['/', ':'])
1077        .next()
1078        .unwrap_or(endpoint)
1079        .to_string()
1080}
1081
1082/// Pump child frames -> parent events until a terminal frame (or cancellation).
1083/// On success, yields the actor's final result text (for session write-back).
1084/// `live_rx` carries in-band frames (steering messages) from the live registry.
1085///
1086/// `escalation_bridge` (#68) is the per-run escalation host bridge CAPTURED BY
1087/// VALUE at spawn time in `execute_external_child` (NOT read live here): when a
1088/// non-bypass child re-proxies an approval request, this owned bridge routes it
1089/// UP to the parent run. Owning it for the call's lifetime is what lets a
1090/// fire-and-forget grandchild that outlives its spawning run still escalate to
1091/// the correct (then-current) parent bridge rather than a stale/overwritten one.
1092async fn drive(
1093    client: &mut dyn bamboo_subagent::ChildLink,
1094    parent_session_id: &str,
1095    child_session_id: &str,
1096    child_attempt: u32,
1097    approval_registry: Option<&super::approval_registry::SharedApprovalRegistry>,
1098    approval_decider: Option<&Arc<dyn ChildApprovalDecider>>,
1099    approval_reviewer: Option<&Arc<dyn ChildApprovalReviewer>>,
1100    escalation_bridge: Option<bamboo_subagent::executor::HostBridge>,
1101    event_tx: &mpsc::Sender<AgentEvent>,
1102    cancel_token: &CancellationToken,
1103    live_rx: &mut mpsc::UnboundedReceiver<ParentFrame>,
1104    first_frame_timeout: Option<Duration>,
1105) -> crate::runtime::runner::Result<Option<String>> {
1106    // First-frame watchdog: a live worker emits its first frame (run-started /
1107    // first token) within seconds; total silence past the deadline means the
1108    // worker is dead (e.g. a pooled worker that exited right after checkout), so
1109    // its Run sits queued forever. We trip ONLY before the first frame — once any
1110    // frame arrives the worker is proven live and a legitimately long run (a slow
1111    // tool between tokens) never trips it.
1112    let mut got_first_frame = false;
1113    let mut first_frame_watch = first_frame_timeout.map(|d| Box::pin(tokio::time::sleep(d)));
1114    loop {
1115        tokio::select! {
1116            _ = cancel_token.cancelled() => {
1117                // fall through to the cancel handling below
1118                break;
1119            }
1120            _ = async {
1121                match first_frame_watch.as_mut() {
1122                    Some(s) => s.as_mut().await,
1123                    None => std::future::pending::<()>().await,
1124                }
1125            }, if !got_first_frame => {
1126                return Err(AgentError::WorkerUnresponsive(format!(
1127                    "child {child_session_id} produced no frame within {:?}",
1128                    first_frame_timeout.unwrap_or_default()
1129                )));
1130            }
1131            Some(frame) = live_rx.recv() => {
1132                // Forward in-band steering to the worker over the existing WS.
1133                if client.send(frame).await.is_err() {
1134                    tracing::warn!("live steering frame could not be sent; connection failing");
1135                }
1136            }
1137            frame = client.next_frame() => {
1138                // Any frame (event / approval / terminal / close / error) proves
1139                // the worker responded — disarm the first-frame watchdog.
1140                got_first_frame = true;
1141                first_frame_watch = None;
1142                match frame {
1143                    Ok(Some(ChildFrame::Event { event })) => {
1144                        // AgentEvent is serialized verbatim on the wire (zero mapping).
1145                        if let Ok(ev) = serde_json::from_value::<AgentEvent>(event) {
1146                            let _ = event_tx.send(ev).await;
1147                        }
1148                    }
1149                    Ok(Some(ChildFrame::ApprovalRequest { id, body })) => {
1150                        // Phase 2: a worker proxied a gated-tool approval back to
1151                        // the host. The WORKER side is live — its executor installs
1152                        // a per-run task-local `ApprovalProxy` (subagent_worker.rs)
1153                        // that calls `host.approval_call`, so this frame arrives
1154                        // when a child hits `ConfirmationRequired`.
1155                        if let Some(reviewer) = approval_reviewer
1156                            .cloned()
1157                            .or_else(child_approval_reviewer)
1158                        {
1159                            // Phase 6, Part B: a BYPASSED parent worker
1160                            // model-reviews its children's forced-ask (dangerous)
1161                            // actions. The review is an LLM call, so run it OFF
1162                            // the frame pump in a spawned task and deliver the
1163                            // verdict async via the live channel — the pump keeps
1164                            // forwarding events and the agent loop never blocks. A
1165                            // timeout denies a hung review so the child can't hang.
1166                            let child = child_session_id.to_string();
1167                            let parent = parent_session_id.to_string();
1168                            let req_id = id.clone();
1169                            let body = body.clone();
1170                            let registry = approval_registry.cloned();
1171                            tokio::spawn(async move {
1172                                let approved = tokio::time::timeout(
1173                                    CHILD_APPROVAL_TIMEOUT,
1174                                    reviewer.review(&parent, &child, &body),
1175                                )
1176                                .await
1177                                .unwrap_or(false);
1178                                super::live::deliver_approval_scoped(
1179                                    registry.as_ref(),
1180                                    &child,
1181                                    child_attempt,
1182                                    &req_id,
1183                                    approved,
1184                                );
1185                            });
1186                        } else if approval_decider.is_some() {
1187                            // A decider is wired (policy / auto): decide promptly
1188                            // and reply inline. (Must not block the pump — see the
1189                            // `ChildApprovalDecider` doc.)
1190                            let approved =
1191                                decide_child_approval(approval_decider, child_session_id, &body)
1192                                    .await;
1193                            if client
1194                                .send(ParentFrame::ApprovalReply { id, approved })
1195                                .await
1196                                .is_err()
1197                            {
1198                                tracing::warn!(
1199                                    "failed to answer approval_request; connection failing"
1200                                );
1201                            }
1202                        } else if let Some(host) = escalation_bridge.clone() {
1203                            // Non-bypass WORKER: ESCALATE up our own actor link
1204                            // (re-proxy) so the request chains to our parent — and
1205                            // up every level until a bypass level (model-review) or
1206                            // the top orchestrator (human) decides. Off-loop so the
1207                            // pump never blocks; relay the reply down to the child.
1208                            let child = child_session_id.to_string();
1209                            let req_id = id.clone();
1210                            let body = body.clone();
1211                            let registry = approval_registry.cloned();
1212                            tokio::spawn(async move {
1213                                let approved = match tokio::time::timeout(
1214                                    CHILD_APPROVAL_TIMEOUT,
1215                                    host.approval_call(body),
1216                                )
1217                                .await
1218                                {
1219                                    Ok(Ok(reply)) => reply
1220                                        .get("approved")
1221                                        .and_then(|v| v.as_bool())
1222                                        .unwrap_or(false),
1223                                    // Transport error or timeout ⇒ fail closed.
1224                                    _ => false,
1225                                };
1226                                super::live::deliver_approval_scoped(
1227                                    registry.as_ref(),
1228                                    &child,
1229                                    child_attempt,
1230                                    &req_id,
1231                                    approved,
1232                                );
1233                            });
1234                        } else {
1235                            // Top orchestrator (no escalation bridge): human-in-the-
1236                            // loop. Surface the request on the parent's event stream
1237                            // and DEFER — the decision arrives out-of-band via
1238                            // `live::deliver_approval(child, request_id, approved)`
1239                            // (→ this child's `live_rx` → forwarded to the worker
1240                            // above). A timeout denies a never-answered request so
1241                            // it can't hang the child forever.
1242                            let (tool_name, permission, resource) =
1243                                approval_request_fields(&body);
1244                            // Register the pending request BEFORE surfacing it so
1245                            // the external handler's `deliver_approval_checked` can
1246                            // correlate an out-of-band POST against a genuine
1247                            // human-loop request (and consume it one-shot).
1248                            let (approval_version, approval_created_at) =
1249                                super::live::register_pending_approval_observed(
1250                                approval_registry,
1251                                parent_session_id,
1252                                child_session_id,
1253                                child_attempt,
1254                                &id,
1255                                &tool_name,
1256                                &permission,
1257                                &resource,
1258                                event_tx.clone(),
1259                            );
1260                            if approval_version == 0 {
1261                                super::live::deliver_approval_scoped(
1262                                    approval_registry,
1263                                    child_session_id,
1264                                    child_attempt,
1265                                    &id,
1266                                    false,
1267                                );
1268                                let _ = event_tx
1269                                    .send(AgentEvent::ChildApprovalChanged {
1270                                        parent_session_id: parent_session_id.to_string(),
1271                                        child_session_id: child_session_id.to_string(),
1272                                        child_attempt,
1273                                        request_id: id,
1274                                        version: 1,
1275                                        status: "delivery_failed".to_string(),
1276                                        reason: Some("persistence_failed".to_string()),
1277                                        tool_name,
1278                                        permission,
1279                                        resource,
1280                                        created_at: approval_created_at,
1281                                        resolved_at: Some(chrono::Utc::now().to_rfc3339()),
1282                                    })
1283                                    .await;
1284                                continue;
1285                            }
1286                            let _ = event_tx.send(AgentEvent::ChildApprovalChanged {
1287                                parent_session_id: parent_session_id.to_string(),
1288                                child_session_id: child_session_id.to_string(),
1289                                child_attempt,
1290                                request_id: id.clone(),
1291                                version: approval_version,
1292                                status: "pending".to_string(),
1293                                reason: None,
1294                                tool_name: tool_name.clone(),
1295                                permission: permission.clone(),
1296                                resource: resource.clone(),
1297                                created_at: approval_created_at,
1298                                resolved_at: None,
1299                            }).await;
1300                            let _ = event_tx
1301                                .send(AgentEvent::ChildApprovalRequested {
1302                                    child_session_id: child_session_id.to_string(),
1303                                    request_id: id.clone(),
1304                                    tool_name,
1305                                    permission,
1306                                    resource,
1307                                })
1308                                .await;
1309                            let child = child_session_id.to_string();
1310                            let approval_registry = approval_registry.cloned();
1311                            tokio::spawn(async move {
1312                                tokio::time::sleep(CHILD_APPROVAL_TIMEOUT).await;
1313                                // Deny only if still pending: a one-shot consume so
1314                                // we don't double-deliver if the human already
1315                                // answered (the POST took it), and so a late POST
1316                                // after this fires finds nothing pending.
1317                                if super::live::expire_pending_approval(approval_registry.as_ref(), &child, &id) {
1318                                    super::live::deliver_approval_scoped(
1319                                        approval_registry.as_ref(),
1320                                        &child,
1321                                        child_attempt,
1322                                        &id,
1323                                        false,
1324                                    );
1325                                }
1326                            });
1327                        }
1328                    }
1329                    Ok(Some(ChildFrame::Terminal { status, result, error, .. })) => {
1330                        return match status {
1331                            TerminalStatus::Completed => Ok(result),
1332                            TerminalStatus::Cancelled => Err(AgentError::Cancelled),
1333                            TerminalStatus::Error => Err(AgentError::LLM(
1334                                error.unwrap_or_else(|| "actor child errored".to_string()),
1335                            )),
1336                            // The suspend/resume round-trip (host re-dispatch of a
1337                            // nested parent) is not wired here yet; a worker in
1338                            // this build never emits Suspended, so this is
1339                            // unreachable in practice.
1340                            TerminalStatus::Suspended => Err(AgentError::LLM(
1341                                "nested sub-agent suspend received but resume transport is not wired"
1342                                    .to_string(),
1343                            )),
1344                        };
1345                    }
1346                    Ok(None) => {
1347                        return Err(AgentError::LLM(
1348                            "actor child closed before terminal".to_string(),
1349                        ));
1350                    }
1351                    Err(e) => {
1352                        return Err(AgentError::LLM(format!("actor transport error: {e}")));
1353                    }
1354                }
1355            }
1356        }
1357    }
1358
1359    // Only reached on cancellation: ask the child to stop (best-effort), then report cancelled.
1360    let _ = client.send(ParentFrame::Cancel).await;
1361    Err(AgentError::Cancelled)
1362}
1363
1364/// The assignment text = the child session's latest user message (falls back to its title).
1365fn extract_assignment(session: &Session) -> String {
1366    session
1367        .messages
1368        .iter()
1369        .rev()
1370        .find(|m| matches!(m.role, Role::User))
1371        .map(|m| m.content.clone())
1372        .unwrap_or_else(|| {
1373            session
1374                .metadata
1375                .get("title")
1376                .cloned()
1377                .unwrap_or_else(|| "Execute task".to_string())
1378        })
1379}
1380
1381#[cfg(test)]
1382mod tests {
1383    use super::*;
1384
1385    fn spec_with(
1386        role: &str,
1387        provider: &str,
1388        model: &str,
1389        workspace: Option<&str>,
1390        disabled: Option<Vec<&str>>,
1391    ) -> ProvisionSpec {
1392        let mut spec = ProvisionSpec::new(
1393            ChildIdentity {
1394                child_id: "c".into(),
1395                parent_id: None,
1396                project_key: None,
1397                role: role.into(),
1398                depth: 0,
1399            },
1400            ExecutorSpec::Echo,
1401            "/tmp/fab".into(),
1402        );
1403        spec.workspace = workspace.map(|w| w.to_string());
1404        spec.model = Some(ModelRefSpec {
1405            provider: provider.into(),
1406            model: model.into(),
1407        });
1408        spec.disabled_tools = disabled.map(|d| d.into_iter().map(String::from).collect());
1409        spec
1410    }
1411
1412    #[test]
1413    fn fingerprint_matches_interchangeable_children() {
1414        // Same role/provider/model/workspace and equal tool sets (order-insensitive)
1415        // are interchangeable on one warm worker — and differ only in child_id.
1416        let a = spec_with(
1417            "explorer",
1418            "p",
1419            "m",
1420            Some("/ws"),
1421            Some(vec!["Bash", "Edit"]),
1422        );
1423        let mut b = spec_with(
1424            "explorer",
1425            "p",
1426            "m",
1427            Some("/ws"),
1428            Some(vec!["Edit", "Bash"]),
1429        );
1430        b.identity.child_id = "other".into();
1431        assert_eq!(
1432            ActorChildRunner::fingerprint(&a),
1433            ActorChildRunner::fingerprint(&b)
1434        );
1435    }
1436
1437    #[test]
1438    fn fingerprint_separates_distinct_runtimes() {
1439        let base = spec_with("explorer", "p", "m", Some("/ws"), None);
1440        let base_fp = ActorChildRunner::fingerprint(&base);
1441        // Each axis that is baked into the worker must split the pool bucket.
1442        assert_ne!(
1443            base_fp,
1444            ActorChildRunner::fingerprint(&spec_with("writer", "p", "m", Some("/ws"), None))
1445        );
1446        assert_ne!(
1447            base_fp,
1448            ActorChildRunner::fingerprint(&spec_with("explorer", "p2", "m", Some("/ws"), None))
1449        );
1450        assert_ne!(
1451            base_fp,
1452            ActorChildRunner::fingerprint(&spec_with("explorer", "p", "m2", Some("/ws"), None))
1453        );
1454        assert_ne!(
1455            base_fp,
1456            ActorChildRunner::fingerprint(&spec_with("explorer", "p", "m", Some("/ws2"), None))
1457        );
1458        assert_ne!(
1459            base_fp,
1460            ActorChildRunner::fingerprint(&spec_with(
1461                "explorer",
1462                "p",
1463                "m",
1464                Some("/ws"),
1465                Some(vec!["Bash"])
1466            ))
1467        );
1468    }
1469
1470    #[test]
1471    fn fingerprint_splits_on_baked_capabilities() {
1472        // Every capability baked once at provision time must split the pool
1473        // bucket, else a worker baked for one posture gets reused for another
1474        // (e.g. a depth-1 worker re-stamping spawn_depth onto a depth-4 child,
1475        // breaking the depth cap; or a bypass worker reused for a non-bypass one).
1476        let base_fp =
1477            ActorChildRunner::fingerprint(&spec_with("explorer", "p", "m", Some("/ws"), None));
1478
1479        let mut depth = spec_with("explorer", "p", "m", Some("/ws"), None);
1480        depth.identity.depth = 2;
1481        assert_ne!(
1482            base_fp,
1483            ActorChildRunner::fingerprint(&depth),
1484            "depth must split"
1485        );
1486
1487        let mut nested = spec_with("explorer", "p", "m", Some("/ws"), None);
1488        nested.capabilities.nested_spawn = true;
1489        assert_ne!(
1490            base_fp,
1491            ActorChildRunner::fingerprint(&nested),
1492            "nested_spawn must split"
1493        );
1494
1495        let mut bypass = spec_with("explorer", "p", "m", Some("/ws"), None);
1496        bypass.capabilities.bypass = true;
1497        assert_ne!(
1498            base_fp,
1499            ActorChildRunner::fingerprint(&bypass),
1500            "bypass must split"
1501        );
1502
1503        let mut enforce = spec_with("explorer", "p", "m", Some("/ws"), None);
1504        enforce.capabilities.enforce_permissions = true;
1505        assert_ne!(
1506            base_fp,
1507            ActorChildRunner::fingerprint(&enforce),
1508            "enforce_permissions must split"
1509        );
1510
1511        let mut cap = spec_with("explorer", "p", "m", Some("/ws"), None);
1512        cap.capabilities.max_spawn_depth = Some(8);
1513        assert_ne!(
1514            base_fp,
1515            ActorChildRunner::fingerprint(&cap),
1516            "max_spawn_depth must split"
1517        );
1518
1519        // #73 (P1): the worker bakes `no_human_review` from this flag once at
1520        // build(), so it MUST split the pool or a worker baked for one approval
1521        // posture is reused for the opposite one.
1522        let mut nha = spec_with("explorer", "p", "m", Some("/ws"), None);
1523        nha.capabilities.no_human_approver = true;
1524        assert_ne!(
1525            base_fp,
1526            ActorChildRunner::fingerprint(&nha),
1527            "no_human_approver must split"
1528        );
1529
1530        // #71: the read-only Bash checker is baked once at build() from this flag,
1531        // so a guardian reviewer worker must not be reused for an ordinary child.
1532        let mut gro = spec_with("explorer", "p", "m", Some("/ws"), None);
1533        gro.capabilities.guardian_read_only = true;
1534        assert_ne!(
1535            base_fp,
1536            ActorChildRunner::fingerprint(&gro),
1537            "guardian_read_only must split"
1538        );
1539    }
1540
1541    struct StaticDecider(bool);
1542
1543    #[async_trait]
1544    impl ChildApprovalDecider for StaticDecider {
1545        async fn decide(&self, _child: &str, _req: &serde_json::Value) -> bool {
1546            self.0
1547        }
1548    }
1549
1550    struct RecordingReviewer {
1551        reviewed: mpsc::UnboundedSender<(String, String, serde_json::Value)>,
1552    }
1553
1554    #[async_trait]
1555    impl ChildApprovalReviewer for RecordingReviewer {
1556        async fn review(&self, parent: &str, child: &str, request: &serde_json::Value) -> bool {
1557            let _ = self
1558                .reviewed
1559                .send((parent.to_string(), child.to_string(), request.clone()));
1560            true
1561        }
1562    }
1563
1564    // ---- first-frame watchdog (dead-pooled-worker recovery) -----------------
1565
1566    /// A link that never yields a frame — models a worker that died (or never
1567    /// subscribed) so its Run sits queued with no server.
1568    struct SilentLink;
1569    #[async_trait]
1570    impl bamboo_subagent::ChildLink for SilentLink {
1571        async fn send(&mut self, _: ParentFrame) -> bamboo_subagent::TransportResult<()> {
1572            Ok(())
1573        }
1574        async fn next_frame(&mut self) -> bamboo_subagent::TransportResult<Option<ChildFrame>> {
1575            std::future::pending().await
1576        }
1577    }
1578
1579    /// A link that immediately yields one terminal frame (a healthy fast worker).
1580    struct InstantTerminalLink {
1581        done: bool,
1582    }
1583
1584    struct ApprovalThenTerminalLink {
1585        step: u8,
1586    }
1587
1588    #[async_trait]
1589    impl bamboo_subagent::ChildLink for ApprovalThenTerminalLink {
1590        async fn send(&mut self, _: ParentFrame) -> bamboo_subagent::TransportResult<()> {
1591            Ok(())
1592        }
1593
1594        async fn next_frame(&mut self) -> bamboo_subagent::TransportResult<Option<ChildFrame>> {
1595            self.step += 1;
1596            match self.step {
1597                1 => Ok(Some(ChildFrame::ApprovalRequest {
1598                    id: "approval-1".into(),
1599                    body: serde_json::json!({
1600                        "tool_name": "Bash",
1601                        "permission": "execute",
1602                        "resource": "rm -rf target",
1603                        "permission_request": {"reason_code": "hard_dangerous"}
1604                    }),
1605                })),
1606                2 => Ok(Some(ChildFrame::Terminal {
1607                    status: TerminalStatus::Completed,
1608                    result: Some("done".into()),
1609                    error: None,
1610                    transcript: vec![],
1611                })),
1612                _ => std::future::pending().await,
1613            }
1614        }
1615    }
1616
1617    #[tokio::test]
1618    async fn drive_routes_forced_ask_to_parent_reviewer_without_human_event() {
1619        let (event_tx, mut event_rx) = mpsc::channel::<AgentEvent>(8);
1620        let (review_tx, mut review_rx) = mpsc::unbounded_channel();
1621        let reviewer: Arc<dyn ChildApprovalReviewer> = Arc::new(RecordingReviewer {
1622            reviewed: review_tx,
1623        });
1624        let cancel = CancellationToken::new();
1625        let (_live_tx, mut live_rx) = mpsc::unbounded_channel::<ParentFrame>();
1626        let mut link = ApprovalThenTerminalLink { step: 0 };
1627
1628        let result = drive(
1629            &mut link,
1630            "parent-reviewer",
1631            "child-reviewer",
1632            0,
1633            None,
1634            None,
1635            Some(&reviewer),
1636            None,
1637            &event_tx,
1638            &cancel,
1639            &mut live_rx,
1640            None,
1641        )
1642        .await;
1643
1644        assert_eq!(result.ok().flatten().as_deref(), Some("done"));
1645        let (parent, child, body) = tokio::time::timeout(Duration::from_secs(1), review_rx.recv())
1646            .await
1647            .expect("reviewer should be invoked off-loop")
1648            .expect("review channel should remain open");
1649        assert_eq!(parent, "parent-reviewer");
1650        assert_eq!(child, "child-reviewer");
1651        assert_eq!(
1652            body.pointer("/permission_request/reason_code")
1653                .and_then(serde_json::Value::as_str),
1654            Some("hard_dangerous")
1655        );
1656        assert!(
1657            event_rx.try_recv().is_err(),
1658            "must not emit a human-review event"
1659        );
1660    }
1661    #[async_trait]
1662    impl bamboo_subagent::ChildLink for InstantTerminalLink {
1663        async fn send(&mut self, _: ParentFrame) -> bamboo_subagent::TransportResult<()> {
1664            Ok(())
1665        }
1666        async fn next_frame(&mut self) -> bamboo_subagent::TransportResult<Option<ChildFrame>> {
1667            if self.done {
1668                std::future::pending().await
1669            } else {
1670                self.done = true;
1671                Ok(Some(ChildFrame::Terminal {
1672                    status: TerminalStatus::Completed,
1673                    result: Some("done".into()),
1674                    error: None,
1675                    transcript: vec![],
1676                }))
1677            }
1678        }
1679    }
1680
1681    #[tokio::test]
1682    async fn drive_trips_first_frame_watchdog_on_a_silent_worker() {
1683        let (event_tx, _rx) = mpsc::channel::<AgentEvent>(8);
1684        let cancel = CancellationToken::new();
1685        let (_live_tx, mut live_rx) = mpsc::unbounded_channel::<ParentFrame>();
1686        let mut link = SilentLink;
1687        let r = drive(
1688            &mut link,
1689            "parent-x",
1690            "child-x",
1691            0,
1692            None,
1693            None,
1694            None,
1695            None,
1696            &event_tx,
1697            &cancel,
1698            &mut live_rx,
1699            Some(Duration::from_millis(100)),
1700        )
1701        .await;
1702        assert!(
1703            matches!(r, Err(AgentError::WorkerUnresponsive(_))),
1704            "a silent worker must trip the first-frame watchdog, got {r:?}"
1705        );
1706    }
1707
1708    #[tokio::test]
1709    async fn drive_does_not_trip_when_a_frame_arrives() {
1710        let (event_tx, _rx) = mpsc::channel::<AgentEvent>(8);
1711        let cancel = CancellationToken::new();
1712        let (_live_tx, mut live_rx) = mpsc::unbounded_channel::<ParentFrame>();
1713        let mut link = InstantTerminalLink { done: false };
1714        // Even a tiny timeout must NOT trip: the terminal frame arrives first and
1715        // disarms the watchdog.
1716        let r = drive(
1717            &mut link,
1718            "parent-y",
1719            "child-y",
1720            0,
1721            None,
1722            None,
1723            None,
1724            None,
1725            &event_tx,
1726            &cancel,
1727            &mut live_rx,
1728            Some(Duration::from_millis(50)),
1729        )
1730        .await;
1731        assert_eq!(r.ok().flatten().as_deref(), Some("done"));
1732    }
1733
1734    #[tokio::test]
1735    async fn child_approval_fails_closed_without_decider() {
1736        // No decider wired ⇒ the host denies (safe default), unchanged behavior.
1737        let body = serde_json::json!({"tool_name":"Bash","permission":"run","resource":"rm -rf /"});
1738        assert!(!decide_child_approval(None, "child-1", &body).await);
1739    }
1740
1741    #[tokio::test]
1742    async fn child_approval_honors_wired_decider() {
1743        let body =
1744            serde_json::json!({"tool_name":"Write","permission":"write","resource":"/tmp/x"});
1745        let approve: Arc<dyn ChildApprovalDecider> = Arc::new(StaticDecider(true));
1746        let deny: Arc<dyn ChildApprovalDecider> = Arc::new(StaticDecider(false));
1747        assert!(decide_child_approval(Some(&approve), "child-1", &body).await);
1748        assert!(!decide_child_approval(Some(&deny), "child-1", &body).await);
1749    }
1750
1751    #[test]
1752    fn approval_request_fields_extracts_and_defaults() {
1753        let full = serde_json::json!({"tool_name":"Bash","permission":"run","resource":"ls"});
1754        assert_eq!(
1755            approval_request_fields(&full),
1756            ("Bash".to_string(), "run".to_string(), "ls".to_string())
1757        );
1758        // Missing fields default to empty strings.
1759        let partial = serde_json::json!({"tool_name":"Write"});
1760        assert_eq!(
1761            approval_request_fields(&partial),
1762            ("Write".to_string(), String::new(), String::new())
1763        );
1764    }
1765
1766    // ---- #193: remote placement routing -------------------------------------
1767
1768    use crate::runtime::execution::SpawnJob;
1769    use bamboo_agent_core::Session;
1770
1771    /// A runner with a BOGUS worker_bin (`/bin/false`): a local spawn here would
1772    /// FAIL, so a passing remote test proves the remote path never spawns.
1773    fn bogus_runner(placements: HashMap<String, ResolvedRemotePlacement>) -> ActorChildRunner {
1774        ActorChildRunner::new(
1775            "test-actor".into(),
1776            PathBuf::from("/bin/false"),
1777            vec![],
1778            std::env::temp_dir().join("bamboo-test-fab-193"),
1779            ExecutorSpec::Echo,
1780            vec![],
1781            "anthropic".into(),
1782            4,
1783        )
1784        .with_remote_placements(placements)
1785    }
1786
1787    /// A child session of the given role (the role rides `subagent_type`, the
1788    /// path build_spec + the remote lookup both read).
1789    fn session_of_role(role: &str, assignment: &str) -> Session {
1790        let mut s = Session::new("child-1", "test-model");
1791        s.metadata
1792            .insert("subagent_type".to_string(), role.to_string());
1793        s.add_message(bamboo_agent_core::Message::user(assignment));
1794        s
1795    }
1796
1797    fn job_for(child: &str) -> SpawnJob {
1798        SpawnJob {
1799            parent_session_id: "parent-1".into(),
1800            child_session_id: child.into(),
1801            model: String::new(),
1802            disabled_tools: None,
1803        }
1804    }
1805
1806    #[test]
1807    fn build_spec_sets_remote_placement_for_matching_role() {
1808        let mut placements = HashMap::new();
1809        placements.insert(
1810            "explorer".to_string(),
1811            ResolvedRemotePlacement {
1812                endpoint: "wss://gpu-host:8443".into(),
1813                token: Some("T-secret".into()),
1814                ca_cert_file: None,
1815                host_label: None,
1816            },
1817        );
1818        let runner = bogus_runner(placements);
1819
1820        // Matching role -> Placement::Remote + the bearer on the secrets envelope.
1821        let s = session_of_role("explorer", "do the thing");
1822        let spec = runner.build_spec(&s, &job_for("child-1"));
1823        match &spec.placement {
1824            Placement::Remote { endpoint } => assert_eq!(endpoint, "wss://gpu-host:8443"),
1825            other => panic!("expected Remote, got {other:?}"),
1826        }
1827        assert_eq!(spec.secrets.worker_auth_token.as_deref(), Some("T-secret"));
1828    }
1829
1830    #[test]
1831    fn build_spec_leaves_local_for_unmatched_role() {
1832        let mut placements = HashMap::new();
1833        placements.insert(
1834            "explorer".to_string(),
1835            ResolvedRemotePlacement {
1836                endpoint: "wss://gpu-host:8443".into(),
1837                token: Some("T".into()),
1838                ca_cert_file: None,
1839                host_label: None,
1840            },
1841        );
1842        let runner = bogus_runner(placements);
1843
1844        // A DIFFERENT role keeps the default Local placement + no bearer.
1845        let s = session_of_role("writer", "do the thing");
1846        let spec = runner.build_spec(&s, &job_for("child-1"));
1847        assert_eq!(spec.placement, Placement::Local);
1848        assert!(spec.secrets.worker_auth_token.is_none());
1849    }
1850
1851    #[test]
1852    fn build_spec_local_when_no_placements() {
1853        let runner = bogus_runner(HashMap::new());
1854        let s = session_of_role("explorer", "do the thing");
1855        let spec = runner.build_spec(&s, &job_for("child-1"));
1856        assert_eq!(spec.placement, Placement::Local);
1857        assert!(spec.secrets.worker_auth_token.is_none());
1858    }
1859
1860    #[test]
1861    fn placement_metadata_stamps_remote_and_schedulable_not_local() {
1862        // Local children carry no stamp — the DTO defaults them to the backend host.
1863        assert_eq!(placement_metadata(&Placement::Local, None), None);
1864
1865        // Remote, no node label → host derived from the endpoint.
1866        let r = placement_metadata(
1867            &Placement::Remote {
1868                endpoint: "wss://10.0.0.5:8443/stream".into(),
1869            },
1870            None,
1871        )
1872        .unwrap();
1873        assert!(r.contains(r#""kind":"remote""#), "{r}");
1874        assert!(r.contains(r#""host":"10.0.0.5""#), "{r}");
1875
1876        // A cluster node's label (its metadata) OVERRIDES the raw endpoint host.
1877        let labeled = placement_metadata(
1878            &Placement::Remote {
1879                endpoint: "ws://169.254.230.101:8899".into(),
1880            },
1881            Some("mini"),
1882        )
1883        .unwrap();
1884        assert!(labeled.contains(r#""host":"mini""#), "{labeled}");
1885
1886        // Schedulable → {kind:"remote", host:<node label, else pool>}.
1887        let s = placement_metadata(
1888            &Placement::Schedulable {
1889                pool: "explorers".into(),
1890            },
1891            Some("mini"),
1892        )
1893        .unwrap();
1894        assert!(s.contains(r#""kind":"remote""#), "{s}");
1895        assert!(s.contains(r#""host":"mini""#), "{s}");
1896
1897        // The stamp round-trips through the storage placement type.
1898        let p: bamboo_storage::SessionPlacement = serde_json::from_str(&labeled).unwrap();
1899        assert_eq!(p.kind, "remote");
1900        assert_eq!(p.host, "mini");
1901    }
1902
1903    /// End-to-end remote run through `execute_external_child`: a resident worker
1904    /// (Bearer-gated `WsServer` + `EchoExecutor`) serves the role; the runner is
1905    /// built with a `remote_placements` entry pointing at it AND a BOGUS
1906    /// worker_bin (`/bin/false`). A passing test proves the remote path CONNECTS
1907    /// to the resident worker and NEVER spawns (a spawn would fail on /bin/false),
1908    /// and that a terminal/echo result flows back.
1909    #[tokio::test]
1910    async fn execute_external_child_routes_role_to_remote_worker_without_spawning() {
1911        // 1. Stand up the resident worker on loopback with a required bearer.
1912        let token = "remote-test-token";
1913        let server = bamboo_subagent::transport::WsServer::bind_with_token(
1914            (std::net::Ipv4Addr::LOCALHOST, 0).into(),
1915            Some(token.to_string()),
1916        )
1917        .await
1918        .expect("bind resident worker");
1919        let endpoint = server.ws_endpoint(); // ws://127.0.0.1:<port>
1920        let srv = tokio::spawn(async move {
1921            // serve() loops connection-after-connection; the test exits, dropping it.
1922            let _ = server
1923                .serve(Arc::new(bamboo_subagent::executor::EchoExecutor))
1924                .await;
1925        });
1926
1927        // 2. Build the runner: role "explorer" pinned remote, bogus worker_bin.
1928        let mut placements = HashMap::new();
1929        placements.insert(
1930            "explorer".to_string(),
1931            ResolvedRemotePlacement {
1932                endpoint: endpoint.clone(),
1933                token: Some(token.to_string()),
1934                ca_cert_file: None,
1935                host_label: Some("mini-e2e".into()), // node label, surfaced on the badge
1936            },
1937        );
1938        let runner = bogus_runner(placements);
1939
1940        // 3. Drive a real run for that role.
1941        let mut session = session_of_role("explorer", "hello remote");
1942        let job = job_for("child-1");
1943        let (event_tx, mut event_rx) = mpsc::channel::<AgentEvent>(64);
1944        let cancel = CancellationToken::new();
1945
1946        let result = tokio::time::timeout(
1947            Duration::from_secs(10),
1948            runner.execute_external_child(&mut session, &job, event_tx, cancel),
1949        )
1950        .await
1951        .expect("run did not hang")
1952        .expect("remote run succeeded (connected to resident worker, did not spawn)");
1953
1954        let _ = result;
1955        // The EchoExecutor's reply is written back onto the child session as an
1956        // assistant message — proof a terminal result flowed back over the link.
1957        let last = session
1958            .messages
1959            .iter()
1960            .rev()
1961            .find(|m| matches!(m.role, Role::Assistant))
1962            .expect("an assistant reply was written back");
1963        assert!(
1964            last.content.contains("echo:"),
1965            "expected echo reply, got {:?}",
1966            last.content
1967        );
1968
1969        // A remote run must stamp WHICH machine it ran on onto the child session
1970        // (mirrored to the UI badge) using the placement's node label.
1971        let placement = session
1972            .metadata
1973            .get("placement")
1974            .expect("remote child session stamped with a placement");
1975        assert!(placement.contains(r#""kind":"remote""#), "{placement}");
1976        assert!(placement.contains(r#""host":"mini-e2e""#), "{placement}");
1977
1978        // Drain a couple of streamed events to confirm the event pipe carried the
1979        // worker's tokens too (best-effort; the reply assertion above is primary).
1980        let mut saw_event = false;
1981        while let Ok(Some(_ev)) =
1982            tokio::time::timeout(Duration::from_millis(50), event_rx.recv()).await
1983        {
1984            saw_event = true;
1985        }
1986        let _ = saw_event;
1987
1988        srv.abort();
1989    }
1990
1991    // ---- #181 (P2b): schedulable placement routing --------------------------
1992
1993    /// A bogus-worker_bin runner carrying SCHEDULABLE placements (and optionally
1994    /// remote ones, to test precedence). A local spawn here would fail on
1995    /// `/bin/false`, so a passing schedulable test proves no subprocess spawned.
1996    fn bogus_sched_runner(
1997        remote: HashMap<String, ResolvedRemotePlacement>,
1998        sched: HashMap<String, ResolvedSchedulablePlacement>,
1999    ) -> ActorChildRunner {
2000        ActorChildRunner::new(
2001            "test-actor".into(),
2002            PathBuf::from("/bin/false"),
2003            vec![],
2004            std::env::temp_dir().join("bamboo-test-fab-181"),
2005            ExecutorSpec::Echo,
2006            vec![],
2007            "anthropic".into(),
2008            4,
2009        )
2010        .with_remote_placements(remote)
2011        .with_schedulable_placements(sched)
2012    }
2013
2014    fn sched_placement(
2015        pool: &str,
2016        _registry_url: impl Into<String>,
2017    ) -> ResolvedSchedulablePlacement {
2018        ResolvedSchedulablePlacement {
2019            pool: pool.into(),
2020            host_label: None,
2021        }
2022    }
2023
2024    #[test]
2025    fn build_spec_sets_schedulable_placement_for_matching_role() {
2026        let mut sched = HashMap::new();
2027        sched.insert(
2028            "explorer".to_string(),
2029            sched_placement("gpu-pool", "unused"),
2030        );
2031        let runner = bogus_sched_runner(HashMap::new(), sched);
2032
2033        let s = session_of_role("explorer", "do the thing");
2034        let spec = runner.build_spec(&s, &job_for("child-1"));
2035        match &spec.placement {
2036            Placement::Schedulable { pool } => assert_eq!(pool, "gpu-pool"),
2037            other => panic!("expected Schedulable, got {other:?}"),
2038        }
2039        // No per-placement bearer now — the bus connection carries the bus token.
2040        assert!(spec.secrets.worker_auth_token.is_none());
2041    }
2042
2043    #[test]
2044    fn build_spec_remote_wins_when_role_in_both_maps() {
2045        // A role present in BOTH remote_placements and schedulable_placements must
2046        // resolve to the FIXED remote placement (documented precedence).
2047        let mut remote = HashMap::new();
2048        remote.insert(
2049            "explorer".to_string(),
2050            ResolvedRemotePlacement {
2051                endpoint: "wss://fixed-host:8443".into(),
2052                token: Some("T-remote".into()),
2053                ca_cert_file: None,
2054                host_label: None,
2055            },
2056        );
2057        let mut sched = HashMap::new();
2058        sched.insert(
2059            "explorer".to_string(),
2060            sched_placement("gpu-pool", "https://control-plane:9562"),
2061        );
2062        let runner = bogus_sched_runner(remote, sched);
2063
2064        let s = session_of_role("explorer", "do the thing");
2065        let spec = runner.build_spec(&s, &job_for("child-1"));
2066        match &spec.placement {
2067            Placement::Remote { endpoint } => assert_eq!(endpoint, "wss://fixed-host:8443"),
2068            other => panic!("expected Remote (precedence), got {other:?}"),
2069        }
2070        assert_eq!(spec.secrets.worker_auth_token.as_deref(), Some("T-remote"));
2071    }
2072
2073    #[test]
2074    fn build_spec_local_for_unmatched_schedulable_role() {
2075        let mut sched = HashMap::new();
2076        sched.insert(
2077            "explorer".to_string(),
2078            sched_placement("gpu-pool", "https://control-plane:9562"),
2079        );
2080        let runner = bogus_sched_runner(HashMap::new(), sched);
2081        let s = session_of_role("writer", "do the thing");
2082        let spec = runner.build_spec(&s, &job_for("child-1"));
2083        assert_eq!(spec.placement, Placement::Local);
2084        assert!(spec.secrets.worker_auth_token.is_none());
2085    }
2086
2087    /// The full role → resolved-placement → badge-host chain: a child routed to a
2088    /// remote/schedulable placement carrying a cluster node's `host_label` stamps
2089    /// that label; without a label it falls back to the endpoint host / pool; a
2090    /// Local child gets no stamp (the DTO defaults it to the backend host).
2091    #[test]
2092    fn placement_stamp_uses_node_label_for_remote_and_schedulable() {
2093        // Remote WITH a node label → {remote, <label>}, overriding the raw IP.
2094        let mut remote = HashMap::new();
2095        remote.insert(
2096            "explorer".to_string(),
2097            ResolvedRemotePlacement {
2098                endpoint: "ws://169.254.230.101:8899".into(),
2099                token: None,
2100                ca_cert_file: None,
2101                host_label: Some("mini".into()),
2102            },
2103        );
2104        let runner = bogus_runner(remote);
2105        let spec = runner.build_spec(&session_of_role("explorer", "go"), &job_for("c1"));
2106        let stamp = runner
2107            .placement_stamp_for(&spec)
2108            .expect("remote child is stamped");
2109        assert!(stamp.contains(r#""kind":"remote""#), "{stamp}");
2110        assert!(stamp.contains(r#""host":"mini""#), "{stamp}");
2111
2112        // Remote WITHOUT a node label → falls back to the endpoint host.
2113        let mut remote_nolabel = HashMap::new();
2114        remote_nolabel.insert(
2115            "explorer".to_string(),
2116            ResolvedRemotePlacement {
2117                endpoint: "ws://169.254.230.101:8899".into(),
2118                token: None,
2119                ca_cert_file: None,
2120                host_label: None,
2121            },
2122        );
2123        let r2 = bogus_runner(remote_nolabel);
2124        let spec2 = r2.build_spec(&session_of_role("explorer", "go"), &job_for("c1"));
2125        assert!(r2
2126            .placement_stamp_for(&spec2)
2127            .unwrap()
2128            .contains(r#""host":"169.254.230.101""#));
2129
2130        // Schedulable WITH a node label → {remote, <label>} (a node, not a pool name).
2131        let mut sched = HashMap::new();
2132        sched.insert(
2133            "mac-mini-monitor".to_string(),
2134            ResolvedSchedulablePlacement {
2135                pool: "mac-mini-monitor".into(),
2136                host_label: Some("mini".into()),
2137            },
2138        );
2139        let sr = bogus_sched_runner(HashMap::new(), sched);
2140        let spec3 = sr.build_spec(&session_of_role("mac-mini-monitor", "go"), &job_for("c1"));
2141        let stamp3 = sr
2142            .placement_stamp_for(&spec3)
2143            .expect("scheduled child is stamped");
2144        assert!(stamp3.contains(r#""kind":"remote""#), "{stamp3}");
2145        assert!(stamp3.contains(r#""host":"mini""#), "{stamp3}");
2146
2147        // A Local (unmatched) child gets NO stamp.
2148        let local = bogus_runner(HashMap::new());
2149        let spec4 = local.build_spec(&session_of_role("writer", "go"), &job_for("c1"));
2150        assert_eq!(local.placement_stamp_for(&spec4), None);
2151    }
2152
2153    // ---- #181: schedulable selection over the BUS (Phase 3 cutover) ----------
2154
2155    async fn start_bus() -> (String, tempfile::TempDir) {
2156        let dir = tempfile::tempdir().unwrap();
2157        let core = std::sync::Arc::new(bamboo_broker::BrokerCore::new(dir.path()));
2158        let server = std::sync::Arc::new(bamboo_broker::BrokerServer::new(core, "t"));
2159        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
2160        let addr = listener.local_addr().unwrap();
2161        tokio::spawn(async move {
2162            let _ = server.serve(listener).await;
2163        });
2164        (format!("ws://{addr}"), dir)
2165    }
2166
2167    async fn join_pool(endpoint: &str, id: &str, pool: &str) -> bamboo_broker::BrokerClient {
2168        let mut c = bamboo_broker::BrokerClient::connect(
2169            endpoint,
2170            bamboo_subagent::AgentRef {
2171                session_id: id.into(),
2172                role: Some(pool.into()),
2173            },
2174            "t",
2175        )
2176        .await
2177        .unwrap();
2178        c.subscribe().await.unwrap();
2179        c
2180    }
2181
2182    fn sched_runner_on_bus(endpoint: &str, child_role: &str, pool: &str) -> ActorChildRunner {
2183        let mut sched = HashMap::new();
2184        sched.insert(child_role.to_string(), sched_placement(pool, "unused"));
2185        bogus_sched_runner(HashMap::new(), sched).with_bus(Some(bamboo_subagent::BusEndpoint {
2186            endpoint: endpoint.into(),
2187            token: "t".into(),
2188        }))
2189    }
2190
2191    #[tokio::test]
2192    async fn resolve_schedulable_picks_a_live_bus_worker() {
2193        let (endpoint, _dir) = start_bus().await;
2194        let _w = join_pool(&endpoint, "w-gpu", "gpu-pool").await;
2195        let runner = sched_runner_on_bus(&endpoint, "explorer", "gpu-pool");
2196
2197        let mailbox = runner
2198            .resolve_schedulable_worker("explorer")
2199            .await
2200            .expect("a live pool worker is found on the bus");
2201        assert_eq!(mailbox, "w-gpu");
2202    }
2203
2204    #[tokio::test]
2205    async fn resolve_schedulable_round_robins_over_pool_workers() {
2206        let (endpoint, _dir) = start_bus().await;
2207        let _a = join_pool(&endpoint, "w-a", "gpu-pool").await;
2208        let _b = join_pool(&endpoint, "w-b", "gpu-pool").await;
2209        let runner = sched_runner_on_bus(&endpoint, "explorer", "gpu-pool");
2210
2211        // Successive resolves spread across both connected workers.
2212        let mut picked = std::collections::HashSet::new();
2213        for _ in 0..6 {
2214            picked.insert(runner.resolve_schedulable_worker("explorer").await.unwrap());
2215        }
2216        assert_eq!(
2217            picked,
2218            ["w-a".to_string(), "w-b".to_string()].into_iter().collect(),
2219            "round-robin must cover every connected pool worker"
2220        );
2221    }
2222
2223    #[tokio::test]
2224    async fn resolve_schedulable_errors_on_empty_pool() {
2225        let (endpoint, _dir) = start_bus().await;
2226        // No worker subscribes to "gpu-pool".
2227        let runner = sched_runner_on_bus(&endpoint, "explorer", "gpu-pool");
2228
2229        let err = runner
2230            .resolve_schedulable_worker("explorer")
2231            .await
2232            .expect_err("an empty pool is terminal — no local fallback")
2233            .to_string();
2234        assert!(err.contains("no live worker in pool"), "got: {err}");
2235        assert!(err.contains("NOT spawning"), "got: {err}");
2236    }
2237
2238    /// FULL schedulable run over the bus: a worker SERVING `EchoExecutor` joins the
2239    /// pool by role; `execute_external_child` with a Schedulable placement resolves
2240    /// it from the bus (no local subprocess — the worker_bin is `/bin/false`),
2241    /// drives the run, gets the echo back, AND stamps the child session with the
2242    /// pool's cluster-node label — `{kind:remote, host:"mini"}`. The end-to-end
2243    /// analogue of the live `mac-mini-monitor`→mini run.
2244    #[tokio::test]
2245    async fn execute_external_child_runs_schedulable_over_bus_and_stamps_node_label() {
2246        let (endpoint, _dir) = start_bus().await;
2247
2248        // A bus worker SERVING runs (not just presence), joined to the pool by role.
2249        let ep = endpoint.clone();
2250        let worker = tokio::spawn(async move {
2251            let _ = bamboo_broker::serve_executor(
2252                &ep,
2253                bamboo_subagent::AgentRef {
2254                    session_id: "mmm-worker".into(),
2255                    role: Some("mac-mini-monitor".into()),
2256                },
2257                "t",
2258                std::sync::Arc::new(bamboo_subagent::executor::EchoExecutor),
2259            )
2260            .await;
2261        });
2262
2263        // Wait until the worker is visible on the bus so the pool is non-empty
2264        // when execute_external_child resolves it (serve_executor connects async).
2265        let mut probe = bamboo_broker::BrokerClient::connect(
2266            &endpoint,
2267            bamboo_subagent::AgentRef {
2268                session_id: "probe".into(),
2269                role: None,
2270            },
2271            "t",
2272        )
2273        .await
2274        .unwrap();
2275        let mut ready = false;
2276        for _ in 0..100 {
2277            if probe
2278                .list_connected("mac-mini-monitor")
2279                .await
2280                .unwrap()
2281                .iter()
2282                .any(|id| id == "mmm-worker")
2283            {
2284                ready = true;
2285                break;
2286            }
2287            tokio::time::sleep(Duration::from_millis(30)).await;
2288        }
2289        assert!(ready, "worker never joined the pool");
2290
2291        // Runner: child role → schedulable pool "mac-mini-monitor" carrying the
2292        // cluster node's label "mini"; bogus worker_bin so any local spawn fails.
2293        let mut sched = HashMap::new();
2294        sched.insert(
2295            "mac-mini-monitor".to_string(),
2296            ResolvedSchedulablePlacement {
2297                pool: "mac-mini-monitor".into(),
2298                host_label: Some("mini".into()),
2299            },
2300        );
2301        let runner = bogus_sched_runner(HashMap::new(), sched).with_bus(Some(
2302            bamboo_subagent::BusEndpoint {
2303                endpoint: endpoint.clone(),
2304                token: "t".into(),
2305            },
2306        ));
2307
2308        let mut session = session_of_role("mac-mini-monitor", "hello scheduled");
2309        let job = job_for("child-1");
2310        let (event_tx, _rx) = mpsc::channel::<AgentEvent>(64);
2311        let cancel = CancellationToken::new();
2312
2313        tokio::time::timeout(
2314            Duration::from_secs(10),
2315            runner.execute_external_child(&mut session, &job, event_tx, cancel),
2316        )
2317        .await
2318        .expect("run did not hang")
2319        .expect("schedulable run succeeded over the bus (no local spawn)");
2320
2321        // Echo reply flowed back — proves it routed to the bus worker, not local.
2322        let last = session
2323            .messages
2324            .iter()
2325            .rev()
2326            .find(|m| matches!(m.role, Role::Assistant))
2327            .expect("an assistant reply was written back");
2328        assert!(last.content.contains("echo:"), "got {:?}", last.content);
2329
2330        // ...and the child is stamped with the pool's cluster-node label.
2331        let placement = session
2332            .metadata
2333            .get("placement")
2334            .expect("scheduled child session stamped with a placement");
2335        assert!(placement.contains(r#""kind":"remote""#), "{placement}");
2336        assert!(placement.contains(r#""host":"mini""#), "{placement}");
2337
2338        worker.abort();
2339    }
2340}