Skip to main content

bamboo_engine/external_agents/
actor_adapter.rs

1//! Actor external child runner.
2//!
3//! Runs a child session as an independent **actor**: a separate OS process with its own
4//! isolated context, speaking the `bamboo-subagent` WebSocket protocol. This is the
5//! engine-side adapter on the `wants_external` seam: it spawns the worker binary, waits for
6//! it to self-register into the Tier-1 file fabric, connects, sends the assignment, and
7//! forwards the child's `AgentEvent`s back onto the parent's `event_tx`.
8//!
9//! The built-in **local actor** instance of this runner is the default runtime for
10//! every sub-agent (the in-process runtime was removed). The expert `externalAgents`
11//! tables can additionally route specific roles to other actor/a2a agents.
12
13use std::collections::HashMap;
14use std::path::{Path, PathBuf};
15use std::sync::Arc;
16use std::time::Duration;
17
18use async_trait::async_trait;
19use bamboo_agent_core::{AgentError, AgentEvent, Role, Session};
20use bamboo_domain::poison::PoisonRecover;
21use tokio::sync::mpsc;
22use tokio_util::sync::CancellationToken;
23
24use bamboo_subagent::fleet::{spawn_worker_on_bus, SpawnedChild};
25use bamboo_subagent::proto::{
26    AgentRecord, ChildFrame, ParentFrame, PermissionPolicyContext, RunSpec, TerminalStatus,
27};
28use bamboo_subagent::provision::{
29    ChildIdentity, ExecutorSpec, ModelRefSpec, Placement, ProvisionSpec, ScopedCredential,
30};
31use bamboo_subagent::transport::{client_config_trusting_cert, ChildClient};
32
33use crate::runtime::execution::{ExternalChildRunner, SpawnJob};
34
35/// Default cap on simultaneously running actor processes.
36pub const DEFAULT_MAX_CONCURRENT_ACTORS: usize = 8;
37
38/// Max nesting depth for direct nested execution (Phase 6). A worker whose
39/// session `spawn_depth` is below this gets its own spawn stack + the real
40/// SubAgent tool; at/over it, neither (and the tool itself refuses). Mirrors
41/// `bamboo_server_tools::DEFAULT_MAX_SPAWN_DEPTH` (kept in sync; engine can't
42/// depend on server-tools). Root orchestrator = 0 ⇒ 4 levels of sub-agents.
43pub const MAX_SPAWN_DEPTH: u32 = 4;
44
45/// Default cap on idle pooled (warm, reusable) workers kept per fingerprint.
46const DEFAULT_MAX_IDLE_PER_KEY: usize = 4;
47
48/// How long a pooled worker waits for its next assignment before reclaiming
49/// itself (must comfortably exceed the gap between sibling spawns).
50const POOLED_IDLE_TIMEOUT_SECS: u64 = 300;
51
52/// Deadline for a local worker's FIRST frame after a Run is dispatched. A warm
53/// worker answers in seconds; a cold spawn within tens. Total silence past this
54/// means the worker is dead (e.g. a pooled worker that exited right after its
55/// liveness check) and its Run is queued with nobody to serve it — trip it so the
56/// runner respawns once instead of hanging forever. Generous, to never false-trip
57/// a slow-but-healthy cold start.
58const WORKER_FIRST_FRAME_TIMEOUT: Duration = Duration::from_secs(60);
59
60/// Plaintext token returned once by the server authority for one Codex run.
61/// `token_id` is non-secret and is the handle used for guaranteed revocation.
62pub struct IssuedCodexRunToken {
63    pub token_id: String,
64    pub token: String,
65}
66
67/// Server-owned authority for Bamboo-as-provider Codex credentials. The engine
68/// only needs mint/revoke; verification remains inside the HTTP server.
69pub trait CodexRunTokenAuthority: Send + Sync + 'static {
70    fn issue(&self, session_id: &str) -> Result<IssuedCodexRunToken, String>;
71    fn revoke(&self, token_id: &str);
72}
73
74struct CodexRunTokenGuard {
75    authority: Arc<dyn CodexRunTokenAuthority>,
76    token_id: String,
77}
78
79impl Drop for CodexRunTokenGuard {
80    fn drop(&mut self) {
81        self.authority.revoke(&self.token_id);
82    }
83}
84
85fn executor_uses_bamboo_codex(executor: &ExecutorSpec) -> bool {
86    matches!(
87        executor,
88        ExecutorSpec::Codex {
89            auth_mode: Some(mode),
90            ..
91        } if mode == "bamboo"
92    ) || matches!(
93        executor,
94        ExecutorSpec::Codex {
95            auth_mode: None,
96            inherit_user_config,
97            ..
98        } if !inherit_user_config.unwrap_or(false)
99    )
100}
101
102fn workspace_is_bamboo_owned(raw: &str) -> bool {
103    let workspace = std::fs::canonicalize(raw).unwrap_or_else(|_| PathBuf::from(raw));
104    let configured_root = bamboo_config::paths::resolve_workspace_root();
105    let configured_root = std::fs::canonicalize(&configured_root).unwrap_or(configured_root);
106    if workspace.starts_with(&configured_root) {
107        return true;
108    }
109
110    // Project worktrees created by Bamboo live under
111    // `<project>/.bamboo/worktree/<name>` and carry the ownership marker used
112    // by the project-worktree lifecycle. A path that merely imitates the
113    // directory shape is not sufficient to bypass Codex's git guard.
114    workspace.ancestors().any(|candidate| {
115        let Some(name) = candidate.file_name().and_then(|name| name.to_str()) else {
116            return false;
117        };
118        let Some(worktree_root) = candidate.parent() else {
119            return false;
120        };
121        if worktree_root.file_name() != Some(std::ffi::OsStr::new("worktree"))
122            || worktree_root.parent().and_then(Path::file_name)
123                != Some(std::ffi::OsStr::new(".bamboo"))
124        {
125            return false;
126        }
127        let marker = worktree_root.join(".bamboo-owned").join(name);
128        std::fs::read_to_string(marker).is_ok_and(|branch| branch == format!("bamboo/{name}"))
129    })
130}
131
132fn build_codex_run_secrets(
133    executor: &ExecutorSpec,
134    authority: Option<Arc<dyn CodexRunTokenAuthority>>,
135    child_session_id: &str,
136) -> Result<
137    (
138        bamboo_subagent::proto::RunSecrets,
139        Option<CodexRunTokenGuard>,
140    ),
141    AgentError,
142> {
143    if !executor_uses_bamboo_codex(executor) {
144        return Ok((bamboo_subagent::proto::RunSecrets::default(), None));
145    }
146
147    let authority = authority.ok_or_else(|| {
148        AgentError::LLM(
149            "Codex auth mode 'bamboo' requires the server per-run token authority".to_string(),
150        )
151    })?;
152    let issued = authority
153        .issue(child_session_id)
154        .map_err(|error| AgentError::LLM(format!("mint Codex per-run provider token: {error}")))?;
155    let guard = CodexRunTokenGuard {
156        authority,
157        token_id: issued.token_id,
158    };
159    Ok((
160        bamboo_subagent::proto::RunSecrets {
161            codex_provider_token: Some(bamboo_subagent::proto::SecretValue::new(issued.token)),
162        },
163        Some(guard),
164    ))
165}
166
167/// A warm worker on the mailbox bus, parked for reuse between runs. It stays
168/// dialed-in + subscribed to `mailbox_id`; the next interchangeable child
169/// delivers its `Run` there instead of spawning a fresh process. Dropping it
170/// kills a local kill-on-drop subprocess; a remote / schedulable handle is
171/// process-less (`kill()` is a no-op — it self-manages via its idle timeout).
172struct PooledWorker {
173    worker: SpawnedChild,
174    /// The bus mailbox this worker subscribes to (where its `Run`s are delivered).
175    mailbox_id: String,
176}
177
178/// A role pinned to a remote resident worker (remote-actor-plan §3.4 / P1.5,
179/// #193), resolved at runner-build time from `SubagentsConfig.remote_placements`:
180/// the env-named bearer is already READ into `token` here (the raw token never
181/// rides the config), and `ca_cert_file` is the path to a PEM pinning a
182/// self-signed worker cert (`None` ⇒ default webpki roots / plaintext `ws://`).
183#[derive(Debug, Clone)]
184pub struct ResolvedRemotePlacement {
185    pub endpoint: String,
186    pub token: Option<String>,
187    pub ca_cert_file: Option<PathBuf>,
188    /// Display name for the machine this role runs on — the matching cluster
189    /// node's `label`/host, surfaced on the UI placement badge. `None` ⇒ derive
190    /// from the endpoint host.
191    pub host_label: Option<String>,
192}
193
194/// A role routed to a SCHEDULED worker (remote-actor-plan §3.4 / P2b, #181),
195/// resolved at runner-build time from `SubagentsConfig.schedulable_placements`.
196/// Names the logical `pool` (= the bus role) whose LIVE connected workers are the
197/// scheduling candidates — the runner picks one via the bus presence query
198/// (`BrokerClient::list_connected`). Phase 3 retired the old HTTP registry, so a
199/// pool is now just a role on the bus.
200#[derive(Debug, Clone)]
201pub struct ResolvedSchedulablePlacement {
202    pub pool: String,
203    /// Display name for the machine this pool's workers run on — the matching
204    /// cluster node's `label`/host, surfaced on the UI placement badge. `None` ⇒
205    /// fall back to the pool name.
206    pub host_label: Option<String>,
207}
208
209/// How `execute_external_child` should obtain its worker connection, decided
210/// once from `spec.placement`. Splits the divergent acquire/connect + retire
211/// logic three ways while the shared middle (Run dispatch, live registration,
212/// drive, close) stays identical. `Local` is the unchanged pre-#193 path;
213/// `Remote` is the unchanged #194 path; `Schedulable` (#181, P2b) is new.
214enum PlacementKind {
215    Local,
216    Remote,
217    Schedulable,
218}
219
220/// Spawns and drives a child session as an independent actor: a `bamboo-subagent` worker process.
221pub struct ActorChildRunner {
222    approval_registry: Option<super::approval_registry::SharedApprovalRegistry>,
223    permission_config: Option<Arc<bamboo_tools::permission::PermissionConfig>>,
224    agent_id: String,
225    worker_bin: PathBuf,
226    worker_args: Vec<String>,
227    fabric_dir: PathBuf,
228    executor: ExecutorSpec,
229    /// Per-provider credentials snapshotted from the parent config at build
230    /// time; the spec carries only the ONE the child's provider needs.
231    credentials: Vec<ScopedCredential>,
232    /// Parent's default provider (used when the child has no explicit one).
233    default_provider: String,
234    /// The mailbox bus to run local children over (the unified transport). Local
235    /// sub-agents require it; `None` only when no broker could be embedded.
236    bus: Option<bamboo_subagent::BusEndpoint>,
237    /// Backpressure: bounds the number of concurrently *running* actors; further
238    /// runs wait for a slot instead of exploding the process table. (Idle pooled
239    /// workers do not hold a slot.)
240    concurrency: std::sync::Arc<tokio::sync::Semaphore>,
241    /// Warm-worker pool keyed by a reuse fingerprint
242    /// (role/provider/model/workspace/disabled-tools/baked-caps). A finished run
243    /// parks its bus worker here so the next interchangeable child reuses it
244    /// (delivers its `Run` to the same mailbox) instead of spawning a fresh
245    /// process — collapsing N sibling sub-agents onto a few warm workers.
246    pool: Arc<tokio::sync::Mutex<HashMap<String, Vec<PooledWorker>>>>,
247    max_idle_per_key: usize,
248    /// Host-side decision for a child's gated-tool approval request (Phase 2).
249    /// `None` ⇒ fail-closed DENY (the safe default). A wired decider (policy or
250    /// human-routing bridge) returns approve/deny over the actor WS.
251    approval_decider: Option<Arc<dyn ChildApprovalDecider>>,
252    /// Off-loop parent-agent reviewer for forced-ask requests. The root server
253    /// wires a session-aware reviewer; nested workers wire their owning model
254    /// reviewer directly into the per-run runner.
255    approval_reviewer: Option<Arc<dyn ChildApprovalReviewer>>,
256    /// Per-run escalation host bridge for non-bypass child-approval routing (#68;
257    /// Phase 6, Part B). The owning worker's `run()` installs its OWN host bridge
258    /// here via `set_escalation_bridge`; `execute_external_child` CAPTURES it at
259    /// grandchild-spawn time and hands the owned value to `drive()`, which uses it
260    /// to RE-PROXY a child's approval request UP to the parent run — chaining up
261    /// every level until a bypass level (model-review) or the top orchestrator
262    /// (human) decides, then relaying the reply back down. Was a process-global
263    /// slot; now per-runner so a fire-and-forget grandchild that OUTLIVES the run
264    /// that spawned it keeps that run's bridge for its whole lifetime instead of
265    /// reading a stale/overwritten global at approval time (→ fail-closed deny).
266    escalation_bridge: Arc<std::sync::Mutex<Option<bamboo_subagent::executor::HostBridge>>>,
267    /// Roles pinned to a REMOTE resident worker (#193), keyed by sub-agent role
268    /// (the child's `subagent_type`). A role present here routes through the
269    /// dedicated remote branch in `execute_external_child` (Bearer-authenticated
270    /// `wss://` connect, no spawn, no pool, no kill) instead of the local
271    /// subprocess + warm-pool path. Empty (the default) = all-local behavior.
272    remote_placements: HashMap<String, ResolvedRemotePlacement>,
273    /// Roles routed to a REGISTRY-SCHEDULED worker (#181, P2b), keyed by sub-agent
274    /// role. A role present here (AND not already in `remote_placements`, which
275    /// wins) routes through the dedicated SCHEDULABLE branch in
276    /// `execute_external_child`: query the registry for live workers in the pool,
277    /// pick one (round-robin), connect over `wss://` — no spawn, no pool, no kill,
278    /// and NO local-subprocess fallback (no live worker ⇒ a clear error). Empty
279    /// (the default) = all-local behavior.
280    schedulable_placements: HashMap<String, ResolvedSchedulablePlacement>,
281    /// Per-pool round-robin cursor for schedulable scheduling (#181, P2b). Bumped
282    /// once per pick so successive sibling spawns SPREAD across a pool's live
283    /// workers instead of all landing on the first candidate. Best-effort spread,
284    /// not a load balancer — the registry's live set can change between picks.
285    schedule_cursor: Arc<std::sync::Mutex<HashMap<String, usize>>>,
286    /// Optional server authority used only by `Codex` in `bamboo` auth mode.
287    codex_run_tokens: Option<Arc<dyn CodexRunTokenAuthority>>,
288}
289
290/// Decides how the host answers a child worker's gated-tool approval request
291/// (Phase 2: child → parent approval delegation). Async so an implementation
292/// can consult a policy. With no decider wired the host replies with a
293/// fail-closed DENY.
294///
295/// NOTE: `decide` is awaited inside the per-child frame pump, so an
296/// implementation must resolve promptly (e.g. a policy lookup). Model-based
297/// review belongs in [`ChildApprovalReviewer`], which runs off-loop and returns
298/// through the live steering channel without stalling the frame pump.
299#[async_trait]
300pub trait ChildApprovalDecider: Send + Sync {
301    /// Decide whether `child_session_id` may perform the gated action described
302    /// by `request` (`{tool_name, permission, resource}`).
303    async fn decide(&self, child_session_id: &str, request: &serde_json::Value) -> bool;
304}
305
306/// Resolve a child approval request to approve/deny. Fail-closed (DENY) when no
307/// decider is wired — the single, testable seam for the host-side decision.
308async fn decide_child_approval(
309    decider: Option<&Arc<dyn ChildApprovalDecider>>,
310    child_session_id: &str,
311    request: &serde_json::Value,
312) -> bool {
313    match decider {
314        Some(decider) => decider.decide(child_session_id, request).await,
315        None => false,
316    }
317}
318
319/// How long a chained parent-agent review may take before the child's gated
320/// tool fails closed (DENY). Bounds an unanswered request so it cannot hang the
321/// worker indefinitely.
322const CHILD_APPROVAL_TIMEOUT: Duration = Duration::from_secs(300);
323
324/// Off-loop reviewer for a child's gated-tool approval request (Phase 6, Part B).
325///
326/// Installed (process-global) by a BYPASSED self-orchestrating worker so its
327/// children's forced-ask (dangerous) gated actions — which still raise
328/// `ConfirmationRequired` even under bypass — get an LLM reasonableness check
329/// rather than a blind pass. `review` is an LLM call: `drive()` invokes it in a
330/// SPAWNED task (NEVER in the frame pump) and delivers the verdict async via the
331/// live channel, so the agent loop is never blocked.
332#[async_trait]
333pub trait ChildApprovalReviewer: Send + Sync {
334    /// Judge whether the gated action `request` (`{tool_name, permission,
335    /// resource}`) is reasonable for `child_session_id`'s task. `true` = approve.
336    async fn review(
337        &self,
338        parent_session_id: &str,
339        child_session_id: &str,
340        request: &serde_json::Value,
341    ) -> bool;
342}
343
344fn child_approval_reviewer_slot() -> &'static std::sync::OnceLock<Arc<dyn ChildApprovalReviewer>> {
345    static SLOT: std::sync::OnceLock<Arc<dyn ChildApprovalReviewer>> = std::sync::OnceLock::new();
346    &SLOT
347}
348
349/// Install the process-global child-approval reviewer (idempotent; first wins).
350pub fn set_child_approval_reviewer(reviewer: Arc<dyn ChildApprovalReviewer>) {
351    let _ = child_approval_reviewer_slot().set(reviewer);
352}
353
354/// The process-global child-approval reviewer, if installed.
355pub fn child_approval_reviewer() -> Option<Arc<dyn ChildApprovalReviewer>> {
356    child_approval_reviewer_slot().get().cloned()
357}
358
359impl ActorChildRunner {
360    #[allow(clippy::too_many_arguments)]
361    pub fn new(
362        agent_id: String,
363        worker_bin: PathBuf,
364        worker_args: Vec<String>,
365        fabric_dir: PathBuf,
366        executor: ExecutorSpec,
367        credentials: Vec<ScopedCredential>,
368        default_provider: String,
369        max_concurrent: usize,
370    ) -> Self {
371        Self {
372            approval_registry: None,
373            permission_config: None,
374            agent_id,
375            worker_bin,
376            worker_args,
377            fabric_dir,
378            executor,
379            credentials,
380            default_provider,
381            bus: None,
382            concurrency: std::sync::Arc::new(tokio::sync::Semaphore::new(max_concurrent.max(1))),
383            pool: Arc::new(tokio::sync::Mutex::new(HashMap::new())),
384            max_idle_per_key: DEFAULT_MAX_IDLE_PER_KEY,
385            approval_decider: None,
386            approval_reviewer: None,
387            escalation_bridge: Arc::new(std::sync::Mutex::new(None)),
388            remote_placements: HashMap::new(),
389            schedulable_placements: HashMap::new(),
390            schedule_cursor: Arc::new(std::sync::Mutex::new(HashMap::new())),
391            codex_run_tokens: None,
392        }
393    }
394
395    pub fn with_approval_registry(
396        mut self,
397        registry: super::approval_registry::SharedApprovalRegistry,
398    ) -> Self {
399        self.approval_registry = Some(registry);
400        self
401    }
402
403    pub fn with_permission_config(
404        mut self,
405        config: Arc<bamboo_tools::permission::PermissionConfig>,
406    ) -> Self {
407        self.permission_config = Some(config);
408        self
409    }
410
411    /// Run children over the mailbox bus (the unified actor+mailbox transport).
412    /// When set, local children dial this bus and are driven by mailbox id; when
413    /// unset they use the legacy direct-WS path. The server passes its in-process
414    /// broker here (`subagents.broker`); tests without a broker leave it unset.
415    pub fn with_bus(mut self, bus: Option<bamboo_subagent::BusEndpoint>) -> Self {
416        self.bus = bus.filter(|b| !b.endpoint.trim().is_empty());
417        self
418    }
419
420    /// Wire the host-side decider for child gated-tool approval requests
421    /// (Phase 2). Without this the host fail-closed DENYs every request.
422    pub fn with_approval_decider(mut self, decider: Arc<dyn ChildApprovalDecider>) -> Self {
423        self.approval_decider = Some(decider);
424        self
425    }
426
427    pub fn with_approval_reviewer(mut self, reviewer: Arc<dyn ChildApprovalReviewer>) -> Self {
428        self.approval_reviewer = Some(reviewer);
429        self
430    }
431
432    pub fn with_codex_run_tokens(
433        mut self,
434        authority: Option<Arc<dyn CodexRunTokenAuthority>>,
435    ) -> Self {
436        self.codex_run_tokens = authority;
437        self
438    }
439
440    /// Pin specific sub-agent roles to remote resident workers (#193). The map
441    /// is keyed by role (`subagent_type`); a child whose role is present connects
442    /// over `wss://` to the resolved endpoint instead of spawning a local
443    /// subprocess. Default (empty) keeps every role on the local path — exactly
444    /// today's behavior.
445    pub fn with_remote_placements(
446        mut self,
447        placements: HashMap<String, ResolvedRemotePlacement>,
448    ) -> Self {
449        self.remote_placements = placements;
450        self
451    }
452
453    /// Route specific sub-agent roles to a registry-SCHEDULED worker (#181, P2b).
454    /// The map is keyed by role (`subagent_type`); a child whose role is present
455    /// (and NOT already pinned by `remote_placements`, which takes precedence) is
456    /// run on a live worker discovered from the registry instead of a local
457    /// subprocess. Default (empty) keeps every role on the local path.
458    pub fn with_schedulable_placements(
459        mut self,
460        placements: HashMap<String, ResolvedSchedulablePlacement>,
461    ) -> Self {
462        self.schedulable_placements = placements;
463        self
464    }
465
466    /// Reuse fingerprint: two children are interchangeable on one warm worker iff
467    /// they share role, provider, model, workspace, disabled-tool set, AND every
468    /// capability the worker BAKES at provision time (`BambooRuntimeExecutor`
469    /// stamps these once and reuses them across runs): nesting depth, nested-spawn
470    /// stack, bypass mode, permission enforcement, and the depth cap. Omitting any
471    /// of these lets the pool hand a run a worker baked for a DIFFERENT posture —
472    /// e.g. a depth-1 worker (with its own spawn stack) reused for a depth-4
473    /// child would re-stamp `spawn_depth=1` and pass the depth-cap check, breaking
474    /// the recursion bound; or a bypass worker reused for a non-bypass child. So
475    /// these MUST split the pool bucket. Everything else (assignment, history) is
476    /// shipped per-run in the `RunSpec` and does not affect the fingerprint.
477    /// Reuse fingerprint (role/provider/model/workspace/disabled-tools/baked
478    /// caps): two children with the same fingerprint are interchangeable on one
479    /// warm worker, so they share a pool bucket. Any axis the worker bakes ONCE
480    /// at provision time MUST be in here, else a worker baked for one posture
481    /// gets reused for another (see the `fingerprint_*` tests).
482    fn fingerprint(spec: &ProvisionSpec) -> String {
483        let role = spec.identity.role.as_str();
484        let (provider, model) = spec
485            .model
486            .as_ref()
487            .map(|m| (m.provider.as_str(), m.model.as_str()))
488            .unwrap_or(("", ""));
489        let workspace = spec.workspace.as_deref().unwrap_or("");
490        let mut tools = spec.disabled_tools.clone().unwrap_or_default();
491        tools.sort();
492        let caps = &spec.capabilities;
493        // The worker constructs its executor exactly once. In particular,
494        // Codex exec and app-server workers are not interchangeable.
495        let executor = serde_json::to_string(&spec.executor).unwrap_or_default();
496        format!(
497            "{role}\u{1}{provider}\u{1}{model}\u{1}{workspace}\u{1}{}\u{1}d={}\u{1}ns={}\u{1}by={}\u{1}ep={}\u{1}md={}\u{1}nha={}\u{1}gro={}\u{1}executor={executor}",
498            tools.join(","),
499            spec.identity.depth,
500            caps.nested_spawn,
501            caps.bypass,
502            caps.enforce_permissions,
503            caps.max_spawn_depth.unwrap_or(0),
504            // #73 review (P1): a worker bakes `no_human_review` ONCE from this flag
505            // at build() and never re-reads it per run, so the pool MUST NOT hand a
506            // worker baked for one approval posture to a run of the opposite one —
507            // else a scheduled-root worker reused for an interactive child would
508            // silently model-review instead of asking the human (and vice-versa,
509            // reintroducing the 300s-deny). Split the bucket on it.
510            caps.no_human_approver,
511            // #71: the read-only Bash checker is baked once at build() from this
512            // flag, so a guardian-reviewer worker must NOT be reused for an
513            // ordinary child (which expects unrestricted Bash), and vice-versa.
514            caps.guardian_read_only,
515        )
516    }
517
518    /// Check out a warm bus worker for `key`, reusing a live parked one if any,
519    /// else spawning a fresh one that dials the bus. The returned worker is OWNED
520    /// by the caller for the run's duration (checkout removes it from the pool, so
521    /// a concurrent sibling gets a different worker or spawns its own — one run per
522    /// worker at a time, matching the pre-bus pool semantics).
523    async fn acquire_bus_worker(
524        &self,
525        key: &str,
526        spec: &ProvisionSpec,
527    ) -> crate::runtime::runner::Result<PooledWorker> {
528        // Drain the bucket, skipping (and reaping) any worker whose process exited
529        // while parked. A live one is handed straight out for reuse.
530        loop {
531            let candidate = {
532                let mut pool = self.pool.lock().await;
533                pool.get_mut(key).and_then(|bucket| bucket.pop())
534            };
535            let Some(mut candidate) = candidate else {
536                break;
537            };
538            if candidate.worker.is_alive() {
539                return Ok(candidate);
540            }
541            candidate.worker.kill().await;
542        }
543
544        let spawned = spawn_worker_on_bus(&self.worker_bin, &self.worker_args, spec)
545            .await
546            .map_err(|e| AgentError::LLM(format!("actor spawn (bus) failed: {e}")))?;
547        let mailbox_id = spawned.record.agent_id.clone();
548        Ok(PooledWorker {
549            worker: spawned,
550            mailbox_id,
551        })
552    }
553
554    /// Park a warm bus worker for reuse after a clean run; if its bucket is full
555    /// (or it died), kill it instead. The worker stays dialed-in + subscribed
556    /// while parked, so a reusing child just delivers a new `Run` to its mailbox.
557    async fn release_bus_worker(&self, key: &str, mut worker: PooledWorker) {
558        if !worker.worker.is_alive() {
559            worker.worker.kill().await;
560            return;
561        }
562        let mut pool = self.pool.lock().await;
563        let bucket = pool.entry(key.to_string()).or_default();
564        if bucket.len() >= self.max_idle_per_key {
565            drop(pool);
566            worker.worker.kill().await;
567            return;
568        }
569        bucket.push(worker);
570    }
571
572    /// Assemble the parent-resolved provisioning document for this child.
573    fn build_spec(&self, session: &Session, job: &SpawnJob) -> ProvisionSpec {
574        let mut spec = ProvisionSpec::new(
575            ChildIdentity {
576                child_id: job.child_session_id.clone(),
577                parent_id: Some(job.parent_session_id.clone()),
578                project_key: None,
579                role: session
580                    .metadata
581                    .get("subagent_type")
582                    .cloned()
583                    .unwrap_or_else(|| "worker".to_string()),
584                // The child session already carries the correct depth
585                // (create_child_action's new_child_of did parent.spawn_depth+1);
586                // stamp it so the worker can re-establish it on its run session
587                // and enforce the max-depth cap across the actor boundary.
588                depth: session.spawn_depth,
589            },
590            self.executor.clone(),
591            self.fabric_dir.to_string_lossy().into_owned(),
592        );
593        spec.workspace = session.workspace.clone();
594        if let ExecutorSpec::Codex {
595            workspace_owned, ..
596        } = &mut spec.executor
597        {
598            *workspace_owned = Some(
599                spec.workspace
600                    .as_deref()
601                    .is_some_and(workspace_is_bamboo_owned),
602            );
603        }
604        // Unified transport: when a bus is configured, the child dials it (no
605        // listen socket / file discovery) and the parent drives it by mailbox id.
606        spec.bus = self.bus.clone();
607        // Final model: the session's pinned model_ref (create.model / routing already applied),
608        // falling back to the job's bare model on the parent's default provider.
609        spec.model = session
610            .model_ref
611            .as_ref()
612            .map(|r| ModelRefSpec {
613                provider: r.provider.clone(),
614                model: r.model.clone(),
615            })
616            .or_else(|| {
617                let m = job.model.trim();
618                (!m.is_empty()).then(|| ModelRefSpec {
619                    provider: self.default_provider.clone(),
620                    model: m.to_string(),
621                })
622            });
623        spec.disabled_tools = job.disabled_tools.clone();
624        match &spec.executor {
625            // Codex auth is independent of the session's normal Bamboo model
626            // provider. Inherit/API-key/Bamboo modes need no credential-store
627            // secret at provisioning; custom mode gets exactly its referenced
628            // key. This prevents an unrelated upstream provider key from
629            // reaching a Codex worker that only needs a per-run bcx1_ token.
630            ExecutorSpec::Codex {
631                auth_mode,
632                provider_key_ref,
633                ..
634            } => {
635                if auth_mode.as_deref() == Some("custom") {
636                    if let Some(reference) = provider_key_ref {
637                        if let Some(credential) = self.credentials.iter().find(|credential| {
638                            credential.credential_ref.as_deref() == Some(reference)
639                        }) {
640                            spec.secrets.provider_credentials.push(credential.clone());
641                        } else {
642                            tracing::warn!(
643                                "actor child {}: custom Codex credential reference '{}' did not resolve",
644                                job.child_session_id,
645                                reference
646                            );
647                        }
648                    } else {
649                        tracing::warn!(
650                            "actor child {}: custom Codex executor has no credential reference",
651                            job.child_session_id
652                        );
653                    }
654                }
655            }
656            // Other executors keep the existing least-privilege contract: only
657            // the credential for the child session's selected provider.
658            _ => {
659                let provider = spec
660                    .model
661                    .as_ref()
662                    .map(|model| model.provider.as_str())
663                    .filter(|provider| !provider.trim().is_empty())
664                    .unwrap_or(&self.default_provider);
665                if let Some(credential) = self
666                    .credentials
667                    .iter()
668                    .find(|credential| credential.provider == provider)
669                {
670                    spec.secrets.provider_credentials.push(credential.clone());
671                } else {
672                    tracing::warn!(
673                        "actor child {}: no credential found for provider '{}'",
674                        job.child_session_id,
675                        provider
676                    );
677                }
678            }
679        }
680        // Phase 6 (direct nested execution): a worker BELOW the depth cap may
681        // orchestrate its OWN children — on startup it builds its own spawn
682        // stack and runs the real SubAgent tool (no host proxy). The cap (the
683        // SubAgent tool refuses to spawn at/over `max_spawn_depth`) bounds the
684        // recursion. Driven purely by the child's depth, so it auto-propagates
685        // down the tree without any extra config threading.
686        spec.capabilities.nested_spawn = session.spawn_depth < MAX_SPAWN_DEPTH;
687        spec.capabilities.max_spawn_depth = Some(MAX_SPAWN_DEPTH);
688        // #69: activate child-approval review. Sub-agents enforce permissions so
689        // their DANGEROUS actions (the worker uses a HIGH threshold) reach the
690        // parent for review — escalated to the human, or model-reviewed off-loop
691        // when the parent is in bypass. The worker installs no checker without
692        // this, so the whole review chain would otherwise stay dormant.
693        spec.capabilities.enforce_permissions = true;
694        // Propagate "bypass permissions" so a self-orchestrating worker knows it
695        // is a bypassed parent and installs the off-loop model-reviewer for its
696        // children's forced-ask actions (Phase 6, Part B). The child session
697        // already carries the inherited flag (create_child_action seeds it).
698        spec.capabilities.bypass = session
699            .agent_runtime_state
700            .as_ref()
701            .is_some_and(|s| s.bypass_permissions);
702        // #73: propagate "no interactive human approver" (headless / scheduled /
703        // deployed root, inherited by the child session). When set, the worker's
704        // per-run approval proxy model-reviews a gated action locally instead of
705        // escalating to a human who will never answer (which would 300s-deny).
706        spec.capabilities.no_human_approver = session
707            .agent_runtime_state
708            .as_ref()
709            .is_some_and(|s| s.no_human_approver);
710        // #71: mark a READ-ONLY Guardian reviewer so the worker installs the
711        // read-only Bash allowlist checker. The reviewer is spawned by
712        // `spawn_guardian_review` with `subagent_type == "guardian"` (the SAME
713        // marker the completion coordinator branches on to parse the verdict) AND
714        // the `guardian_read_only_disabled_tools` denylist. Keyed off that role
715        // marker (already read above to set `identity.role`), so it rides the same
716        // session-metadata path the denylist/subagent_type use — no new wire seam.
717        // Without this the worker keeps an UNRESTRICTED Bash, so the reviewer could
718        // still `rm -rf` / `git push` / `curl | sh`, defeating "read-only".
719        spec.capabilities.guardian_read_only =
720            session.metadata.get("subagent_type").map(String::as_str) == Some("guardian");
721        if spec.capabilities.guardian_read_only {
722            if let ExecutorSpec::Codex {
723                permission_profile, ..
724            } = &mut spec.executor
725            {
726                *permission_profile = Some("read-only".to_string());
727            }
728        }
729        // #193: route this role to a REMOTE resident worker when one is pinned.
730        // `spec.identity.role` was just computed from `subagent_type` above; a
731        // match flips the placement to Remote and rides the worker's bearer on the
732        // scoped secrets envelope (TLS handshake / Authorization header only — the
733        // token is never logged). No match leaves the default `Placement::Local`,
734        // so the local path is byte-for-byte unchanged for every non-pinned role.
735        if let Some(placement) = self.remote_placements.get(spec.identity.role.as_str()) {
736            spec.placement = Placement::Remote {
737                endpoint: placement.endpoint.clone(),
738            };
739            spec.secrets.worker_auth_token = placement.token.clone();
740        } else if let Some(placement) = self.schedulable_placements.get(spec.identity.role.as_str())
741        {
742            // #181 (P2b): route this role to a SCHEDULED worker — ONLY when it is
743            // NOT already pinned to a fixed remote endpoint (the `else if` makes
744            // remote_placements take precedence for a role in both). The concrete
745            // worker is picked at run time in `execute_external_child` from the bus
746            // (a live connected worker of the pool role). No per-placement bearer
747            // now — the bus connection uses the bus token. No match in either map
748            // leaves the default `Placement::Local`.
749            spec.placement = Placement::Schedulable {
750                pool: placement.pool.clone(),
751            };
752        }
753        spec
754    }
755
756    /// The `metadata["placement"]` JSON to stamp on a child from its resolved
757    /// placement, preferring the matching cluster node's `host_label` (its
758    /// operator label/host) over the raw endpoint/pool. `None` for a Local child
759    /// (the DTO defaults it to the backend's own host). Split out of
760    /// `execute_external_child` so the role→placement→host resolution is unit-testable.
761    fn placement_stamp_for(&self, spec: &ProvisionSpec) -> Option<String> {
762        let host_label = match &spec.placement {
763            Placement::Remote { .. } => self
764                .remote_placements
765                .get(spec.identity.role.as_str())
766                .and_then(|p| p.host_label.as_deref()),
767            Placement::Schedulable { .. } => self
768                .schedulable_placements
769                .get(spec.identity.role.as_str())
770                .and_then(|p| p.host_label.as_deref()),
771            Placement::Local => None,
772        };
773        placement_metadata(&spec.placement, host_label)
774    }
775
776    /// Pick a live worker for a SCHEDULABLE role from the BUS (#181, Phase 3):
777    /// ask the broker which actors are connected serving the pool role (presence
778    /// is connection-truth — no HTTP registry, no leases, no connect-fail
779    /// failover), then round-robin one per resolve for spread. Returns the chosen
780    /// worker's mailbox id. An empty pool ⇒ a terminal `AgentError` — NEVER a
781    /// local-subprocess fallback (that would silently defeat the placement).
782    async fn resolve_schedulable_worker(
783        &self,
784        role: &str,
785    ) -> std::result::Result<String, AgentError> {
786        let pool = self
787            .schedulable_placements
788            .get(role)
789            .ok_or_else(|| {
790                AgentError::LLM(format!(
791                    "schedulable placement for role '{role}' vanished before scheduling"
792                ))
793            })?
794            .pool
795            .clone();
796        let bus = self.bus.as_ref().ok_or_else(|| {
797            AgentError::LLM(format!(
798                "schedulable role '{role}': no mailbox bus configured (subagents.broker)"
799            ))
800        })?;
801
802        // Ask the BUS who is connected serving the pool role — presence is
803        // connection-truth (no HTTP registry, no leases, no stale-record failover).
804        let mut q = bamboo_broker::BrokerClient::connect(
805            &bus.endpoint,
806            bamboo_subagent::AgentRef {
807                session_id: format!("sched-q-{role}"),
808                role: None,
809            },
810            &bus.token,
811        )
812        .await
813        .map_err(|e| {
814            AgentError::LLM(format!(
815                "schedulable role '{role}': bus connect failed: {e}"
816            ))
817        })?;
818        let candidates = q.list_connected(&pool).await.map_err(|e| {
819            AgentError::LLM(format!(
820                "schedulable role '{role}': bus presence query failed: {e}"
821            ))
822        })?;
823
824        if candidates.is_empty() {
825            return Err(AgentError::LLM(format!(
826                "schedulable role '{role}': no live worker in pool '{pool}' on the bus \
827                 (NOT spawning a local subprocess — a schedulable role has no local fallback)"
828            )));
829        }
830
831        // Round-robin: advance a per-pool cursor once per resolve so successive
832        // sibling spawns spread across the connected pool workers. No failover
833        // needed — a listed worker is connected NOW (the bus only lists live
834        // subscribers), so there is no stale-but-leased candidate to skip.
835        let idx = {
836            let mut cursors = self.schedule_cursor.lock().recover_poison();
837            let cursor = cursors.entry(pool.clone()).or_insert(0);
838            let i = *cursor % candidates.len();
839            *cursor = cursor.wrapping_add(1);
840            i
841        };
842        Ok(candidates[idx].clone())
843    }
844}
845
846#[async_trait]
847impl ExternalChildRunner for ActorChildRunner {
848    async fn should_handle(&self, session: &Session) -> bool {
849        session.metadata.get("runtime.kind") == Some(&"external".to_string())
850            && session.metadata.get("external.protocol") == Some(&"actor".to_string())
851            && session.metadata.get("external.agent_id") == Some(&self.agent_id)
852    }
853
854    fn set_escalation_bridge(&self, bridge: Option<bamboo_subagent::executor::HostBridge>) {
855        *self.escalation_bridge.lock().recover_poison() = bridge;
856    }
857
858    async fn execute_external_child(
859        &self,
860        session: &mut Session,
861        job: &SpawnJob,
862        event_tx: mpsc::Sender<AgentEvent>,
863        cancel_token: CancellationToken,
864    ) -> crate::runtime::runner::Result<()> {
865        // #68 CORRECTNESS CRUX: capture the per-run escalation bridge HERE, at the
866        // moment this grandchild is spawned — while the parent run's bridge is
867        // still in our slot — into an owned local handed to `drive()` for this
868        // grandchild's whole lifetime. A fire-and-forget grandchild that OUTLIVES
869        // the run that spawned it must NOT re-read `self.escalation_bridge` at
870        // approval time: by then `run()` may have cleared/overwritten it (a worker
871        // serves runs sequentially), and re-proxying through a closed bridge
872        // fail-closed denies. Capturing at spawn pins the right bridge per run.
873        let escalation = self.escalation_bridge.lock().recover_poison().clone();
874        let assignment = extract_assignment(session);
875        let mut spec = self.build_spec(session, job);
876        // Mark the worker reusable + give it an idle timeout so it self-reaps if
877        // orphaned. Warm bus workers are pooled per fingerprint and reused.
878        spec.reusable = true;
879        if spec.limits.idle_timeout_secs.is_none() {
880            spec.limits.idle_timeout_secs = Some(POOLED_IDLE_TIMEOUT_SECS);
881        }
882        let pool_key = Self::fingerprint(&spec);
883
884        // The recommended provider URL is deliberately parent-loopback. A
885        // resident remote worker would interpret 127.0.0.1 as itself, not this
886        // server, so reject that ambiguous deployment instead of minting a
887        // credential that can never authenticate to the intended parent.
888        if executor_uses_bamboo_codex(&spec.executor) && !matches!(spec.placement, Placement::Local)
889        {
890            return Err(AgentError::LLM(
891                "Codex auth mode 'bamboo' requires local actor placement; use custom mode with a reachable URL for remote workers"
892                    .to_string(),
893            ));
894        }
895        // Rehydration: the child session in the parent's store is the actor's
896        // durable state. Ship the full conversation so a reactivation
897        // (send_message / update / rerun) carries its history. A reused worker is
898        // stateless between runs, so this is also what isolates each child's
899        // context on a shared process.
900        let messages: Vec<serde_json::Value> = session
901            .messages
902            .iter()
903            .filter_map(|m| serde_json::to_value(m).ok())
904            .collect();
905        // Policy is captured per activation (not only when a worker is
906        // provisioned), so reused local workers and resident remote/broker
907        // workers observe the latest durable revision and bypass flag at the
908        // next run boundary. Session grants are intentionally not inherited.
909        let permission_policy = self.permission_config.as_ref().and_then(|config| {
910            serde_json::to_value(config.to_serializable())
911                .ok()
912                .map(|policy| PermissionPolicyContext {
913                    revision: config.policy_revision(),
914                    bypass_permissions: session
915                        .agent_runtime_state
916                        .as_ref()
917                        .is_some_and(|state| state.bypass_permissions),
918                    session_id: session.id.clone(),
919                    workspace_path: session.workspace.clone(),
920                    inherit_session_grants: false,
921                    policy,
922                })
923        });
924
925        // Backpressure: hold a concurrency slot for the lifetime of the *run*
926        // (cancellation still proceeds — the cancel branch in drive() runs while
927        // we hold the permit). Released when this fn returns, i.e. once the worker
928        // is parked back into the pool, so idle workers don't pin slots.
929        let _slot = self
930            .concurrency
931            .acquire()
932            .await
933            .map_err(|_| AgentError::LLM("actor concurrency limiter closed".to_string()))?;
934
935        // Bamboo-as-provider credentials are minted at the activation boundary,
936        // after backpressure admits the run and never at worker provisioning.
937        // This is load-bearing for warm workers: a parked process must never
938        // retain a token from its previous run. The guard revokes on every return
939        // path (success, error, cancellation, dispatch failure, or first-frame
940        // retry exhaustion).
941        let (run_secrets, _codex_token_guard) = build_codex_run_secrets(
942            &spec.executor,
943            self.codex_run_tokens.clone(),
944            &job.child_session_id,
945        )?;
946
947        // Split LOCAL (spawn + warm-pool) from the two process-less remote paths
948        // ONLY at the divergent spots — acquire/connect here and the park/retire at
949        // the end. Everything between (Run dispatch, live-actor registration,
950        // drive, the close) is identical for all three. `kind` is the single guard.
951        //   - Local       (#0):  byte-for-byte the pre-#193 reuse-or-spawn path.
952        //   - Remote       (#194): connect to a FIXED resident endpoint, no spawn.
953        //   - Schedulable  (#181): resolve a live worker from the registry, connect.
954        let kind = match spec.placement {
955            Placement::Remote { .. } => PlacementKind::Remote,
956            Placement::Schedulable { .. } => PlacementKind::Schedulable,
957            Placement::Local => PlacementKind::Local,
958        };
959        let remote = !matches!(kind, PlacementKind::Local);
960
961        // Stamp WHICH machine this child runs on onto its session metadata, so the
962        // UI can show it (mirrored into the session index → SessionSummary.placement).
963        // Only remote/scheduled placements need a stamp — a Local child falls through
964        // to the DTO default (this backend's own host). Persisted by the caller with
965        // the rest of the child session after we return.
966        if let Some(placement_meta) = self.placement_stamp_for(&spec) {
967            session
968                .metadata
969                .insert("placement".to_string(), placement_meta);
970        }
971
972        // Retry-once loop: a pooled local worker can die between its liveness
973        // check and handling the Run (a tiny TOCTOU window) — its Run then sits
974        // queued with no server. The first-frame watchdog in `drive` surfaces that
975        // as `WorkerUnresponsive`; we reap the dead worker and re-acquire ONCE
976        // (which spawns fresh / reuses the next live one). Remote/schedulable have
977        // no spawn fallback, so they never retry.
978        let mut attempt = 0u8;
979        let (result, actor) = loop {
980            let (actor, mut client) = match kind {
981                PlacementKind::Remote => {
982                    // REMOTE branch: connect to a resident worker. No spawn, no pool
983                    // touch, no drain. We do not own the worker, so a connect failure
984                    // has NO respawn fallback — it is a clear, terminal error.
985                    let placement = self
986                        .remote_placements
987                        .get(spec.identity.role.as_str())
988                        .ok_or_else(|| {
989                            AgentError::LLM(format!(
990                                "remote placement for role '{}' vanished before connect",
991                                spec.identity.role
992                            ))
993                        })?;
994                    let endpoint = placement.endpoint.clone();
995                    // Build the TLS trust: a pinned CA pins a self-signed worker cert;
996                    // otherwise default webpki roots (or plaintext for `ws://`).
997                    let trust_cfg = match placement.ca_cert_file.as_deref() {
998                        Some(path) => Some(client_config_trusting_cert(path).map_err(|e| {
999                            AgentError::LLM(format!(
1000                                "remote worker CA cert '{}': {e}",
1001                                path.display()
1002                            ))
1003                        })?),
1004                        None => None,
1005                    };
1006                    let client = ChildClient::connect_with_auth_tls(
1007                        &endpoint,
1008                        placement.token.as_deref(),
1009                        trust_cfg,
1010                    )
1011                    .await
1012                    .map_err(|e| {
1013                        AgentError::LLM(format!("remote actor connect to '{endpoint}' failed: {e}"))
1014                    })?;
1015                    // Process-less handle so live-actor registration (in-band steering)
1016                    // works exactly as for a local worker; `kill()` is a no-op.
1017                    let record = AgentRecord {
1018                        agent_id: job.child_session_id.clone(),
1019                        role: spec.identity.role.clone(),
1020                        labels: Vec::new(),
1021                        endpoint: endpoint.clone(),
1022                        pid: 0,
1023                        version: String::new(),
1024                        started_at: chrono::Utc::now(),
1025                        lease_expires_at: chrono::Utc::now(),
1026                    };
1027                    let _ = endpoint;
1028                    let actor = PooledWorker {
1029                        worker: SpawnedChild::remote(record),
1030                        mailbox_id: job.child_session_id.clone(),
1031                    };
1032                    let client: Box<dyn bamboo_subagent::ChildLink> = Box::new(client);
1033                    (actor, client)
1034                }
1035                PlacementKind::Schedulable => {
1036                    // SCHEDULABLE branch (#181): pick a LIVE worker of the pool role
1037                    // from the BUS (presence = connection-truth; no HTTP registry, no
1038                    // leases, no failover) and drive it by mailbox id. The pool worker
1039                    // stays connected and is reused next time. No spawn, no kill, NO
1040                    // local fallback — an empty pool is a terminal error (raised in
1041                    // resolve_schedulable_worker).
1042                    let bus = self.bus.as_ref().ok_or_else(|| {
1043                        AgentError::LLM(
1044                            "schedulable sub-agents require a mailbox bus (subagents.broker)"
1045                                .to_string(),
1046                        )
1047                    })?;
1048                    let mailbox_id = self
1049                        .resolve_schedulable_worker(spec.identity.role.as_str())
1050                        .await?;
1051                    let parent = bamboo_subagent::AgentRef {
1052                        session_id: format!("p-{}", job.child_session_id),
1053                        role: None,
1054                    };
1055                    let link = bamboo_broker::BrokerChildLink::connect(
1056                        &bus.endpoint,
1057                        parent,
1058                        &bus.token,
1059                        mailbox_id.clone(),
1060                    )
1061                    .await
1062                    .map_err(|e| {
1063                        AgentError::LLM(format!(
1064                            "schedulable link connect to '{mailbox_id}' failed: {e}"
1065                        ))
1066                    })?;
1067                    // Process-less handle — a bus-resident pool worker is never ours to
1068                    // kill (remote ⇒ dropped, not pooled, after the run).
1069                    let actor = PooledWorker {
1070                        worker: SpawnedChild::remote(AgentRecord {
1071                            agent_id: mailbox_id.clone(),
1072                            role: spec.identity.role.clone(),
1073                            labels: Vec::new(),
1074                            endpoint: bus.endpoint.clone(),
1075                            pid: 0,
1076                            version: String::new(),
1077                            started_at: chrono::Utc::now(),
1078                            lease_expires_at: chrono::Utc::now(),
1079                        }),
1080                        mailbox_id,
1081                    };
1082                    let client: Box<dyn bamboo_subagent::ChildLink> = Box::new(link);
1083                    (actor, client)
1084                }
1085                PlacementKind::Local => {
1086                    // LOCAL = the mailbox bus (the unified transport): check out a warm
1087                    // pooled worker (reuse a live parked one, else spawn fresh) and
1088                    // drive it by mailbox id — no listen socket, no file discovery, no
1089                    // respawn-on-connect-miss (the broker queues the Run until the
1090                    // worker handles it). The legacy direct-WS path was retired; the bus
1091                    // is required.
1092                    let bus = self.bus.as_ref().ok_or_else(|| {
1093                        AgentError::LLM(
1094                            "local sub-agents require a mailbox bus (subagents.broker); none is \
1095                         configured and the bus could not be embedded"
1096                                .to_string(),
1097                        )
1098                    })?;
1099                    let actor = self.acquire_bus_worker(&pool_key, &spec).await?;
1100                    let parent = bamboo_subagent::AgentRef {
1101                        session_id: format!("p-{}", job.child_session_id),
1102                        role: None,
1103                    };
1104                    let link = bamboo_broker::BrokerChildLink::connect(
1105                        &bus.endpoint,
1106                        parent,
1107                        &bus.token,
1108                        actor.mailbox_id.clone(),
1109                    )
1110                    .await
1111                    .map_err(|e| {
1112                        AgentError::LLM(format!("broker child link connect failed: {e}"))
1113                    })?;
1114                    let client: Box<dyn bamboo_subagent::ChildLink> = Box::new(link);
1115                    (actor, client)
1116                }
1117            };
1118
1119            if let Err(e) = client
1120                .send(ParentFrame::Run(RunSpec {
1121                    // Cloned (not moved) so a retry can re-dispatch to a fresh worker.
1122                    assignment: assignment.clone(),
1123                    reasoning_effort: None,
1124                    permission_policy: permission_policy.clone(),
1125                    messages: messages.clone(),
1126                    secrets: run_secrets.clone(),
1127                }))
1128                .await
1129            {
1130                if !remote {
1131                    actor.worker.kill().await;
1132                }
1133                return Err(AgentError::LLM(format!("actor run dispatch failed: {e}")));
1134            }
1135
1136            // Register as a live actor so send_message (running, no interrupt) can
1137            // steer this child in-band over the existing WS connection. The guard
1138            // unregisters on every exit path.
1139            let (live_tx, mut live_rx) = mpsc::unbounded_channel::<ParentFrame>();
1140            let live_guard = super::live::register(
1141                &job.child_session_id,
1142                live_tx,
1143                attempt as u32,
1144                self.approval_registry.clone(),
1145            );
1146
1147            let result = drive(
1148                &mut *client,
1149                &job.parent_session_id,
1150                &job.child_session_id,
1151                attempt as u32,
1152                self.approval_registry.as_ref(),
1153                self.approval_decider.as_ref(),
1154                self.approval_reviewer.as_ref(),
1155                escalation.clone(),
1156                &event_tx,
1157                &cancel_token,
1158                &mut live_rx,
1159                // First-frame watchdog for EVERY placement: a wedged-but-connected
1160                // worker (subscribed ≠ serving — e.g. stuck on a prior LLM call) emits
1161                // no first frame; without a deadline drive() blocks forever. Bounding it
1162                // turns the "running-but-unresponsive" hang into a recoverable
1163                // WorkerUnresponsive (reap+respawn local / re-pick schedulable / error
1164                // on a fixed remote endpoint).
1165                Some(WORKER_FIRST_FRAME_TIMEOUT),
1166            )
1167            .await;
1168            // Unregister IMMEDIATELY: after drive returns nobody consumes live_rx,
1169            // so a send_message landing in the close/park window below must see
1170            // "not live" and take the durable-queue fallback instead of vanishing.
1171            // (Even if one slipped in earlier, send_message also appends it to the
1172            // durable transcript, so the next activation still rehydrates it.)
1173            drop(live_guard);
1174            // Close the parent link (dropping it closes our broker connection; the
1175            // worker stays dialed-in + subscribed, ready for its next Run).
1176            drop(client);
1177
1178            // No first frame ⇒ the worker is wedged. Recover ONCE before giving up:
1179            //   - Local: reap the dead pooled worker + respawn.
1180            //   - Schedulable: not ours to kill — drop it and re-select a live pool
1181            //     member (a wedged worker must not fail the run when the pool has others).
1182            //   - Remote: a FIXED endpoint has no alternative — fall through to a bounded
1183            //     WorkerUnresponsive error (far better than the previous infinite hang).
1184            if attempt == 0 && matches!(result, Err(AgentError::WorkerUnresponsive(_))) {
1185                match kind {
1186                    PlacementKind::Local => {
1187                        tracing::warn!(
1188                        "actor child {} got no first frame; reaping the worker and respawning once",
1189                        job.child_session_id
1190                    );
1191                        actor.worker.kill().await;
1192                        attempt += 1;
1193                        continue;
1194                    }
1195                    PlacementKind::Schedulable => {
1196                        tracing::warn!(
1197                        "scheduled actor child {} got no first frame; re-selecting a pool worker",
1198                        job.child_session_id
1199                    );
1200                        drop(actor);
1201                        attempt += 1;
1202                        continue;
1203                    }
1204                    PlacementKind::Remote => {}
1205                }
1206            }
1207            break (result, actor);
1208        };
1209
1210        // Park the warm worker for reuse on a clean run, or kill it on
1211        // error/cancel (a wedged worker must not be reused). Remote / schedulable
1212        // workers are registry-managed — never ours to pool/kill, just drop.
1213        if remote {
1214            drop(actor);
1215        } else {
1216            match &result {
1217                Ok(_) => self.release_bus_worker(&pool_key, actor).await,
1218                Err(_) => actor.worker.kill().await,
1219            }
1220        }
1221
1222        // Write-back: persist the actor's final reply onto the child session so
1223        // the transcript survives and the NEXT activation sees it as history.
1224        // (run_child_spawn saves the session right after we return.)
1225        match result {
1226            Ok(Some(text)) => {
1227                if !text.is_empty() {
1228                    session.add_message(bamboo_agent_core::Message::assistant(text, None));
1229                }
1230                Ok(())
1231            }
1232            Ok(None) => Ok(()),
1233            Err(e) => Err(e),
1234        }
1235    }
1236}
1237
1238/// The `{kind,host}` placement descriptor stamped onto a child session's metadata
1239/// under `"placement"` — read back by the storage index → `SessionSummary.placement`
1240/// → the UI's machine badge. `None` for `Local` (those fall through to the DTO's
1241/// default of this backend's own host). The value is a JSON string matching
1242/// `bamboo_storage::SessionPlacement { kind, host }`.
1243fn placement_metadata(placement: &Placement, host_label: Option<&str>) -> Option<String> {
1244    // Prefer the cluster node's own label/host (its metadata) when the placement
1245    // maps to a node; else fall back to the raw endpoint host / pool name.
1246    let value = match placement {
1247        Placement::Local => return None,
1248        Placement::Remote { endpoint } => serde_json::json!({
1249            "kind": "remote",
1250            "host": host_label.map(str::to_string).unwrap_or_else(|| host_of_endpoint(endpoint)),
1251        }),
1252        Placement::Schedulable { pool } => serde_json::json!({
1253            "kind": "remote",
1254            "host": host_label.unwrap_or(pool),
1255        }),
1256    };
1257    serde_json::to_string(&value).ok()
1258}
1259
1260/// Extract the host from a `ws[s]://host:port[/path]` bus endpoint, for display.
1261fn host_of_endpoint(endpoint: &str) -> String {
1262    endpoint
1263        .trim()
1264        .trim_start_matches("wss://")
1265        .trim_start_matches("ws://")
1266        .split(['/', ':'])
1267        .next()
1268        .unwrap_or(endpoint)
1269        .to_string()
1270}
1271
1272/// Pump child frames -> parent events until a terminal frame (or cancellation).
1273/// On success, yields the actor's final result text (for session write-back).
1274/// `live_rx` carries in-band frames (steering messages) from the live registry.
1275///
1276/// `escalation_bridge` (#68) is the per-run escalation host bridge CAPTURED BY
1277/// VALUE at spawn time in `execute_external_child` (NOT read live here): when a
1278/// non-bypass child re-proxies an approval request, this owned bridge routes it
1279/// UP to the parent run. Owning it for the call's lifetime is what lets a
1280/// fire-and-forget grandchild that outlives its spawning run still escalate to
1281/// the correct (then-current) parent bridge rather than a stale/overwritten one.
1282async fn drive(
1283    client: &mut dyn bamboo_subagent::ChildLink,
1284    parent_session_id: &str,
1285    child_session_id: &str,
1286    child_attempt: u32,
1287    approval_registry: Option<&super::approval_registry::SharedApprovalRegistry>,
1288    approval_decider: Option<&Arc<dyn ChildApprovalDecider>>,
1289    approval_reviewer: Option<&Arc<dyn ChildApprovalReviewer>>,
1290    escalation_bridge: Option<bamboo_subagent::executor::HostBridge>,
1291    event_tx: &mpsc::Sender<AgentEvent>,
1292    cancel_token: &CancellationToken,
1293    live_rx: &mut mpsc::UnboundedReceiver<ParentFrame>,
1294    first_frame_timeout: Option<Duration>,
1295) -> crate::runtime::runner::Result<Option<String>> {
1296    // First-frame watchdog: a live worker emits its first frame (run-started /
1297    // first token) within seconds; total silence past the deadline means the
1298    // worker is dead (e.g. a pooled worker that exited right after checkout), so
1299    // its Run sits queued forever. We trip ONLY before the first frame — once any
1300    // frame arrives the worker is proven live and a legitimately long run (a slow
1301    // tool between tokens) never trips it.
1302    let mut got_first_frame = false;
1303    let mut first_frame_watch = first_frame_timeout.map(|d| Box::pin(tokio::time::sleep(d)));
1304    loop {
1305        tokio::select! {
1306            _ = cancel_token.cancelled() => {
1307                // fall through to the cancel handling below
1308                break;
1309            }
1310            _ = async {
1311                match first_frame_watch.as_mut() {
1312                    Some(s) => s.as_mut().await,
1313                    None => std::future::pending::<()>().await,
1314                }
1315            }, if !got_first_frame => {
1316                return Err(AgentError::WorkerUnresponsive(format!(
1317                    "child {child_session_id} produced no frame within {:?}",
1318                    first_frame_timeout.unwrap_or_default()
1319                )));
1320            }
1321            Some(frame) = live_rx.recv() => {
1322                // Forward in-band steering to the worker over the existing WS.
1323                if client.send(frame).await.is_err() {
1324                    tracing::warn!("live steering frame could not be sent; connection failing");
1325                }
1326            }
1327            frame = client.next_frame() => {
1328                // Any frame (event / approval / terminal / close / error) proves
1329                // the worker responded — disarm the first-frame watchdog.
1330                got_first_frame = true;
1331                first_frame_watch = None;
1332                match frame {
1333                    Ok(Some(ChildFrame::Event { event })) => {
1334                        // AgentEvent is serialized verbatim on the wire (zero mapping).
1335                        if let Ok(ev) = serde_json::from_value::<AgentEvent>(event) {
1336                            let _ = event_tx.send(ev).await;
1337                        }
1338                    }
1339                    Ok(Some(ChildFrame::ApprovalRequest { id, body })) => {
1340                        // Phase 2: a worker proxied a gated-tool approval back to
1341                        // the host. The WORKER side is live — its executor installs
1342                        // a per-run task-local `ApprovalProxy` (subagent_worker.rs)
1343                        // that calls `host.approval_call`, so this frame arrives
1344                        // when a child hits `ConfirmationRequired`.
1345                        if let Some(reviewer) = approval_reviewer
1346                            .cloned()
1347                            .or_else(child_approval_reviewer)
1348                        {
1349                            // Phase 6, Part B: a BYPASSED parent worker
1350                            // model-reviews its children's forced-ask (dangerous)
1351                            // actions. The review is an LLM call, so run it OFF
1352                            // the frame pump in a spawned task and deliver the
1353                            // verdict async via the live channel — the pump keeps
1354                            // forwarding events and the agent loop never blocks. A
1355                            // timeout denies a hung review so the child can't hang.
1356                            let child = child_session_id.to_string();
1357                            let parent = parent_session_id.to_string();
1358                            let req_id = id.clone();
1359                            let body = body.clone();
1360                            let registry = approval_registry.cloned();
1361                            tokio::spawn(async move {
1362                                let approved = tokio::time::timeout(
1363                                    CHILD_APPROVAL_TIMEOUT,
1364                                    reviewer.review(&parent, &child, &body),
1365                                )
1366                                .await
1367                                .unwrap_or(false);
1368                                super::live::deliver_approval_scoped(
1369                                    registry.as_ref(),
1370                                    &child,
1371                                    child_attempt,
1372                                    &req_id,
1373                                    approved,
1374                                );
1375                            });
1376                        } else if approval_decider.is_some() {
1377                            // A decider is wired (policy / auto): decide promptly
1378                            // and reply inline. (Must not block the pump — see the
1379                            // `ChildApprovalDecider` doc.)
1380                            let approved =
1381                                decide_child_approval(approval_decider, child_session_id, &body)
1382                                    .await;
1383                            if client
1384                                .send(ParentFrame::ApprovalReply { id, approved })
1385                                .await
1386                                .is_err()
1387                            {
1388                                tracing::warn!(
1389                                    "failed to answer approval_request; connection failing"
1390                                );
1391                            }
1392                        } else if let Some(host) = escalation_bridge.clone() {
1393                            // Non-bypass WORKER: ESCALATE up our own actor link
1394                            // (re-proxy) so the request chains to our parent — and
1395                            // up every level until a bypass level or the top
1396                            // orchestrator's model reviewer decides. With no such
1397                            // reviewer the top level fails closed. Off-loop so the
1398                            // pump never blocks; relay the reply down to the child.
1399                            let child = child_session_id.to_string();
1400                            let req_id = id.clone();
1401                            let body = body.clone();
1402                            let registry = approval_registry.cloned();
1403                            tokio::spawn(async move {
1404                                let approved = match tokio::time::timeout(
1405                                    CHILD_APPROVAL_TIMEOUT,
1406                                    host.approval_call(body),
1407                                )
1408                                .await
1409                                {
1410                                    Ok(Ok(reply)) => reply
1411                                        .get("approved")
1412                                        .and_then(|v| v.as_bool())
1413                                        .unwrap_or(false),
1414                                    // Transport error or timeout ⇒ fail closed.
1415                                    _ => false,
1416                                };
1417                                super::live::deliver_approval_scoped(
1418                                    registry.as_ref(),
1419                                    &child,
1420                                    child_attempt,
1421                                    &req_id,
1422                                    approved,
1423                                );
1424                            });
1425                        } else {
1426                            // There is no parent-agent reviewer or upstream actor
1427                            // to own this decision. Never open a manual/UI approval
1428                            // path: forced-ask is parent-reviewed or fail-closed.
1429                            tracing::warn!(
1430                                parent_session_id,
1431                                child_session_id,
1432                                request_id = %id,
1433                                "forced-ask request has no parent-agent reviewer; denying"
1434                            );
1435                            if client
1436                                .send(ParentFrame::ApprovalReply {
1437                                    id,
1438                                    approved: false,
1439                                })
1440                                .await
1441                                .is_err()
1442                            {
1443                                tracing::warn!(
1444                                    "failed to send fail-closed approval reply; connection failing"
1445                                );
1446                            }
1447                        }
1448                    }
1449                    Ok(Some(ChildFrame::Terminal { status, result, error, .. })) => {
1450                        return match status {
1451                            TerminalStatus::Completed => Ok(result),
1452                            TerminalStatus::Cancelled => Err(AgentError::Cancelled),
1453                            TerminalStatus::Error => Err(AgentError::LLM(
1454                                error.unwrap_or_else(|| "actor child errored".to_string()),
1455                            )),
1456                            // The suspend/resume round-trip (host re-dispatch of a
1457                            // nested parent) is not wired here yet; a worker in
1458                            // this build never emits Suspended, so this is
1459                            // unreachable in practice.
1460                            TerminalStatus::Suspended => Err(AgentError::LLM(
1461                                "nested sub-agent suspend received but resume transport is not wired"
1462                                    .to_string(),
1463                            )),
1464                        };
1465                    }
1466                    Ok(None) => {
1467                        return Err(AgentError::LLM(
1468                            "actor child closed before terminal".to_string(),
1469                        ));
1470                    }
1471                    Err(e) => {
1472                        return Err(AgentError::LLM(format!("actor transport error: {e}")));
1473                    }
1474                }
1475            }
1476        }
1477    }
1478
1479    // Only reached on cancellation: ask the child to stop (best-effort), then report cancelled.
1480    let _ = client.send(ParentFrame::Cancel).await;
1481    Err(AgentError::Cancelled)
1482}
1483
1484/// The assignment text = the child session's latest user message (falls back to its title).
1485fn extract_assignment(session: &Session) -> String {
1486    session
1487        .messages
1488        .iter()
1489        .rev()
1490        .find(|m| matches!(m.role, Role::User))
1491        .map(|m| m.content.clone())
1492        .unwrap_or_else(|| {
1493            session
1494                .metadata
1495                .get("title")
1496                .cloned()
1497                .unwrap_or_else(|| "Execute task".to_string())
1498        })
1499}
1500
1501#[cfg(test)]
1502mod tests {
1503    use super::*;
1504
1505    #[derive(Default)]
1506    struct RecordingCodexTokenAuthority {
1507        issued_for: std::sync::Mutex<Vec<String>>,
1508        revoked: std::sync::Mutex<Vec<String>>,
1509    }
1510
1511    impl CodexRunTokenAuthority for RecordingCodexTokenAuthority {
1512        fn issue(&self, session_id: &str) -> Result<IssuedCodexRunToken, String> {
1513            self.issued_for
1514                .lock()
1515                .expect("issued fixture lock")
1516                .push(session_id.to_string());
1517            Ok(IssuedCodexRunToken {
1518                token_id: format!("id-{session_id}"),
1519                token: format!("bcx1_secret-{session_id}"),
1520            })
1521        }
1522
1523        fn revoke(&self, token_id: &str) {
1524            self.revoked
1525                .lock()
1526                .expect("revoked fixture lock")
1527                .push(token_id.to_string());
1528        }
1529    }
1530
1531    fn codex_executor(auth_mode: Option<&str>, inherit_user_config: Option<bool>) -> ExecutorSpec {
1532        let bamboo_mode = auth_mode == Some("bamboo")
1533            || (auth_mode.is_none() && !inherit_user_config.unwrap_or(false));
1534        ExecutorSpec::Codex {
1535            binary: None,
1536            model: None,
1537            mode: None,
1538            sandbox: None,
1539            inherit_user_config,
1540            auth_mode: auth_mode.map(str::to_string),
1541            base_url: bamboo_mode.then(|| "http://127.0.0.1:9562/openai/v1".to_string()),
1542            wire_api: Some("responses".to_string()),
1543            provider_key_ref: None,
1544            forward_env: None,
1545            approval_policy: None,
1546            network_access: None,
1547            allow_danger_bypass: None,
1548            permission_profile: None,
1549            workspace_owned: None,
1550        }
1551    }
1552
1553    #[test]
1554    fn only_bamboo_managed_non_git_workspaces_are_marked_owned() {
1555        let project = tempfile::tempdir().unwrap();
1556        let managed = project.path().join(".bamboo/worktree/child-571");
1557        std::fs::create_dir_all(&managed).unwrap();
1558        assert!(!workspace_is_bamboo_owned(managed.to_str().unwrap()));
1559        let marker = project
1560            .path()
1561            .join(".bamboo/worktree/.bamboo-owned/child-571");
1562        std::fs::create_dir_all(marker.parent().unwrap()).unwrap();
1563        std::fs::write(&marker, "bamboo/child-571").unwrap();
1564        assert!(workspace_is_bamboo_owned(managed.to_str().unwrap()));
1565        let nested = managed.join("nested/path");
1566        std::fs::create_dir_all(&nested).unwrap();
1567        assert!(workspace_is_bamboo_owned(nested.to_str().unwrap()));
1568
1569        let arbitrary = tempfile::tempdir().unwrap();
1570        assert!(!workspace_is_bamboo_owned(
1571            arbitrary.path().to_str().unwrap()
1572        ));
1573    }
1574
1575    #[test]
1576    fn bamboo_codex_token_is_per_run_redacted_and_revoked_on_guard_drop() {
1577        let authority = Arc::new(RecordingCodexTokenAuthority::default());
1578        let authority_dyn: Arc<dyn CodexRunTokenAuthority> = authority.clone();
1579
1580        let (secrets, guard) = build_codex_run_secrets(
1581            &codex_executor(Some("bamboo"), None),
1582            Some(authority_dyn),
1583            "child-570",
1584        )
1585        .unwrap();
1586
1587        let token = secrets
1588            .codex_provider_token
1589            .as_ref()
1590            .expect("bamboo mode mints a token");
1591        assert_eq!(token.expose(), "bcx1_secret-child-570");
1592        assert!(!format!("{token:?}").contains("secret-child-570"));
1593        assert_eq!(
1594            authority.issued_for.lock().unwrap().as_slice(),
1595            ["child-570"]
1596        );
1597        assert!(authority.revoked.lock().unwrap().is_empty());
1598
1599        drop(guard);
1600        assert_eq!(
1601            authority.revoked.lock().unwrap().as_slice(),
1602            ["id-child-570"]
1603        );
1604    }
1605
1606    #[test]
1607    fn non_bamboo_codex_never_mints_and_bamboo_fails_closed_without_authority() {
1608        let authority = Arc::new(RecordingCodexTokenAuthority::default());
1609        let authority_dyn: Arc<dyn CodexRunTokenAuthority> = authority.clone();
1610        let (secrets, guard) = build_codex_run_secrets(
1611            &codex_executor(Some("custom"), None),
1612            Some(authority_dyn),
1613            "child-custom",
1614        )
1615        .unwrap();
1616        assert!(secrets.codex_provider_token.is_none());
1617        assert!(guard.is_none());
1618        assert!(authority.issued_for.lock().unwrap().is_empty());
1619
1620        let error = build_codex_run_secrets(
1621            &codex_executor(Some("bamboo"), None),
1622            None,
1623            "child-no-authority",
1624        )
1625        .err()
1626        .expect("bamboo mode without an authority must fail closed");
1627        assert!(error.to_string().contains("per-run token authority"));
1628    }
1629
1630    #[test]
1631    fn codex_provisioning_never_leaks_the_session_provider_credential() {
1632        let credentials = vec![ScopedCredential {
1633            provider: "openai".to_string(),
1634            api_key: "upstream-secret-must-not-cross".to_string(),
1635            base_url: None,
1636            provider_type: Some("openai".to_string()),
1637            credential_ref: Some("provider.openai.api_key".to_string()),
1638        }];
1639
1640        for (mode, label) in [
1641            (Some("inherit"), "inherit"),
1642            (Some("api_key"), "api_key"),
1643            (Some("bamboo"), "bamboo"),
1644            (None, "default-bamboo"),
1645        ] {
1646            let runner = ActorChildRunner::new(
1647                format!("codex-{label}-test"),
1648                PathBuf::from("/bin/false"),
1649                Vec::new(),
1650                std::env::temp_dir().join(format!("bamboo-codex-{label}-570")),
1651                codex_executor(mode, None),
1652                credentials.clone(),
1653                "openai".to_string(),
1654                1,
1655            );
1656            let mut session = Session::new(format!("child-{label}"), "model");
1657            session.add_message(bamboo_agent_core::Message::user("test"));
1658            let spec = runner.build_spec(
1659                &session,
1660                &crate::runtime::execution::SpawnJob {
1661                    parent_session_id: "parent".to_string(),
1662                    child_session_id: format!("child-{label}"),
1663                    model: "gpt-5.4".to_string(),
1664                    disabled_tools: None,
1665                },
1666            );
1667            assert!(
1668                spec.secrets.provider_credentials.is_empty(),
1669                "{label} Codex must not receive the session provider key"
1670            );
1671        }
1672    }
1673
1674    #[test]
1675    fn non_codex_provisioning_still_receives_only_its_selected_provider_credential() {
1676        let credentials = vec![
1677            ScopedCredential {
1678                provider: "openai".to_string(),
1679                api_key: "selected-openai-secret".to_string(),
1680                base_url: None,
1681                provider_type: Some("openai".to_string()),
1682                credential_ref: Some("provider.openai.api_key".to_string()),
1683            },
1684            ScopedCredential {
1685                provider: "other".to_string(),
1686                api_key: "unrelated-secret".to_string(),
1687                base_url: None,
1688                provider_type: Some("openai".to_string()),
1689                credential_ref: Some("provider.other.api_key".to_string()),
1690            },
1691        ];
1692        let runner = ActorChildRunner::new(
1693            "echo-test".to_string(),
1694            PathBuf::from("/bin/false"),
1695            Vec::new(),
1696            std::env::temp_dir().join("bamboo-echo-provider-570"),
1697            ExecutorSpec::Echo,
1698            credentials,
1699            "openai".to_string(),
1700            1,
1701        );
1702        let spec = runner.build_spec(
1703            &Session::new("child-echo", "model"),
1704            &crate::runtime::execution::SpawnJob {
1705                parent_session_id: "parent".to_string(),
1706                child_session_id: "child-echo".to_string(),
1707                model: "gpt-5.4".to_string(),
1708                disabled_tools: None,
1709            },
1710        );
1711
1712        assert_eq!(spec.secrets.provider_credentials.len(), 1);
1713        assert_eq!(
1714            spec.secrets.provider_credentials[0].api_key,
1715            "selected-openai-secret"
1716        );
1717    }
1718
1719    #[test]
1720    fn custom_codex_provisioning_scopes_only_the_referenced_credential() {
1721        let mut executor = codex_executor(Some("custom"), None);
1722        if let ExecutorSpec::Codex {
1723            base_url,
1724            provider_key_ref,
1725            ..
1726        } = &mut executor
1727        {
1728            *base_url = Some("https://provider.example/v1".to_string());
1729            *provider_key_ref = Some("provider.custom.api_key".to_string());
1730        }
1731        let credentials = vec![
1732            ScopedCredential {
1733                provider: "openai".to_string(),
1734                api_key: "session-provider-secret".to_string(),
1735                base_url: None,
1736                provider_type: Some("openai".to_string()),
1737                credential_ref: Some("provider.openai.api_key".to_string()),
1738            },
1739            ScopedCredential {
1740                provider: "custom".to_string(),
1741                api_key: "selected-secret".to_string(),
1742                base_url: None,
1743                provider_type: Some("openai".to_string()),
1744                credential_ref: Some("provider.custom.api_key".to_string()),
1745            },
1746            ScopedCredential {
1747                provider: "other".to_string(),
1748                api_key: "unrelated-secret".to_string(),
1749                base_url: None,
1750                provider_type: Some("openai".to_string()),
1751                credential_ref: Some("provider.other.api_key".to_string()),
1752            },
1753        ];
1754        let runner = ActorChildRunner::new(
1755            "codex-test".to_string(),
1756            PathBuf::from("/bin/false"),
1757            Vec::new(),
1758            std::env::temp_dir().join("bamboo-codex-570"),
1759            executor,
1760            credentials,
1761            "openai".to_string(),
1762            1,
1763        );
1764        let mut session = Session::new("child-custom", "model");
1765        session.add_message(bamboo_agent_core::Message::user("test"));
1766        let spec = runner.build_spec(
1767            &session,
1768            &crate::runtime::execution::SpawnJob {
1769                parent_session_id: "parent".to_string(),
1770                child_session_id: "child-custom".to_string(),
1771                model: "gpt-5.4".to_string(),
1772                disabled_tools: None,
1773            },
1774        );
1775
1776        assert_eq!(spec.secrets.provider_credentials.len(), 1);
1777        assert_eq!(
1778            spec.secrets.provider_credentials[0]
1779                .credential_ref
1780                .as_deref(),
1781            Some("provider.custom.api_key")
1782        );
1783        assert_eq!(
1784            spec.secrets.provider_credentials[0].api_key,
1785            "selected-secret"
1786        );
1787    }
1788
1789    fn spec_with(
1790        role: &str,
1791        provider: &str,
1792        model: &str,
1793        workspace: Option<&str>,
1794        disabled: Option<Vec<&str>>,
1795    ) -> ProvisionSpec {
1796        let mut spec = ProvisionSpec::new(
1797            ChildIdentity {
1798                child_id: "c".into(),
1799                parent_id: None,
1800                project_key: None,
1801                role: role.into(),
1802                depth: 0,
1803            },
1804            ExecutorSpec::Echo,
1805            "/tmp/fab".into(),
1806        );
1807        spec.workspace = workspace.map(|w| w.to_string());
1808        spec.model = Some(ModelRefSpec {
1809            provider: provider.into(),
1810            model: model.into(),
1811        });
1812        spec.disabled_tools = disabled.map(|d| d.into_iter().map(String::from).collect());
1813        spec
1814    }
1815
1816    #[test]
1817    fn fingerprint_matches_interchangeable_children() {
1818        // Same role/provider/model/workspace and equal tool sets (order-insensitive)
1819        // are interchangeable on one warm worker — and differ only in child_id.
1820        let a = spec_with(
1821            "explorer",
1822            "p",
1823            "m",
1824            Some("/ws"),
1825            Some(vec!["Bash", "Edit"]),
1826        );
1827        let mut b = spec_with(
1828            "explorer",
1829            "p",
1830            "m",
1831            Some("/ws"),
1832            Some(vec!["Edit", "Bash"]),
1833        );
1834        b.identity.child_id = "other".into();
1835        assert_eq!(
1836            ActorChildRunner::fingerprint(&a),
1837            ActorChildRunner::fingerprint(&b)
1838        );
1839    }
1840
1841    #[test]
1842    fn fingerprint_separates_distinct_runtimes() {
1843        let base = spec_with("explorer", "p", "m", Some("/ws"), None);
1844        let base_fp = ActorChildRunner::fingerprint(&base);
1845        // Each axis that is baked into the worker must split the pool bucket.
1846        assert_ne!(
1847            base_fp,
1848            ActorChildRunner::fingerprint(&spec_with("writer", "p", "m", Some("/ws"), None))
1849        );
1850        assert_ne!(
1851            base_fp,
1852            ActorChildRunner::fingerprint(&spec_with("explorer", "p2", "m", Some("/ws"), None))
1853        );
1854        assert_ne!(
1855            base_fp,
1856            ActorChildRunner::fingerprint(&spec_with("explorer", "p", "m2", Some("/ws"), None))
1857        );
1858        assert_ne!(
1859            base_fp,
1860            ActorChildRunner::fingerprint(&spec_with("explorer", "p", "m", Some("/ws2"), None))
1861        );
1862        assert_ne!(
1863            base_fp,
1864            ActorChildRunner::fingerprint(&spec_with(
1865                "explorer",
1866                "p",
1867                "m",
1868                Some("/ws"),
1869                Some(vec!["Bash"])
1870            ))
1871        );
1872    }
1873
1874    #[test]
1875    fn fingerprint_splits_on_baked_capabilities() {
1876        // Every capability baked once at provision time must split the pool
1877        // bucket, else a worker baked for one posture gets reused for another
1878        // (e.g. a depth-1 worker re-stamping spawn_depth onto a depth-4 child,
1879        // breaking the depth cap; or a bypass worker reused for a non-bypass one).
1880        let base_fp =
1881            ActorChildRunner::fingerprint(&spec_with("explorer", "p", "m", Some("/ws"), None));
1882
1883        let mut depth = spec_with("explorer", "p", "m", Some("/ws"), None);
1884        depth.identity.depth = 2;
1885        assert_ne!(
1886            base_fp,
1887            ActorChildRunner::fingerprint(&depth),
1888            "depth must split"
1889        );
1890
1891        let mut nested = spec_with("explorer", "p", "m", Some("/ws"), None);
1892        nested.capabilities.nested_spawn = true;
1893        assert_ne!(
1894            base_fp,
1895            ActorChildRunner::fingerprint(&nested),
1896            "nested_spawn must split"
1897        );
1898
1899        let mut bypass = spec_with("explorer", "p", "m", Some("/ws"), None);
1900        bypass.capabilities.bypass = true;
1901        assert_ne!(
1902            base_fp,
1903            ActorChildRunner::fingerprint(&bypass),
1904            "bypass must split"
1905        );
1906
1907        let mut enforce = spec_with("explorer", "p", "m", Some("/ws"), None);
1908        enforce.capabilities.enforce_permissions = true;
1909        assert_ne!(
1910            base_fp,
1911            ActorChildRunner::fingerprint(&enforce),
1912            "enforce_permissions must split"
1913        );
1914
1915        let mut cap = spec_with("explorer", "p", "m", Some("/ws"), None);
1916        cap.capabilities.max_spawn_depth = Some(8);
1917        assert_ne!(
1918            base_fp,
1919            ActorChildRunner::fingerprint(&cap),
1920            "max_spawn_depth must split"
1921        );
1922
1923        // #73 (P1): the worker bakes `no_human_review` from this flag once at
1924        // build(), so it MUST split the pool or a worker baked for one approval
1925        // posture is reused for the opposite one.
1926        let mut nha = spec_with("explorer", "p", "m", Some("/ws"), None);
1927        nha.capabilities.no_human_approver = true;
1928        assert_ne!(
1929            base_fp,
1930            ActorChildRunner::fingerprint(&nha),
1931            "no_human_approver must split"
1932        );
1933
1934        // #71: the read-only Bash checker is baked once at build() from this flag,
1935        // so a guardian reviewer worker must not be reused for an ordinary child.
1936        let mut gro = spec_with("explorer", "p", "m", Some("/ws"), None);
1937        gro.capabilities.guardian_read_only = true;
1938        assert_ne!(
1939            base_fp,
1940            ActorChildRunner::fingerprint(&gro),
1941            "guardian_read_only must split"
1942        );
1943    }
1944
1945    #[test]
1946    fn fingerprint_splits_codex_exec_and_app_server_workers() {
1947        let mut exec = spec_with("explorer", "p", "m", Some("/ws"), None);
1948        exec.executor = codex_executor(Some("inherit"), None);
1949        let mut app_server = exec.clone();
1950        if let ExecutorSpec::Codex { mode, .. } = &mut app_server.executor {
1951            *mode = Some("app_server".to_string());
1952        }
1953        assert_ne!(
1954            ActorChildRunner::fingerprint(&exec),
1955            ActorChildRunner::fingerprint(&app_server)
1956        );
1957    }
1958
1959    struct StaticDecider(bool);
1960
1961    #[async_trait]
1962    impl ChildApprovalDecider for StaticDecider {
1963        async fn decide(&self, _child: &str, _req: &serde_json::Value) -> bool {
1964            self.0
1965        }
1966    }
1967
1968    struct RecordingReviewer {
1969        reviewed: mpsc::UnboundedSender<(String, String, serde_json::Value)>,
1970    }
1971
1972    #[async_trait]
1973    impl ChildApprovalReviewer for RecordingReviewer {
1974        async fn review(&self, parent: &str, child: &str, request: &serde_json::Value) -> bool {
1975            let _ = self
1976                .reviewed
1977                .send((parent.to_string(), child.to_string(), request.clone()));
1978            true
1979        }
1980    }
1981
1982    // ---- first-frame watchdog (dead-pooled-worker recovery) -----------------
1983
1984    /// A link that never yields a frame — models a worker that died (or never
1985    /// subscribed) so its Run sits queued with no server.
1986    struct SilentLink;
1987    #[async_trait]
1988    impl bamboo_subagent::ChildLink for SilentLink {
1989        async fn send(&mut self, _: ParentFrame) -> bamboo_subagent::TransportResult<()> {
1990            Ok(())
1991        }
1992        async fn next_frame(&mut self) -> bamboo_subagent::TransportResult<Option<ChildFrame>> {
1993            std::future::pending().await
1994        }
1995    }
1996
1997    /// A link that immediately yields one terminal frame (a healthy fast worker).
1998    struct InstantTerminalLink {
1999        done: bool,
2000    }
2001
2002    struct ApprovalRoundTripLink {
2003        step: u8,
2004        approval_reply: Option<(String, bool)>,
2005    }
2006
2007    #[async_trait]
2008    impl bamboo_subagent::ChildLink for ApprovalRoundTripLink {
2009        async fn send(&mut self, frame: ParentFrame) -> bamboo_subagent::TransportResult<()> {
2010            if let ParentFrame::ApprovalReply { id, approved } = frame {
2011                self.approval_reply = Some((id, approved));
2012                self.step = 2;
2013            }
2014            Ok(())
2015        }
2016
2017        async fn next_frame(&mut self) -> bamboo_subagent::TransportResult<Option<ChildFrame>> {
2018            match self.step {
2019                0 => {
2020                    self.step = 1;
2021                    Ok(Some(ChildFrame::ApprovalRequest {
2022                        id: "approval-1".into(),
2023                        body: serde_json::json!({
2024                            "tool_name": "Bash",
2025                            "permission": "execute",
2026                            "resource": "rm -rf target",
2027                            "permission_request": {"reason_code": "hard_dangerous"}
2028                        }),
2029                    }))
2030                }
2031                1 => std::future::pending().await,
2032                2 => {
2033                    self.step = 3;
2034                    Ok(Some(ChildFrame::Terminal {
2035                        status: TerminalStatus::Completed,
2036                        result: Some("done".into()),
2037                        error: None,
2038                        transcript: vec![],
2039                    }))
2040                }
2041                _ => std::future::pending().await,
2042            }
2043        }
2044    }
2045
2046    #[tokio::test]
2047    async fn drive_routes_forced_ask_to_parent_reviewer_without_human_event() {
2048        let (event_tx, mut event_rx) = mpsc::channel::<AgentEvent>(8);
2049        let (review_tx, mut review_rx) = mpsc::unbounded_channel();
2050        let reviewer: Arc<dyn ChildApprovalReviewer> = Arc::new(RecordingReviewer {
2051            reviewed: review_tx,
2052        });
2053        let cancel = CancellationToken::new();
2054        let (live_tx, mut live_rx) = mpsc::unbounded_channel::<ParentFrame>();
2055        let live_guard = crate::external_agents::live::register("child-reviewer", live_tx, 0, None);
2056        let mut link = ApprovalRoundTripLink {
2057            step: 0,
2058            approval_reply: None,
2059        };
2060
2061        let result = tokio::time::timeout(
2062            Duration::from_secs(1),
2063            drive(
2064                &mut link,
2065                "parent-reviewer",
2066                "child-reviewer",
2067                0,
2068                None,
2069                None,
2070                Some(&reviewer),
2071                None,
2072                &event_tx,
2073                &cancel,
2074                &mut live_rx,
2075                None,
2076            ),
2077        )
2078        .await
2079        .expect("worker must receive the reviewer verdict before terminating");
2080
2081        assert_eq!(result.ok().flatten().as_deref(), Some("done"));
2082        assert_eq!(
2083            link.approval_reply,
2084            Some(("approval-1".to_string(), true)),
2085            "reviewer verdict must traverse the live route back to the worker"
2086        );
2087        let (parent, child, body) = tokio::time::timeout(Duration::from_secs(1), review_rx.recv())
2088            .await
2089            .expect("reviewer should be invoked off-loop")
2090            .expect("review channel should remain open");
2091        assert_eq!(parent, "parent-reviewer");
2092        assert_eq!(child, "child-reviewer");
2093        assert_eq!(
2094            body.pointer("/permission_request/reason_code")
2095                .and_then(serde_json::Value::as_str),
2096            Some("hard_dangerous")
2097        );
2098        assert!(
2099            event_rx.try_recv().is_err(),
2100            "must not emit a human-review event"
2101        );
2102        drop(live_guard);
2103    }
2104
2105    #[tokio::test]
2106    async fn drive_denies_forced_ask_without_parent_reviewer_or_manual_event() {
2107        let (event_tx, mut event_rx) = mpsc::channel::<AgentEvent>(8);
2108        let cancel = CancellationToken::new();
2109        let (_live_tx, mut live_rx) = mpsc::unbounded_channel::<ParentFrame>();
2110        let mut link = ApprovalRoundTripLink {
2111            step: 0,
2112            approval_reply: None,
2113        };
2114
2115        let result = tokio::time::timeout(
2116            Duration::from_secs(1),
2117            drive(
2118                &mut link,
2119                "parent-no-reviewer",
2120                "child-no-reviewer",
2121                0,
2122                None,
2123                None,
2124                None,
2125                None,
2126                &event_tx,
2127                &cancel,
2128                &mut live_rx,
2129                None,
2130            ),
2131        )
2132        .await
2133        .expect("fail-closed reply must unblock the child immediately");
2134
2135        assert_eq!(result.ok().flatten().as_deref(), Some("done"));
2136        assert_eq!(link.approval_reply, Some(("approval-1".to_string(), false)));
2137        assert!(
2138            event_rx.try_recv().is_err(),
2139            "missing parent review must not surface a manual approval event"
2140        );
2141    }
2142    #[async_trait]
2143    impl bamboo_subagent::ChildLink for InstantTerminalLink {
2144        async fn send(&mut self, _: ParentFrame) -> bamboo_subagent::TransportResult<()> {
2145            Ok(())
2146        }
2147        async fn next_frame(&mut self) -> bamboo_subagent::TransportResult<Option<ChildFrame>> {
2148            if self.done {
2149                std::future::pending().await
2150            } else {
2151                self.done = true;
2152                Ok(Some(ChildFrame::Terminal {
2153                    status: TerminalStatus::Completed,
2154                    result: Some("done".into()),
2155                    error: None,
2156                    transcript: vec![],
2157                }))
2158            }
2159        }
2160    }
2161
2162    #[tokio::test]
2163    async fn drive_trips_first_frame_watchdog_on_a_silent_worker() {
2164        let (event_tx, _rx) = mpsc::channel::<AgentEvent>(8);
2165        let cancel = CancellationToken::new();
2166        let (_live_tx, mut live_rx) = mpsc::unbounded_channel::<ParentFrame>();
2167        let mut link = SilentLink;
2168        let r = drive(
2169            &mut link,
2170            "parent-x",
2171            "child-x",
2172            0,
2173            None,
2174            None,
2175            None,
2176            None,
2177            &event_tx,
2178            &cancel,
2179            &mut live_rx,
2180            Some(Duration::from_millis(100)),
2181        )
2182        .await;
2183        assert!(
2184            matches!(r, Err(AgentError::WorkerUnresponsive(_))),
2185            "a silent worker must trip the first-frame watchdog, got {r:?}"
2186        );
2187    }
2188
2189    #[tokio::test]
2190    async fn drive_does_not_trip_when_a_frame_arrives() {
2191        let (event_tx, _rx) = mpsc::channel::<AgentEvent>(8);
2192        let cancel = CancellationToken::new();
2193        let (_live_tx, mut live_rx) = mpsc::unbounded_channel::<ParentFrame>();
2194        let mut link = InstantTerminalLink { done: false };
2195        // Even a tiny timeout must NOT trip: the terminal frame arrives first and
2196        // disarms the watchdog.
2197        let r = drive(
2198            &mut link,
2199            "parent-y",
2200            "child-y",
2201            0,
2202            None,
2203            None,
2204            None,
2205            None,
2206            &event_tx,
2207            &cancel,
2208            &mut live_rx,
2209            Some(Duration::from_millis(50)),
2210        )
2211        .await;
2212        assert_eq!(r.ok().flatten().as_deref(), Some("done"));
2213    }
2214
2215    #[tokio::test]
2216    async fn child_approval_fails_closed_without_decider() {
2217        // No decider wired ⇒ the host denies (safe default), unchanged behavior.
2218        let body = serde_json::json!({"tool_name":"Bash","permission":"run","resource":"rm -rf /"});
2219        assert!(!decide_child_approval(None, "child-1", &body).await);
2220    }
2221
2222    #[tokio::test]
2223    async fn child_approval_honors_wired_decider() {
2224        let body =
2225            serde_json::json!({"tool_name":"Write","permission":"write","resource":"/tmp/x"});
2226        let approve: Arc<dyn ChildApprovalDecider> = Arc::new(StaticDecider(true));
2227        let deny: Arc<dyn ChildApprovalDecider> = Arc::new(StaticDecider(false));
2228        assert!(decide_child_approval(Some(&approve), "child-1", &body).await);
2229        assert!(!decide_child_approval(Some(&deny), "child-1", &body).await);
2230    }
2231
2232    // ---- #193: remote placement routing -------------------------------------
2233
2234    use crate::runtime::execution::SpawnJob;
2235    use bamboo_agent_core::Session;
2236
2237    /// A runner with a BOGUS worker_bin (`/bin/false`): a local spawn here would
2238    /// FAIL, so a passing remote test proves the remote path never spawns.
2239    fn bogus_runner(placements: HashMap<String, ResolvedRemotePlacement>) -> ActorChildRunner {
2240        ActorChildRunner::new(
2241            "test-actor".into(),
2242            PathBuf::from("/bin/false"),
2243            vec![],
2244            std::env::temp_dir().join("bamboo-test-fab-193"),
2245            ExecutorSpec::Echo,
2246            vec![],
2247            "anthropic".into(),
2248            4,
2249        )
2250        .with_remote_placements(placements)
2251    }
2252
2253    /// A child session of the given role (the role rides `subagent_type`, the
2254    /// path build_spec + the remote lookup both read).
2255    fn session_of_role(role: &str, assignment: &str) -> Session {
2256        let mut s = Session::new("child-1", "test-model");
2257        s.metadata
2258            .insert("subagent_type".to_string(), role.to_string());
2259        s.add_message(bamboo_agent_core::Message::user(assignment));
2260        s
2261    }
2262
2263    fn job_for(child: &str) -> SpawnJob {
2264        SpawnJob {
2265            parent_session_id: "parent-1".into(),
2266            child_session_id: child.into(),
2267            model: String::new(),
2268            disabled_tools: None,
2269        }
2270    }
2271
2272    #[derive(Default)]
2273    struct RecordingChildSessionPort {
2274        saved: std::sync::Mutex<Option<Session>>,
2275    }
2276
2277    impl RecordingChildSessionPort {
2278        fn saved_child(&self) -> Session {
2279            self.saved
2280                .lock()
2281                .expect("saved-child fixture lock")
2282                .clone()
2283                .expect("create_child_action must save the child")
2284        }
2285    }
2286
2287    #[async_trait]
2288    impl crate::session_app::child_session::ChildSessionPort for RecordingChildSessionPort {
2289        async fn load_root_session(
2290            &self,
2291            _root_id: &str,
2292        ) -> Result<Session, crate::session_app::child_session::ChildSessionError> {
2293            unreachable!("create_child_action does not load the root")
2294        }
2295
2296        async fn load_child_for_parent(
2297            &self,
2298            _parent_id: &str,
2299            _child_id: &str,
2300        ) -> Result<Session, crate::session_app::child_session::ChildSessionError> {
2301            unreachable!("create_child_action does not reload the child")
2302        }
2303
2304        async fn save_child_session(
2305            &self,
2306            child: &mut Session,
2307        ) -> Result<(), crate::session_app::child_session::ChildSessionError> {
2308            *self.saved.lock().expect("saved-child fixture lock") = Some(child.clone());
2309            Ok(())
2310        }
2311
2312        async fn save_child_session_authoritative_flags(
2313            &self,
2314            _child: &mut Session,
2315        ) -> Result<(), crate::session_app::child_session::ChildSessionError> {
2316            unreachable!("new-child creation uses the ordinary save")
2317        }
2318
2319        async fn is_child_running(&self, _child_id: &str) -> bool {
2320            false
2321        }
2322
2323        async fn list_children(
2324            &self,
2325            _parent_id: &str,
2326        ) -> Vec<crate::session_app::child_session::ChildSessionEntry> {
2327            Vec::new()
2328        }
2329
2330        async fn enqueue_child_run(
2331            &self,
2332            _parent: &Session,
2333            _child: &Session,
2334        ) -> Result<(), crate::session_app::child_session::ChildSessionError> {
2335            unreachable!("fixture creates the child with auto_run=false")
2336        }
2337
2338        async fn cancel_child_run_and_wait(
2339            &self,
2340            _child_id: &str,
2341        ) -> Result<(), crate::session_app::child_session::ChildSessionError> {
2342            unreachable!("create_child_action does not cancel")
2343        }
2344
2345        async fn delete_child_session(
2346            &self,
2347            _parent_id: &str,
2348            _child_id: &str,
2349        ) -> Result<
2350            crate::session_app::child_session::DeleteChildResult,
2351            crate::session_app::child_session::ChildSessionError,
2352        > {
2353            unreachable!("create_child_action does not delete")
2354        }
2355
2356        async fn get_child_runner_info(
2357            &self,
2358            _child_id: &str,
2359        ) -> Option<crate::session_app::child_session::ChildRunnerInfo> {
2360            None
2361        }
2362
2363        async fn register_parent_wait_for_child(
2364            &self,
2365            _parent_session_id: &str,
2366            _child_session_id: &str,
2367            _tool_call_id: Option<&str>,
2368        ) -> Result<(), crate::session_app::child_session::ChildSessionError> {
2369            unreachable!("create_child_action does not register a wait")
2370        }
2371
2372        async fn register_parent_wait_for_children(
2373            &self,
2374            _parent_session_id: &str,
2375            _child_session_ids: &[String],
2376            _policy: bamboo_domain::session::runtime_state::ChildWaitPolicy,
2377        ) -> Result<usize, crate::session_app::child_session::ChildSessionError> {
2378            unreachable!("create_child_action does not register a wait")
2379        }
2380
2381        async fn active_child_ids(&self, _parent_session_id: &str) -> Vec<String> {
2382            Vec::new()
2383        }
2384
2385        async fn find_resident_child(
2386            &self,
2387            _root_session_id: &str,
2388            _resident_name: &str,
2389        ) -> Option<String> {
2390            None
2391        }
2392
2393        async fn ensure_child_indexed(&self, _child_session_id: &str) {}
2394    }
2395
2396    #[test]
2397    fn build_spec_sets_remote_placement_for_matching_role() {
2398        let mut placements = HashMap::new();
2399        placements.insert(
2400            "explorer".to_string(),
2401            ResolvedRemotePlacement {
2402                endpoint: "wss://gpu-host:8443".into(),
2403                token: Some("T-secret".into()),
2404                ca_cert_file: None,
2405                host_label: None,
2406            },
2407        );
2408        let runner = bogus_runner(placements);
2409
2410        // Matching role -> Placement::Remote + the bearer on the secrets envelope.
2411        let s = session_of_role("explorer", "do the thing");
2412        let spec = runner.build_spec(&s, &job_for("child-1"));
2413        match &spec.placement {
2414            Placement::Remote { endpoint } => assert_eq!(endpoint, "wss://gpu-host:8443"),
2415            other => panic!("expected Remote, got {other:?}"),
2416        }
2417        assert_eq!(spec.secrets.worker_auth_token.as_deref(), Some("T-secret"));
2418    }
2419
2420    #[test]
2421    fn build_spec_leaves_local_for_unmatched_role() {
2422        let mut placements = HashMap::new();
2423        placements.insert(
2424            "explorer".to_string(),
2425            ResolvedRemotePlacement {
2426                endpoint: "wss://gpu-host:8443".into(),
2427                token: Some("T".into()),
2428                ca_cert_file: None,
2429                host_label: None,
2430            },
2431        );
2432        let runner = bogus_runner(placements);
2433
2434        // A DIFFERENT role keeps the default Local placement + no bearer.
2435        let s = session_of_role("writer", "do the thing");
2436        let spec = runner.build_spec(&s, &job_for("child-1"));
2437        assert_eq!(spec.placement, Placement::Local);
2438        assert!(spec.secrets.worker_auth_token.is_none());
2439    }
2440
2441    #[test]
2442    fn build_spec_local_when_no_placements() {
2443        let runner = bogus_runner(HashMap::new());
2444        let s = session_of_role("explorer", "do the thing");
2445        let spec = runner.build_spec(&s, &job_for("child-1"));
2446        assert_eq!(spec.placement, Placement::Local);
2447        assert!(spec.secrets.worker_auth_token.is_none());
2448    }
2449
2450    #[tokio::test]
2451    async fn build_spec_preserves_inherited_bypass_for_child_worker() {
2452        // Exercise the real creation path instead of pre-seeding the child by
2453        // hand: a bypassed parent's posture must survive both child creation and
2454        // the actor provisioning boundary.
2455        let runner = bogus_runner(HashMap::new());
2456        let mut parent = Session::new("parent-bypass", "test-model");
2457        parent
2458            .agent_runtime_state
2459            .get_or_insert_with(bamboo_domain::AgentRuntimeState::default)
2460            .bypass_permissions = true;
2461        let workspace = tempfile::tempdir().expect("workspace fixture");
2462        let port = RecordingChildSessionPort::default();
2463        let child_id = format!("child-bypass-{}", uuid::Uuid::new_v4());
2464        crate::session_app::child_session::create_child_action(
2465            &port,
2466            crate::session_app::child_session::CreateChildInput {
2467                parent_session: parent,
2468                child_id: child_id.clone(),
2469                title: "Bypassed child".to_string(),
2470                responsibility: "Run ordinary commands".to_string(),
2471                assignment_prompt: "run an ordinary command".to_string(),
2472                subagent_type: "explorer".to_string(),
2473                workspace: workspace.path().to_string_lossy().into_owned(),
2474                model_override: None,
2475                model_ref_override: None,
2476                runtime_metadata: HashMap::new(),
2477                auto_run: false,
2478                reasoning_effort: None,
2479                lifecycle: None,
2480                resident_name: None,
2481                resident_context: None,
2482                disabled_tools: None,
2483                context_fork: None,
2484            },
2485        )
2486        .await
2487        .expect("create inherited-bypass child");
2488        let child = port.saved_child();
2489
2490        assert!(
2491            child
2492                .agent_runtime_state
2493                .as_ref()
2494                .is_some_and(|state| state.bypass_permissions),
2495            "create_child_action must inherit bypass from the parent"
2496        );
2497
2498        let spec = runner.build_spec(&child, &job_for(&child_id));
2499
2500        assert!(spec.capabilities.bypass, "child worker must inherit bypass");
2501        assert!(
2502            spec.capabilities.enforce_permissions,
2503            "forced-ask evaluation must remain active under bypass"
2504        );
2505    }
2506
2507    #[test]
2508    fn placement_metadata_stamps_remote_and_schedulable_not_local() {
2509        // Local children carry no stamp — the DTO defaults them to the backend host.
2510        assert_eq!(placement_metadata(&Placement::Local, None), None);
2511
2512        // Remote, no node label → host derived from the endpoint.
2513        let r = placement_metadata(
2514            &Placement::Remote {
2515                endpoint: "wss://10.0.0.5:8443/stream".into(),
2516            },
2517            None,
2518        )
2519        .unwrap();
2520        assert!(r.contains(r#""kind":"remote""#), "{r}");
2521        assert!(r.contains(r#""host":"10.0.0.5""#), "{r}");
2522
2523        // A cluster node's label (its metadata) OVERRIDES the raw endpoint host.
2524        let labeled = placement_metadata(
2525            &Placement::Remote {
2526                endpoint: "ws://169.254.230.101:8899".into(),
2527            },
2528            Some("mini"),
2529        )
2530        .unwrap();
2531        assert!(labeled.contains(r#""host":"mini""#), "{labeled}");
2532
2533        // Schedulable → {kind:"remote", host:<node label, else pool>}.
2534        let s = placement_metadata(
2535            &Placement::Schedulable {
2536                pool: "explorers".into(),
2537            },
2538            Some("mini"),
2539        )
2540        .unwrap();
2541        assert!(s.contains(r#""kind":"remote""#), "{s}");
2542        assert!(s.contains(r#""host":"mini""#), "{s}");
2543
2544        // The stamp round-trips through the storage placement type.
2545        let p: bamboo_storage::SessionPlacement = serde_json::from_str(&labeled).unwrap();
2546        assert_eq!(p.kind, "remote");
2547        assert_eq!(p.host, "mini");
2548    }
2549
2550    /// End-to-end remote run through `execute_external_child`: a resident worker
2551    /// (Bearer-gated `WsServer` + `EchoExecutor`) serves the role; the runner is
2552    /// built with a `remote_placements` entry pointing at it AND a BOGUS
2553    /// worker_bin (`/bin/false`). A passing test proves the remote path CONNECTS
2554    /// to the resident worker and NEVER spawns (a spawn would fail on /bin/false),
2555    /// and that a terminal/echo result flows back.
2556    #[tokio::test]
2557    async fn execute_external_child_routes_role_to_remote_worker_without_spawning() {
2558        // 1. Stand up the resident worker on loopback with a required bearer.
2559        let token = "remote-test-token";
2560        let server = bamboo_subagent::transport::WsServer::bind_with_token(
2561            (std::net::Ipv4Addr::LOCALHOST, 0).into(),
2562            Some(token.to_string()),
2563        )
2564        .await
2565        .expect("bind resident worker");
2566        let endpoint = server.ws_endpoint(); // ws://127.0.0.1:<port>
2567        let srv = tokio::spawn(async move {
2568            // serve() loops connection-after-connection; the test exits, dropping it.
2569            let _ = server
2570                .serve(Arc::new(bamboo_subagent::executor::EchoExecutor))
2571                .await;
2572        });
2573
2574        // 2. Build the runner: role "explorer" pinned remote, bogus worker_bin.
2575        let mut placements = HashMap::new();
2576        placements.insert(
2577            "explorer".to_string(),
2578            ResolvedRemotePlacement {
2579                endpoint: endpoint.clone(),
2580                token: Some(token.to_string()),
2581                ca_cert_file: None,
2582                host_label: Some("mini-e2e".into()), // node label, surfaced on the badge
2583            },
2584        );
2585        let runner = bogus_runner(placements);
2586
2587        // 3. Drive a real run for that role.
2588        let mut session = session_of_role("explorer", "hello remote");
2589        let job = job_for("child-1");
2590        let (event_tx, mut event_rx) = mpsc::channel::<AgentEvent>(64);
2591        let cancel = CancellationToken::new();
2592
2593        let result = tokio::time::timeout(
2594            Duration::from_secs(10),
2595            runner.execute_external_child(&mut session, &job, event_tx, cancel),
2596        )
2597        .await
2598        .expect("run did not hang")
2599        .expect("remote run succeeded (connected to resident worker, did not spawn)");
2600
2601        let _ = result;
2602        // The EchoExecutor's reply is written back onto the child session as an
2603        // assistant message — proof a terminal result flowed back over the link.
2604        let last = session
2605            .messages
2606            .iter()
2607            .rev()
2608            .find(|m| matches!(m.role, Role::Assistant))
2609            .expect("an assistant reply was written back");
2610        assert!(
2611            last.content.contains("echo:"),
2612            "expected echo reply, got {:?}",
2613            last.content
2614        );
2615
2616        // A remote run must stamp WHICH machine it ran on onto the child session
2617        // (mirrored to the UI badge) using the placement's node label.
2618        let placement = session
2619            .metadata
2620            .get("placement")
2621            .expect("remote child session stamped with a placement");
2622        assert!(placement.contains(r#""kind":"remote""#), "{placement}");
2623        assert!(placement.contains(r#""host":"mini-e2e""#), "{placement}");
2624
2625        // Drain a couple of streamed events to confirm the event pipe carried the
2626        // worker's tokens too (best-effort; the reply assertion above is primary).
2627        let mut saw_event = false;
2628        while let Ok(Some(_ev)) =
2629            tokio::time::timeout(Duration::from_millis(50), event_rx.recv()).await
2630        {
2631            saw_event = true;
2632        }
2633        let _ = saw_event;
2634
2635        srv.abort();
2636    }
2637
2638    // ---- #181 (P2b): schedulable placement routing --------------------------
2639
2640    /// A bogus-worker_bin runner carrying SCHEDULABLE placements (and optionally
2641    /// remote ones, to test precedence). A local spawn here would fail on
2642    /// `/bin/false`, so a passing schedulable test proves no subprocess spawned.
2643    fn bogus_sched_runner(
2644        remote: HashMap<String, ResolvedRemotePlacement>,
2645        sched: HashMap<String, ResolvedSchedulablePlacement>,
2646    ) -> ActorChildRunner {
2647        ActorChildRunner::new(
2648            "test-actor".into(),
2649            PathBuf::from("/bin/false"),
2650            vec![],
2651            std::env::temp_dir().join("bamboo-test-fab-181"),
2652            ExecutorSpec::Echo,
2653            vec![],
2654            "anthropic".into(),
2655            4,
2656        )
2657        .with_remote_placements(remote)
2658        .with_schedulable_placements(sched)
2659    }
2660
2661    fn sched_placement(
2662        pool: &str,
2663        _registry_url: impl Into<String>,
2664    ) -> ResolvedSchedulablePlacement {
2665        ResolvedSchedulablePlacement {
2666            pool: pool.into(),
2667            host_label: None,
2668        }
2669    }
2670
2671    #[test]
2672    fn build_spec_sets_schedulable_placement_for_matching_role() {
2673        let mut sched = HashMap::new();
2674        sched.insert(
2675            "explorer".to_string(),
2676            sched_placement("gpu-pool", "unused"),
2677        );
2678        let runner = bogus_sched_runner(HashMap::new(), sched);
2679
2680        let s = session_of_role("explorer", "do the thing");
2681        let spec = runner.build_spec(&s, &job_for("child-1"));
2682        match &spec.placement {
2683            Placement::Schedulable { pool } => assert_eq!(pool, "gpu-pool"),
2684            other => panic!("expected Schedulable, got {other:?}"),
2685        }
2686        // No per-placement bearer now — the bus connection carries the bus token.
2687        assert!(spec.secrets.worker_auth_token.is_none());
2688    }
2689
2690    #[test]
2691    fn build_spec_remote_wins_when_role_in_both_maps() {
2692        // A role present in BOTH remote_placements and schedulable_placements must
2693        // resolve to the FIXED remote placement (documented precedence).
2694        let mut remote = HashMap::new();
2695        remote.insert(
2696            "explorer".to_string(),
2697            ResolvedRemotePlacement {
2698                endpoint: "wss://fixed-host:8443".into(),
2699                token: Some("T-remote".into()),
2700                ca_cert_file: None,
2701                host_label: None,
2702            },
2703        );
2704        let mut sched = HashMap::new();
2705        sched.insert(
2706            "explorer".to_string(),
2707            sched_placement("gpu-pool", "https://control-plane:9562"),
2708        );
2709        let runner = bogus_sched_runner(remote, sched);
2710
2711        let s = session_of_role("explorer", "do the thing");
2712        let spec = runner.build_spec(&s, &job_for("child-1"));
2713        match &spec.placement {
2714            Placement::Remote { endpoint } => assert_eq!(endpoint, "wss://fixed-host:8443"),
2715            other => panic!("expected Remote (precedence), got {other:?}"),
2716        }
2717        assert_eq!(spec.secrets.worker_auth_token.as_deref(), Some("T-remote"));
2718    }
2719
2720    #[test]
2721    fn build_spec_local_for_unmatched_schedulable_role() {
2722        let mut sched = HashMap::new();
2723        sched.insert(
2724            "explorer".to_string(),
2725            sched_placement("gpu-pool", "https://control-plane:9562"),
2726        );
2727        let runner = bogus_sched_runner(HashMap::new(), sched);
2728        let s = session_of_role("writer", "do the thing");
2729        let spec = runner.build_spec(&s, &job_for("child-1"));
2730        assert_eq!(spec.placement, Placement::Local);
2731        assert!(spec.secrets.worker_auth_token.is_none());
2732    }
2733
2734    /// The full role → resolved-placement → badge-host chain: a child routed to a
2735    /// remote/schedulable placement carrying a cluster node's `host_label` stamps
2736    /// that label; without a label it falls back to the endpoint host / pool; a
2737    /// Local child gets no stamp (the DTO defaults it to the backend host).
2738    #[test]
2739    fn placement_stamp_uses_node_label_for_remote_and_schedulable() {
2740        // Remote WITH a node label → {remote, <label>}, overriding the raw IP.
2741        let mut remote = HashMap::new();
2742        remote.insert(
2743            "explorer".to_string(),
2744            ResolvedRemotePlacement {
2745                endpoint: "ws://169.254.230.101:8899".into(),
2746                token: None,
2747                ca_cert_file: None,
2748                host_label: Some("mini".into()),
2749            },
2750        );
2751        let runner = bogus_runner(remote);
2752        let spec = runner.build_spec(&session_of_role("explorer", "go"), &job_for("c1"));
2753        let stamp = runner
2754            .placement_stamp_for(&spec)
2755            .expect("remote child is stamped");
2756        assert!(stamp.contains(r#""kind":"remote""#), "{stamp}");
2757        assert!(stamp.contains(r#""host":"mini""#), "{stamp}");
2758
2759        // Remote WITHOUT a node label → falls back to the endpoint host.
2760        let mut remote_nolabel = HashMap::new();
2761        remote_nolabel.insert(
2762            "explorer".to_string(),
2763            ResolvedRemotePlacement {
2764                endpoint: "ws://169.254.230.101:8899".into(),
2765                token: None,
2766                ca_cert_file: None,
2767                host_label: None,
2768            },
2769        );
2770        let r2 = bogus_runner(remote_nolabel);
2771        let spec2 = r2.build_spec(&session_of_role("explorer", "go"), &job_for("c1"));
2772        assert!(r2
2773            .placement_stamp_for(&spec2)
2774            .unwrap()
2775            .contains(r#""host":"169.254.230.101""#));
2776
2777        // Schedulable WITH a node label → {remote, <label>} (a node, not a pool name).
2778        let mut sched = HashMap::new();
2779        sched.insert(
2780            "mac-mini-monitor".to_string(),
2781            ResolvedSchedulablePlacement {
2782                pool: "mac-mini-monitor".into(),
2783                host_label: Some("mini".into()),
2784            },
2785        );
2786        let sr = bogus_sched_runner(HashMap::new(), sched);
2787        let spec3 = sr.build_spec(&session_of_role("mac-mini-monitor", "go"), &job_for("c1"));
2788        let stamp3 = sr
2789            .placement_stamp_for(&spec3)
2790            .expect("scheduled child is stamped");
2791        assert!(stamp3.contains(r#""kind":"remote""#), "{stamp3}");
2792        assert!(stamp3.contains(r#""host":"mini""#), "{stamp3}");
2793
2794        // A Local (unmatched) child gets NO stamp.
2795        let local = bogus_runner(HashMap::new());
2796        let spec4 = local.build_spec(&session_of_role("writer", "go"), &job_for("c1"));
2797        assert_eq!(local.placement_stamp_for(&spec4), None);
2798    }
2799
2800    // ---- #181: schedulable selection over the BUS (Phase 3 cutover) ----------
2801
2802    async fn start_bus() -> (String, tempfile::TempDir) {
2803        let dir = tempfile::tempdir().unwrap();
2804        let core = std::sync::Arc::new(bamboo_broker::BrokerCore::new(dir.path()));
2805        let server = std::sync::Arc::new(bamboo_broker::BrokerServer::new(core, "t"));
2806        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
2807        let addr = listener.local_addr().unwrap();
2808        tokio::spawn(async move {
2809            let _ = server.serve(listener).await;
2810        });
2811        (format!("ws://{addr}"), dir)
2812    }
2813
2814    async fn join_pool(endpoint: &str, id: &str, pool: &str) -> bamboo_broker::BrokerClient {
2815        let mut c = bamboo_broker::BrokerClient::connect(
2816            endpoint,
2817            bamboo_subagent::AgentRef {
2818                session_id: id.into(),
2819                role: Some(pool.into()),
2820            },
2821            "t",
2822        )
2823        .await
2824        .unwrap();
2825        c.subscribe().await.unwrap();
2826        c
2827    }
2828
2829    fn sched_runner_on_bus(endpoint: &str, child_role: &str, pool: &str) -> ActorChildRunner {
2830        let mut sched = HashMap::new();
2831        sched.insert(child_role.to_string(), sched_placement(pool, "unused"));
2832        bogus_sched_runner(HashMap::new(), sched).with_bus(Some(bamboo_subagent::BusEndpoint {
2833            endpoint: endpoint.into(),
2834            token: "t".into(),
2835        }))
2836    }
2837
2838    #[tokio::test]
2839    async fn resolve_schedulable_picks_a_live_bus_worker() {
2840        let (endpoint, _dir) = start_bus().await;
2841        let _w = join_pool(&endpoint, "w-gpu", "gpu-pool").await;
2842        let runner = sched_runner_on_bus(&endpoint, "explorer", "gpu-pool");
2843
2844        let mailbox = runner
2845            .resolve_schedulable_worker("explorer")
2846            .await
2847            .expect("a live pool worker is found on the bus");
2848        assert_eq!(mailbox, "w-gpu");
2849    }
2850
2851    #[tokio::test]
2852    async fn resolve_schedulable_round_robins_over_pool_workers() {
2853        let (endpoint, _dir) = start_bus().await;
2854        let _a = join_pool(&endpoint, "w-a", "gpu-pool").await;
2855        let _b = join_pool(&endpoint, "w-b", "gpu-pool").await;
2856        let runner = sched_runner_on_bus(&endpoint, "explorer", "gpu-pool");
2857
2858        // Successive resolves spread across both connected workers.
2859        let mut picked = std::collections::HashSet::new();
2860        for _ in 0..6 {
2861            picked.insert(runner.resolve_schedulable_worker("explorer").await.unwrap());
2862        }
2863        assert_eq!(
2864            picked,
2865            ["w-a".to_string(), "w-b".to_string()].into_iter().collect(),
2866            "round-robin must cover every connected pool worker"
2867        );
2868    }
2869
2870    #[tokio::test]
2871    async fn resolve_schedulable_errors_on_empty_pool() {
2872        let (endpoint, _dir) = start_bus().await;
2873        // No worker subscribes to "gpu-pool".
2874        let runner = sched_runner_on_bus(&endpoint, "explorer", "gpu-pool");
2875
2876        let err = runner
2877            .resolve_schedulable_worker("explorer")
2878            .await
2879            .expect_err("an empty pool is terminal — no local fallback")
2880            .to_string();
2881        assert!(err.contains("no live worker in pool"), "got: {err}");
2882        assert!(err.contains("NOT spawning"), "got: {err}");
2883    }
2884
2885    /// FULL schedulable run over the bus: a worker SERVING `EchoExecutor` joins the
2886    /// pool by role; `execute_external_child` with a Schedulable placement resolves
2887    /// it from the bus (no local subprocess — the worker_bin is `/bin/false`),
2888    /// drives the run, gets the echo back, AND stamps the child session with the
2889    /// pool's cluster-node label — `{kind:remote, host:"mini"}`. The end-to-end
2890    /// analogue of the live `mac-mini-monitor`→mini run.
2891    #[tokio::test]
2892    async fn execute_external_child_runs_schedulable_over_bus_and_stamps_node_label() {
2893        let (endpoint, _dir) = start_bus().await;
2894
2895        // A bus worker SERVING runs (not just presence), joined to the pool by role.
2896        let ep = endpoint.clone();
2897        let worker = tokio::spawn(async move {
2898            let _ = bamboo_broker::serve_executor(
2899                &ep,
2900                bamboo_subagent::AgentRef {
2901                    session_id: "mmm-worker".into(),
2902                    role: Some("mac-mini-monitor".into()),
2903                },
2904                "t",
2905                std::sync::Arc::new(bamboo_subagent::executor::EchoExecutor),
2906            )
2907            .await;
2908        });
2909
2910        // Wait until the worker is visible on the bus so the pool is non-empty
2911        // when execute_external_child resolves it (serve_executor connects async).
2912        let mut probe = bamboo_broker::BrokerClient::connect(
2913            &endpoint,
2914            bamboo_subagent::AgentRef {
2915                session_id: "probe".into(),
2916                role: None,
2917            },
2918            "t",
2919        )
2920        .await
2921        .unwrap();
2922        let mut ready = false;
2923        for _ in 0..100 {
2924            if probe
2925                .list_connected("mac-mini-monitor")
2926                .await
2927                .unwrap()
2928                .iter()
2929                .any(|id| id == "mmm-worker")
2930            {
2931                ready = true;
2932                break;
2933            }
2934            tokio::time::sleep(Duration::from_millis(30)).await;
2935        }
2936        assert!(ready, "worker never joined the pool");
2937
2938        // Runner: child role → schedulable pool "mac-mini-monitor" carrying the
2939        // cluster node's label "mini"; bogus worker_bin so any local spawn fails.
2940        let mut sched = HashMap::new();
2941        sched.insert(
2942            "mac-mini-monitor".to_string(),
2943            ResolvedSchedulablePlacement {
2944                pool: "mac-mini-monitor".into(),
2945                host_label: Some("mini".into()),
2946            },
2947        );
2948        let runner = bogus_sched_runner(HashMap::new(), sched).with_bus(Some(
2949            bamboo_subagent::BusEndpoint {
2950                endpoint: endpoint.clone(),
2951                token: "t".into(),
2952            },
2953        ));
2954
2955        let mut session = session_of_role("mac-mini-monitor", "hello scheduled");
2956        let job = job_for("child-1");
2957        let (event_tx, _rx) = mpsc::channel::<AgentEvent>(64);
2958        let cancel = CancellationToken::new();
2959
2960        tokio::time::timeout(
2961            Duration::from_secs(10),
2962            runner.execute_external_child(&mut session, &job, event_tx, cancel),
2963        )
2964        .await
2965        .expect("run did not hang")
2966        .expect("schedulable run succeeded over the bus (no local spawn)");
2967
2968        // Echo reply flowed back — proves it routed to the bus worker, not local.
2969        let last = session
2970            .messages
2971            .iter()
2972            .rev()
2973            .find(|m| matches!(m.role, Role::Assistant))
2974            .expect("an assistant reply was written back");
2975        assert!(last.content.contains("echo:"), "got {:?}", last.content);
2976
2977        // ...and the child is stamped with the pool's cluster-node label.
2978        let placement = session
2979            .metadata
2980            .get("placement")
2981            .expect("scheduled child session stamped with a placement");
2982        assert!(placement.contains(r#""kind":"remote""#), "{placement}");
2983        assert!(placement.contains(r#""host":"mini""#), "{placement}");
2984
2985        worker.abort();
2986    }
2987}