bamboo_engine/external_agents/actor_adapter.rs
1//! Actor external child runner.
2//!
3//! Runs a child session as an independent **actor**: a separate OS process with its own
4//! isolated context, speaking the `bamboo-subagent` WebSocket protocol. This is the
5//! engine-side adapter on the `wants_external` seam: it spawns the worker binary, waits for
6//! it to self-register into the Tier-1 file fabric, connects, sends the assignment, and
7//! forwards the child's `AgentEvent`s back onto the parent's `event_tx`.
8//!
9//! The built-in **local actor** instance of this runner is the default runtime for
10//! every sub-agent (the in-process runtime was removed). The expert `externalAgents`
11//! tables can additionally route specific roles to other actor/a2a agents.
12
13use std::collections::HashMap;
14use std::path::PathBuf;
15use std::sync::Arc;
16use std::time::Duration;
17
18use async_trait::async_trait;
19use bamboo_agent_core::{AgentError, AgentEvent, Role, Session};
20use bamboo_domain::poison::PoisonRecover;
21use tokio::sync::mpsc;
22use tokio_util::sync::CancellationToken;
23
24use bamboo_subagent::fleet::{spawn_worker_on_bus, SpawnedChild};
25use bamboo_subagent::proto::{AgentRecord, ChildFrame, ParentFrame, RunSpec, TerminalStatus};
26use bamboo_subagent::provision::{
27 ChildIdentity, ExecutorSpec, ModelRefSpec, Placement, ProvisionSpec, ScopedCredential,
28};
29use bamboo_subagent::transport::{client_config_trusting_cert, ChildClient};
30
31use crate::runtime::execution::{ExternalChildRunner, SpawnJob};
32
33/// Default cap on simultaneously running actor processes.
34pub const DEFAULT_MAX_CONCURRENT_ACTORS: usize = 8;
35
36/// Max nesting depth for direct nested execution (Phase 6). A worker whose
37/// session `spawn_depth` is below this gets its own spawn stack + the real
38/// SubAgent tool; at/over it, neither (and the tool itself refuses). Mirrors
39/// `bamboo_server_tools::DEFAULT_MAX_SPAWN_DEPTH` (kept in sync; engine can't
40/// depend on server-tools). Root orchestrator = 0 ⇒ 4 levels of sub-agents.
41pub const MAX_SPAWN_DEPTH: u32 = 4;
42
43/// Default cap on idle pooled (warm, reusable) workers kept per fingerprint.
44const DEFAULT_MAX_IDLE_PER_KEY: usize = 4;
45
46/// How long a pooled worker waits for its next assignment before reclaiming
47/// itself (must comfortably exceed the gap between sibling spawns).
48const POOLED_IDLE_TIMEOUT_SECS: u64 = 300;
49
50/// Deadline for a local worker's FIRST frame after a Run is dispatched. A warm
51/// worker answers in seconds; a cold spawn within tens. Total silence past this
52/// means the worker is dead (e.g. a pooled worker that exited right after its
53/// liveness check) and its Run is queued with nobody to serve it — trip it so the
54/// runner respawns once instead of hanging forever. Generous, to never false-trip
55/// a slow-but-healthy cold start.
56const WORKER_FIRST_FRAME_TIMEOUT: Duration = Duration::from_secs(60);
57
58/// A warm worker on the mailbox bus, parked for reuse between runs. It stays
59/// dialed-in + subscribed to `mailbox_id`; the next interchangeable child
60/// delivers its `Run` there instead of spawning a fresh process. Dropping it
61/// kills a local kill-on-drop subprocess; a remote / schedulable handle is
62/// process-less (`kill()` is a no-op — it self-manages via its idle timeout).
63struct PooledWorker {
64 worker: SpawnedChild,
65 /// The bus mailbox this worker subscribes to (where its `Run`s are delivered).
66 mailbox_id: String,
67}
68
69/// A role pinned to a remote resident worker (remote-actor-plan §3.4 / P1.5,
70/// #193), resolved at runner-build time from `SubagentsConfig.remote_placements`:
71/// the env-named bearer is already READ into `token` here (the raw token never
72/// rides the config), and `ca_cert_file` is the path to a PEM pinning a
73/// self-signed worker cert (`None` ⇒ default webpki roots / plaintext `ws://`).
74#[derive(Debug, Clone)]
75pub struct ResolvedRemotePlacement {
76 pub endpoint: String,
77 pub token: Option<String>,
78 pub ca_cert_file: Option<PathBuf>,
79 /// Display name for the machine this role runs on — the matching cluster
80 /// node's `label`/host, surfaced on the UI placement badge. `None` ⇒ derive
81 /// from the endpoint host.
82 pub host_label: Option<String>,
83}
84
85/// A role routed to a SCHEDULED worker (remote-actor-plan §3.4 / P2b, #181),
86/// resolved at runner-build time from `SubagentsConfig.schedulable_placements`.
87/// Names the logical `pool` (= the bus role) whose LIVE connected workers are the
88/// scheduling candidates — the runner picks one via the bus presence query
89/// (`BrokerClient::list_connected`). Phase 3 retired the old HTTP registry, so a
90/// pool is now just a role on the bus.
91#[derive(Debug, Clone)]
92pub struct ResolvedSchedulablePlacement {
93 pub pool: String,
94 /// Display name for the machine this pool's workers run on — the matching
95 /// cluster node's `label`/host, surfaced on the UI placement badge. `None` ⇒
96 /// fall back to the pool name.
97 pub host_label: Option<String>,
98}
99
100/// How `execute_external_child` should obtain its worker connection, decided
101/// once from `spec.placement`. Splits the divergent acquire/connect + retire
102/// logic three ways while the shared middle (Run dispatch, live registration,
103/// drive, close) stays identical. `Local` is the unchanged pre-#193 path;
104/// `Remote` is the unchanged #194 path; `Schedulable` (#181, P2b) is new.
105enum PlacementKind {
106 Local,
107 Remote,
108 Schedulable,
109}
110
111/// Spawns and drives a child session as an independent actor: a `bamboo-subagent` worker process.
112pub struct ActorChildRunner {
113 approval_registry: Option<super::approval_registry::SharedApprovalRegistry>,
114 agent_id: String,
115 worker_bin: PathBuf,
116 worker_args: Vec<String>,
117 fabric_dir: PathBuf,
118 executor: ExecutorSpec,
119 /// Per-provider credentials snapshotted from the parent config at build
120 /// time; the spec carries only the ONE the child's provider needs.
121 credentials: Vec<ScopedCredential>,
122 /// Parent's default provider (used when the child has no explicit one).
123 default_provider: String,
124 /// The mailbox bus to run local children over (the unified transport). Local
125 /// sub-agents require it; `None` only when no broker could be embedded.
126 bus: Option<bamboo_subagent::BusEndpoint>,
127 /// Backpressure: bounds the number of concurrently *running* actors; further
128 /// runs wait for a slot instead of exploding the process table. (Idle pooled
129 /// workers do not hold a slot.)
130 concurrency: std::sync::Arc<tokio::sync::Semaphore>,
131 /// Warm-worker pool keyed by a reuse fingerprint
132 /// (role/provider/model/workspace/disabled-tools/baked-caps). A finished run
133 /// parks its bus worker here so the next interchangeable child reuses it
134 /// (delivers its `Run` to the same mailbox) instead of spawning a fresh
135 /// process — collapsing N sibling sub-agents onto a few warm workers.
136 pool: Arc<tokio::sync::Mutex<HashMap<String, Vec<PooledWorker>>>>,
137 max_idle_per_key: usize,
138 /// Host-side decision for a child's gated-tool approval request (Phase 2).
139 /// `None` ⇒ fail-closed DENY (the safe default). A wired decider (policy or
140 /// human-routing bridge) returns approve/deny over the actor WS.
141 approval_decider: Option<Arc<dyn ChildApprovalDecider>>,
142 /// Per-run escalation host bridge for non-bypass child-approval routing (#68;
143 /// Phase 6, Part B). The owning worker's `run()` installs its OWN host bridge
144 /// here via `set_escalation_bridge`; `execute_external_child` CAPTURES it at
145 /// grandchild-spawn time and hands the owned value to `drive()`, which uses it
146 /// to RE-PROXY a child's approval request UP to the parent run — chaining up
147 /// every level until a bypass level (model-review) or the top orchestrator
148 /// (human) decides, then relaying the reply back down. Was a process-global
149 /// slot; now per-runner so a fire-and-forget grandchild that OUTLIVES the run
150 /// that spawned it keeps that run's bridge for its whole lifetime instead of
151 /// reading a stale/overwritten global at approval time (→ fail-closed deny).
152 escalation_bridge: Arc<std::sync::Mutex<Option<bamboo_subagent::executor::HostBridge>>>,
153 /// Roles pinned to a REMOTE resident worker (#193), keyed by sub-agent role
154 /// (the child's `subagent_type`). A role present here routes through the
155 /// dedicated remote branch in `execute_external_child` (Bearer-authenticated
156 /// `wss://` connect, no spawn, no pool, no kill) instead of the local
157 /// subprocess + warm-pool path. Empty (the default) = all-local behavior.
158 remote_placements: HashMap<String, ResolvedRemotePlacement>,
159 /// Roles routed to a REGISTRY-SCHEDULED worker (#181, P2b), keyed by sub-agent
160 /// role. A role present here (AND not already in `remote_placements`, which
161 /// wins) routes through the dedicated SCHEDULABLE branch in
162 /// `execute_external_child`: query the registry for live workers in the pool,
163 /// pick one (round-robin), connect over `wss://` — no spawn, no pool, no kill,
164 /// and NO local-subprocess fallback (no live worker ⇒ a clear error). Empty
165 /// (the default) = all-local behavior.
166 schedulable_placements: HashMap<String, ResolvedSchedulablePlacement>,
167 /// Per-pool round-robin cursor for schedulable scheduling (#181, P2b). Bumped
168 /// once per pick so successive sibling spawns SPREAD across a pool's live
169 /// workers instead of all landing on the first candidate. Best-effort spread,
170 /// not a load balancer — the registry's live set can change between picks.
171 schedule_cursor: Arc<std::sync::Mutex<HashMap<String, usize>>>,
172}
173
174/// Decides how the host answers a child worker's gated-tool approval request
175/// (Phase 2: child → parent approval delegation). Async so an implementation
176/// can consult a policy or defer to a human. With no decider wired the host
177/// replies with a fail-closed DENY.
178///
179/// NOTE: `decide` is awaited inside the per-child frame pump, so an
180/// implementation must resolve promptly (e.g. a policy lookup). A human-in-the-
181/// loop decision that may block indefinitely should instead be delivered
182/// out-of-band as a `ParentFrame::ApprovalReply` via the live steering channel
183/// (`super::live`), which `drive()` already forwards to the worker without
184/// stalling the pump.
185#[async_trait]
186pub trait ChildApprovalDecider: Send + Sync {
187 /// Decide whether `child_session_id` may perform the gated action described
188 /// by `request` (`{tool_name, permission, resource}`).
189 async fn decide(&self, child_session_id: &str, request: &serde_json::Value) -> bool;
190}
191
192/// Resolve a child approval request to approve/deny. Fail-closed (DENY) when no
193/// decider is wired — the single, testable seam for the host-side decision.
194async fn decide_child_approval(
195 decider: Option<&Arc<dyn ChildApprovalDecider>>,
196 child_session_id: &str,
197 request: &serde_json::Value,
198) -> bool {
199 match decider {
200 Some(decider) => decider.decide(child_session_id, request).await,
201 None => false,
202 }
203}
204
205/// How long the host waits for a human approval decision before failing the
206/// child's gated tool closed (DENY). Bounds an unanswered request so it can't
207/// hang the worker indefinitely.
208const CHILD_APPROVAL_TIMEOUT: Duration = Duration::from_secs(300);
209
210/// Extract `(tool_name, permission, resource)` from a worker's approval request
211/// body (`{tool_name, permission, resource}`); missing fields default to empty.
212fn approval_request_fields(body: &serde_json::Value) -> (String, String, String) {
213 let field = |k: &str| {
214 body.get(k)
215 .and_then(|v| v.as_str())
216 .unwrap_or("")
217 .to_string()
218 };
219 (field("tool_name"), field("permission"), field("resource"))
220}
221
222/// Off-loop reviewer for a child's gated-tool approval request (Phase 6, Part B).
223///
224/// Installed (process-global) by a BYPASSED self-orchestrating worker so its
225/// children's forced-ask (dangerous) gated actions — which still raise
226/// `ConfirmationRequired` even under bypass — get an LLM reasonableness check
227/// rather than a blind pass. `review` is an LLM call: `drive()` invokes it in a
228/// SPAWNED task (NEVER in the frame pump) and delivers the verdict async via the
229/// live channel, so the agent loop is never blocked.
230#[async_trait]
231pub trait ChildApprovalReviewer: Send + Sync {
232 /// Judge whether the gated action `request` (`{tool_name, permission,
233 /// resource}`) is reasonable for `child_session_id`'s task. `true` = approve.
234 async fn review(&self, child_session_id: &str, request: &serde_json::Value) -> bool;
235}
236
237fn child_approval_reviewer_slot() -> &'static std::sync::OnceLock<Arc<dyn ChildApprovalReviewer>> {
238 static SLOT: std::sync::OnceLock<Arc<dyn ChildApprovalReviewer>> = std::sync::OnceLock::new();
239 &SLOT
240}
241
242/// Install the process-global child-approval reviewer (idempotent; first wins).
243pub fn set_child_approval_reviewer(reviewer: Arc<dyn ChildApprovalReviewer>) {
244 let _ = child_approval_reviewer_slot().set(reviewer);
245}
246
247/// The process-global child-approval reviewer, if installed.
248pub fn child_approval_reviewer() -> Option<Arc<dyn ChildApprovalReviewer>> {
249 child_approval_reviewer_slot().get().cloned()
250}
251
252impl ActorChildRunner {
253 #[allow(clippy::too_many_arguments)]
254 pub fn new(
255 agent_id: String,
256 worker_bin: PathBuf,
257 worker_args: Vec<String>,
258 fabric_dir: PathBuf,
259 executor: ExecutorSpec,
260 credentials: Vec<ScopedCredential>,
261 default_provider: String,
262 max_concurrent: usize,
263 ) -> Self {
264 Self {
265 approval_registry: None,
266 agent_id,
267 worker_bin,
268 worker_args,
269 fabric_dir,
270 executor,
271 credentials,
272 default_provider,
273 bus: None,
274 concurrency: std::sync::Arc::new(tokio::sync::Semaphore::new(max_concurrent.max(1))),
275 pool: Arc::new(tokio::sync::Mutex::new(HashMap::new())),
276 max_idle_per_key: DEFAULT_MAX_IDLE_PER_KEY,
277 approval_decider: None,
278 escalation_bridge: Arc::new(std::sync::Mutex::new(None)),
279 remote_placements: HashMap::new(),
280 schedulable_placements: HashMap::new(),
281 schedule_cursor: Arc::new(std::sync::Mutex::new(HashMap::new())),
282 }
283 }
284
285 pub fn with_approval_registry(
286 mut self,
287 registry: super::approval_registry::SharedApprovalRegistry,
288 ) -> Self {
289 self.approval_registry = Some(registry);
290 self
291 }
292
293 /// Run children over the mailbox bus (the unified actor+mailbox transport).
294 /// When set, local children dial this bus and are driven by mailbox id; when
295 /// unset they use the legacy direct-WS path. The server passes its in-process
296 /// broker here (`subagents.broker`); tests without a broker leave it unset.
297 pub fn with_bus(mut self, bus: Option<bamboo_subagent::BusEndpoint>) -> Self {
298 self.bus = bus.filter(|b| !b.endpoint.trim().is_empty());
299 self
300 }
301
302 /// Wire the host-side decider for child gated-tool approval requests
303 /// (Phase 2). Without this the host fail-closed DENYs every request.
304 pub fn with_approval_decider(mut self, decider: Arc<dyn ChildApprovalDecider>) -> Self {
305 self.approval_decider = Some(decider);
306 self
307 }
308
309 /// Pin specific sub-agent roles to remote resident workers (#193). The map
310 /// is keyed by role (`subagent_type`); a child whose role is present connects
311 /// over `wss://` to the resolved endpoint instead of spawning a local
312 /// subprocess. Default (empty) keeps every role on the local path — exactly
313 /// today's behavior.
314 pub fn with_remote_placements(
315 mut self,
316 placements: HashMap<String, ResolvedRemotePlacement>,
317 ) -> Self {
318 self.remote_placements = placements;
319 self
320 }
321
322 /// Route specific sub-agent roles to a registry-SCHEDULED worker (#181, P2b).
323 /// The map is keyed by role (`subagent_type`); a child whose role is present
324 /// (and NOT already pinned by `remote_placements`, which takes precedence) is
325 /// run on a live worker discovered from the registry instead of a local
326 /// subprocess. Default (empty) keeps every role on the local path.
327 pub fn with_schedulable_placements(
328 mut self,
329 placements: HashMap<String, ResolvedSchedulablePlacement>,
330 ) -> Self {
331 self.schedulable_placements = placements;
332 self
333 }
334
335 /// Reuse fingerprint: two children are interchangeable on one warm worker iff
336 /// they share role, provider, model, workspace, disabled-tool set, AND every
337 /// capability the worker BAKES at provision time (`BambooRuntimeExecutor`
338 /// stamps these once and reuses them across runs): nesting depth, nested-spawn
339 /// stack, bypass mode, permission enforcement, and the depth cap. Omitting any
340 /// of these lets the pool hand a run a worker baked for a DIFFERENT posture —
341 /// e.g. a depth-1 worker (with its own spawn stack) reused for a depth-4
342 /// child would re-stamp `spawn_depth=1` and pass the depth-cap check, breaking
343 /// the recursion bound; or a bypass worker reused for a non-bypass child. So
344 /// these MUST split the pool bucket. Everything else (assignment, history) is
345 /// shipped per-run in the `RunSpec` and does not affect the fingerprint.
346 /// Reuse fingerprint (role/provider/model/workspace/disabled-tools/baked
347 /// caps): two children with the same fingerprint are interchangeable on one
348 /// warm worker, so they share a pool bucket. Any axis the worker bakes ONCE
349 /// at provision time MUST be in here, else a worker baked for one posture
350 /// gets reused for another (see the `fingerprint_*` tests).
351 fn fingerprint(spec: &ProvisionSpec) -> String {
352 let role = spec.identity.role.as_str();
353 let (provider, model) = spec
354 .model
355 .as_ref()
356 .map(|m| (m.provider.as_str(), m.model.as_str()))
357 .unwrap_or(("", ""));
358 let workspace = spec.workspace.as_deref().unwrap_or("");
359 let mut tools = spec.disabled_tools.clone().unwrap_or_default();
360 tools.sort();
361 let caps = &spec.capabilities;
362 format!(
363 "{role}\u{1}{provider}\u{1}{model}\u{1}{workspace}\u{1}{}\u{1}d={}\u{1}ns={}\u{1}by={}\u{1}ep={}\u{1}md={}\u{1}nha={}\u{1}gro={}",
364 tools.join(","),
365 spec.identity.depth,
366 caps.nested_spawn,
367 caps.bypass,
368 caps.enforce_permissions,
369 caps.max_spawn_depth.unwrap_or(0),
370 // #73 review (P1): a worker bakes `no_human_review` ONCE from this flag
371 // at build() and never re-reads it per run, so the pool MUST NOT hand a
372 // worker baked for one approval posture to a run of the opposite one —
373 // else a scheduled-root worker reused for an interactive child would
374 // silently model-review instead of asking the human (and vice-versa,
375 // reintroducing the 300s-deny). Split the bucket on it.
376 caps.no_human_approver,
377 // #71: the read-only Bash checker is baked once at build() from this
378 // flag, so a guardian-reviewer worker must NOT be reused for an
379 // ordinary child (which expects unrestricted Bash), and vice-versa.
380 caps.guardian_read_only,
381 )
382 }
383
384 /// Check out a warm bus worker for `key`, reusing a live parked one if any,
385 /// else spawning a fresh one that dials the bus. The returned worker is OWNED
386 /// by the caller for the run's duration (checkout removes it from the pool, so
387 /// a concurrent sibling gets a different worker or spawns its own — one run per
388 /// worker at a time, matching the pre-bus pool semantics).
389 async fn acquire_bus_worker(
390 &self,
391 key: &str,
392 spec: &ProvisionSpec,
393 ) -> crate::runtime::runner::Result<PooledWorker> {
394 // Drain the bucket, skipping (and reaping) any worker whose process exited
395 // while parked. A live one is handed straight out for reuse.
396 loop {
397 let candidate = {
398 let mut pool = self.pool.lock().await;
399 pool.get_mut(key).and_then(|bucket| bucket.pop())
400 };
401 let Some(mut candidate) = candidate else {
402 break;
403 };
404 if candidate.worker.is_alive() {
405 return Ok(candidate);
406 }
407 candidate.worker.kill().await;
408 }
409
410 let spawned = spawn_worker_on_bus(&self.worker_bin, &self.worker_args, spec)
411 .await
412 .map_err(|e| AgentError::LLM(format!("actor spawn (bus) failed: {e}")))?;
413 let mailbox_id = spawned.record.agent_id.clone();
414 Ok(PooledWorker {
415 worker: spawned,
416 mailbox_id,
417 })
418 }
419
420 /// Park a warm bus worker for reuse after a clean run; if its bucket is full
421 /// (or it died), kill it instead. The worker stays dialed-in + subscribed
422 /// while parked, so a reusing child just delivers a new `Run` to its mailbox.
423 async fn release_bus_worker(&self, key: &str, mut worker: PooledWorker) {
424 if !worker.worker.is_alive() {
425 worker.worker.kill().await;
426 return;
427 }
428 let mut pool = self.pool.lock().await;
429 let bucket = pool.entry(key.to_string()).or_default();
430 if bucket.len() >= self.max_idle_per_key {
431 drop(pool);
432 worker.worker.kill().await;
433 return;
434 }
435 bucket.push(worker);
436 }
437
438 /// Assemble the parent-resolved provisioning document for this child.
439 fn build_spec(&self, session: &Session, job: &SpawnJob) -> ProvisionSpec {
440 let mut spec = ProvisionSpec::new(
441 ChildIdentity {
442 child_id: job.child_session_id.clone(),
443 parent_id: Some(job.parent_session_id.clone()),
444 project_key: None,
445 role: session
446 .metadata
447 .get("subagent_type")
448 .cloned()
449 .unwrap_or_else(|| "worker".to_string()),
450 // The child session already carries the correct depth
451 // (create_child_action's new_child_of did parent.spawn_depth+1);
452 // stamp it so the worker can re-establish it on its run session
453 // and enforce the max-depth cap across the actor boundary.
454 depth: session.spawn_depth,
455 },
456 self.executor.clone(),
457 self.fabric_dir.to_string_lossy().into_owned(),
458 );
459 spec.workspace = session.workspace.clone();
460 // Unified transport: when a bus is configured, the child dials it (no
461 // listen socket / file discovery) and the parent drives it by mailbox id.
462 spec.bus = self.bus.clone();
463 // Final model: the session's pinned model_ref (create.model / routing already applied),
464 // falling back to the job's bare model on the parent's default provider.
465 spec.model = session
466 .model_ref
467 .as_ref()
468 .map(|r| ModelRefSpec {
469 provider: r.provider.clone(),
470 model: r.model.clone(),
471 })
472 .or_else(|| {
473 let m = job.model.trim();
474 (!m.is_empty()).then(|| ModelRefSpec {
475 provider: self.default_provider.clone(),
476 model: m.to_string(),
477 })
478 });
479 spec.disabled_tools = job.disabled_tools.clone();
480 // Least-privilege secrets: only the credential for the child's provider.
481 let provider = spec
482 .model
483 .as_ref()
484 .map(|m| m.provider.as_str())
485 .filter(|p| !p.trim().is_empty())
486 .unwrap_or(&self.default_provider);
487 if let Some(cred) = self.credentials.iter().find(|c| c.provider == provider) {
488 spec.secrets.provider_credentials.push(cred.clone());
489 } else {
490 tracing::warn!(
491 "actor child {}: no credential found for provider '{}'",
492 job.child_session_id,
493 provider
494 );
495 }
496 // Phase 6 (direct nested execution): a worker BELOW the depth cap may
497 // orchestrate its OWN children — on startup it builds its own spawn
498 // stack and runs the real SubAgent tool (no host proxy). The cap (the
499 // SubAgent tool refuses to spawn at/over `max_spawn_depth`) bounds the
500 // recursion. Driven purely by the child's depth, so it auto-propagates
501 // down the tree without any extra config threading.
502 spec.capabilities.nested_spawn = session.spawn_depth < MAX_SPAWN_DEPTH;
503 spec.capabilities.max_spawn_depth = Some(MAX_SPAWN_DEPTH);
504 // #69: activate child-approval review. Sub-agents enforce permissions so
505 // their DANGEROUS actions (the worker uses a HIGH threshold) reach the
506 // parent for review — escalated to the human, or model-reviewed off-loop
507 // when the parent is in bypass. The worker installs no checker without
508 // this, so the whole review chain would otherwise stay dormant.
509 spec.capabilities.enforce_permissions = true;
510 // Propagate "bypass permissions" so a self-orchestrating worker knows it
511 // is a bypassed parent and installs the off-loop model-reviewer for its
512 // children's forced-ask actions (Phase 6, Part B). The child session
513 // already carries the inherited flag (create_child_action seeds it).
514 spec.capabilities.bypass = session
515 .agent_runtime_state
516 .as_ref()
517 .is_some_and(|s| s.bypass_permissions);
518 // #73: propagate "no interactive human approver" (headless / scheduled /
519 // deployed root, inherited by the child session). When set, the worker's
520 // per-run approval proxy model-reviews a gated action locally instead of
521 // escalating to a human who will never answer (which would 300s-deny).
522 spec.capabilities.no_human_approver = session
523 .agent_runtime_state
524 .as_ref()
525 .is_some_and(|s| s.no_human_approver);
526 // #71: mark a READ-ONLY Guardian reviewer so the worker installs the
527 // read-only Bash allowlist checker. The reviewer is spawned by
528 // `spawn_guardian_review` with `subagent_type == "guardian"` (the SAME
529 // marker the completion coordinator branches on to parse the verdict) AND
530 // the `guardian_read_only_disabled_tools` denylist. Keyed off that role
531 // marker (already read above to set `identity.role`), so it rides the same
532 // session-metadata path the denylist/subagent_type use — no new wire seam.
533 // Without this the worker keeps an UNRESTRICTED Bash, so the reviewer could
534 // still `rm -rf` / `git push` / `curl | sh`, defeating "read-only".
535 spec.capabilities.guardian_read_only =
536 session.metadata.get("subagent_type").map(String::as_str) == Some("guardian");
537 // #193: route this role to a REMOTE resident worker when one is pinned.
538 // `spec.identity.role` was just computed from `subagent_type` above; a
539 // match flips the placement to Remote and rides the worker's bearer on the
540 // scoped secrets envelope (TLS handshake / Authorization header only — the
541 // token is never logged). No match leaves the default `Placement::Local`,
542 // so the local path is byte-for-byte unchanged for every non-pinned role.
543 if let Some(placement) = self.remote_placements.get(spec.identity.role.as_str()) {
544 spec.placement = Placement::Remote {
545 endpoint: placement.endpoint.clone(),
546 };
547 spec.secrets.worker_auth_token = placement.token.clone();
548 } else if let Some(placement) = self.schedulable_placements.get(spec.identity.role.as_str())
549 {
550 // #181 (P2b): route this role to a SCHEDULED worker — ONLY when it is
551 // NOT already pinned to a fixed remote endpoint (the `else if` makes
552 // remote_placements take precedence for a role in both). The concrete
553 // worker is picked at run time in `execute_external_child` from the bus
554 // (a live connected worker of the pool role). No per-placement bearer
555 // now — the bus connection uses the bus token. No match in either map
556 // leaves the default `Placement::Local`.
557 spec.placement = Placement::Schedulable {
558 pool: placement.pool.clone(),
559 };
560 }
561 spec
562 }
563
564 /// The `metadata["placement"]` JSON to stamp on a child from its resolved
565 /// placement, preferring the matching cluster node's `host_label` (its
566 /// operator label/host) over the raw endpoint/pool. `None` for a Local child
567 /// (the DTO defaults it to the backend's own host). Split out of
568 /// `execute_external_child` so the role→placement→host resolution is unit-testable.
569 fn placement_stamp_for(&self, spec: &ProvisionSpec) -> Option<String> {
570 let host_label = match &spec.placement {
571 Placement::Remote { .. } => self
572 .remote_placements
573 .get(spec.identity.role.as_str())
574 .and_then(|p| p.host_label.as_deref()),
575 Placement::Schedulable { .. } => self
576 .schedulable_placements
577 .get(spec.identity.role.as_str())
578 .and_then(|p| p.host_label.as_deref()),
579 Placement::Local => None,
580 };
581 placement_metadata(&spec.placement, host_label)
582 }
583
584 /// Pick a live worker for a SCHEDULABLE role from the BUS (#181, Phase 3):
585 /// ask the broker which actors are connected serving the pool role (presence
586 /// is connection-truth — no HTTP registry, no leases, no connect-fail
587 /// failover), then round-robin one per resolve for spread. Returns the chosen
588 /// worker's mailbox id. An empty pool ⇒ a terminal `AgentError` — NEVER a
589 /// local-subprocess fallback (that would silently defeat the placement).
590 async fn resolve_schedulable_worker(
591 &self,
592 role: &str,
593 ) -> std::result::Result<String, AgentError> {
594 let pool = self
595 .schedulable_placements
596 .get(role)
597 .ok_or_else(|| {
598 AgentError::LLM(format!(
599 "schedulable placement for role '{role}' vanished before scheduling"
600 ))
601 })?
602 .pool
603 .clone();
604 let bus = self.bus.as_ref().ok_or_else(|| {
605 AgentError::LLM(format!(
606 "schedulable role '{role}': no mailbox bus configured (subagents.broker)"
607 ))
608 })?;
609
610 // Ask the BUS who is connected serving the pool role — presence is
611 // connection-truth (no HTTP registry, no leases, no stale-record failover).
612 let mut q = bamboo_broker::BrokerClient::connect(
613 &bus.endpoint,
614 bamboo_subagent::AgentRef {
615 session_id: format!("sched-q-{role}"),
616 role: None,
617 },
618 &bus.token,
619 )
620 .await
621 .map_err(|e| {
622 AgentError::LLM(format!(
623 "schedulable role '{role}': bus connect failed: {e}"
624 ))
625 })?;
626 let candidates = q.list_connected(&pool).await.map_err(|e| {
627 AgentError::LLM(format!(
628 "schedulable role '{role}': bus presence query failed: {e}"
629 ))
630 })?;
631
632 if candidates.is_empty() {
633 return Err(AgentError::LLM(format!(
634 "schedulable role '{role}': no live worker in pool '{pool}' on the bus \
635 (NOT spawning a local subprocess — a schedulable role has no local fallback)"
636 )));
637 }
638
639 // Round-robin: advance a per-pool cursor once per resolve so successive
640 // sibling spawns spread across the connected pool workers. No failover
641 // needed — a listed worker is connected NOW (the bus only lists live
642 // subscribers), so there is no stale-but-leased candidate to skip.
643 let idx = {
644 let mut cursors = self.schedule_cursor.lock().recover_poison();
645 let cursor = cursors.entry(pool.clone()).or_insert(0);
646 let i = *cursor % candidates.len();
647 *cursor = cursor.wrapping_add(1);
648 i
649 };
650 Ok(candidates[idx].clone())
651 }
652}
653
654#[async_trait]
655impl ExternalChildRunner for ActorChildRunner {
656 async fn should_handle(&self, session: &Session) -> bool {
657 session.metadata.get("runtime.kind") == Some(&"external".to_string())
658 && session.metadata.get("external.protocol") == Some(&"actor".to_string())
659 && session.metadata.get("external.agent_id") == Some(&self.agent_id)
660 }
661
662 fn set_escalation_bridge(&self, bridge: Option<bamboo_subagent::executor::HostBridge>) {
663 *self.escalation_bridge.lock().recover_poison() = bridge;
664 }
665
666 async fn execute_external_child(
667 &self,
668 session: &mut Session,
669 job: &SpawnJob,
670 event_tx: mpsc::Sender<AgentEvent>,
671 cancel_token: CancellationToken,
672 ) -> crate::runtime::runner::Result<()> {
673 // #68 CORRECTNESS CRUX: capture the per-run escalation bridge HERE, at the
674 // moment this grandchild is spawned — while the parent run's bridge is
675 // still in our slot — into an owned local handed to `drive()` for this
676 // grandchild's whole lifetime. A fire-and-forget grandchild that OUTLIVES
677 // the run that spawned it must NOT re-read `self.escalation_bridge` at
678 // approval time: by then `run()` may have cleared/overwritten it (a worker
679 // serves runs sequentially), and re-proxying through a closed bridge
680 // fail-closed denies. Capturing at spawn pins the right bridge per run.
681 let escalation = self.escalation_bridge.lock().recover_poison().clone();
682 let assignment = extract_assignment(session);
683 let mut spec = self.build_spec(session, job);
684 // Mark the worker reusable + give it an idle timeout so it self-reaps if
685 // orphaned. Warm bus workers are pooled per fingerprint and reused.
686 spec.reusable = true;
687 if spec.limits.idle_timeout_secs.is_none() {
688 spec.limits.idle_timeout_secs = Some(POOLED_IDLE_TIMEOUT_SECS);
689 }
690 let pool_key = Self::fingerprint(&spec);
691 // Rehydration: the child session in the parent's store is the actor's
692 // durable state. Ship the full conversation so a reactivation
693 // (send_message / update / rerun) carries its history. A reused worker is
694 // stateless between runs, so this is also what isolates each child's
695 // context on a shared process.
696 let messages: Vec<serde_json::Value> = session
697 .messages
698 .iter()
699 .filter_map(|m| serde_json::to_value(m).ok())
700 .collect();
701
702 // Backpressure: hold a concurrency slot for the lifetime of the *run*
703 // (cancellation still proceeds — the cancel branch in drive() runs while
704 // we hold the permit). Released when this fn returns, i.e. once the worker
705 // is parked back into the pool, so idle workers don't pin slots.
706 let _slot = self
707 .concurrency
708 .acquire()
709 .await
710 .map_err(|_| AgentError::LLM("actor concurrency limiter closed".to_string()))?;
711
712 // Split LOCAL (spawn + warm-pool) from the two process-less remote paths
713 // ONLY at the divergent spots — acquire/connect here and the park/retire at
714 // the end. Everything between (Run dispatch, live-actor registration,
715 // drive, the close) is identical for all three. `kind` is the single guard.
716 // - Local (#0): byte-for-byte the pre-#193 reuse-or-spawn path.
717 // - Remote (#194): connect to a FIXED resident endpoint, no spawn.
718 // - Schedulable (#181): resolve a live worker from the registry, connect.
719 let kind = match spec.placement {
720 Placement::Remote { .. } => PlacementKind::Remote,
721 Placement::Schedulable { .. } => PlacementKind::Schedulable,
722 Placement::Local => PlacementKind::Local,
723 };
724 let remote = !matches!(kind, PlacementKind::Local);
725
726 // Stamp WHICH machine this child runs on onto its session metadata, so the
727 // UI can show it (mirrored into the session index → SessionSummary.placement).
728 // Only remote/scheduled placements need a stamp — a Local child falls through
729 // to the DTO default (this backend's own host). Persisted by the caller with
730 // the rest of the child session after we return.
731 if let Some(placement_meta) = self.placement_stamp_for(&spec) {
732 session
733 .metadata
734 .insert("placement".to_string(), placement_meta);
735 }
736
737 // Retry-once loop: a pooled local worker can die between its liveness
738 // check and handling the Run (a tiny TOCTOU window) — its Run then sits
739 // queued with no server. The first-frame watchdog in `drive` surfaces that
740 // as `WorkerUnresponsive`; we reap the dead worker and re-acquire ONCE
741 // (which spawns fresh / reuses the next live one). Remote/schedulable have
742 // no spawn fallback, so they never retry.
743 let mut attempt = 0u8;
744 let (result, actor) = loop {
745 let (actor, mut client) = match kind {
746 PlacementKind::Remote => {
747 // REMOTE branch: connect to a resident worker. No spawn, no pool
748 // touch, no drain. We do not own the worker, so a connect failure
749 // has NO respawn fallback — it is a clear, terminal error.
750 let placement = self
751 .remote_placements
752 .get(spec.identity.role.as_str())
753 .ok_or_else(|| {
754 AgentError::LLM(format!(
755 "remote placement for role '{}' vanished before connect",
756 spec.identity.role
757 ))
758 })?;
759 let endpoint = placement.endpoint.clone();
760 // Build the TLS trust: a pinned CA pins a self-signed worker cert;
761 // otherwise default webpki roots (or plaintext for `ws://`).
762 let trust_cfg = match placement.ca_cert_file.as_deref() {
763 Some(path) => Some(client_config_trusting_cert(path).map_err(|e| {
764 AgentError::LLM(format!(
765 "remote worker CA cert '{}': {e}",
766 path.display()
767 ))
768 })?),
769 None => None,
770 };
771 let client = ChildClient::connect_with_auth_tls(
772 &endpoint,
773 placement.token.as_deref(),
774 trust_cfg,
775 )
776 .await
777 .map_err(|e| {
778 AgentError::LLM(format!("remote actor connect to '{endpoint}' failed: {e}"))
779 })?;
780 // Process-less handle so live-actor registration (in-band steering)
781 // works exactly as for a local worker; `kill()` is a no-op.
782 let record = AgentRecord {
783 agent_id: job.child_session_id.clone(),
784 role: spec.identity.role.clone(),
785 labels: Vec::new(),
786 endpoint: endpoint.clone(),
787 pid: 0,
788 version: String::new(),
789 started_at: chrono::Utc::now(),
790 lease_expires_at: chrono::Utc::now(),
791 };
792 let _ = endpoint;
793 let actor = PooledWorker {
794 worker: SpawnedChild::remote(record),
795 mailbox_id: job.child_session_id.clone(),
796 };
797 let client: Box<dyn bamboo_subagent::ChildLink> = Box::new(client);
798 (actor, client)
799 }
800 PlacementKind::Schedulable => {
801 // SCHEDULABLE branch (#181): pick a LIVE worker of the pool role
802 // from the BUS (presence = connection-truth; no HTTP registry, no
803 // leases, no failover) and drive it by mailbox id. The pool worker
804 // stays connected and is reused next time. No spawn, no kill, NO
805 // local fallback — an empty pool is a terminal error (raised in
806 // resolve_schedulable_worker).
807 let bus = self.bus.as_ref().ok_or_else(|| {
808 AgentError::LLM(
809 "schedulable sub-agents require a mailbox bus (subagents.broker)"
810 .to_string(),
811 )
812 })?;
813 let mailbox_id = self
814 .resolve_schedulable_worker(spec.identity.role.as_str())
815 .await?;
816 let parent = bamboo_subagent::AgentRef {
817 session_id: format!("p-{}", job.child_session_id),
818 role: None,
819 };
820 let link = bamboo_broker::BrokerChildLink::connect(
821 &bus.endpoint,
822 parent,
823 &bus.token,
824 mailbox_id.clone(),
825 )
826 .await
827 .map_err(|e| {
828 AgentError::LLM(format!(
829 "schedulable link connect to '{mailbox_id}' failed: {e}"
830 ))
831 })?;
832 // Process-less handle — a bus-resident pool worker is never ours to
833 // kill (remote ⇒ dropped, not pooled, after the run).
834 let actor = PooledWorker {
835 worker: SpawnedChild::remote(AgentRecord {
836 agent_id: mailbox_id.clone(),
837 role: spec.identity.role.clone(),
838 labels: Vec::new(),
839 endpoint: bus.endpoint.clone(),
840 pid: 0,
841 version: String::new(),
842 started_at: chrono::Utc::now(),
843 lease_expires_at: chrono::Utc::now(),
844 }),
845 mailbox_id,
846 };
847 let client: Box<dyn bamboo_subagent::ChildLink> = Box::new(link);
848 (actor, client)
849 }
850 PlacementKind::Local => {
851 // LOCAL = the mailbox bus (the unified transport): check out a warm
852 // pooled worker (reuse a live parked one, else spawn fresh) and
853 // drive it by mailbox id — no listen socket, no file discovery, no
854 // respawn-on-connect-miss (the broker queues the Run until the
855 // worker handles it). The legacy direct-WS path was retired; the bus
856 // is required.
857 let bus = self.bus.as_ref().ok_or_else(|| {
858 AgentError::LLM(
859 "local sub-agents require a mailbox bus (subagents.broker); none is \
860 configured and the bus could not be embedded"
861 .to_string(),
862 )
863 })?;
864 let actor = self.acquire_bus_worker(&pool_key, &spec).await?;
865 let parent = bamboo_subagent::AgentRef {
866 session_id: format!("p-{}", job.child_session_id),
867 role: None,
868 };
869 let link = bamboo_broker::BrokerChildLink::connect(
870 &bus.endpoint,
871 parent,
872 &bus.token,
873 actor.mailbox_id.clone(),
874 )
875 .await
876 .map_err(|e| {
877 AgentError::LLM(format!("broker child link connect failed: {e}"))
878 })?;
879 let client: Box<dyn bamboo_subagent::ChildLink> = Box::new(link);
880 (actor, client)
881 }
882 };
883
884 if let Err(e) = client
885 .send(ParentFrame::Run(RunSpec {
886 // Cloned (not moved) so a retry can re-dispatch to a fresh worker.
887 assignment: assignment.clone(),
888 reasoning_effort: None,
889 messages: messages.clone(),
890 }))
891 .await
892 {
893 if !remote {
894 actor.worker.kill().await;
895 }
896 return Err(AgentError::LLM(format!("actor run dispatch failed: {e}")));
897 }
898
899 // Register as a live actor so send_message (running, no interrupt) can
900 // steer this child in-band over the existing WS connection. The guard
901 // unregisters on every exit path.
902 let (live_tx, mut live_rx) = mpsc::unbounded_channel::<ParentFrame>();
903 let live_guard = super::live::register(
904 &job.child_session_id,
905 live_tx,
906 attempt as u32,
907 self.approval_registry.clone(),
908 );
909
910 let result = drive(
911 &mut *client,
912 &job.parent_session_id,
913 &job.child_session_id,
914 attempt as u32,
915 self.approval_registry.as_ref(),
916 self.approval_decider.as_ref(),
917 escalation.clone(),
918 &event_tx,
919 &cancel_token,
920 &mut live_rx,
921 // First-frame watchdog for EVERY placement: a wedged-but-connected
922 // worker (subscribed ≠ serving — e.g. stuck on a prior LLM call) emits
923 // no first frame; without a deadline drive() blocks forever. Bounding it
924 // turns the "running-but-unresponsive" hang into a recoverable
925 // WorkerUnresponsive (reap+respawn local / re-pick schedulable / error
926 // on a fixed remote endpoint).
927 Some(WORKER_FIRST_FRAME_TIMEOUT),
928 )
929 .await;
930 // Unregister IMMEDIATELY: after drive returns nobody consumes live_rx,
931 // so a send_message landing in the close/park window below must see
932 // "not live" and take the durable-queue fallback instead of vanishing.
933 // (Even if one slipped in earlier, send_message also appends it to the
934 // durable transcript, so the next activation still rehydrates it.)
935 drop(live_guard);
936 // Close the parent link (dropping it closes our broker connection; the
937 // worker stays dialed-in + subscribed, ready for its next Run).
938 drop(client);
939
940 // No first frame ⇒ the worker is wedged. Recover ONCE before giving up:
941 // - Local: reap the dead pooled worker + respawn.
942 // - Schedulable: not ours to kill — drop it and re-select a live pool
943 // member (a wedged worker must not fail the run when the pool has others).
944 // - Remote: a FIXED endpoint has no alternative — fall through to a bounded
945 // WorkerUnresponsive error (far better than the previous infinite hang).
946 if attempt == 0 && matches!(result, Err(AgentError::WorkerUnresponsive(_))) {
947 match kind {
948 PlacementKind::Local => {
949 tracing::warn!(
950 "actor child {} got no first frame; reaping the worker and respawning once",
951 job.child_session_id
952 );
953 actor.worker.kill().await;
954 attempt += 1;
955 continue;
956 }
957 PlacementKind::Schedulable => {
958 tracing::warn!(
959 "scheduled actor child {} got no first frame; re-selecting a pool worker",
960 job.child_session_id
961 );
962 drop(actor);
963 attempt += 1;
964 continue;
965 }
966 PlacementKind::Remote => {}
967 }
968 }
969 break (result, actor);
970 };
971
972 // Park the warm worker for reuse on a clean run, or kill it on
973 // error/cancel (a wedged worker must not be reused). Remote / schedulable
974 // workers are registry-managed — never ours to pool/kill, just drop.
975 if remote {
976 drop(actor);
977 } else {
978 match &result {
979 Ok(_) => self.release_bus_worker(&pool_key, actor).await,
980 Err(_) => actor.worker.kill().await,
981 }
982 }
983
984 // Write-back: persist the actor's final reply onto the child session so
985 // the transcript survives and the NEXT activation sees it as history.
986 // (run_child_spawn saves the session right after we return.)
987 match result {
988 Ok(Some(text)) => {
989 if !text.is_empty() {
990 session.add_message(bamboo_agent_core::Message::assistant(text, None));
991 }
992 Ok(())
993 }
994 Ok(None) => Ok(()),
995 Err(e) => Err(e),
996 }
997 }
998}
999
1000/// The `{kind,host}` placement descriptor stamped onto a child session's metadata
1001/// under `"placement"` — read back by the storage index → `SessionSummary.placement`
1002/// → the UI's machine badge. `None` for `Local` (those fall through to the DTO's
1003/// default of this backend's own host). The value is a JSON string matching
1004/// `bamboo_storage::SessionPlacement { kind, host }`.
1005fn placement_metadata(placement: &Placement, host_label: Option<&str>) -> Option<String> {
1006 // Prefer the cluster node's own label/host (its metadata) when the placement
1007 // maps to a node; else fall back to the raw endpoint host / pool name.
1008 let value = match placement {
1009 Placement::Local => return None,
1010 Placement::Remote { endpoint } => serde_json::json!({
1011 "kind": "remote",
1012 "host": host_label.map(str::to_string).unwrap_or_else(|| host_of_endpoint(endpoint)),
1013 }),
1014 Placement::Schedulable { pool } => serde_json::json!({
1015 "kind": "remote",
1016 "host": host_label.unwrap_or(pool),
1017 }),
1018 };
1019 serde_json::to_string(&value).ok()
1020}
1021
1022/// Extract the host from a `ws[s]://host:port[/path]` bus endpoint, for display.
1023fn host_of_endpoint(endpoint: &str) -> String {
1024 endpoint
1025 .trim()
1026 .trim_start_matches("wss://")
1027 .trim_start_matches("ws://")
1028 .split(['/', ':'])
1029 .next()
1030 .unwrap_or(endpoint)
1031 .to_string()
1032}
1033
1034/// Pump child frames -> parent events until a terminal frame (or cancellation).
1035/// On success, yields the actor's final result text (for session write-back).
1036/// `live_rx` carries in-band frames (steering messages) from the live registry.
1037///
1038/// `escalation_bridge` (#68) is the per-run escalation host bridge CAPTURED BY
1039/// VALUE at spawn time in `execute_external_child` (NOT read live here): when a
1040/// non-bypass child re-proxies an approval request, this owned bridge routes it
1041/// UP to the parent run. Owning it for the call's lifetime is what lets a
1042/// fire-and-forget grandchild that outlives its spawning run still escalate to
1043/// the correct (then-current) parent bridge rather than a stale/overwritten one.
1044async fn drive(
1045 client: &mut dyn bamboo_subagent::ChildLink,
1046 parent_session_id: &str,
1047 child_session_id: &str,
1048 child_attempt: u32,
1049 approval_registry: Option<&super::approval_registry::SharedApprovalRegistry>,
1050 approval_decider: Option<&Arc<dyn ChildApprovalDecider>>,
1051 escalation_bridge: Option<bamboo_subagent::executor::HostBridge>,
1052 event_tx: &mpsc::Sender<AgentEvent>,
1053 cancel_token: &CancellationToken,
1054 live_rx: &mut mpsc::UnboundedReceiver<ParentFrame>,
1055 first_frame_timeout: Option<Duration>,
1056) -> crate::runtime::runner::Result<Option<String>> {
1057 // First-frame watchdog: a live worker emits its first frame (run-started /
1058 // first token) within seconds; total silence past the deadline means the
1059 // worker is dead (e.g. a pooled worker that exited right after checkout), so
1060 // its Run sits queued forever. We trip ONLY before the first frame — once any
1061 // frame arrives the worker is proven live and a legitimately long run (a slow
1062 // tool between tokens) never trips it.
1063 let mut got_first_frame = false;
1064 let mut first_frame_watch = first_frame_timeout.map(|d| Box::pin(tokio::time::sleep(d)));
1065 loop {
1066 tokio::select! {
1067 _ = cancel_token.cancelled() => {
1068 // fall through to the cancel handling below
1069 break;
1070 }
1071 _ = async {
1072 match first_frame_watch.as_mut() {
1073 Some(s) => s.as_mut().await,
1074 None => std::future::pending::<()>().await,
1075 }
1076 }, if !got_first_frame => {
1077 return Err(AgentError::WorkerUnresponsive(format!(
1078 "child {child_session_id} produced no frame within {:?}",
1079 first_frame_timeout.unwrap_or_default()
1080 )));
1081 }
1082 Some(frame) = live_rx.recv() => {
1083 // Forward in-band steering to the worker over the existing WS.
1084 if client.send(frame).await.is_err() {
1085 tracing::warn!("live steering frame could not be sent; connection failing");
1086 }
1087 }
1088 frame = client.next_frame() => {
1089 // Any frame (event / approval / terminal / close / error) proves
1090 // the worker responded — disarm the first-frame watchdog.
1091 got_first_frame = true;
1092 first_frame_watch = None;
1093 match frame {
1094 Ok(Some(ChildFrame::Event { event })) => {
1095 // AgentEvent is serialized verbatim on the wire (zero mapping).
1096 if let Ok(ev) = serde_json::from_value::<AgentEvent>(event) {
1097 let _ = event_tx.send(ev).await;
1098 }
1099 }
1100 Ok(Some(ChildFrame::ApprovalRequest { id, body })) => {
1101 // Phase 2: a worker proxied a gated-tool approval back to
1102 // the host. The WORKER side is live — its executor installs
1103 // a per-run task-local `ApprovalProxy` (subagent_worker.rs)
1104 // that calls `host.approval_call`, so this frame arrives
1105 // when a child hits `ConfirmationRequired`.
1106 if let Some(reviewer) = child_approval_reviewer() {
1107 // Phase 6, Part B: a BYPASSED parent worker
1108 // model-reviews its children's forced-ask (dangerous)
1109 // actions. The review is an LLM call, so run it OFF
1110 // the frame pump in a spawned task and deliver the
1111 // verdict async via the live channel — the pump keeps
1112 // forwarding events and the agent loop never blocks. A
1113 // timeout denies a hung review so the child can't hang.
1114 let child = child_session_id.to_string();
1115 let req_id = id.clone();
1116 let body = body.clone();
1117 let registry = approval_registry.cloned();
1118 tokio::spawn(async move {
1119 let approved = tokio::time::timeout(
1120 CHILD_APPROVAL_TIMEOUT,
1121 reviewer.review(&child, &body),
1122 )
1123 .await
1124 .unwrap_or(false);
1125 super::live::deliver_approval_scoped(
1126 registry.as_ref(),
1127 &child,
1128 child_attempt,
1129 &req_id,
1130 approved,
1131 );
1132 });
1133 } else if approval_decider.is_some() {
1134 // A decider is wired (policy / auto): decide promptly
1135 // and reply inline. (Must not block the pump — see the
1136 // `ChildApprovalDecider` doc.)
1137 let approved =
1138 decide_child_approval(approval_decider, child_session_id, &body)
1139 .await;
1140 if client
1141 .send(ParentFrame::ApprovalReply { id, approved })
1142 .await
1143 .is_err()
1144 {
1145 tracing::warn!(
1146 "failed to answer approval_request; connection failing"
1147 );
1148 }
1149 } else if let Some(host) = escalation_bridge.clone() {
1150 // Non-bypass WORKER: ESCALATE up our own actor link
1151 // (re-proxy) so the request chains to our parent — and
1152 // up every level until a bypass level (model-review) or
1153 // the top orchestrator (human) decides. Off-loop so the
1154 // pump never blocks; relay the reply down to the child.
1155 let child = child_session_id.to_string();
1156 let req_id = id.clone();
1157 let body = body.clone();
1158 let registry = approval_registry.cloned();
1159 tokio::spawn(async move {
1160 let approved = match tokio::time::timeout(
1161 CHILD_APPROVAL_TIMEOUT,
1162 host.approval_call(body),
1163 )
1164 .await
1165 {
1166 Ok(Ok(reply)) => reply
1167 .get("approved")
1168 .and_then(|v| v.as_bool())
1169 .unwrap_or(false),
1170 // Transport error or timeout ⇒ fail closed.
1171 _ => false,
1172 };
1173 super::live::deliver_approval_scoped(
1174 registry.as_ref(),
1175 &child,
1176 child_attempt,
1177 &req_id,
1178 approved,
1179 );
1180 });
1181 } else {
1182 // Top orchestrator (no escalation bridge): human-in-the-
1183 // loop. Surface the request on the parent's event stream
1184 // and DEFER — the decision arrives out-of-band via
1185 // `live::deliver_approval(child, request_id, approved)`
1186 // (→ this child's `live_rx` → forwarded to the worker
1187 // above). A timeout denies a never-answered request so
1188 // it can't hang the child forever.
1189 let (tool_name, permission, resource) =
1190 approval_request_fields(&body);
1191 // Register the pending request BEFORE surfacing it so
1192 // the external handler's `deliver_approval_checked` can
1193 // correlate an out-of-band POST against a genuine
1194 // human-loop request (and consume it one-shot).
1195 let (approval_version, approval_created_at) =
1196 super::live::register_pending_approval_observed(
1197 approval_registry,
1198 parent_session_id,
1199 child_session_id,
1200 child_attempt,
1201 &id,
1202 &tool_name,
1203 &permission,
1204 &resource,
1205 event_tx.clone(),
1206 );
1207 if approval_version == 0 {
1208 super::live::deliver_approval_scoped(
1209 approval_registry,
1210 child_session_id,
1211 child_attempt,
1212 &id,
1213 false,
1214 );
1215 let _ = event_tx
1216 .send(AgentEvent::ChildApprovalChanged {
1217 parent_session_id: parent_session_id.to_string(),
1218 child_session_id: child_session_id.to_string(),
1219 child_attempt,
1220 request_id: id,
1221 version: 1,
1222 status: "delivery_failed".to_string(),
1223 reason: Some("persistence_failed".to_string()),
1224 tool_name,
1225 permission,
1226 resource,
1227 created_at: approval_created_at,
1228 resolved_at: Some(chrono::Utc::now().to_rfc3339()),
1229 })
1230 .await;
1231 continue;
1232 }
1233 let _ = event_tx.send(AgentEvent::ChildApprovalChanged {
1234 parent_session_id: parent_session_id.to_string(),
1235 child_session_id: child_session_id.to_string(),
1236 child_attempt,
1237 request_id: id.clone(),
1238 version: approval_version,
1239 status: "pending".to_string(),
1240 reason: None,
1241 tool_name: tool_name.clone(),
1242 permission: permission.clone(),
1243 resource: resource.clone(),
1244 created_at: approval_created_at,
1245 resolved_at: None,
1246 }).await;
1247 let _ = event_tx
1248 .send(AgentEvent::ChildApprovalRequested {
1249 child_session_id: child_session_id.to_string(),
1250 request_id: id.clone(),
1251 tool_name,
1252 permission,
1253 resource,
1254 })
1255 .await;
1256 let child = child_session_id.to_string();
1257 let approval_registry = approval_registry.cloned();
1258 tokio::spawn(async move {
1259 tokio::time::sleep(CHILD_APPROVAL_TIMEOUT).await;
1260 // Deny only if still pending: a one-shot consume so
1261 // we don't double-deliver if the human already
1262 // answered (the POST took it), and so a late POST
1263 // after this fires finds nothing pending.
1264 if super::live::expire_pending_approval(approval_registry.as_ref(), &child, &id) {
1265 super::live::deliver_approval_scoped(
1266 approval_registry.as_ref(),
1267 &child,
1268 child_attempt,
1269 &id,
1270 false,
1271 );
1272 }
1273 });
1274 }
1275 }
1276 Ok(Some(ChildFrame::Terminal { status, result, error, .. })) => {
1277 return match status {
1278 TerminalStatus::Completed => Ok(result),
1279 TerminalStatus::Cancelled => Err(AgentError::Cancelled),
1280 TerminalStatus::Error => Err(AgentError::LLM(
1281 error.unwrap_or_else(|| "actor child errored".to_string()),
1282 )),
1283 // The suspend/resume round-trip (host re-dispatch of a
1284 // nested parent) is not wired here yet; a worker in
1285 // this build never emits Suspended, so this is
1286 // unreachable in practice.
1287 TerminalStatus::Suspended => Err(AgentError::LLM(
1288 "nested sub-agent suspend received but resume transport is not wired"
1289 .to_string(),
1290 )),
1291 };
1292 }
1293 Ok(None) => {
1294 return Err(AgentError::LLM(
1295 "actor child closed before terminal".to_string(),
1296 ));
1297 }
1298 Err(e) => {
1299 return Err(AgentError::LLM(format!("actor transport error: {e}")));
1300 }
1301 }
1302 }
1303 }
1304 }
1305
1306 // Only reached on cancellation: ask the child to stop (best-effort), then report cancelled.
1307 let _ = client.send(ParentFrame::Cancel).await;
1308 Err(AgentError::Cancelled)
1309}
1310
1311/// The assignment text = the child session's latest user message (falls back to its title).
1312fn extract_assignment(session: &Session) -> String {
1313 session
1314 .messages
1315 .iter()
1316 .rev()
1317 .find(|m| matches!(m.role, Role::User))
1318 .map(|m| m.content.clone())
1319 .unwrap_or_else(|| {
1320 session
1321 .metadata
1322 .get("title")
1323 .cloned()
1324 .unwrap_or_else(|| "Execute task".to_string())
1325 })
1326}
1327
1328#[cfg(test)]
1329mod tests {
1330 use super::*;
1331
1332 fn spec_with(
1333 role: &str,
1334 provider: &str,
1335 model: &str,
1336 workspace: Option<&str>,
1337 disabled: Option<Vec<&str>>,
1338 ) -> ProvisionSpec {
1339 let mut spec = ProvisionSpec::new(
1340 ChildIdentity {
1341 child_id: "c".into(),
1342 parent_id: None,
1343 project_key: None,
1344 role: role.into(),
1345 depth: 0,
1346 },
1347 ExecutorSpec::Echo,
1348 "/tmp/fab".into(),
1349 );
1350 spec.workspace = workspace.map(|w| w.to_string());
1351 spec.model = Some(ModelRefSpec {
1352 provider: provider.into(),
1353 model: model.into(),
1354 });
1355 spec.disabled_tools = disabled.map(|d| d.into_iter().map(String::from).collect());
1356 spec
1357 }
1358
1359 #[test]
1360 fn fingerprint_matches_interchangeable_children() {
1361 // Same role/provider/model/workspace and equal tool sets (order-insensitive)
1362 // are interchangeable on one warm worker — and differ only in child_id.
1363 let a = spec_with(
1364 "explorer",
1365 "p",
1366 "m",
1367 Some("/ws"),
1368 Some(vec!["Bash", "Edit"]),
1369 );
1370 let mut b = spec_with(
1371 "explorer",
1372 "p",
1373 "m",
1374 Some("/ws"),
1375 Some(vec!["Edit", "Bash"]),
1376 );
1377 b.identity.child_id = "other".into();
1378 assert_eq!(
1379 ActorChildRunner::fingerprint(&a),
1380 ActorChildRunner::fingerprint(&b)
1381 );
1382 }
1383
1384 #[test]
1385 fn fingerprint_separates_distinct_runtimes() {
1386 let base = spec_with("explorer", "p", "m", Some("/ws"), None);
1387 let base_fp = ActorChildRunner::fingerprint(&base);
1388 // Each axis that is baked into the worker must split the pool bucket.
1389 assert_ne!(
1390 base_fp,
1391 ActorChildRunner::fingerprint(&spec_with("writer", "p", "m", Some("/ws"), None))
1392 );
1393 assert_ne!(
1394 base_fp,
1395 ActorChildRunner::fingerprint(&spec_with("explorer", "p2", "m", Some("/ws"), None))
1396 );
1397 assert_ne!(
1398 base_fp,
1399 ActorChildRunner::fingerprint(&spec_with("explorer", "p", "m2", Some("/ws"), None))
1400 );
1401 assert_ne!(
1402 base_fp,
1403 ActorChildRunner::fingerprint(&spec_with("explorer", "p", "m", Some("/ws2"), None))
1404 );
1405 assert_ne!(
1406 base_fp,
1407 ActorChildRunner::fingerprint(&spec_with(
1408 "explorer",
1409 "p",
1410 "m",
1411 Some("/ws"),
1412 Some(vec!["Bash"])
1413 ))
1414 );
1415 }
1416
1417 #[test]
1418 fn fingerprint_splits_on_baked_capabilities() {
1419 // Every capability baked once at provision time must split the pool
1420 // bucket, else a worker baked for one posture gets reused for another
1421 // (e.g. a depth-1 worker re-stamping spawn_depth onto a depth-4 child,
1422 // breaking the depth cap; or a bypass worker reused for a non-bypass one).
1423 let base_fp =
1424 ActorChildRunner::fingerprint(&spec_with("explorer", "p", "m", Some("/ws"), None));
1425
1426 let mut depth = spec_with("explorer", "p", "m", Some("/ws"), None);
1427 depth.identity.depth = 2;
1428 assert_ne!(
1429 base_fp,
1430 ActorChildRunner::fingerprint(&depth),
1431 "depth must split"
1432 );
1433
1434 let mut nested = spec_with("explorer", "p", "m", Some("/ws"), None);
1435 nested.capabilities.nested_spawn = true;
1436 assert_ne!(
1437 base_fp,
1438 ActorChildRunner::fingerprint(&nested),
1439 "nested_spawn must split"
1440 );
1441
1442 let mut bypass = spec_with("explorer", "p", "m", Some("/ws"), None);
1443 bypass.capabilities.bypass = true;
1444 assert_ne!(
1445 base_fp,
1446 ActorChildRunner::fingerprint(&bypass),
1447 "bypass must split"
1448 );
1449
1450 let mut enforce = spec_with("explorer", "p", "m", Some("/ws"), None);
1451 enforce.capabilities.enforce_permissions = true;
1452 assert_ne!(
1453 base_fp,
1454 ActorChildRunner::fingerprint(&enforce),
1455 "enforce_permissions must split"
1456 );
1457
1458 let mut cap = spec_with("explorer", "p", "m", Some("/ws"), None);
1459 cap.capabilities.max_spawn_depth = Some(8);
1460 assert_ne!(
1461 base_fp,
1462 ActorChildRunner::fingerprint(&cap),
1463 "max_spawn_depth must split"
1464 );
1465
1466 // #73 (P1): the worker bakes `no_human_review` from this flag once at
1467 // build(), so it MUST split the pool or a worker baked for one approval
1468 // posture is reused for the opposite one.
1469 let mut nha = spec_with("explorer", "p", "m", Some("/ws"), None);
1470 nha.capabilities.no_human_approver = true;
1471 assert_ne!(
1472 base_fp,
1473 ActorChildRunner::fingerprint(&nha),
1474 "no_human_approver must split"
1475 );
1476
1477 // #71: the read-only Bash checker is baked once at build() from this flag,
1478 // so a guardian reviewer worker must not be reused for an ordinary child.
1479 let mut gro = spec_with("explorer", "p", "m", Some("/ws"), None);
1480 gro.capabilities.guardian_read_only = true;
1481 assert_ne!(
1482 base_fp,
1483 ActorChildRunner::fingerprint(&gro),
1484 "guardian_read_only must split"
1485 );
1486 }
1487
1488 struct StaticDecider(bool);
1489
1490 #[async_trait]
1491 impl ChildApprovalDecider for StaticDecider {
1492 async fn decide(&self, _child: &str, _req: &serde_json::Value) -> bool {
1493 self.0
1494 }
1495 }
1496
1497 // ---- first-frame watchdog (dead-pooled-worker recovery) -----------------
1498
1499 /// A link that never yields a frame — models a worker that died (or never
1500 /// subscribed) so its Run sits queued with no server.
1501 struct SilentLink;
1502 #[async_trait]
1503 impl bamboo_subagent::ChildLink for SilentLink {
1504 async fn send(&mut self, _: ParentFrame) -> bamboo_subagent::TransportResult<()> {
1505 Ok(())
1506 }
1507 async fn next_frame(&mut self) -> bamboo_subagent::TransportResult<Option<ChildFrame>> {
1508 std::future::pending().await
1509 }
1510 }
1511
1512 /// A link that immediately yields one terminal frame (a healthy fast worker).
1513 struct InstantTerminalLink {
1514 done: bool,
1515 }
1516 #[async_trait]
1517 impl bamboo_subagent::ChildLink for InstantTerminalLink {
1518 async fn send(&mut self, _: ParentFrame) -> bamboo_subagent::TransportResult<()> {
1519 Ok(())
1520 }
1521 async fn next_frame(&mut self) -> bamboo_subagent::TransportResult<Option<ChildFrame>> {
1522 if self.done {
1523 std::future::pending().await
1524 } else {
1525 self.done = true;
1526 Ok(Some(ChildFrame::Terminal {
1527 status: TerminalStatus::Completed,
1528 result: Some("done".into()),
1529 error: None,
1530 transcript: vec![],
1531 }))
1532 }
1533 }
1534 }
1535
1536 #[tokio::test]
1537 async fn drive_trips_first_frame_watchdog_on_a_silent_worker() {
1538 let (event_tx, _rx) = mpsc::channel::<AgentEvent>(8);
1539 let cancel = CancellationToken::new();
1540 let (_live_tx, mut live_rx) = mpsc::unbounded_channel::<ParentFrame>();
1541 let mut link = SilentLink;
1542 let r = drive(
1543 &mut link,
1544 "parent-x",
1545 "child-x",
1546 0,
1547 None,
1548 None,
1549 None,
1550 &event_tx,
1551 &cancel,
1552 &mut live_rx,
1553 Some(Duration::from_millis(100)),
1554 )
1555 .await;
1556 assert!(
1557 matches!(r, Err(AgentError::WorkerUnresponsive(_))),
1558 "a silent worker must trip the first-frame watchdog, got {r:?}"
1559 );
1560 }
1561
1562 #[tokio::test]
1563 async fn drive_does_not_trip_when_a_frame_arrives() {
1564 let (event_tx, _rx) = mpsc::channel::<AgentEvent>(8);
1565 let cancel = CancellationToken::new();
1566 let (_live_tx, mut live_rx) = mpsc::unbounded_channel::<ParentFrame>();
1567 let mut link = InstantTerminalLink { done: false };
1568 // Even a tiny timeout must NOT trip: the terminal frame arrives first and
1569 // disarms the watchdog.
1570 let r = drive(
1571 &mut link,
1572 "parent-y",
1573 "child-y",
1574 0,
1575 None,
1576 None,
1577 None,
1578 &event_tx,
1579 &cancel,
1580 &mut live_rx,
1581 Some(Duration::from_millis(50)),
1582 )
1583 .await;
1584 assert_eq!(r.ok().flatten().as_deref(), Some("done"));
1585 }
1586
1587 #[tokio::test]
1588 async fn child_approval_fails_closed_without_decider() {
1589 // No decider wired ⇒ the host denies (safe default), unchanged behavior.
1590 let body = serde_json::json!({"tool_name":"Bash","permission":"run","resource":"rm -rf /"});
1591 assert!(!decide_child_approval(None, "child-1", &body).await);
1592 }
1593
1594 #[tokio::test]
1595 async fn child_approval_honors_wired_decider() {
1596 let body =
1597 serde_json::json!({"tool_name":"Write","permission":"write","resource":"/tmp/x"});
1598 let approve: Arc<dyn ChildApprovalDecider> = Arc::new(StaticDecider(true));
1599 let deny: Arc<dyn ChildApprovalDecider> = Arc::new(StaticDecider(false));
1600 assert!(decide_child_approval(Some(&approve), "child-1", &body).await);
1601 assert!(!decide_child_approval(Some(&deny), "child-1", &body).await);
1602 }
1603
1604 #[test]
1605 fn approval_request_fields_extracts_and_defaults() {
1606 let full = serde_json::json!({"tool_name":"Bash","permission":"run","resource":"ls"});
1607 assert_eq!(
1608 approval_request_fields(&full),
1609 ("Bash".to_string(), "run".to_string(), "ls".to_string())
1610 );
1611 // Missing fields default to empty strings.
1612 let partial = serde_json::json!({"tool_name":"Write"});
1613 assert_eq!(
1614 approval_request_fields(&partial),
1615 ("Write".to_string(), String::new(), String::new())
1616 );
1617 }
1618
1619 // ---- #193: remote placement routing -------------------------------------
1620
1621 use crate::runtime::execution::SpawnJob;
1622 use bamboo_agent_core::Session;
1623
1624 /// A runner with a BOGUS worker_bin (`/bin/false`): a local spawn here would
1625 /// FAIL, so a passing remote test proves the remote path never spawns.
1626 fn bogus_runner(placements: HashMap<String, ResolvedRemotePlacement>) -> ActorChildRunner {
1627 ActorChildRunner::new(
1628 "test-actor".into(),
1629 PathBuf::from("/bin/false"),
1630 vec![],
1631 std::env::temp_dir().join("bamboo-test-fab-193"),
1632 ExecutorSpec::Echo,
1633 vec![],
1634 "anthropic".into(),
1635 4,
1636 )
1637 .with_remote_placements(placements)
1638 }
1639
1640 /// A child session of the given role (the role rides `subagent_type`, the
1641 /// path build_spec + the remote lookup both read).
1642 fn session_of_role(role: &str, assignment: &str) -> Session {
1643 let mut s = Session::new("child-1", "test-model");
1644 s.metadata
1645 .insert("subagent_type".to_string(), role.to_string());
1646 s.add_message(bamboo_agent_core::Message::user(assignment));
1647 s
1648 }
1649
1650 fn job_for(child: &str) -> SpawnJob {
1651 SpawnJob {
1652 parent_session_id: "parent-1".into(),
1653 child_session_id: child.into(),
1654 model: String::new(),
1655 disabled_tools: None,
1656 }
1657 }
1658
1659 #[test]
1660 fn build_spec_sets_remote_placement_for_matching_role() {
1661 let mut placements = HashMap::new();
1662 placements.insert(
1663 "explorer".to_string(),
1664 ResolvedRemotePlacement {
1665 endpoint: "wss://gpu-host:8443".into(),
1666 token: Some("T-secret".into()),
1667 ca_cert_file: None,
1668 host_label: None,
1669 },
1670 );
1671 let runner = bogus_runner(placements);
1672
1673 // Matching role -> Placement::Remote + the bearer on the secrets envelope.
1674 let s = session_of_role("explorer", "do the thing");
1675 let spec = runner.build_spec(&s, &job_for("child-1"));
1676 match &spec.placement {
1677 Placement::Remote { endpoint } => assert_eq!(endpoint, "wss://gpu-host:8443"),
1678 other => panic!("expected Remote, got {other:?}"),
1679 }
1680 assert_eq!(spec.secrets.worker_auth_token.as_deref(), Some("T-secret"));
1681 }
1682
1683 #[test]
1684 fn build_spec_leaves_local_for_unmatched_role() {
1685 let mut placements = HashMap::new();
1686 placements.insert(
1687 "explorer".to_string(),
1688 ResolvedRemotePlacement {
1689 endpoint: "wss://gpu-host:8443".into(),
1690 token: Some("T".into()),
1691 ca_cert_file: None,
1692 host_label: None,
1693 },
1694 );
1695 let runner = bogus_runner(placements);
1696
1697 // A DIFFERENT role keeps the default Local placement + no bearer.
1698 let s = session_of_role("writer", "do the thing");
1699 let spec = runner.build_spec(&s, &job_for("child-1"));
1700 assert_eq!(spec.placement, Placement::Local);
1701 assert!(spec.secrets.worker_auth_token.is_none());
1702 }
1703
1704 #[test]
1705 fn build_spec_local_when_no_placements() {
1706 let runner = bogus_runner(HashMap::new());
1707 let s = session_of_role("explorer", "do the thing");
1708 let spec = runner.build_spec(&s, &job_for("child-1"));
1709 assert_eq!(spec.placement, Placement::Local);
1710 assert!(spec.secrets.worker_auth_token.is_none());
1711 }
1712
1713 #[test]
1714 fn placement_metadata_stamps_remote_and_schedulable_not_local() {
1715 // Local children carry no stamp — the DTO defaults them to the backend host.
1716 assert_eq!(placement_metadata(&Placement::Local, None), None);
1717
1718 // Remote, no node label → host derived from the endpoint.
1719 let r = placement_metadata(
1720 &Placement::Remote {
1721 endpoint: "wss://10.0.0.5:8443/stream".into(),
1722 },
1723 None,
1724 )
1725 .unwrap();
1726 assert!(r.contains(r#""kind":"remote""#), "{r}");
1727 assert!(r.contains(r#""host":"10.0.0.5""#), "{r}");
1728
1729 // A cluster node's label (its metadata) OVERRIDES the raw endpoint host.
1730 let labeled = placement_metadata(
1731 &Placement::Remote {
1732 endpoint: "ws://169.254.230.101:8899".into(),
1733 },
1734 Some("mini"),
1735 )
1736 .unwrap();
1737 assert!(labeled.contains(r#""host":"mini""#), "{labeled}");
1738
1739 // Schedulable → {kind:"remote", host:<node label, else pool>}.
1740 let s = placement_metadata(
1741 &Placement::Schedulable {
1742 pool: "explorers".into(),
1743 },
1744 Some("mini"),
1745 )
1746 .unwrap();
1747 assert!(s.contains(r#""kind":"remote""#), "{s}");
1748 assert!(s.contains(r#""host":"mini""#), "{s}");
1749
1750 // The stamp round-trips through the storage placement type.
1751 let p: bamboo_storage::SessionPlacement = serde_json::from_str(&labeled).unwrap();
1752 assert_eq!(p.kind, "remote");
1753 assert_eq!(p.host, "mini");
1754 }
1755
1756 /// End-to-end remote run through `execute_external_child`: a resident worker
1757 /// (Bearer-gated `WsServer` + `EchoExecutor`) serves the role; the runner is
1758 /// built with a `remote_placements` entry pointing at it AND a BOGUS
1759 /// worker_bin (`/bin/false`). A passing test proves the remote path CONNECTS
1760 /// to the resident worker and NEVER spawns (a spawn would fail on /bin/false),
1761 /// and that a terminal/echo result flows back.
1762 #[tokio::test]
1763 async fn execute_external_child_routes_role_to_remote_worker_without_spawning() {
1764 // 1. Stand up the resident worker on loopback with a required bearer.
1765 let token = "remote-test-token";
1766 let server = bamboo_subagent::transport::WsServer::bind_with_token(
1767 (std::net::Ipv4Addr::LOCALHOST, 0).into(),
1768 Some(token.to_string()),
1769 )
1770 .await
1771 .expect("bind resident worker");
1772 let endpoint = server.ws_endpoint(); // ws://127.0.0.1:<port>
1773 let srv = tokio::spawn(async move {
1774 // serve() loops connection-after-connection; the test exits, dropping it.
1775 let _ = server
1776 .serve(Arc::new(bamboo_subagent::executor::EchoExecutor))
1777 .await;
1778 });
1779
1780 // 2. Build the runner: role "explorer" pinned remote, bogus worker_bin.
1781 let mut placements = HashMap::new();
1782 placements.insert(
1783 "explorer".to_string(),
1784 ResolvedRemotePlacement {
1785 endpoint: endpoint.clone(),
1786 token: Some(token.to_string()),
1787 ca_cert_file: None,
1788 host_label: Some("mini-e2e".into()), // node label, surfaced on the badge
1789 },
1790 );
1791 let runner = bogus_runner(placements);
1792
1793 // 3. Drive a real run for that role.
1794 let mut session = session_of_role("explorer", "hello remote");
1795 let job = job_for("child-1");
1796 let (event_tx, mut event_rx) = mpsc::channel::<AgentEvent>(64);
1797 let cancel = CancellationToken::new();
1798
1799 let result = tokio::time::timeout(
1800 Duration::from_secs(10),
1801 runner.execute_external_child(&mut session, &job, event_tx, cancel),
1802 )
1803 .await
1804 .expect("run did not hang")
1805 .expect("remote run succeeded (connected to resident worker, did not spawn)");
1806
1807 let _ = result;
1808 // The EchoExecutor's reply is written back onto the child session as an
1809 // assistant message — proof a terminal result flowed back over the link.
1810 let last = session
1811 .messages
1812 .iter()
1813 .rev()
1814 .find(|m| matches!(m.role, Role::Assistant))
1815 .expect("an assistant reply was written back");
1816 assert!(
1817 last.content.contains("echo:"),
1818 "expected echo reply, got {:?}",
1819 last.content
1820 );
1821
1822 // A remote run must stamp WHICH machine it ran on onto the child session
1823 // (mirrored to the UI badge) using the placement's node label.
1824 let placement = session
1825 .metadata
1826 .get("placement")
1827 .expect("remote child session stamped with a placement");
1828 assert!(placement.contains(r#""kind":"remote""#), "{placement}");
1829 assert!(placement.contains(r#""host":"mini-e2e""#), "{placement}");
1830
1831 // Drain a couple of streamed events to confirm the event pipe carried the
1832 // worker's tokens too (best-effort; the reply assertion above is primary).
1833 let mut saw_event = false;
1834 while let Ok(Some(_ev)) =
1835 tokio::time::timeout(Duration::from_millis(50), event_rx.recv()).await
1836 {
1837 saw_event = true;
1838 }
1839 let _ = saw_event;
1840
1841 srv.abort();
1842 }
1843
1844 // ---- #181 (P2b): schedulable placement routing --------------------------
1845
1846 /// A bogus-worker_bin runner carrying SCHEDULABLE placements (and optionally
1847 /// remote ones, to test precedence). A local spawn here would fail on
1848 /// `/bin/false`, so a passing schedulable test proves no subprocess spawned.
1849 fn bogus_sched_runner(
1850 remote: HashMap<String, ResolvedRemotePlacement>,
1851 sched: HashMap<String, ResolvedSchedulablePlacement>,
1852 ) -> ActorChildRunner {
1853 ActorChildRunner::new(
1854 "test-actor".into(),
1855 PathBuf::from("/bin/false"),
1856 vec![],
1857 std::env::temp_dir().join("bamboo-test-fab-181"),
1858 ExecutorSpec::Echo,
1859 vec![],
1860 "anthropic".into(),
1861 4,
1862 )
1863 .with_remote_placements(remote)
1864 .with_schedulable_placements(sched)
1865 }
1866
1867 fn sched_placement(
1868 pool: &str,
1869 _registry_url: impl Into<String>,
1870 ) -> ResolvedSchedulablePlacement {
1871 ResolvedSchedulablePlacement {
1872 pool: pool.into(),
1873 host_label: None,
1874 }
1875 }
1876
1877 #[test]
1878 fn build_spec_sets_schedulable_placement_for_matching_role() {
1879 let mut sched = HashMap::new();
1880 sched.insert(
1881 "explorer".to_string(),
1882 sched_placement("gpu-pool", "unused"),
1883 );
1884 let runner = bogus_sched_runner(HashMap::new(), sched);
1885
1886 let s = session_of_role("explorer", "do the thing");
1887 let spec = runner.build_spec(&s, &job_for("child-1"));
1888 match &spec.placement {
1889 Placement::Schedulable { pool } => assert_eq!(pool, "gpu-pool"),
1890 other => panic!("expected Schedulable, got {other:?}"),
1891 }
1892 // No per-placement bearer now — the bus connection carries the bus token.
1893 assert!(spec.secrets.worker_auth_token.is_none());
1894 }
1895
1896 #[test]
1897 fn build_spec_remote_wins_when_role_in_both_maps() {
1898 // A role present in BOTH remote_placements and schedulable_placements must
1899 // resolve to the FIXED remote placement (documented precedence).
1900 let mut remote = HashMap::new();
1901 remote.insert(
1902 "explorer".to_string(),
1903 ResolvedRemotePlacement {
1904 endpoint: "wss://fixed-host:8443".into(),
1905 token: Some("T-remote".into()),
1906 ca_cert_file: None,
1907 host_label: None,
1908 },
1909 );
1910 let mut sched = HashMap::new();
1911 sched.insert(
1912 "explorer".to_string(),
1913 sched_placement("gpu-pool", "https://control-plane:9562"),
1914 );
1915 let runner = bogus_sched_runner(remote, sched);
1916
1917 let s = session_of_role("explorer", "do the thing");
1918 let spec = runner.build_spec(&s, &job_for("child-1"));
1919 match &spec.placement {
1920 Placement::Remote { endpoint } => assert_eq!(endpoint, "wss://fixed-host:8443"),
1921 other => panic!("expected Remote (precedence), got {other:?}"),
1922 }
1923 assert_eq!(spec.secrets.worker_auth_token.as_deref(), Some("T-remote"));
1924 }
1925
1926 #[test]
1927 fn build_spec_local_for_unmatched_schedulable_role() {
1928 let mut sched = HashMap::new();
1929 sched.insert(
1930 "explorer".to_string(),
1931 sched_placement("gpu-pool", "https://control-plane:9562"),
1932 );
1933 let runner = bogus_sched_runner(HashMap::new(), sched);
1934 let s = session_of_role("writer", "do the thing");
1935 let spec = runner.build_spec(&s, &job_for("child-1"));
1936 assert_eq!(spec.placement, Placement::Local);
1937 assert!(spec.secrets.worker_auth_token.is_none());
1938 }
1939
1940 /// The full role → resolved-placement → badge-host chain: a child routed to a
1941 /// remote/schedulable placement carrying a cluster node's `host_label` stamps
1942 /// that label; without a label it falls back to the endpoint host / pool; a
1943 /// Local child gets no stamp (the DTO defaults it to the backend host).
1944 #[test]
1945 fn placement_stamp_uses_node_label_for_remote_and_schedulable() {
1946 // Remote WITH a node label → {remote, <label>}, overriding the raw IP.
1947 let mut remote = HashMap::new();
1948 remote.insert(
1949 "explorer".to_string(),
1950 ResolvedRemotePlacement {
1951 endpoint: "ws://169.254.230.101:8899".into(),
1952 token: None,
1953 ca_cert_file: None,
1954 host_label: Some("mini".into()),
1955 },
1956 );
1957 let runner = bogus_runner(remote);
1958 let spec = runner.build_spec(&session_of_role("explorer", "go"), &job_for("c1"));
1959 let stamp = runner
1960 .placement_stamp_for(&spec)
1961 .expect("remote child is stamped");
1962 assert!(stamp.contains(r#""kind":"remote""#), "{stamp}");
1963 assert!(stamp.contains(r#""host":"mini""#), "{stamp}");
1964
1965 // Remote WITHOUT a node label → falls back to the endpoint host.
1966 let mut remote_nolabel = HashMap::new();
1967 remote_nolabel.insert(
1968 "explorer".to_string(),
1969 ResolvedRemotePlacement {
1970 endpoint: "ws://169.254.230.101:8899".into(),
1971 token: None,
1972 ca_cert_file: None,
1973 host_label: None,
1974 },
1975 );
1976 let r2 = bogus_runner(remote_nolabel);
1977 let spec2 = r2.build_spec(&session_of_role("explorer", "go"), &job_for("c1"));
1978 assert!(r2
1979 .placement_stamp_for(&spec2)
1980 .unwrap()
1981 .contains(r#""host":"169.254.230.101""#));
1982
1983 // Schedulable WITH a node label → {remote, <label>} (a node, not a pool name).
1984 let mut sched = HashMap::new();
1985 sched.insert(
1986 "mac-mini-monitor".to_string(),
1987 ResolvedSchedulablePlacement {
1988 pool: "mac-mini-monitor".into(),
1989 host_label: Some("mini".into()),
1990 },
1991 );
1992 let sr = bogus_sched_runner(HashMap::new(), sched);
1993 let spec3 = sr.build_spec(&session_of_role("mac-mini-monitor", "go"), &job_for("c1"));
1994 let stamp3 = sr
1995 .placement_stamp_for(&spec3)
1996 .expect("scheduled child is stamped");
1997 assert!(stamp3.contains(r#""kind":"remote""#), "{stamp3}");
1998 assert!(stamp3.contains(r#""host":"mini""#), "{stamp3}");
1999
2000 // A Local (unmatched) child gets NO stamp.
2001 let local = bogus_runner(HashMap::new());
2002 let spec4 = local.build_spec(&session_of_role("writer", "go"), &job_for("c1"));
2003 assert_eq!(local.placement_stamp_for(&spec4), None);
2004 }
2005
2006 // ---- #181: schedulable selection over the BUS (Phase 3 cutover) ----------
2007
2008 async fn start_bus() -> (String, tempfile::TempDir) {
2009 let dir = tempfile::tempdir().unwrap();
2010 let core = std::sync::Arc::new(bamboo_broker::BrokerCore::new(dir.path()));
2011 let server = std::sync::Arc::new(bamboo_broker::BrokerServer::new(core, "t"));
2012 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
2013 let addr = listener.local_addr().unwrap();
2014 tokio::spawn(async move {
2015 let _ = server.serve(listener).await;
2016 });
2017 (format!("ws://{addr}"), dir)
2018 }
2019
2020 async fn join_pool(endpoint: &str, id: &str, pool: &str) -> bamboo_broker::BrokerClient {
2021 let mut c = bamboo_broker::BrokerClient::connect(
2022 endpoint,
2023 bamboo_subagent::AgentRef {
2024 session_id: id.into(),
2025 role: Some(pool.into()),
2026 },
2027 "t",
2028 )
2029 .await
2030 .unwrap();
2031 c.subscribe().await.unwrap();
2032 c
2033 }
2034
2035 fn sched_runner_on_bus(endpoint: &str, child_role: &str, pool: &str) -> ActorChildRunner {
2036 let mut sched = HashMap::new();
2037 sched.insert(child_role.to_string(), sched_placement(pool, "unused"));
2038 bogus_sched_runner(HashMap::new(), sched).with_bus(Some(bamboo_subagent::BusEndpoint {
2039 endpoint: endpoint.into(),
2040 token: "t".into(),
2041 }))
2042 }
2043
2044 #[tokio::test]
2045 async fn resolve_schedulable_picks_a_live_bus_worker() {
2046 let (endpoint, _dir) = start_bus().await;
2047 let _w = join_pool(&endpoint, "w-gpu", "gpu-pool").await;
2048 let runner = sched_runner_on_bus(&endpoint, "explorer", "gpu-pool");
2049
2050 let mailbox = runner
2051 .resolve_schedulable_worker("explorer")
2052 .await
2053 .expect("a live pool worker is found on the bus");
2054 assert_eq!(mailbox, "w-gpu");
2055 }
2056
2057 #[tokio::test]
2058 async fn resolve_schedulable_round_robins_over_pool_workers() {
2059 let (endpoint, _dir) = start_bus().await;
2060 let _a = join_pool(&endpoint, "w-a", "gpu-pool").await;
2061 let _b = join_pool(&endpoint, "w-b", "gpu-pool").await;
2062 let runner = sched_runner_on_bus(&endpoint, "explorer", "gpu-pool");
2063
2064 // Successive resolves spread across both connected workers.
2065 let mut picked = std::collections::HashSet::new();
2066 for _ in 0..6 {
2067 picked.insert(runner.resolve_schedulable_worker("explorer").await.unwrap());
2068 }
2069 assert_eq!(
2070 picked,
2071 ["w-a".to_string(), "w-b".to_string()].into_iter().collect(),
2072 "round-robin must cover every connected pool worker"
2073 );
2074 }
2075
2076 #[tokio::test]
2077 async fn resolve_schedulable_errors_on_empty_pool() {
2078 let (endpoint, _dir) = start_bus().await;
2079 // No worker subscribes to "gpu-pool".
2080 let runner = sched_runner_on_bus(&endpoint, "explorer", "gpu-pool");
2081
2082 let err = runner
2083 .resolve_schedulable_worker("explorer")
2084 .await
2085 .expect_err("an empty pool is terminal — no local fallback")
2086 .to_string();
2087 assert!(err.contains("no live worker in pool"), "got: {err}");
2088 assert!(err.contains("NOT spawning"), "got: {err}");
2089 }
2090
2091 /// FULL schedulable run over the bus: a worker SERVING `EchoExecutor` joins the
2092 /// pool by role; `execute_external_child` with a Schedulable placement resolves
2093 /// it from the bus (no local subprocess — the worker_bin is `/bin/false`),
2094 /// drives the run, gets the echo back, AND stamps the child session with the
2095 /// pool's cluster-node label — `{kind:remote, host:"mini"}`. The end-to-end
2096 /// analogue of the live `mac-mini-monitor`→mini run.
2097 #[tokio::test]
2098 async fn execute_external_child_runs_schedulable_over_bus_and_stamps_node_label() {
2099 let (endpoint, _dir) = start_bus().await;
2100
2101 // A bus worker SERVING runs (not just presence), joined to the pool by role.
2102 let ep = endpoint.clone();
2103 let worker = tokio::spawn(async move {
2104 let _ = bamboo_broker::serve_executor(
2105 &ep,
2106 bamboo_subagent::AgentRef {
2107 session_id: "mmm-worker".into(),
2108 role: Some("mac-mini-monitor".into()),
2109 },
2110 "t",
2111 std::sync::Arc::new(bamboo_subagent::executor::EchoExecutor),
2112 )
2113 .await;
2114 });
2115
2116 // Wait until the worker is visible on the bus so the pool is non-empty
2117 // when execute_external_child resolves it (serve_executor connects async).
2118 let mut probe = bamboo_broker::BrokerClient::connect(
2119 &endpoint,
2120 bamboo_subagent::AgentRef {
2121 session_id: "probe".into(),
2122 role: None,
2123 },
2124 "t",
2125 )
2126 .await
2127 .unwrap();
2128 let mut ready = false;
2129 for _ in 0..100 {
2130 if probe
2131 .list_connected("mac-mini-monitor")
2132 .await
2133 .unwrap()
2134 .iter()
2135 .any(|id| id == "mmm-worker")
2136 {
2137 ready = true;
2138 break;
2139 }
2140 tokio::time::sleep(Duration::from_millis(30)).await;
2141 }
2142 assert!(ready, "worker never joined the pool");
2143
2144 // Runner: child role → schedulable pool "mac-mini-monitor" carrying the
2145 // cluster node's label "mini"; bogus worker_bin so any local spawn fails.
2146 let mut sched = HashMap::new();
2147 sched.insert(
2148 "mac-mini-monitor".to_string(),
2149 ResolvedSchedulablePlacement {
2150 pool: "mac-mini-monitor".into(),
2151 host_label: Some("mini".into()),
2152 },
2153 );
2154 let runner = bogus_sched_runner(HashMap::new(), sched).with_bus(Some(
2155 bamboo_subagent::BusEndpoint {
2156 endpoint: endpoint.clone(),
2157 token: "t".into(),
2158 },
2159 ));
2160
2161 let mut session = session_of_role("mac-mini-monitor", "hello scheduled");
2162 let job = job_for("child-1");
2163 let (event_tx, _rx) = mpsc::channel::<AgentEvent>(64);
2164 let cancel = CancellationToken::new();
2165
2166 tokio::time::timeout(
2167 Duration::from_secs(10),
2168 runner.execute_external_child(&mut session, &job, event_tx, cancel),
2169 )
2170 .await
2171 .expect("run did not hang")
2172 .expect("schedulable run succeeded over the bus (no local spawn)");
2173
2174 // Echo reply flowed back — proves it routed to the bus worker, not local.
2175 let last = session
2176 .messages
2177 .iter()
2178 .rev()
2179 .find(|m| matches!(m.role, Role::Assistant))
2180 .expect("an assistant reply was written back");
2181 assert!(last.content.contains("echo:"), "got {:?}", last.content);
2182
2183 // ...and the child is stamped with the pool's cluster-node label.
2184 let placement = session
2185 .metadata
2186 .get("placement")
2187 .expect("scheduled child session stamped with a placement");
2188 assert!(placement.contains(r#""kind":"remote""#), "{placement}");
2189 assert!(placement.contains(r#""host":"mini""#), "{placement}");
2190
2191 worker.abort();
2192 }
2193}