bamboo_engine/external_agents/actor_adapter.rs
1//! Actor external child runner.
2//!
3//! Runs a child session as an independent **actor**: a separate OS process with its own
4//! isolated context, speaking the `bamboo-subagent` WebSocket protocol. This is the
5//! engine-side adapter on the `wants_external` seam: it spawns the worker binary, waits for
6//! it to self-register into the Tier-1 file fabric, connects, sends the assignment, and
7//! forwards the child's `AgentEvent`s back onto the parent's `event_tx`.
8//!
9//! The built-in **local actor** instance of this runner is the default runtime for
10//! every sub-agent (the in-process runtime was removed). The expert `externalAgents`
11//! tables can additionally route specific roles to other actor/a2a agents.
12
13use std::collections::HashMap;
14use std::path::PathBuf;
15use std::sync::Arc;
16use std::time::Duration;
17
18use async_trait::async_trait;
19use bamboo_agent_core::{AgentError, AgentEvent, Role, Session};
20use bamboo_domain::poison::PoisonRecover;
21use tokio::sync::mpsc;
22use tokio_util::sync::CancellationToken;
23
24use bamboo_subagent::fleet::{spawn_worker_on_bus, SpawnedChild};
25use bamboo_subagent::proto::{AgentRecord, ChildFrame, ParentFrame, RunSpec, TerminalStatus};
26use bamboo_subagent::provision::{
27 ChildIdentity, ExecutorSpec, ModelRefSpec, Placement, ProvisionSpec, ScopedCredential,
28};
29use bamboo_subagent::transport::{client_config_trusting_cert, ChildClient};
30
31use crate::runtime::execution::{ExternalChildRunner, SpawnJob};
32
33/// Default cap on simultaneously running actor processes.
34pub const DEFAULT_MAX_CONCURRENT_ACTORS: usize = 8;
35
36/// Max nesting depth for direct nested execution (Phase 6). A worker whose
37/// session `spawn_depth` is below this gets its own spawn stack + the real
38/// SubAgent tool; at/over it, neither (and the tool itself refuses). Mirrors
39/// `bamboo_server_tools::DEFAULT_MAX_SPAWN_DEPTH` (kept in sync; engine can't
40/// depend on server-tools). Root orchestrator = 0 ⇒ 4 levels of sub-agents.
41pub const MAX_SPAWN_DEPTH: u32 = 4;
42
43/// Default cap on idle pooled (warm, reusable) workers kept per fingerprint.
44const DEFAULT_MAX_IDLE_PER_KEY: usize = 4;
45
46/// How long a pooled worker waits for its next assignment before reclaiming
47/// itself (must comfortably exceed the gap between sibling spawns).
48const POOLED_IDLE_TIMEOUT_SECS: u64 = 300;
49
50/// Deadline for a local worker's FIRST frame after a Run is dispatched. A warm
51/// worker answers in seconds; a cold spawn within tens. Total silence past this
52/// means the worker is dead (e.g. a pooled worker that exited right after its
53/// liveness check) and its Run is queued with nobody to serve it — trip it so the
54/// runner respawns once instead of hanging forever. Generous, to never false-trip
55/// a slow-but-healthy cold start.
56const WORKER_FIRST_FRAME_TIMEOUT: Duration = Duration::from_secs(60);
57
58/// A warm worker on the mailbox bus, parked for reuse between runs. It stays
59/// dialed-in + subscribed to `mailbox_id`; the next interchangeable child
60/// delivers its `Run` there instead of spawning a fresh process. Dropping it
61/// kills a local kill-on-drop subprocess; a remote / schedulable handle is
62/// process-less (`kill()` is a no-op — it self-manages via its idle timeout).
63struct PooledWorker {
64 worker: SpawnedChild,
65 /// The bus mailbox this worker subscribes to (where its `Run`s are delivered).
66 mailbox_id: String,
67}
68
69/// A role pinned to a remote resident worker (remote-actor-plan §3.4 / P1.5,
70/// #193), resolved at runner-build time from `SubagentsConfig.remote_placements`:
71/// the env-named bearer is already READ into `token` here (the raw token never
72/// rides the config), and `ca_cert_file` is the path to a PEM pinning a
73/// self-signed worker cert (`None` ⇒ default webpki roots / plaintext `ws://`).
74#[derive(Debug, Clone)]
75pub struct ResolvedRemotePlacement {
76 pub endpoint: String,
77 pub token: Option<String>,
78 pub ca_cert_file: Option<PathBuf>,
79 /// Display name for the machine this role runs on — the matching cluster
80 /// node's `label`/host, surfaced on the UI placement badge. `None` ⇒ derive
81 /// from the endpoint host.
82 pub host_label: Option<String>,
83}
84
85/// A role routed to a SCHEDULED worker (remote-actor-plan §3.4 / P2b, #181),
86/// resolved at runner-build time from `SubagentsConfig.schedulable_placements`.
87/// Names the logical `pool` (= the bus role) whose LIVE connected workers are the
88/// scheduling candidates — the runner picks one via the bus presence query
89/// (`BrokerClient::list_connected`). Phase 3 retired the old HTTP registry, so a
90/// pool is now just a role on the bus.
91#[derive(Debug, Clone)]
92pub struct ResolvedSchedulablePlacement {
93 pub pool: String,
94 /// Display name for the machine this pool's workers run on — the matching
95 /// cluster node's `label`/host, surfaced on the UI placement badge. `None` ⇒
96 /// fall back to the pool name.
97 pub host_label: Option<String>,
98}
99
100/// How `execute_external_child` should obtain its worker connection, decided
101/// once from `spec.placement`. Splits the divergent acquire/connect + retire
102/// logic three ways while the shared middle (Run dispatch, live registration,
103/// drive, close) stays identical. `Local` is the unchanged pre-#193 path;
104/// `Remote` is the unchanged #194 path; `Schedulable` (#181, P2b) is new.
105enum PlacementKind {
106 Local,
107 Remote,
108 Schedulable,
109}
110
111/// Spawns and drives a child session as an independent actor: a `bamboo-subagent` worker process.
112pub struct ActorChildRunner {
113 agent_id: String,
114 worker_bin: PathBuf,
115 worker_args: Vec<String>,
116 fabric_dir: PathBuf,
117 executor: ExecutorSpec,
118 /// Per-provider credentials snapshotted from the parent config at build
119 /// time; the spec carries only the ONE the child's provider needs.
120 credentials: Vec<ScopedCredential>,
121 /// Parent's default provider (used when the child has no explicit one).
122 default_provider: String,
123 /// The mailbox bus to run local children over (the unified transport). Local
124 /// sub-agents require it; `None` only when no broker could be embedded.
125 bus: Option<bamboo_subagent::BusEndpoint>,
126 /// Backpressure: bounds the number of concurrently *running* actors; further
127 /// runs wait for a slot instead of exploding the process table. (Idle pooled
128 /// workers do not hold a slot.)
129 concurrency: std::sync::Arc<tokio::sync::Semaphore>,
130 /// Warm-worker pool keyed by a reuse fingerprint
131 /// (role/provider/model/workspace/disabled-tools/baked-caps). A finished run
132 /// parks its bus worker here so the next interchangeable child reuses it
133 /// (delivers its `Run` to the same mailbox) instead of spawning a fresh
134 /// process — collapsing N sibling sub-agents onto a few warm workers.
135 pool: Arc<tokio::sync::Mutex<HashMap<String, Vec<PooledWorker>>>>,
136 max_idle_per_key: usize,
137 /// Host-side decision for a child's gated-tool approval request (Phase 2).
138 /// `None` ⇒ fail-closed DENY (the safe default). A wired decider (policy or
139 /// human-routing bridge) returns approve/deny over the actor WS.
140 approval_decider: Option<Arc<dyn ChildApprovalDecider>>,
141 /// Per-run escalation host bridge for non-bypass child-approval routing (#68;
142 /// Phase 6, Part B). The owning worker's `run()` installs its OWN host bridge
143 /// here via `set_escalation_bridge`; `execute_external_child` CAPTURES it at
144 /// grandchild-spawn time and hands the owned value to `drive()`, which uses it
145 /// to RE-PROXY a child's approval request UP to the parent run — chaining up
146 /// every level until a bypass level (model-review) or the top orchestrator
147 /// (human) decides, then relaying the reply back down. Was a process-global
148 /// slot; now per-runner so a fire-and-forget grandchild that OUTLIVES the run
149 /// that spawned it keeps that run's bridge for its whole lifetime instead of
150 /// reading a stale/overwritten global at approval time (→ fail-closed deny).
151 escalation_bridge: Arc<std::sync::Mutex<Option<bamboo_subagent::executor::HostBridge>>>,
152 /// Roles pinned to a REMOTE resident worker (#193), keyed by sub-agent role
153 /// (the child's `subagent_type`). A role present here routes through the
154 /// dedicated remote branch in `execute_external_child` (Bearer-authenticated
155 /// `wss://` connect, no spawn, no pool, no kill) instead of the local
156 /// subprocess + warm-pool path. Empty (the default) = all-local behavior.
157 remote_placements: HashMap<String, ResolvedRemotePlacement>,
158 /// Roles routed to a REGISTRY-SCHEDULED worker (#181, P2b), keyed by sub-agent
159 /// role. A role present here (AND not already in `remote_placements`, which
160 /// wins) routes through the dedicated SCHEDULABLE branch in
161 /// `execute_external_child`: query the registry for live workers in the pool,
162 /// pick one (round-robin), connect over `wss://` — no spawn, no pool, no kill,
163 /// and NO local-subprocess fallback (no live worker ⇒ a clear error). Empty
164 /// (the default) = all-local behavior.
165 schedulable_placements: HashMap<String, ResolvedSchedulablePlacement>,
166 /// Per-pool round-robin cursor for schedulable scheduling (#181, P2b). Bumped
167 /// once per pick so successive sibling spawns SPREAD across a pool's live
168 /// workers instead of all landing on the first candidate. Best-effort spread,
169 /// not a load balancer — the registry's live set can change between picks.
170 schedule_cursor: Arc<std::sync::Mutex<HashMap<String, usize>>>,
171}
172
173/// Decides how the host answers a child worker's gated-tool approval request
174/// (Phase 2: child → parent approval delegation). Async so an implementation
175/// can consult a policy or defer to a human. With no decider wired the host
176/// replies with a fail-closed DENY.
177///
178/// NOTE: `decide` is awaited inside the per-child frame pump, so an
179/// implementation must resolve promptly (e.g. a policy lookup). A human-in-the-
180/// loop decision that may block indefinitely should instead be delivered
181/// out-of-band as a `ParentFrame::ApprovalReply` via the live steering channel
182/// (`super::live`), which `drive()` already forwards to the worker without
183/// stalling the pump.
184#[async_trait]
185pub trait ChildApprovalDecider: Send + Sync {
186 /// Decide whether `child_session_id` may perform the gated action described
187 /// by `request` (`{tool_name, permission, resource}`).
188 async fn decide(&self, child_session_id: &str, request: &serde_json::Value) -> bool;
189}
190
191/// Resolve a child approval request to approve/deny. Fail-closed (DENY) when no
192/// decider is wired — the single, testable seam for the host-side decision.
193async fn decide_child_approval(
194 decider: Option<&Arc<dyn ChildApprovalDecider>>,
195 child_session_id: &str,
196 request: &serde_json::Value,
197) -> bool {
198 match decider {
199 Some(decider) => decider.decide(child_session_id, request).await,
200 None => false,
201 }
202}
203
204/// How long the host waits for a human approval decision before failing the
205/// child's gated tool closed (DENY). Bounds an unanswered request so it can't
206/// hang the worker indefinitely.
207const CHILD_APPROVAL_TIMEOUT: Duration = Duration::from_secs(300);
208
209/// Extract `(tool_name, permission, resource)` from a worker's approval request
210/// body (`{tool_name, permission, resource}`); missing fields default to empty.
211fn approval_request_fields(body: &serde_json::Value) -> (String, String, String) {
212 let field = |k: &str| {
213 body.get(k)
214 .and_then(|v| v.as_str())
215 .unwrap_or("")
216 .to_string()
217 };
218 (field("tool_name"), field("permission"), field("resource"))
219}
220
221/// Off-loop reviewer for a child's gated-tool approval request (Phase 6, Part B).
222///
223/// Installed (process-global) by a BYPASSED self-orchestrating worker so its
224/// children's forced-ask (dangerous) gated actions — which still raise
225/// `ConfirmationRequired` even under bypass — get an LLM reasonableness check
226/// rather than a blind pass. `review` is an LLM call: `drive()` invokes it in a
227/// SPAWNED task (NEVER in the frame pump) and delivers the verdict async via the
228/// live channel, so the agent loop is never blocked.
229#[async_trait]
230pub trait ChildApprovalReviewer: Send + Sync {
231 /// Judge whether the gated action `request` (`{tool_name, permission,
232 /// resource}`) is reasonable for `child_session_id`'s task. `true` = approve.
233 async fn review(&self, child_session_id: &str, request: &serde_json::Value) -> bool;
234}
235
236fn child_approval_reviewer_slot() -> &'static std::sync::OnceLock<Arc<dyn ChildApprovalReviewer>> {
237 static SLOT: std::sync::OnceLock<Arc<dyn ChildApprovalReviewer>> = std::sync::OnceLock::new();
238 &SLOT
239}
240
241/// Install the process-global child-approval reviewer (idempotent; first wins).
242pub fn set_child_approval_reviewer(reviewer: Arc<dyn ChildApprovalReviewer>) {
243 let _ = child_approval_reviewer_slot().set(reviewer);
244}
245
246/// The process-global child-approval reviewer, if installed.
247pub fn child_approval_reviewer() -> Option<Arc<dyn ChildApprovalReviewer>> {
248 child_approval_reviewer_slot().get().cloned()
249}
250
251impl ActorChildRunner {
252 #[allow(clippy::too_many_arguments)]
253 pub fn new(
254 agent_id: String,
255 worker_bin: PathBuf,
256 worker_args: Vec<String>,
257 fabric_dir: PathBuf,
258 executor: ExecutorSpec,
259 credentials: Vec<ScopedCredential>,
260 default_provider: String,
261 max_concurrent: usize,
262 ) -> Self {
263 Self {
264 agent_id,
265 worker_bin,
266 worker_args,
267 fabric_dir,
268 executor,
269 credentials,
270 default_provider,
271 bus: None,
272 concurrency: std::sync::Arc::new(tokio::sync::Semaphore::new(max_concurrent.max(1))),
273 pool: Arc::new(tokio::sync::Mutex::new(HashMap::new())),
274 max_idle_per_key: DEFAULT_MAX_IDLE_PER_KEY,
275 approval_decider: None,
276 escalation_bridge: Arc::new(std::sync::Mutex::new(None)),
277 remote_placements: HashMap::new(),
278 schedulable_placements: HashMap::new(),
279 schedule_cursor: Arc::new(std::sync::Mutex::new(HashMap::new())),
280 }
281 }
282
283 /// Run children over the mailbox bus (the unified actor+mailbox transport).
284 /// When set, local children dial this bus and are driven by mailbox id; when
285 /// unset they use the legacy direct-WS path. The server passes its in-process
286 /// broker here (`subagents.broker`); tests without a broker leave it unset.
287 pub fn with_bus(mut self, bus: Option<bamboo_subagent::BusEndpoint>) -> Self {
288 self.bus = bus.filter(|b| !b.endpoint.trim().is_empty());
289 self
290 }
291
292 /// Wire the host-side decider for child gated-tool approval requests
293 /// (Phase 2). Without this the host fail-closed DENYs every request.
294 pub fn with_approval_decider(mut self, decider: Arc<dyn ChildApprovalDecider>) -> Self {
295 self.approval_decider = Some(decider);
296 self
297 }
298
299 /// Pin specific sub-agent roles to remote resident workers (#193). The map
300 /// is keyed by role (`subagent_type`); a child whose role is present connects
301 /// over `wss://` to the resolved endpoint instead of spawning a local
302 /// subprocess. Default (empty) keeps every role on the local path — exactly
303 /// today's behavior.
304 pub fn with_remote_placements(
305 mut self,
306 placements: HashMap<String, ResolvedRemotePlacement>,
307 ) -> Self {
308 self.remote_placements = placements;
309 self
310 }
311
312 /// Route specific sub-agent roles to a registry-SCHEDULED worker (#181, P2b).
313 /// The map is keyed by role (`subagent_type`); a child whose role is present
314 /// (and NOT already pinned by `remote_placements`, which takes precedence) is
315 /// run on a live worker discovered from the registry instead of a local
316 /// subprocess. Default (empty) keeps every role on the local path.
317 pub fn with_schedulable_placements(
318 mut self,
319 placements: HashMap<String, ResolvedSchedulablePlacement>,
320 ) -> Self {
321 self.schedulable_placements = placements;
322 self
323 }
324
325 /// Reuse fingerprint: two children are interchangeable on one warm worker iff
326 /// they share role, provider, model, workspace, disabled-tool set, AND every
327 /// capability the worker BAKES at provision time (`BambooRuntimeExecutor`
328 /// stamps these once and reuses them across runs): nesting depth, nested-spawn
329 /// stack, bypass mode, permission enforcement, and the depth cap. Omitting any
330 /// of these lets the pool hand a run a worker baked for a DIFFERENT posture —
331 /// e.g. a depth-1 worker (with its own spawn stack) reused for a depth-4
332 /// child would re-stamp `spawn_depth=1` and pass the depth-cap check, breaking
333 /// the recursion bound; or a bypass worker reused for a non-bypass child. So
334 /// these MUST split the pool bucket. Everything else (assignment, history) is
335 /// shipped per-run in the `RunSpec` and does not affect the fingerprint.
336 /// Reuse fingerprint (role/provider/model/workspace/disabled-tools/baked
337 /// caps): two children with the same fingerprint are interchangeable on one
338 /// warm worker, so they share a pool bucket. Any axis the worker bakes ONCE
339 /// at provision time MUST be in here, else a worker baked for one posture
340 /// gets reused for another (see the `fingerprint_*` tests).
341 fn fingerprint(spec: &ProvisionSpec) -> String {
342 let role = spec.identity.role.as_str();
343 let (provider, model) = spec
344 .model
345 .as_ref()
346 .map(|m| (m.provider.as_str(), m.model.as_str()))
347 .unwrap_or(("", ""));
348 let workspace = spec.workspace.as_deref().unwrap_or("");
349 let mut tools = spec.disabled_tools.clone().unwrap_or_default();
350 tools.sort();
351 let caps = &spec.capabilities;
352 format!(
353 "{role}\u{1}{provider}\u{1}{model}\u{1}{workspace}\u{1}{}\u{1}d={}\u{1}ns={}\u{1}by={}\u{1}ep={}\u{1}md={}\u{1}nha={}\u{1}gro={}",
354 tools.join(","),
355 spec.identity.depth,
356 caps.nested_spawn,
357 caps.bypass,
358 caps.enforce_permissions,
359 caps.max_spawn_depth.unwrap_or(0),
360 // #73 review (P1): a worker bakes `no_human_review` ONCE from this flag
361 // at build() and never re-reads it per run, so the pool MUST NOT hand a
362 // worker baked for one approval posture to a run of the opposite one —
363 // else a scheduled-root worker reused for an interactive child would
364 // silently model-review instead of asking the human (and vice-versa,
365 // reintroducing the 300s-deny). Split the bucket on it.
366 caps.no_human_approver,
367 // #71: the read-only Bash checker is baked once at build() from this
368 // flag, so a guardian-reviewer worker must NOT be reused for an
369 // ordinary child (which expects unrestricted Bash), and vice-versa.
370 caps.guardian_read_only,
371 )
372 }
373
374 /// Check out a warm bus worker for `key`, reusing a live parked one if any,
375 /// else spawning a fresh one that dials the bus. The returned worker is OWNED
376 /// by the caller for the run's duration (checkout removes it from the pool, so
377 /// a concurrent sibling gets a different worker or spawns its own — one run per
378 /// worker at a time, matching the pre-bus pool semantics).
379 async fn acquire_bus_worker(
380 &self,
381 key: &str,
382 spec: &ProvisionSpec,
383 ) -> crate::runtime::runner::Result<PooledWorker> {
384 // Drain the bucket, skipping (and reaping) any worker whose process exited
385 // while parked. A live one is handed straight out for reuse.
386 loop {
387 let candidate = {
388 let mut pool = self.pool.lock().await;
389 pool.get_mut(key).and_then(|bucket| bucket.pop())
390 };
391 let Some(mut candidate) = candidate else {
392 break;
393 };
394 if candidate.worker.is_alive() {
395 return Ok(candidate);
396 }
397 candidate.worker.kill().await;
398 }
399
400 let spawned = spawn_worker_on_bus(&self.worker_bin, &self.worker_args, spec)
401 .await
402 .map_err(|e| AgentError::LLM(format!("actor spawn (bus) failed: {e}")))?;
403 let mailbox_id = spawned.record.agent_id.clone();
404 Ok(PooledWorker {
405 worker: spawned,
406 mailbox_id,
407 })
408 }
409
410 /// Park a warm bus worker for reuse after a clean run; if its bucket is full
411 /// (or it died), kill it instead. The worker stays dialed-in + subscribed
412 /// while parked, so a reusing child just delivers a new `Run` to its mailbox.
413 async fn release_bus_worker(&self, key: &str, mut worker: PooledWorker) {
414 if !worker.worker.is_alive() {
415 worker.worker.kill().await;
416 return;
417 }
418 let mut pool = self.pool.lock().await;
419 let bucket = pool.entry(key.to_string()).or_default();
420 if bucket.len() >= self.max_idle_per_key {
421 drop(pool);
422 worker.worker.kill().await;
423 return;
424 }
425 bucket.push(worker);
426 }
427
428 /// Assemble the parent-resolved provisioning document for this child.
429 fn build_spec(&self, session: &Session, job: &SpawnJob) -> ProvisionSpec {
430 let mut spec = ProvisionSpec::new(
431 ChildIdentity {
432 child_id: job.child_session_id.clone(),
433 parent_id: Some(job.parent_session_id.clone()),
434 project_key: None,
435 role: session
436 .metadata
437 .get("subagent_type")
438 .cloned()
439 .unwrap_or_else(|| "worker".to_string()),
440 // The child session already carries the correct depth
441 // (create_child_action's new_child_of did parent.spawn_depth+1);
442 // stamp it so the worker can re-establish it on its run session
443 // and enforce the max-depth cap across the actor boundary.
444 depth: session.spawn_depth,
445 },
446 self.executor.clone(),
447 self.fabric_dir.to_string_lossy().into_owned(),
448 );
449 spec.workspace = session.workspace.clone();
450 // Unified transport: when a bus is configured, the child dials it (no
451 // listen socket / file discovery) and the parent drives it by mailbox id.
452 spec.bus = self.bus.clone();
453 // Final model: the session's pinned model_ref (create.model / routing already applied),
454 // falling back to the job's bare model on the parent's default provider.
455 spec.model = session
456 .model_ref
457 .as_ref()
458 .map(|r| ModelRefSpec {
459 provider: r.provider.clone(),
460 model: r.model.clone(),
461 })
462 .or_else(|| {
463 let m = job.model.trim();
464 (!m.is_empty()).then(|| ModelRefSpec {
465 provider: self.default_provider.clone(),
466 model: m.to_string(),
467 })
468 });
469 spec.disabled_tools = job.disabled_tools.clone();
470 // Least-privilege secrets: only the credential for the child's provider.
471 let provider = spec
472 .model
473 .as_ref()
474 .map(|m| m.provider.as_str())
475 .filter(|p| !p.trim().is_empty())
476 .unwrap_or(&self.default_provider);
477 if let Some(cred) = self.credentials.iter().find(|c| c.provider == provider) {
478 spec.secrets.provider_credentials.push(cred.clone());
479 } else {
480 tracing::warn!(
481 "actor child {}: no credential found for provider '{}'",
482 job.child_session_id,
483 provider
484 );
485 }
486 // Phase 6 (direct nested execution): a worker BELOW the depth cap may
487 // orchestrate its OWN children — on startup it builds its own spawn
488 // stack and runs the real SubAgent tool (no host proxy). The cap (the
489 // SubAgent tool refuses to spawn at/over `max_spawn_depth`) bounds the
490 // recursion. Driven purely by the child's depth, so it auto-propagates
491 // down the tree without any extra config threading.
492 spec.capabilities.nested_spawn = session.spawn_depth < MAX_SPAWN_DEPTH;
493 spec.capabilities.max_spawn_depth = Some(MAX_SPAWN_DEPTH);
494 // #69: activate child-approval review. Sub-agents enforce permissions so
495 // their DANGEROUS actions (the worker uses a HIGH threshold) reach the
496 // parent for review — escalated to the human, or model-reviewed off-loop
497 // when the parent is in bypass. The worker installs no checker without
498 // this, so the whole review chain would otherwise stay dormant.
499 spec.capabilities.enforce_permissions = true;
500 // Propagate "bypass permissions" so a self-orchestrating worker knows it
501 // is a bypassed parent and installs the off-loop model-reviewer for its
502 // children's forced-ask actions (Phase 6, Part B). The child session
503 // already carries the inherited flag (create_child_action seeds it).
504 spec.capabilities.bypass = session
505 .agent_runtime_state
506 .as_ref()
507 .is_some_and(|s| s.bypass_permissions);
508 // #73: propagate "no interactive human approver" (headless / scheduled /
509 // deployed root, inherited by the child session). When set, the worker's
510 // per-run approval proxy model-reviews a gated action locally instead of
511 // escalating to a human who will never answer (which would 300s-deny).
512 spec.capabilities.no_human_approver = session
513 .agent_runtime_state
514 .as_ref()
515 .is_some_and(|s| s.no_human_approver);
516 // #71: mark a READ-ONLY Guardian reviewer so the worker installs the
517 // read-only Bash allowlist checker. The reviewer is spawned by
518 // `spawn_guardian_review` with `subagent_type == "guardian"` (the SAME
519 // marker the completion coordinator branches on to parse the verdict) AND
520 // the `guardian_read_only_disabled_tools` denylist. Keyed off that role
521 // marker (already read above to set `identity.role`), so it rides the same
522 // session-metadata path the denylist/subagent_type use — no new wire seam.
523 // Without this the worker keeps an UNRESTRICTED Bash, so the reviewer could
524 // still `rm -rf` / `git push` / `curl | sh`, defeating "read-only".
525 spec.capabilities.guardian_read_only =
526 session.metadata.get("subagent_type").map(String::as_str) == Some("guardian");
527 // #193: route this role to a REMOTE resident worker when one is pinned.
528 // `spec.identity.role` was just computed from `subagent_type` above; a
529 // match flips the placement to Remote and rides the worker's bearer on the
530 // scoped secrets envelope (TLS handshake / Authorization header only — the
531 // token is never logged). No match leaves the default `Placement::Local`,
532 // so the local path is byte-for-byte unchanged for every non-pinned role.
533 if let Some(placement) = self.remote_placements.get(spec.identity.role.as_str()) {
534 spec.placement = Placement::Remote {
535 endpoint: placement.endpoint.clone(),
536 };
537 spec.secrets.worker_auth_token = placement.token.clone();
538 } else if let Some(placement) = self.schedulable_placements.get(spec.identity.role.as_str())
539 {
540 // #181 (P2b): route this role to a SCHEDULED worker — ONLY when it is
541 // NOT already pinned to a fixed remote endpoint (the `else if` makes
542 // remote_placements take precedence for a role in both). The concrete
543 // worker is picked at run time in `execute_external_child` from the bus
544 // (a live connected worker of the pool role). No per-placement bearer
545 // now — the bus connection uses the bus token. No match in either map
546 // leaves the default `Placement::Local`.
547 spec.placement = Placement::Schedulable {
548 pool: placement.pool.clone(),
549 };
550 }
551 spec
552 }
553
554 /// The `metadata["placement"]` JSON to stamp on a child from its resolved
555 /// placement, preferring the matching cluster node's `host_label` (its
556 /// operator label/host) over the raw endpoint/pool. `None` for a Local child
557 /// (the DTO defaults it to the backend's own host). Split out of
558 /// `execute_external_child` so the role→placement→host resolution is unit-testable.
559 fn placement_stamp_for(&self, spec: &ProvisionSpec) -> Option<String> {
560 let host_label = match &spec.placement {
561 Placement::Remote { .. } => self
562 .remote_placements
563 .get(spec.identity.role.as_str())
564 .and_then(|p| p.host_label.as_deref()),
565 Placement::Schedulable { .. } => self
566 .schedulable_placements
567 .get(spec.identity.role.as_str())
568 .and_then(|p| p.host_label.as_deref()),
569 Placement::Local => None,
570 };
571 placement_metadata(&spec.placement, host_label)
572 }
573
574 /// Pick a live worker for a SCHEDULABLE role from the BUS (#181, Phase 3):
575 /// ask the broker which actors are connected serving the pool role (presence
576 /// is connection-truth — no HTTP registry, no leases, no connect-fail
577 /// failover), then round-robin one per resolve for spread. Returns the chosen
578 /// worker's mailbox id. An empty pool ⇒ a terminal `AgentError` — NEVER a
579 /// local-subprocess fallback (that would silently defeat the placement).
580 async fn resolve_schedulable_worker(
581 &self,
582 role: &str,
583 ) -> std::result::Result<String, AgentError> {
584 let pool = self
585 .schedulable_placements
586 .get(role)
587 .ok_or_else(|| {
588 AgentError::LLM(format!(
589 "schedulable placement for role '{role}' vanished before scheduling"
590 ))
591 })?
592 .pool
593 .clone();
594 let bus = self.bus.as_ref().ok_or_else(|| {
595 AgentError::LLM(format!(
596 "schedulable role '{role}': no mailbox bus configured (subagents.broker)"
597 ))
598 })?;
599
600 // Ask the BUS who is connected serving the pool role — presence is
601 // connection-truth (no HTTP registry, no leases, no stale-record failover).
602 let mut q = bamboo_broker::BrokerClient::connect(
603 &bus.endpoint,
604 bamboo_subagent::AgentRef {
605 session_id: format!("sched-q-{role}"),
606 role: None,
607 },
608 &bus.token,
609 )
610 .await
611 .map_err(|e| {
612 AgentError::LLM(format!(
613 "schedulable role '{role}': bus connect failed: {e}"
614 ))
615 })?;
616 let candidates = q.list_connected(&pool).await.map_err(|e| {
617 AgentError::LLM(format!(
618 "schedulable role '{role}': bus presence query failed: {e}"
619 ))
620 })?;
621
622 if candidates.is_empty() {
623 return Err(AgentError::LLM(format!(
624 "schedulable role '{role}': no live worker in pool '{pool}' on the bus \
625 (NOT spawning a local subprocess — a schedulable role has no local fallback)"
626 )));
627 }
628
629 // Round-robin: advance a per-pool cursor once per resolve so successive
630 // sibling spawns spread across the connected pool workers. No failover
631 // needed — a listed worker is connected NOW (the bus only lists live
632 // subscribers), so there is no stale-but-leased candidate to skip.
633 let idx = {
634 let mut cursors = self.schedule_cursor.lock().recover_poison();
635 let cursor = cursors.entry(pool.clone()).or_insert(0);
636 let i = *cursor % candidates.len();
637 *cursor = cursor.wrapping_add(1);
638 i
639 };
640 Ok(candidates[idx].clone())
641 }
642}
643
644#[async_trait]
645impl ExternalChildRunner for ActorChildRunner {
646 async fn should_handle(&self, session: &Session) -> bool {
647 session.metadata.get("runtime.kind") == Some(&"external".to_string())
648 && session.metadata.get("external.protocol") == Some(&"actor".to_string())
649 && session.metadata.get("external.agent_id") == Some(&self.agent_id)
650 }
651
652 fn set_escalation_bridge(&self, bridge: Option<bamboo_subagent::executor::HostBridge>) {
653 *self.escalation_bridge.lock().recover_poison() = bridge;
654 }
655
656 async fn execute_external_child(
657 &self,
658 session: &mut Session,
659 job: &SpawnJob,
660 event_tx: mpsc::Sender<AgentEvent>,
661 cancel_token: CancellationToken,
662 ) -> crate::runtime::runner::Result<()> {
663 // #68 CORRECTNESS CRUX: capture the per-run escalation bridge HERE, at the
664 // moment this grandchild is spawned — while the parent run's bridge is
665 // still in our slot — into an owned local handed to `drive()` for this
666 // grandchild's whole lifetime. A fire-and-forget grandchild that OUTLIVES
667 // the run that spawned it must NOT re-read `self.escalation_bridge` at
668 // approval time: by then `run()` may have cleared/overwritten it (a worker
669 // serves runs sequentially), and re-proxying through a closed bridge
670 // fail-closed denies. Capturing at spawn pins the right bridge per run.
671 let escalation = self.escalation_bridge.lock().recover_poison().clone();
672 let assignment = extract_assignment(session);
673 let mut spec = self.build_spec(session, job);
674 // Mark the worker reusable + give it an idle timeout so it self-reaps if
675 // orphaned. Warm bus workers are pooled per fingerprint and reused.
676 spec.reusable = true;
677 if spec.limits.idle_timeout_secs.is_none() {
678 spec.limits.idle_timeout_secs = Some(POOLED_IDLE_TIMEOUT_SECS);
679 }
680 let pool_key = Self::fingerprint(&spec);
681 // Rehydration: the child session in the parent's store is the actor's
682 // durable state. Ship the full conversation so a reactivation
683 // (send_message / update / rerun) carries its history. A reused worker is
684 // stateless between runs, so this is also what isolates each child's
685 // context on a shared process.
686 let messages: Vec<serde_json::Value> = session
687 .messages
688 .iter()
689 .filter_map(|m| serde_json::to_value(m).ok())
690 .collect();
691
692 // Backpressure: hold a concurrency slot for the lifetime of the *run*
693 // (cancellation still proceeds — the cancel branch in drive() runs while
694 // we hold the permit). Released when this fn returns, i.e. once the worker
695 // is parked back into the pool, so idle workers don't pin slots.
696 let _slot = self
697 .concurrency
698 .acquire()
699 .await
700 .map_err(|_| AgentError::LLM("actor concurrency limiter closed".to_string()))?;
701
702 // Split LOCAL (spawn + warm-pool) from the two process-less remote paths
703 // ONLY at the divergent spots — acquire/connect here and the park/retire at
704 // the end. Everything between (Run dispatch, live-actor registration,
705 // drive, the close) is identical for all three. `kind` is the single guard.
706 // - Local (#0): byte-for-byte the pre-#193 reuse-or-spawn path.
707 // - Remote (#194): connect to a FIXED resident endpoint, no spawn.
708 // - Schedulable (#181): resolve a live worker from the registry, connect.
709 let kind = match spec.placement {
710 Placement::Remote { .. } => PlacementKind::Remote,
711 Placement::Schedulable { .. } => PlacementKind::Schedulable,
712 Placement::Local => PlacementKind::Local,
713 };
714 let remote = !matches!(kind, PlacementKind::Local);
715
716 // Stamp WHICH machine this child runs on onto its session metadata, so the
717 // UI can show it (mirrored into the session index → SessionSummary.placement).
718 // Only remote/scheduled placements need a stamp — a Local child falls through
719 // to the DTO default (this backend's own host). Persisted by the caller with
720 // the rest of the child session after we return.
721 if let Some(placement_meta) = self.placement_stamp_for(&spec) {
722 session
723 .metadata
724 .insert("placement".to_string(), placement_meta);
725 }
726
727 // Retry-once loop: a pooled local worker can die between its liveness
728 // check and handling the Run (a tiny TOCTOU window) — its Run then sits
729 // queued with no server. The first-frame watchdog in `drive` surfaces that
730 // as `WorkerUnresponsive`; we reap the dead worker and re-acquire ONCE
731 // (which spawns fresh / reuses the next live one). Remote/schedulable have
732 // no spawn fallback, so they never retry.
733 let mut attempt = 0u8;
734 let (result, actor) = loop {
735 let (actor, mut client) = match kind {
736 PlacementKind::Remote => {
737 // REMOTE branch: connect to a resident worker. No spawn, no pool
738 // touch, no drain. We do not own the worker, so a connect failure
739 // has NO respawn fallback — it is a clear, terminal error.
740 let placement = self
741 .remote_placements
742 .get(spec.identity.role.as_str())
743 .ok_or_else(|| {
744 AgentError::LLM(format!(
745 "remote placement for role '{}' vanished before connect",
746 spec.identity.role
747 ))
748 })?;
749 let endpoint = placement.endpoint.clone();
750 // Build the TLS trust: a pinned CA pins a self-signed worker cert;
751 // otherwise default webpki roots (or plaintext for `ws://`).
752 let trust_cfg = match placement.ca_cert_file.as_deref() {
753 Some(path) => Some(client_config_trusting_cert(path).map_err(|e| {
754 AgentError::LLM(format!(
755 "remote worker CA cert '{}': {e}",
756 path.display()
757 ))
758 })?),
759 None => None,
760 };
761 let client = ChildClient::connect_with_auth_tls(
762 &endpoint,
763 placement.token.as_deref(),
764 trust_cfg,
765 )
766 .await
767 .map_err(|e| {
768 AgentError::LLM(format!("remote actor connect to '{endpoint}' failed: {e}"))
769 })?;
770 // Process-less handle so live-actor registration (in-band steering)
771 // works exactly as for a local worker; `kill()` is a no-op.
772 let record = AgentRecord {
773 agent_id: job.child_session_id.clone(),
774 role: spec.identity.role.clone(),
775 labels: Vec::new(),
776 endpoint: endpoint.clone(),
777 pid: 0,
778 version: String::new(),
779 started_at: chrono::Utc::now(),
780 lease_expires_at: chrono::Utc::now(),
781 };
782 let _ = endpoint;
783 let actor = PooledWorker {
784 worker: SpawnedChild::remote(record),
785 mailbox_id: job.child_session_id.clone(),
786 };
787 let client: Box<dyn bamboo_subagent::ChildLink> = Box::new(client);
788 (actor, client)
789 }
790 PlacementKind::Schedulable => {
791 // SCHEDULABLE branch (#181): pick a LIVE worker of the pool role
792 // from the BUS (presence = connection-truth; no HTTP registry, no
793 // leases, no failover) and drive it by mailbox id. The pool worker
794 // stays connected and is reused next time. No spawn, no kill, NO
795 // local fallback — an empty pool is a terminal error (raised in
796 // resolve_schedulable_worker).
797 let bus = self.bus.as_ref().ok_or_else(|| {
798 AgentError::LLM(
799 "schedulable sub-agents require a mailbox bus (subagents.broker)"
800 .to_string(),
801 )
802 })?;
803 let mailbox_id = self
804 .resolve_schedulable_worker(spec.identity.role.as_str())
805 .await?;
806 let parent = bamboo_subagent::AgentRef {
807 session_id: format!("p-{}", job.child_session_id),
808 role: None,
809 };
810 let link = bamboo_broker::BrokerChildLink::connect(
811 &bus.endpoint,
812 parent,
813 &bus.token,
814 mailbox_id.clone(),
815 )
816 .await
817 .map_err(|e| {
818 AgentError::LLM(format!(
819 "schedulable link connect to '{mailbox_id}' failed: {e}"
820 ))
821 })?;
822 // Process-less handle — a bus-resident pool worker is never ours to
823 // kill (remote ⇒ dropped, not pooled, after the run).
824 let actor = PooledWorker {
825 worker: SpawnedChild::remote(AgentRecord {
826 agent_id: mailbox_id.clone(),
827 role: spec.identity.role.clone(),
828 labels: Vec::new(),
829 endpoint: bus.endpoint.clone(),
830 pid: 0,
831 version: String::new(),
832 started_at: chrono::Utc::now(),
833 lease_expires_at: chrono::Utc::now(),
834 }),
835 mailbox_id,
836 };
837 let client: Box<dyn bamboo_subagent::ChildLink> = Box::new(link);
838 (actor, client)
839 }
840 PlacementKind::Local => {
841 // LOCAL = the mailbox bus (the unified transport): check out a warm
842 // pooled worker (reuse a live parked one, else spawn fresh) and
843 // drive it by mailbox id — no listen socket, no file discovery, no
844 // respawn-on-connect-miss (the broker queues the Run until the
845 // worker handles it). The legacy direct-WS path was retired; the bus
846 // is required.
847 let bus = self.bus.as_ref().ok_or_else(|| {
848 AgentError::LLM(
849 "local sub-agents require a mailbox bus (subagents.broker); none is \
850 configured and the bus could not be embedded"
851 .to_string(),
852 )
853 })?;
854 let actor = self.acquire_bus_worker(&pool_key, &spec).await?;
855 let parent = bamboo_subagent::AgentRef {
856 session_id: format!("p-{}", job.child_session_id),
857 role: None,
858 };
859 let link = bamboo_broker::BrokerChildLink::connect(
860 &bus.endpoint,
861 parent,
862 &bus.token,
863 actor.mailbox_id.clone(),
864 )
865 .await
866 .map_err(|e| {
867 AgentError::LLM(format!("broker child link connect failed: {e}"))
868 })?;
869 let client: Box<dyn bamboo_subagent::ChildLink> = Box::new(link);
870 (actor, client)
871 }
872 };
873
874 if let Err(e) = client
875 .send(ParentFrame::Run(RunSpec {
876 // Cloned (not moved) so a retry can re-dispatch to a fresh worker.
877 assignment: assignment.clone(),
878 reasoning_effort: None,
879 messages: messages.clone(),
880 }))
881 .await
882 {
883 if !remote {
884 actor.worker.kill().await;
885 }
886 return Err(AgentError::LLM(format!("actor run dispatch failed: {e}")));
887 }
888
889 // Register as a live actor so send_message (running, no interrupt) can
890 // steer this child in-band over the existing WS connection. The guard
891 // unregisters on every exit path.
892 let (live_tx, mut live_rx) = mpsc::unbounded_channel::<ParentFrame>();
893 let live_guard = super::live::register(&job.child_session_id, live_tx);
894
895 let result = drive(
896 &mut *client,
897 &job.child_session_id,
898 self.approval_decider.as_ref(),
899 escalation.clone(),
900 &event_tx,
901 &cancel_token,
902 &mut live_rx,
903 // First-frame watchdog for EVERY placement: a wedged-but-connected
904 // worker (subscribed ≠ serving — e.g. stuck on a prior LLM call) emits
905 // no first frame; without a deadline drive() blocks forever. Bounding it
906 // turns the "running-but-unresponsive" hang into a recoverable
907 // WorkerUnresponsive (reap+respawn local / re-pick schedulable / error
908 // on a fixed remote endpoint).
909 Some(WORKER_FIRST_FRAME_TIMEOUT),
910 )
911 .await;
912 // Unregister IMMEDIATELY: after drive returns nobody consumes live_rx,
913 // so a send_message landing in the close/park window below must see
914 // "not live" and take the durable-queue fallback instead of vanishing.
915 // (Even if one slipped in earlier, send_message also appends it to the
916 // durable transcript, so the next activation still rehydrates it.)
917 drop(live_guard);
918 // Close the parent link (dropping it closes our broker connection; the
919 // worker stays dialed-in + subscribed, ready for its next Run).
920 drop(client);
921
922 // No first frame ⇒ the worker is wedged. Recover ONCE before giving up:
923 // - Local: reap the dead pooled worker + respawn.
924 // - Schedulable: not ours to kill — drop it and re-select a live pool
925 // member (a wedged worker must not fail the run when the pool has others).
926 // - Remote: a FIXED endpoint has no alternative — fall through to a bounded
927 // WorkerUnresponsive error (far better than the previous infinite hang).
928 if attempt == 0 && matches!(result, Err(AgentError::WorkerUnresponsive(_))) {
929 match kind {
930 PlacementKind::Local => {
931 tracing::warn!(
932 "actor child {} got no first frame; reaping the worker and respawning once",
933 job.child_session_id
934 );
935 actor.worker.kill().await;
936 attempt += 1;
937 continue;
938 }
939 PlacementKind::Schedulable => {
940 tracing::warn!(
941 "scheduled actor child {} got no first frame; re-selecting a pool worker",
942 job.child_session_id
943 );
944 drop(actor);
945 attempt += 1;
946 continue;
947 }
948 PlacementKind::Remote => {}
949 }
950 }
951 break (result, actor);
952 };
953
954 // Park the warm worker for reuse on a clean run, or kill it on
955 // error/cancel (a wedged worker must not be reused). Remote / schedulable
956 // workers are registry-managed — never ours to pool/kill, just drop.
957 if remote {
958 drop(actor);
959 } else {
960 match &result {
961 Ok(_) => self.release_bus_worker(&pool_key, actor).await,
962 Err(_) => actor.worker.kill().await,
963 }
964 }
965
966 // Write-back: persist the actor's final reply onto the child session so
967 // the transcript survives and the NEXT activation sees it as history.
968 // (run_child_spawn saves the session right after we return.)
969 match result {
970 Ok(Some(text)) => {
971 if !text.is_empty() {
972 session.add_message(bamboo_agent_core::Message::assistant(text, None));
973 }
974 Ok(())
975 }
976 Ok(None) => Ok(()),
977 Err(e) => Err(e),
978 }
979 }
980}
981
982/// The `{kind,host}` placement descriptor stamped onto a child session's metadata
983/// under `"placement"` — read back by the storage index → `SessionSummary.placement`
984/// → the UI's machine badge. `None` for `Local` (those fall through to the DTO's
985/// default of this backend's own host). The value is a JSON string matching
986/// `bamboo_storage::SessionPlacement { kind, host }`.
987fn placement_metadata(placement: &Placement, host_label: Option<&str>) -> Option<String> {
988 // Prefer the cluster node's own label/host (its metadata) when the placement
989 // maps to a node; else fall back to the raw endpoint host / pool name.
990 let value = match placement {
991 Placement::Local => return None,
992 Placement::Remote { endpoint } => serde_json::json!({
993 "kind": "remote",
994 "host": host_label.map(str::to_string).unwrap_or_else(|| host_of_endpoint(endpoint)),
995 }),
996 Placement::Schedulable { pool } => serde_json::json!({
997 "kind": "remote",
998 "host": host_label.unwrap_or(pool),
999 }),
1000 };
1001 serde_json::to_string(&value).ok()
1002}
1003
1004/// Extract the host from a `ws[s]://host:port[/path]` bus endpoint, for display.
1005fn host_of_endpoint(endpoint: &str) -> String {
1006 endpoint
1007 .trim()
1008 .trim_start_matches("wss://")
1009 .trim_start_matches("ws://")
1010 .split(['/', ':'])
1011 .next()
1012 .unwrap_or(endpoint)
1013 .to_string()
1014}
1015
1016/// Pump child frames -> parent events until a terminal frame (or cancellation).
1017/// On success, yields the actor's final result text (for session write-back).
1018/// `live_rx` carries in-band frames (steering messages) from the live registry.
1019///
1020/// `escalation_bridge` (#68) is the per-run escalation host bridge CAPTURED BY
1021/// VALUE at spawn time in `execute_external_child` (NOT read live here): when a
1022/// non-bypass child re-proxies an approval request, this owned bridge routes it
1023/// UP to the parent run. Owning it for the call's lifetime is what lets a
1024/// fire-and-forget grandchild that outlives its spawning run still escalate to
1025/// the correct (then-current) parent bridge rather than a stale/overwritten one.
1026async fn drive(
1027 client: &mut dyn bamboo_subagent::ChildLink,
1028 child_session_id: &str,
1029 approval_decider: Option<&Arc<dyn ChildApprovalDecider>>,
1030 escalation_bridge: Option<bamboo_subagent::executor::HostBridge>,
1031 event_tx: &mpsc::Sender<AgentEvent>,
1032 cancel_token: &CancellationToken,
1033 live_rx: &mut mpsc::UnboundedReceiver<ParentFrame>,
1034 first_frame_timeout: Option<Duration>,
1035) -> crate::runtime::runner::Result<Option<String>> {
1036 // First-frame watchdog: a live worker emits its first frame (run-started /
1037 // first token) within seconds; total silence past the deadline means the
1038 // worker is dead (e.g. a pooled worker that exited right after checkout), so
1039 // its Run sits queued forever. We trip ONLY before the first frame — once any
1040 // frame arrives the worker is proven live and a legitimately long run (a slow
1041 // tool between tokens) never trips it.
1042 let mut got_first_frame = false;
1043 let mut first_frame_watch = first_frame_timeout.map(|d| Box::pin(tokio::time::sleep(d)));
1044 loop {
1045 tokio::select! {
1046 _ = cancel_token.cancelled() => {
1047 // fall through to the cancel handling below
1048 break;
1049 }
1050 _ = async {
1051 match first_frame_watch.as_mut() {
1052 Some(s) => s.as_mut().await,
1053 None => std::future::pending::<()>().await,
1054 }
1055 }, if !got_first_frame => {
1056 return Err(AgentError::WorkerUnresponsive(format!(
1057 "child {child_session_id} produced no frame within {:?}",
1058 first_frame_timeout.unwrap_or_default()
1059 )));
1060 }
1061 Some(frame) = live_rx.recv() => {
1062 // Forward in-band steering to the worker over the existing WS.
1063 if client.send(frame).await.is_err() {
1064 tracing::warn!("live steering frame could not be sent; connection failing");
1065 }
1066 }
1067 frame = client.next_frame() => {
1068 // Any frame (event / approval / terminal / close / error) proves
1069 // the worker responded — disarm the first-frame watchdog.
1070 got_first_frame = true;
1071 first_frame_watch = None;
1072 match frame {
1073 Ok(Some(ChildFrame::Event { event })) => {
1074 // AgentEvent is serialized verbatim on the wire (zero mapping).
1075 if let Ok(ev) = serde_json::from_value::<AgentEvent>(event) {
1076 let _ = event_tx.send(ev).await;
1077 }
1078 }
1079 Ok(Some(ChildFrame::ApprovalRequest { id, body })) => {
1080 // Phase 2: a worker proxied a gated-tool approval back to
1081 // the host. The WORKER side is live — its executor installs
1082 // a per-run task-local `ApprovalProxy` (subagent_worker.rs)
1083 // that calls `host.approval_call`, so this frame arrives
1084 // when a child hits `ConfirmationRequired`.
1085 if let Some(reviewer) = child_approval_reviewer() {
1086 // Phase 6, Part B: a BYPASSED parent worker
1087 // model-reviews its children's forced-ask (dangerous)
1088 // actions. The review is an LLM call, so run it OFF
1089 // the frame pump in a spawned task and deliver the
1090 // verdict async via the live channel — the pump keeps
1091 // forwarding events and the agent loop never blocks. A
1092 // timeout denies a hung review so the child can't hang.
1093 let child = child_session_id.to_string();
1094 let req_id = id.clone();
1095 let body = body.clone();
1096 tokio::spawn(async move {
1097 let approved = tokio::time::timeout(
1098 CHILD_APPROVAL_TIMEOUT,
1099 reviewer.review(&child, &body),
1100 )
1101 .await
1102 .unwrap_or(false);
1103 super::live::deliver_approval(&child, &req_id, approved);
1104 });
1105 } else if approval_decider.is_some() {
1106 // A decider is wired (policy / auto): decide promptly
1107 // and reply inline. (Must not block the pump — see the
1108 // `ChildApprovalDecider` doc.)
1109 let approved =
1110 decide_child_approval(approval_decider, child_session_id, &body)
1111 .await;
1112 if client
1113 .send(ParentFrame::ApprovalReply { id, approved })
1114 .await
1115 .is_err()
1116 {
1117 tracing::warn!(
1118 "failed to answer approval_request; connection failing"
1119 );
1120 }
1121 } else if let Some(host) = escalation_bridge.clone() {
1122 // Non-bypass WORKER: ESCALATE up our own actor link
1123 // (re-proxy) so the request chains to our parent — and
1124 // up every level until a bypass level (model-review) or
1125 // the top orchestrator (human) decides. Off-loop so the
1126 // pump never blocks; relay the reply down to the child.
1127 let child = child_session_id.to_string();
1128 let req_id = id.clone();
1129 let body = body.clone();
1130 tokio::spawn(async move {
1131 let approved = match tokio::time::timeout(
1132 CHILD_APPROVAL_TIMEOUT,
1133 host.approval_call(body),
1134 )
1135 .await
1136 {
1137 Ok(Ok(reply)) => reply
1138 .get("approved")
1139 .and_then(|v| v.as_bool())
1140 .unwrap_or(false),
1141 // Transport error or timeout ⇒ fail closed.
1142 _ => false,
1143 };
1144 super::live::deliver_approval(&child, &req_id, approved);
1145 });
1146 } else {
1147 // Top orchestrator (no escalation bridge): human-in-the-
1148 // loop. Surface the request on the parent's event stream
1149 // and DEFER — the decision arrives out-of-band via
1150 // `live::deliver_approval(child, request_id, approved)`
1151 // (→ this child's `live_rx` → forwarded to the worker
1152 // above). A timeout denies a never-answered request so
1153 // it can't hang the child forever.
1154 let (tool_name, permission, resource) =
1155 approval_request_fields(&body);
1156 // Register the pending request BEFORE surfacing it so
1157 // the external handler's `deliver_approval_checked` can
1158 // correlate an out-of-band POST against a genuine
1159 // human-loop request (and consume it one-shot).
1160 super::live::register_pending_approval(child_session_id, &id);
1161 let _ = event_tx
1162 .send(AgentEvent::ChildApprovalRequested {
1163 child_session_id: child_session_id.to_string(),
1164 request_id: id.clone(),
1165 tool_name,
1166 permission,
1167 resource,
1168 })
1169 .await;
1170 let child = child_session_id.to_string();
1171 tokio::spawn(async move {
1172 tokio::time::sleep(CHILD_APPROVAL_TIMEOUT).await;
1173 // Deny only if still pending: a one-shot consume so
1174 // we don't double-deliver if the human already
1175 // answered (the POST took it), and so a late POST
1176 // after this fires finds nothing pending.
1177 if super::live::take_pending_approval(&child, &id) {
1178 super::live::deliver_approval(&child, &id, false);
1179 }
1180 });
1181 }
1182 }
1183 Ok(Some(ChildFrame::Terminal { status, result, error, .. })) => {
1184 return match status {
1185 TerminalStatus::Completed => Ok(result),
1186 TerminalStatus::Cancelled => Err(AgentError::Cancelled),
1187 TerminalStatus::Error => Err(AgentError::LLM(
1188 error.unwrap_or_else(|| "actor child errored".to_string()),
1189 )),
1190 // The suspend/resume round-trip (host re-dispatch of a
1191 // nested parent) is not wired here yet; a worker in
1192 // this build never emits Suspended, so this is
1193 // unreachable in practice.
1194 TerminalStatus::Suspended => Err(AgentError::LLM(
1195 "nested sub-agent suspend received but resume transport is not wired"
1196 .to_string(),
1197 )),
1198 };
1199 }
1200 Ok(None) => {
1201 return Err(AgentError::LLM(
1202 "actor child closed before terminal".to_string(),
1203 ));
1204 }
1205 Err(e) => {
1206 return Err(AgentError::LLM(format!("actor transport error: {e}")));
1207 }
1208 }
1209 }
1210 }
1211 }
1212
1213 // Only reached on cancellation: ask the child to stop (best-effort), then report cancelled.
1214 let _ = client.send(ParentFrame::Cancel).await;
1215 Err(AgentError::Cancelled)
1216}
1217
1218/// The assignment text = the child session's latest user message (falls back to its title).
1219fn extract_assignment(session: &Session) -> String {
1220 session
1221 .messages
1222 .iter()
1223 .rev()
1224 .find(|m| matches!(m.role, Role::User))
1225 .map(|m| m.content.clone())
1226 .unwrap_or_else(|| {
1227 session
1228 .metadata
1229 .get("title")
1230 .cloned()
1231 .unwrap_or_else(|| "Execute task".to_string())
1232 })
1233}
1234
1235#[cfg(test)]
1236mod tests {
1237 use super::*;
1238
1239 fn spec_with(
1240 role: &str,
1241 provider: &str,
1242 model: &str,
1243 workspace: Option<&str>,
1244 disabled: Option<Vec<&str>>,
1245 ) -> ProvisionSpec {
1246 let mut spec = ProvisionSpec::new(
1247 ChildIdentity {
1248 child_id: "c".into(),
1249 parent_id: None,
1250 project_key: None,
1251 role: role.into(),
1252 depth: 0,
1253 },
1254 ExecutorSpec::Echo,
1255 "/tmp/fab".into(),
1256 );
1257 spec.workspace = workspace.map(|w| w.to_string());
1258 spec.model = Some(ModelRefSpec {
1259 provider: provider.into(),
1260 model: model.into(),
1261 });
1262 spec.disabled_tools = disabled.map(|d| d.into_iter().map(String::from).collect());
1263 spec
1264 }
1265
1266 #[test]
1267 fn fingerprint_matches_interchangeable_children() {
1268 // Same role/provider/model/workspace and equal tool sets (order-insensitive)
1269 // are interchangeable on one warm worker — and differ only in child_id.
1270 let a = spec_with(
1271 "explorer",
1272 "p",
1273 "m",
1274 Some("/ws"),
1275 Some(vec!["Bash", "Edit"]),
1276 );
1277 let mut b = spec_with(
1278 "explorer",
1279 "p",
1280 "m",
1281 Some("/ws"),
1282 Some(vec!["Edit", "Bash"]),
1283 );
1284 b.identity.child_id = "other".into();
1285 assert_eq!(
1286 ActorChildRunner::fingerprint(&a),
1287 ActorChildRunner::fingerprint(&b)
1288 );
1289 }
1290
1291 #[test]
1292 fn fingerprint_separates_distinct_runtimes() {
1293 let base = spec_with("explorer", "p", "m", Some("/ws"), None);
1294 let base_fp = ActorChildRunner::fingerprint(&base);
1295 // Each axis that is baked into the worker must split the pool bucket.
1296 assert_ne!(
1297 base_fp,
1298 ActorChildRunner::fingerprint(&spec_with("writer", "p", "m", Some("/ws"), None))
1299 );
1300 assert_ne!(
1301 base_fp,
1302 ActorChildRunner::fingerprint(&spec_with("explorer", "p2", "m", Some("/ws"), None))
1303 );
1304 assert_ne!(
1305 base_fp,
1306 ActorChildRunner::fingerprint(&spec_with("explorer", "p", "m2", Some("/ws"), None))
1307 );
1308 assert_ne!(
1309 base_fp,
1310 ActorChildRunner::fingerprint(&spec_with("explorer", "p", "m", Some("/ws2"), None))
1311 );
1312 assert_ne!(
1313 base_fp,
1314 ActorChildRunner::fingerprint(&spec_with(
1315 "explorer",
1316 "p",
1317 "m",
1318 Some("/ws"),
1319 Some(vec!["Bash"])
1320 ))
1321 );
1322 }
1323
1324 #[test]
1325 fn fingerprint_splits_on_baked_capabilities() {
1326 // Every capability baked once at provision time must split the pool
1327 // bucket, else a worker baked for one posture gets reused for another
1328 // (e.g. a depth-1 worker re-stamping spawn_depth onto a depth-4 child,
1329 // breaking the depth cap; or a bypass worker reused for a non-bypass one).
1330 let base_fp =
1331 ActorChildRunner::fingerprint(&spec_with("explorer", "p", "m", Some("/ws"), None));
1332
1333 let mut depth = spec_with("explorer", "p", "m", Some("/ws"), None);
1334 depth.identity.depth = 2;
1335 assert_ne!(
1336 base_fp,
1337 ActorChildRunner::fingerprint(&depth),
1338 "depth must split"
1339 );
1340
1341 let mut nested = spec_with("explorer", "p", "m", Some("/ws"), None);
1342 nested.capabilities.nested_spawn = true;
1343 assert_ne!(
1344 base_fp,
1345 ActorChildRunner::fingerprint(&nested),
1346 "nested_spawn must split"
1347 );
1348
1349 let mut bypass = spec_with("explorer", "p", "m", Some("/ws"), None);
1350 bypass.capabilities.bypass = true;
1351 assert_ne!(
1352 base_fp,
1353 ActorChildRunner::fingerprint(&bypass),
1354 "bypass must split"
1355 );
1356
1357 let mut enforce = spec_with("explorer", "p", "m", Some("/ws"), None);
1358 enforce.capabilities.enforce_permissions = true;
1359 assert_ne!(
1360 base_fp,
1361 ActorChildRunner::fingerprint(&enforce),
1362 "enforce_permissions must split"
1363 );
1364
1365 let mut cap = spec_with("explorer", "p", "m", Some("/ws"), None);
1366 cap.capabilities.max_spawn_depth = Some(8);
1367 assert_ne!(
1368 base_fp,
1369 ActorChildRunner::fingerprint(&cap),
1370 "max_spawn_depth must split"
1371 );
1372
1373 // #73 (P1): the worker bakes `no_human_review` from this flag once at
1374 // build(), so it MUST split the pool or a worker baked for one approval
1375 // posture is reused for the opposite one.
1376 let mut nha = spec_with("explorer", "p", "m", Some("/ws"), None);
1377 nha.capabilities.no_human_approver = true;
1378 assert_ne!(
1379 base_fp,
1380 ActorChildRunner::fingerprint(&nha),
1381 "no_human_approver must split"
1382 );
1383
1384 // #71: the read-only Bash checker is baked once at build() from this flag,
1385 // so a guardian reviewer worker must not be reused for an ordinary child.
1386 let mut gro = spec_with("explorer", "p", "m", Some("/ws"), None);
1387 gro.capabilities.guardian_read_only = true;
1388 assert_ne!(
1389 base_fp,
1390 ActorChildRunner::fingerprint(&gro),
1391 "guardian_read_only must split"
1392 );
1393 }
1394
1395 struct StaticDecider(bool);
1396
1397 #[async_trait]
1398 impl ChildApprovalDecider for StaticDecider {
1399 async fn decide(&self, _child: &str, _req: &serde_json::Value) -> bool {
1400 self.0
1401 }
1402 }
1403
1404 // ---- first-frame watchdog (dead-pooled-worker recovery) -----------------
1405
1406 /// A link that never yields a frame — models a worker that died (or never
1407 /// subscribed) so its Run sits queued with no server.
1408 struct SilentLink;
1409 #[async_trait]
1410 impl bamboo_subagent::ChildLink for SilentLink {
1411 async fn send(&mut self, _: ParentFrame) -> bamboo_subagent::TransportResult<()> {
1412 Ok(())
1413 }
1414 async fn next_frame(&mut self) -> bamboo_subagent::TransportResult<Option<ChildFrame>> {
1415 std::future::pending().await
1416 }
1417 }
1418
1419 /// A link that immediately yields one terminal frame (a healthy fast worker).
1420 struct InstantTerminalLink {
1421 done: bool,
1422 }
1423 #[async_trait]
1424 impl bamboo_subagent::ChildLink for InstantTerminalLink {
1425 async fn send(&mut self, _: ParentFrame) -> bamboo_subagent::TransportResult<()> {
1426 Ok(())
1427 }
1428 async fn next_frame(&mut self) -> bamboo_subagent::TransportResult<Option<ChildFrame>> {
1429 if self.done {
1430 std::future::pending().await
1431 } else {
1432 self.done = true;
1433 Ok(Some(ChildFrame::Terminal {
1434 status: TerminalStatus::Completed,
1435 result: Some("done".into()),
1436 error: None,
1437 transcript: vec![],
1438 }))
1439 }
1440 }
1441 }
1442
1443 #[tokio::test]
1444 async fn drive_trips_first_frame_watchdog_on_a_silent_worker() {
1445 let (event_tx, _rx) = mpsc::channel::<AgentEvent>(8);
1446 let cancel = CancellationToken::new();
1447 let (_live_tx, mut live_rx) = mpsc::unbounded_channel::<ParentFrame>();
1448 let mut link = SilentLink;
1449 let r = drive(
1450 &mut link,
1451 "child-x",
1452 None,
1453 None,
1454 &event_tx,
1455 &cancel,
1456 &mut live_rx,
1457 Some(Duration::from_millis(100)),
1458 )
1459 .await;
1460 assert!(
1461 matches!(r, Err(AgentError::WorkerUnresponsive(_))),
1462 "a silent worker must trip the first-frame watchdog, got {r:?}"
1463 );
1464 }
1465
1466 #[tokio::test]
1467 async fn drive_does_not_trip_when_a_frame_arrives() {
1468 let (event_tx, _rx) = mpsc::channel::<AgentEvent>(8);
1469 let cancel = CancellationToken::new();
1470 let (_live_tx, mut live_rx) = mpsc::unbounded_channel::<ParentFrame>();
1471 let mut link = InstantTerminalLink { done: false };
1472 // Even a tiny timeout must NOT trip: the terminal frame arrives first and
1473 // disarms the watchdog.
1474 let r = drive(
1475 &mut link,
1476 "child-y",
1477 None,
1478 None,
1479 &event_tx,
1480 &cancel,
1481 &mut live_rx,
1482 Some(Duration::from_millis(50)),
1483 )
1484 .await;
1485 assert_eq!(r.ok().flatten().as_deref(), Some("done"));
1486 }
1487
1488 #[tokio::test]
1489 async fn child_approval_fails_closed_without_decider() {
1490 // No decider wired ⇒ the host denies (safe default), unchanged behavior.
1491 let body = serde_json::json!({"tool_name":"Bash","permission":"run","resource":"rm -rf /"});
1492 assert!(!decide_child_approval(None, "child-1", &body).await);
1493 }
1494
1495 #[tokio::test]
1496 async fn child_approval_honors_wired_decider() {
1497 let body =
1498 serde_json::json!({"tool_name":"Write","permission":"write","resource":"/tmp/x"});
1499 let approve: Arc<dyn ChildApprovalDecider> = Arc::new(StaticDecider(true));
1500 let deny: Arc<dyn ChildApprovalDecider> = Arc::new(StaticDecider(false));
1501 assert!(decide_child_approval(Some(&approve), "child-1", &body).await);
1502 assert!(!decide_child_approval(Some(&deny), "child-1", &body).await);
1503 }
1504
1505 #[test]
1506 fn approval_request_fields_extracts_and_defaults() {
1507 let full = serde_json::json!({"tool_name":"Bash","permission":"run","resource":"ls"});
1508 assert_eq!(
1509 approval_request_fields(&full),
1510 ("Bash".to_string(), "run".to_string(), "ls".to_string())
1511 );
1512 // Missing fields default to empty strings.
1513 let partial = serde_json::json!({"tool_name":"Write"});
1514 assert_eq!(
1515 approval_request_fields(&partial),
1516 ("Write".to_string(), String::new(), String::new())
1517 );
1518 }
1519
1520 // ---- #193: remote placement routing -------------------------------------
1521
1522 use crate::runtime::execution::SpawnJob;
1523 use bamboo_agent_core::Session;
1524
1525 /// A runner with a BOGUS worker_bin (`/bin/false`): a local spawn here would
1526 /// FAIL, so a passing remote test proves the remote path never spawns.
1527 fn bogus_runner(placements: HashMap<String, ResolvedRemotePlacement>) -> ActorChildRunner {
1528 ActorChildRunner::new(
1529 "test-actor".into(),
1530 PathBuf::from("/bin/false"),
1531 vec![],
1532 std::env::temp_dir().join("bamboo-test-fab-193"),
1533 ExecutorSpec::Echo,
1534 vec![],
1535 "anthropic".into(),
1536 4,
1537 )
1538 .with_remote_placements(placements)
1539 }
1540
1541 /// A child session of the given role (the role rides `subagent_type`, the
1542 /// path build_spec + the remote lookup both read).
1543 fn session_of_role(role: &str, assignment: &str) -> Session {
1544 let mut s = Session::new("child-1", "test-model");
1545 s.metadata
1546 .insert("subagent_type".to_string(), role.to_string());
1547 s.add_message(bamboo_agent_core::Message::user(assignment));
1548 s
1549 }
1550
1551 fn job_for(child: &str) -> SpawnJob {
1552 SpawnJob {
1553 parent_session_id: "parent-1".into(),
1554 child_session_id: child.into(),
1555 model: String::new(),
1556 disabled_tools: None,
1557 }
1558 }
1559
1560 #[test]
1561 fn build_spec_sets_remote_placement_for_matching_role() {
1562 let mut placements = HashMap::new();
1563 placements.insert(
1564 "explorer".to_string(),
1565 ResolvedRemotePlacement {
1566 endpoint: "wss://gpu-host:8443".into(),
1567 token: Some("T-secret".into()),
1568 ca_cert_file: None,
1569 host_label: None,
1570 },
1571 );
1572 let runner = bogus_runner(placements);
1573
1574 // Matching role -> Placement::Remote + the bearer on the secrets envelope.
1575 let s = session_of_role("explorer", "do the thing");
1576 let spec = runner.build_spec(&s, &job_for("child-1"));
1577 match &spec.placement {
1578 Placement::Remote { endpoint } => assert_eq!(endpoint, "wss://gpu-host:8443"),
1579 other => panic!("expected Remote, got {other:?}"),
1580 }
1581 assert_eq!(spec.secrets.worker_auth_token.as_deref(), Some("T-secret"));
1582 }
1583
1584 #[test]
1585 fn build_spec_leaves_local_for_unmatched_role() {
1586 let mut placements = HashMap::new();
1587 placements.insert(
1588 "explorer".to_string(),
1589 ResolvedRemotePlacement {
1590 endpoint: "wss://gpu-host:8443".into(),
1591 token: Some("T".into()),
1592 ca_cert_file: None,
1593 host_label: None,
1594 },
1595 );
1596 let runner = bogus_runner(placements);
1597
1598 // A DIFFERENT role keeps the default Local placement + no bearer.
1599 let s = session_of_role("writer", "do the thing");
1600 let spec = runner.build_spec(&s, &job_for("child-1"));
1601 assert_eq!(spec.placement, Placement::Local);
1602 assert!(spec.secrets.worker_auth_token.is_none());
1603 }
1604
1605 #[test]
1606 fn build_spec_local_when_no_placements() {
1607 let runner = bogus_runner(HashMap::new());
1608 let s = session_of_role("explorer", "do the thing");
1609 let spec = runner.build_spec(&s, &job_for("child-1"));
1610 assert_eq!(spec.placement, Placement::Local);
1611 assert!(spec.secrets.worker_auth_token.is_none());
1612 }
1613
1614 #[test]
1615 fn placement_metadata_stamps_remote_and_schedulable_not_local() {
1616 // Local children carry no stamp — the DTO defaults them to the backend host.
1617 assert_eq!(placement_metadata(&Placement::Local, None), None);
1618
1619 // Remote, no node label → host derived from the endpoint.
1620 let r = placement_metadata(
1621 &Placement::Remote {
1622 endpoint: "wss://10.0.0.5:8443/stream".into(),
1623 },
1624 None,
1625 )
1626 .unwrap();
1627 assert!(r.contains(r#""kind":"remote""#), "{r}");
1628 assert!(r.contains(r#""host":"10.0.0.5""#), "{r}");
1629
1630 // A cluster node's label (its metadata) OVERRIDES the raw endpoint host.
1631 let labeled = placement_metadata(
1632 &Placement::Remote {
1633 endpoint: "ws://169.254.230.101:8899".into(),
1634 },
1635 Some("mini"),
1636 )
1637 .unwrap();
1638 assert!(labeled.contains(r#""host":"mini""#), "{labeled}");
1639
1640 // Schedulable → {kind:"remote", host:<node label, else pool>}.
1641 let s = placement_metadata(
1642 &Placement::Schedulable {
1643 pool: "explorers".into(),
1644 },
1645 Some("mini"),
1646 )
1647 .unwrap();
1648 assert!(s.contains(r#""kind":"remote""#), "{s}");
1649 assert!(s.contains(r#""host":"mini""#), "{s}");
1650
1651 // The stamp round-trips through the storage placement type.
1652 let p: bamboo_storage::SessionPlacement = serde_json::from_str(&labeled).unwrap();
1653 assert_eq!(p.kind, "remote");
1654 assert_eq!(p.host, "mini");
1655 }
1656
1657 /// End-to-end remote run through `execute_external_child`: a resident worker
1658 /// (Bearer-gated `WsServer` + `EchoExecutor`) serves the role; the runner is
1659 /// built with a `remote_placements` entry pointing at it AND a BOGUS
1660 /// worker_bin (`/bin/false`). A passing test proves the remote path CONNECTS
1661 /// to the resident worker and NEVER spawns (a spawn would fail on /bin/false),
1662 /// and that a terminal/echo result flows back.
1663 #[tokio::test]
1664 async fn execute_external_child_routes_role_to_remote_worker_without_spawning() {
1665 // 1. Stand up the resident worker on loopback with a required bearer.
1666 let token = "remote-test-token";
1667 let server = bamboo_subagent::transport::WsServer::bind_with_token(
1668 (std::net::Ipv4Addr::LOCALHOST, 0).into(),
1669 Some(token.to_string()),
1670 )
1671 .await
1672 .expect("bind resident worker");
1673 let endpoint = server.ws_endpoint(); // ws://127.0.0.1:<port>
1674 let srv = tokio::spawn(async move {
1675 // serve() loops connection-after-connection; the test exits, dropping it.
1676 let _ = server
1677 .serve(Arc::new(bamboo_subagent::executor::EchoExecutor))
1678 .await;
1679 });
1680
1681 // 2. Build the runner: role "explorer" pinned remote, bogus worker_bin.
1682 let mut placements = HashMap::new();
1683 placements.insert(
1684 "explorer".to_string(),
1685 ResolvedRemotePlacement {
1686 endpoint: endpoint.clone(),
1687 token: Some(token.to_string()),
1688 ca_cert_file: None,
1689 host_label: Some("mini-e2e".into()), // node label, surfaced on the badge
1690 },
1691 );
1692 let runner = bogus_runner(placements);
1693
1694 // 3. Drive a real run for that role.
1695 let mut session = session_of_role("explorer", "hello remote");
1696 let job = job_for("child-1");
1697 let (event_tx, mut event_rx) = mpsc::channel::<AgentEvent>(64);
1698 let cancel = CancellationToken::new();
1699
1700 let result = tokio::time::timeout(
1701 Duration::from_secs(10),
1702 runner.execute_external_child(&mut session, &job, event_tx, cancel),
1703 )
1704 .await
1705 .expect("run did not hang")
1706 .expect("remote run succeeded (connected to resident worker, did not spawn)");
1707
1708 let _ = result;
1709 // The EchoExecutor's reply is written back onto the child session as an
1710 // assistant message — proof a terminal result flowed back over the link.
1711 let last = session
1712 .messages
1713 .iter()
1714 .rev()
1715 .find(|m| matches!(m.role, Role::Assistant))
1716 .expect("an assistant reply was written back");
1717 assert!(
1718 last.content.contains("echo:"),
1719 "expected echo reply, got {:?}",
1720 last.content
1721 );
1722
1723 // A remote run must stamp WHICH machine it ran on onto the child session
1724 // (mirrored to the UI badge) using the placement's node label.
1725 let placement = session
1726 .metadata
1727 .get("placement")
1728 .expect("remote child session stamped with a placement");
1729 assert!(placement.contains(r#""kind":"remote""#), "{placement}");
1730 assert!(placement.contains(r#""host":"mini-e2e""#), "{placement}");
1731
1732 // Drain a couple of streamed events to confirm the event pipe carried the
1733 // worker's tokens too (best-effort; the reply assertion above is primary).
1734 let mut saw_event = false;
1735 while let Ok(Some(_ev)) =
1736 tokio::time::timeout(Duration::from_millis(50), event_rx.recv()).await
1737 {
1738 saw_event = true;
1739 }
1740 let _ = saw_event;
1741
1742 srv.abort();
1743 }
1744
1745 // ---- #181 (P2b): schedulable placement routing --------------------------
1746
1747 /// A bogus-worker_bin runner carrying SCHEDULABLE placements (and optionally
1748 /// remote ones, to test precedence). A local spawn here would fail on
1749 /// `/bin/false`, so a passing schedulable test proves no subprocess spawned.
1750 fn bogus_sched_runner(
1751 remote: HashMap<String, ResolvedRemotePlacement>,
1752 sched: HashMap<String, ResolvedSchedulablePlacement>,
1753 ) -> ActorChildRunner {
1754 ActorChildRunner::new(
1755 "test-actor".into(),
1756 PathBuf::from("/bin/false"),
1757 vec![],
1758 std::env::temp_dir().join("bamboo-test-fab-181"),
1759 ExecutorSpec::Echo,
1760 vec![],
1761 "anthropic".into(),
1762 4,
1763 )
1764 .with_remote_placements(remote)
1765 .with_schedulable_placements(sched)
1766 }
1767
1768 fn sched_placement(
1769 pool: &str,
1770 _registry_url: impl Into<String>,
1771 ) -> ResolvedSchedulablePlacement {
1772 ResolvedSchedulablePlacement {
1773 pool: pool.into(),
1774 host_label: None,
1775 }
1776 }
1777
1778 #[test]
1779 fn build_spec_sets_schedulable_placement_for_matching_role() {
1780 let mut sched = HashMap::new();
1781 sched.insert(
1782 "explorer".to_string(),
1783 sched_placement("gpu-pool", "unused"),
1784 );
1785 let runner = bogus_sched_runner(HashMap::new(), sched);
1786
1787 let s = session_of_role("explorer", "do the thing");
1788 let spec = runner.build_spec(&s, &job_for("child-1"));
1789 match &spec.placement {
1790 Placement::Schedulable { pool } => assert_eq!(pool, "gpu-pool"),
1791 other => panic!("expected Schedulable, got {other:?}"),
1792 }
1793 // No per-placement bearer now — the bus connection carries the bus token.
1794 assert!(spec.secrets.worker_auth_token.is_none());
1795 }
1796
1797 #[test]
1798 fn build_spec_remote_wins_when_role_in_both_maps() {
1799 // A role present in BOTH remote_placements and schedulable_placements must
1800 // resolve to the FIXED remote placement (documented precedence).
1801 let mut remote = HashMap::new();
1802 remote.insert(
1803 "explorer".to_string(),
1804 ResolvedRemotePlacement {
1805 endpoint: "wss://fixed-host:8443".into(),
1806 token: Some("T-remote".into()),
1807 ca_cert_file: None,
1808 host_label: None,
1809 },
1810 );
1811 let mut sched = HashMap::new();
1812 sched.insert(
1813 "explorer".to_string(),
1814 sched_placement("gpu-pool", "https://control-plane:9562"),
1815 );
1816 let runner = bogus_sched_runner(remote, sched);
1817
1818 let s = session_of_role("explorer", "do the thing");
1819 let spec = runner.build_spec(&s, &job_for("child-1"));
1820 match &spec.placement {
1821 Placement::Remote { endpoint } => assert_eq!(endpoint, "wss://fixed-host:8443"),
1822 other => panic!("expected Remote (precedence), got {other:?}"),
1823 }
1824 assert_eq!(spec.secrets.worker_auth_token.as_deref(), Some("T-remote"));
1825 }
1826
1827 #[test]
1828 fn build_spec_local_for_unmatched_schedulable_role() {
1829 let mut sched = HashMap::new();
1830 sched.insert(
1831 "explorer".to_string(),
1832 sched_placement("gpu-pool", "https://control-plane:9562"),
1833 );
1834 let runner = bogus_sched_runner(HashMap::new(), sched);
1835 let s = session_of_role("writer", "do the thing");
1836 let spec = runner.build_spec(&s, &job_for("child-1"));
1837 assert_eq!(spec.placement, Placement::Local);
1838 assert!(spec.secrets.worker_auth_token.is_none());
1839 }
1840
1841 /// The full role → resolved-placement → badge-host chain: a child routed to a
1842 /// remote/schedulable placement carrying a cluster node's `host_label` stamps
1843 /// that label; without a label it falls back to the endpoint host / pool; a
1844 /// Local child gets no stamp (the DTO defaults it to the backend host).
1845 #[test]
1846 fn placement_stamp_uses_node_label_for_remote_and_schedulable() {
1847 // Remote WITH a node label → {remote, <label>}, overriding the raw IP.
1848 let mut remote = HashMap::new();
1849 remote.insert(
1850 "explorer".to_string(),
1851 ResolvedRemotePlacement {
1852 endpoint: "ws://169.254.230.101:8899".into(),
1853 token: None,
1854 ca_cert_file: None,
1855 host_label: Some("mini".into()),
1856 },
1857 );
1858 let runner = bogus_runner(remote);
1859 let spec = runner.build_spec(&session_of_role("explorer", "go"), &job_for("c1"));
1860 let stamp = runner
1861 .placement_stamp_for(&spec)
1862 .expect("remote child is stamped");
1863 assert!(stamp.contains(r#""kind":"remote""#), "{stamp}");
1864 assert!(stamp.contains(r#""host":"mini""#), "{stamp}");
1865
1866 // Remote WITHOUT a node label → falls back to the endpoint host.
1867 let mut remote_nolabel = HashMap::new();
1868 remote_nolabel.insert(
1869 "explorer".to_string(),
1870 ResolvedRemotePlacement {
1871 endpoint: "ws://169.254.230.101:8899".into(),
1872 token: None,
1873 ca_cert_file: None,
1874 host_label: None,
1875 },
1876 );
1877 let r2 = bogus_runner(remote_nolabel);
1878 let spec2 = r2.build_spec(&session_of_role("explorer", "go"), &job_for("c1"));
1879 assert!(r2
1880 .placement_stamp_for(&spec2)
1881 .unwrap()
1882 .contains(r#""host":"169.254.230.101""#));
1883
1884 // Schedulable WITH a node label → {remote, <label>} (a node, not a pool name).
1885 let mut sched = HashMap::new();
1886 sched.insert(
1887 "mac-mini-monitor".to_string(),
1888 ResolvedSchedulablePlacement {
1889 pool: "mac-mini-monitor".into(),
1890 host_label: Some("mini".into()),
1891 },
1892 );
1893 let sr = bogus_sched_runner(HashMap::new(), sched);
1894 let spec3 = sr.build_spec(&session_of_role("mac-mini-monitor", "go"), &job_for("c1"));
1895 let stamp3 = sr
1896 .placement_stamp_for(&spec3)
1897 .expect("scheduled child is stamped");
1898 assert!(stamp3.contains(r#""kind":"remote""#), "{stamp3}");
1899 assert!(stamp3.contains(r#""host":"mini""#), "{stamp3}");
1900
1901 // A Local (unmatched) child gets NO stamp.
1902 let local = bogus_runner(HashMap::new());
1903 let spec4 = local.build_spec(&session_of_role("writer", "go"), &job_for("c1"));
1904 assert_eq!(local.placement_stamp_for(&spec4), None);
1905 }
1906
1907 // ---- #181: schedulable selection over the BUS (Phase 3 cutover) ----------
1908
1909 async fn start_bus() -> (String, tempfile::TempDir) {
1910 let dir = tempfile::tempdir().unwrap();
1911 let core = std::sync::Arc::new(bamboo_broker::BrokerCore::new(dir.path()));
1912 let server = std::sync::Arc::new(bamboo_broker::BrokerServer::new(core, "t"));
1913 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
1914 let addr = listener.local_addr().unwrap();
1915 tokio::spawn(async move {
1916 let _ = server.serve(listener).await;
1917 });
1918 (format!("ws://{addr}"), dir)
1919 }
1920
1921 async fn join_pool(endpoint: &str, id: &str, pool: &str) -> bamboo_broker::BrokerClient {
1922 let mut c = bamboo_broker::BrokerClient::connect(
1923 endpoint,
1924 bamboo_subagent::AgentRef {
1925 session_id: id.into(),
1926 role: Some(pool.into()),
1927 },
1928 "t",
1929 )
1930 .await
1931 .unwrap();
1932 c.subscribe().await.unwrap();
1933 c
1934 }
1935
1936 fn sched_runner_on_bus(endpoint: &str, child_role: &str, pool: &str) -> ActorChildRunner {
1937 let mut sched = HashMap::new();
1938 sched.insert(child_role.to_string(), sched_placement(pool, "unused"));
1939 bogus_sched_runner(HashMap::new(), sched).with_bus(Some(bamboo_subagent::BusEndpoint {
1940 endpoint: endpoint.into(),
1941 token: "t".into(),
1942 }))
1943 }
1944
1945 #[tokio::test]
1946 async fn resolve_schedulable_picks_a_live_bus_worker() {
1947 let (endpoint, _dir) = start_bus().await;
1948 let _w = join_pool(&endpoint, "w-gpu", "gpu-pool").await;
1949 let runner = sched_runner_on_bus(&endpoint, "explorer", "gpu-pool");
1950
1951 let mailbox = runner
1952 .resolve_schedulable_worker("explorer")
1953 .await
1954 .expect("a live pool worker is found on the bus");
1955 assert_eq!(mailbox, "w-gpu");
1956 }
1957
1958 #[tokio::test]
1959 async fn resolve_schedulable_round_robins_over_pool_workers() {
1960 let (endpoint, _dir) = start_bus().await;
1961 let _a = join_pool(&endpoint, "w-a", "gpu-pool").await;
1962 let _b = join_pool(&endpoint, "w-b", "gpu-pool").await;
1963 let runner = sched_runner_on_bus(&endpoint, "explorer", "gpu-pool");
1964
1965 // Successive resolves spread across both connected workers.
1966 let mut picked = std::collections::HashSet::new();
1967 for _ in 0..6 {
1968 picked.insert(runner.resolve_schedulable_worker("explorer").await.unwrap());
1969 }
1970 assert_eq!(
1971 picked,
1972 ["w-a".to_string(), "w-b".to_string()].into_iter().collect(),
1973 "round-robin must cover every connected pool worker"
1974 );
1975 }
1976
1977 #[tokio::test]
1978 async fn resolve_schedulable_errors_on_empty_pool() {
1979 let (endpoint, _dir) = start_bus().await;
1980 // No worker subscribes to "gpu-pool".
1981 let runner = sched_runner_on_bus(&endpoint, "explorer", "gpu-pool");
1982
1983 let err = runner
1984 .resolve_schedulable_worker("explorer")
1985 .await
1986 .expect_err("an empty pool is terminal — no local fallback")
1987 .to_string();
1988 assert!(err.contains("no live worker in pool"), "got: {err}");
1989 assert!(err.contains("NOT spawning"), "got: {err}");
1990 }
1991
1992 /// FULL schedulable run over the bus: a worker SERVING `EchoExecutor` joins the
1993 /// pool by role; `execute_external_child` with a Schedulable placement resolves
1994 /// it from the bus (no local subprocess — the worker_bin is `/bin/false`),
1995 /// drives the run, gets the echo back, AND stamps the child session with the
1996 /// pool's cluster-node label — `{kind:remote, host:"mini"}`. The end-to-end
1997 /// analogue of the live `mac-mini-monitor`→mini run.
1998 #[tokio::test]
1999 async fn execute_external_child_runs_schedulable_over_bus_and_stamps_node_label() {
2000 let (endpoint, _dir) = start_bus().await;
2001
2002 // A bus worker SERVING runs (not just presence), joined to the pool by role.
2003 let ep = endpoint.clone();
2004 let worker = tokio::spawn(async move {
2005 let _ = bamboo_broker::serve_executor(
2006 &ep,
2007 bamboo_subagent::AgentRef {
2008 session_id: "mmm-worker".into(),
2009 role: Some("mac-mini-monitor".into()),
2010 },
2011 "t",
2012 std::sync::Arc::new(bamboo_subagent::executor::EchoExecutor),
2013 )
2014 .await;
2015 });
2016
2017 // Wait until the worker is visible on the bus so the pool is non-empty
2018 // when execute_external_child resolves it (serve_executor connects async).
2019 let mut probe = bamboo_broker::BrokerClient::connect(
2020 &endpoint,
2021 bamboo_subagent::AgentRef {
2022 session_id: "probe".into(),
2023 role: None,
2024 },
2025 "t",
2026 )
2027 .await
2028 .unwrap();
2029 let mut ready = false;
2030 for _ in 0..100 {
2031 if probe
2032 .list_connected("mac-mini-monitor")
2033 .await
2034 .unwrap()
2035 .iter()
2036 .any(|id| id == "mmm-worker")
2037 {
2038 ready = true;
2039 break;
2040 }
2041 tokio::time::sleep(Duration::from_millis(30)).await;
2042 }
2043 assert!(ready, "worker never joined the pool");
2044
2045 // Runner: child role → schedulable pool "mac-mini-monitor" carrying the
2046 // cluster node's label "mini"; bogus worker_bin so any local spawn fails.
2047 let mut sched = HashMap::new();
2048 sched.insert(
2049 "mac-mini-monitor".to_string(),
2050 ResolvedSchedulablePlacement {
2051 pool: "mac-mini-monitor".into(),
2052 host_label: Some("mini".into()),
2053 },
2054 );
2055 let runner = bogus_sched_runner(HashMap::new(), sched).with_bus(Some(
2056 bamboo_subagent::BusEndpoint {
2057 endpoint: endpoint.clone(),
2058 token: "t".into(),
2059 },
2060 ));
2061
2062 let mut session = session_of_role("mac-mini-monitor", "hello scheduled");
2063 let job = job_for("child-1");
2064 let (event_tx, _rx) = mpsc::channel::<AgentEvent>(64);
2065 let cancel = CancellationToken::new();
2066
2067 tokio::time::timeout(
2068 Duration::from_secs(10),
2069 runner.execute_external_child(&mut session, &job, event_tx, cancel),
2070 )
2071 .await
2072 .expect("run did not hang")
2073 .expect("schedulable run succeeded over the bus (no local spawn)");
2074
2075 // Echo reply flowed back — proves it routed to the bus worker, not local.
2076 let last = session
2077 .messages
2078 .iter()
2079 .rev()
2080 .find(|m| matches!(m.role, Role::Assistant))
2081 .expect("an assistant reply was written back");
2082 assert!(last.content.contains("echo:"), "got {:?}", last.content);
2083
2084 // ...and the child is stamped with the pool's cluster-node label.
2085 let placement = session
2086 .metadata
2087 .get("placement")
2088 .expect("scheduled child session stamped with a placement");
2089 assert!(placement.contains(r#""kind":"remote""#), "{placement}");
2090 assert!(placement.contains(r#""host":"mini""#), "{placement}");
2091
2092 worker.abort();
2093 }
2094}