bamboo_engine/external_agents/actor_adapter.rs
1//! Actor external child runner.
2//!
3//! Runs a child session as an independent **actor**: a separate OS process with its own
4//! isolated context, speaking the `bamboo-subagent` WebSocket protocol. This is the
5//! engine-side adapter on the `wants_external` seam: it spawns the worker binary, waits for
6//! it to self-register into the Tier-1 file fabric, connects, sends the assignment, and
7//! forwards the child's `AgentEvent`s back onto the parent's `event_tx`.
8//!
9//! The built-in **local actor** instance of this runner is the default runtime for
10//! every sub-agent (the in-process runtime was removed). The expert `externalAgents`
11//! tables can additionally route specific roles to other actor/a2a agents.
12
13use std::collections::HashMap;
14use std::path::PathBuf;
15use std::sync::Arc;
16use std::time::Duration;
17
18use async_trait::async_trait;
19use bamboo_agent_core::{AgentError, AgentEvent, Role, Session};
20use bamboo_domain::poison::PoisonRecover;
21use tokio::sync::mpsc;
22use tokio_util::sync::CancellationToken;
23
24use bamboo_subagent::fleet::{spawn_worker_on_bus, SpawnedChild};
25use bamboo_subagent::proto::{
26 AgentRecord, ChildFrame, ParentFrame, PermissionPolicyContext, RunSpec, TerminalStatus,
27};
28use bamboo_subagent::provision::{
29 ChildIdentity, ExecutorSpec, ModelRefSpec, Placement, ProvisionSpec, ScopedCredential,
30};
31use bamboo_subagent::transport::{client_config_trusting_cert, ChildClient};
32
33use crate::runtime::execution::{ExternalChildRunner, SpawnJob};
34
35/// Default cap on simultaneously running actor processes.
36pub const DEFAULT_MAX_CONCURRENT_ACTORS: usize = 8;
37
38/// Max nesting depth for direct nested execution (Phase 6). A worker whose
39/// session `spawn_depth` is below this gets its own spawn stack + the real
40/// SubAgent tool; at/over it, neither (and the tool itself refuses). Mirrors
41/// `bamboo_server_tools::DEFAULT_MAX_SPAWN_DEPTH` (kept in sync; engine can't
42/// depend on server-tools). Root orchestrator = 0 ⇒ 4 levels of sub-agents.
43pub const MAX_SPAWN_DEPTH: u32 = 4;
44
45/// Default cap on idle pooled (warm, reusable) workers kept per fingerprint.
46const DEFAULT_MAX_IDLE_PER_KEY: usize = 4;
47
48/// How long a pooled worker waits for its next assignment before reclaiming
49/// itself (must comfortably exceed the gap between sibling spawns).
50const POOLED_IDLE_TIMEOUT_SECS: u64 = 300;
51
52/// Deadline for a local worker's FIRST frame after a Run is dispatched. A warm
53/// worker answers in seconds; a cold spawn within tens. Total silence past this
54/// means the worker is dead (e.g. a pooled worker that exited right after its
55/// liveness check) and its Run is queued with nobody to serve it — trip it so the
56/// runner respawns once instead of hanging forever. Generous, to never false-trip
57/// a slow-but-healthy cold start.
58const WORKER_FIRST_FRAME_TIMEOUT: Duration = Duration::from_secs(60);
59
60/// A warm worker on the mailbox bus, parked for reuse between runs. It stays
61/// dialed-in + subscribed to `mailbox_id`; the next interchangeable child
62/// delivers its `Run` there instead of spawning a fresh process. Dropping it
63/// kills a local kill-on-drop subprocess; a remote / schedulable handle is
64/// process-less (`kill()` is a no-op — it self-manages via its idle timeout).
65struct PooledWorker {
66 worker: SpawnedChild,
67 /// The bus mailbox this worker subscribes to (where its `Run`s are delivered).
68 mailbox_id: String,
69}
70
71/// A role pinned to a remote resident worker (remote-actor-plan §3.4 / P1.5,
72/// #193), resolved at runner-build time from `SubagentsConfig.remote_placements`:
73/// the env-named bearer is already READ into `token` here (the raw token never
74/// rides the config), and `ca_cert_file` is the path to a PEM pinning a
75/// self-signed worker cert (`None` ⇒ default webpki roots / plaintext `ws://`).
76#[derive(Debug, Clone)]
77pub struct ResolvedRemotePlacement {
78 pub endpoint: String,
79 pub token: Option<String>,
80 pub ca_cert_file: Option<PathBuf>,
81 /// Display name for the machine this role runs on — the matching cluster
82 /// node's `label`/host, surfaced on the UI placement badge. `None` ⇒ derive
83 /// from the endpoint host.
84 pub host_label: Option<String>,
85}
86
87/// A role routed to a SCHEDULED worker (remote-actor-plan §3.4 / P2b, #181),
88/// resolved at runner-build time from `SubagentsConfig.schedulable_placements`.
89/// Names the logical `pool` (= the bus role) whose LIVE connected workers are the
90/// scheduling candidates — the runner picks one via the bus presence query
91/// (`BrokerClient::list_connected`). Phase 3 retired the old HTTP registry, so a
92/// pool is now just a role on the bus.
93#[derive(Debug, Clone)]
94pub struct ResolvedSchedulablePlacement {
95 pub pool: String,
96 /// Display name for the machine this pool's workers run on — the matching
97 /// cluster node's `label`/host, surfaced on the UI placement badge. `None` ⇒
98 /// fall back to the pool name.
99 pub host_label: Option<String>,
100}
101
102/// How `execute_external_child` should obtain its worker connection, decided
103/// once from `spec.placement`. Splits the divergent acquire/connect + retire
104/// logic three ways while the shared middle (Run dispatch, live registration,
105/// drive, close) stays identical. `Local` is the unchanged pre-#193 path;
106/// `Remote` is the unchanged #194 path; `Schedulable` (#181, P2b) is new.
107enum PlacementKind {
108 Local,
109 Remote,
110 Schedulable,
111}
112
113/// Spawns and drives a child session as an independent actor: a `bamboo-subagent` worker process.
114pub struct ActorChildRunner {
115 approval_registry: Option<super::approval_registry::SharedApprovalRegistry>,
116 permission_config: Option<Arc<bamboo_tools::permission::PermissionConfig>>,
117 agent_id: String,
118 worker_bin: PathBuf,
119 worker_args: Vec<String>,
120 fabric_dir: PathBuf,
121 executor: ExecutorSpec,
122 /// Per-provider credentials snapshotted from the parent config at build
123 /// time; the spec carries only the ONE the child's provider needs.
124 credentials: Vec<ScopedCredential>,
125 /// Parent's default provider (used when the child has no explicit one).
126 default_provider: String,
127 /// The mailbox bus to run local children over (the unified transport). Local
128 /// sub-agents require it; `None` only when no broker could be embedded.
129 bus: Option<bamboo_subagent::BusEndpoint>,
130 /// Backpressure: bounds the number of concurrently *running* actors; further
131 /// runs wait for a slot instead of exploding the process table. (Idle pooled
132 /// workers do not hold a slot.)
133 concurrency: std::sync::Arc<tokio::sync::Semaphore>,
134 /// Warm-worker pool keyed by a reuse fingerprint
135 /// (role/provider/model/workspace/disabled-tools/baked-caps). A finished run
136 /// parks its bus worker here so the next interchangeable child reuses it
137 /// (delivers its `Run` to the same mailbox) instead of spawning a fresh
138 /// process — collapsing N sibling sub-agents onto a few warm workers.
139 pool: Arc<tokio::sync::Mutex<HashMap<String, Vec<PooledWorker>>>>,
140 max_idle_per_key: usize,
141 /// Host-side decision for a child's gated-tool approval request (Phase 2).
142 /// `None` ⇒ fail-closed DENY (the safe default). A wired decider (policy or
143 /// human-routing bridge) returns approve/deny over the actor WS.
144 approval_decider: Option<Arc<dyn ChildApprovalDecider>>,
145 /// Off-loop parent-agent reviewer for forced-ask requests. The root server
146 /// wires a session-aware reviewer; nested workers wire their owning model
147 /// reviewer directly into the per-run runner.
148 approval_reviewer: Option<Arc<dyn ChildApprovalReviewer>>,
149 /// Per-run escalation host bridge for non-bypass child-approval routing (#68;
150 /// Phase 6, Part B). The owning worker's `run()` installs its OWN host bridge
151 /// here via `set_escalation_bridge`; `execute_external_child` CAPTURES it at
152 /// grandchild-spawn time and hands the owned value to `drive()`, which uses it
153 /// to RE-PROXY a child's approval request UP to the parent run — chaining up
154 /// every level until a bypass level (model-review) or the top orchestrator
155 /// (human) decides, then relaying the reply back down. Was a process-global
156 /// slot; now per-runner so a fire-and-forget grandchild that OUTLIVES the run
157 /// that spawned it keeps that run's bridge for its whole lifetime instead of
158 /// reading a stale/overwritten global at approval time (→ fail-closed deny).
159 escalation_bridge: Arc<std::sync::Mutex<Option<bamboo_subagent::executor::HostBridge>>>,
160 /// Roles pinned to a REMOTE resident worker (#193), keyed by sub-agent role
161 /// (the child's `subagent_type`). A role present here routes through the
162 /// dedicated remote branch in `execute_external_child` (Bearer-authenticated
163 /// `wss://` connect, no spawn, no pool, no kill) instead of the local
164 /// subprocess + warm-pool path. Empty (the default) = all-local behavior.
165 remote_placements: HashMap<String, ResolvedRemotePlacement>,
166 /// Roles routed to a REGISTRY-SCHEDULED worker (#181, P2b), keyed by sub-agent
167 /// role. A role present here (AND not already in `remote_placements`, which
168 /// wins) routes through the dedicated SCHEDULABLE branch in
169 /// `execute_external_child`: query the registry for live workers in the pool,
170 /// pick one (round-robin), connect over `wss://` — no spawn, no pool, no kill,
171 /// and NO local-subprocess fallback (no live worker ⇒ a clear error). Empty
172 /// (the default) = all-local behavior.
173 schedulable_placements: HashMap<String, ResolvedSchedulablePlacement>,
174 /// Per-pool round-robin cursor for schedulable scheduling (#181, P2b). Bumped
175 /// once per pick so successive sibling spawns SPREAD across a pool's live
176 /// workers instead of all landing on the first candidate. Best-effort spread,
177 /// not a load balancer — the registry's live set can change between picks.
178 schedule_cursor: Arc<std::sync::Mutex<HashMap<String, usize>>>,
179}
180
181/// Decides how the host answers a child worker's gated-tool approval request
182/// (Phase 2: child → parent approval delegation). Async so an implementation
183/// can consult a policy or defer to a human. With no decider wired the host
184/// replies with a fail-closed DENY.
185///
186/// NOTE: `decide` is awaited inside the per-child frame pump, so an
187/// implementation must resolve promptly (e.g. a policy lookup). A human-in-the-
188/// loop decision that may block indefinitely should instead be delivered
189/// out-of-band as a `ParentFrame::ApprovalReply` via the live steering channel
190/// (`super::live`), which `drive()` already forwards to the worker without
191/// stalling the pump.
192#[async_trait]
193pub trait ChildApprovalDecider: Send + Sync {
194 /// Decide whether `child_session_id` may perform the gated action described
195 /// by `request` (`{tool_name, permission, resource}`).
196 async fn decide(&self, child_session_id: &str, request: &serde_json::Value) -> bool;
197}
198
199/// Resolve a child approval request to approve/deny. Fail-closed (DENY) when no
200/// decider is wired — the single, testable seam for the host-side decision.
201async fn decide_child_approval(
202 decider: Option<&Arc<dyn ChildApprovalDecider>>,
203 child_session_id: &str,
204 request: &serde_json::Value,
205) -> bool {
206 match decider {
207 Some(decider) => decider.decide(child_session_id, request).await,
208 None => false,
209 }
210}
211
212/// How long the host waits for a human approval decision before failing the
213/// child's gated tool closed (DENY). Bounds an unanswered request so it can't
214/// hang the worker indefinitely.
215const CHILD_APPROVAL_TIMEOUT: Duration = Duration::from_secs(300);
216
217/// Extract `(tool_name, permission, resource)` from a worker's approval request
218/// body (`{tool_name, permission, resource}`); missing fields default to empty.
219fn approval_request_fields(body: &serde_json::Value) -> (String, String, String) {
220 let field = |k: &str| {
221 body.get(k)
222 .and_then(|v| v.as_str())
223 .unwrap_or("")
224 .to_string()
225 };
226 (field("tool_name"), field("permission"), field("resource"))
227}
228
229/// Off-loop reviewer for a child's gated-tool approval request (Phase 6, Part B).
230///
231/// Installed (process-global) by a BYPASSED self-orchestrating worker so its
232/// children's forced-ask (dangerous) gated actions — which still raise
233/// `ConfirmationRequired` even under bypass — get an LLM reasonableness check
234/// rather than a blind pass. `review` is an LLM call: `drive()` invokes it in a
235/// SPAWNED task (NEVER in the frame pump) and delivers the verdict async via the
236/// live channel, so the agent loop is never blocked.
237#[async_trait]
238pub trait ChildApprovalReviewer: Send + Sync {
239 /// Judge whether the gated action `request` (`{tool_name, permission,
240 /// resource}`) is reasonable for `child_session_id`'s task. `true` = approve.
241 async fn review(
242 &self,
243 parent_session_id: &str,
244 child_session_id: &str,
245 request: &serde_json::Value,
246 ) -> bool;
247}
248
249fn child_approval_reviewer_slot() -> &'static std::sync::OnceLock<Arc<dyn ChildApprovalReviewer>> {
250 static SLOT: std::sync::OnceLock<Arc<dyn ChildApprovalReviewer>> = std::sync::OnceLock::new();
251 &SLOT
252}
253
254/// Install the process-global child-approval reviewer (idempotent; first wins).
255pub fn set_child_approval_reviewer(reviewer: Arc<dyn ChildApprovalReviewer>) {
256 let _ = child_approval_reviewer_slot().set(reviewer);
257}
258
259/// The process-global child-approval reviewer, if installed.
260pub fn child_approval_reviewer() -> Option<Arc<dyn ChildApprovalReviewer>> {
261 child_approval_reviewer_slot().get().cloned()
262}
263
264impl ActorChildRunner {
265 #[allow(clippy::too_many_arguments)]
266 pub fn new(
267 agent_id: String,
268 worker_bin: PathBuf,
269 worker_args: Vec<String>,
270 fabric_dir: PathBuf,
271 executor: ExecutorSpec,
272 credentials: Vec<ScopedCredential>,
273 default_provider: String,
274 max_concurrent: usize,
275 ) -> Self {
276 Self {
277 approval_registry: None,
278 permission_config: None,
279 agent_id,
280 worker_bin,
281 worker_args,
282 fabric_dir,
283 executor,
284 credentials,
285 default_provider,
286 bus: None,
287 concurrency: std::sync::Arc::new(tokio::sync::Semaphore::new(max_concurrent.max(1))),
288 pool: Arc::new(tokio::sync::Mutex::new(HashMap::new())),
289 max_idle_per_key: DEFAULT_MAX_IDLE_PER_KEY,
290 approval_decider: None,
291 approval_reviewer: None,
292 escalation_bridge: Arc::new(std::sync::Mutex::new(None)),
293 remote_placements: HashMap::new(),
294 schedulable_placements: HashMap::new(),
295 schedule_cursor: Arc::new(std::sync::Mutex::new(HashMap::new())),
296 }
297 }
298
299 pub fn with_approval_registry(
300 mut self,
301 registry: super::approval_registry::SharedApprovalRegistry,
302 ) -> Self {
303 self.approval_registry = Some(registry);
304 self
305 }
306
307 pub fn with_permission_config(
308 mut self,
309 config: Arc<bamboo_tools::permission::PermissionConfig>,
310 ) -> Self {
311 self.permission_config = Some(config);
312 self
313 }
314
315 /// Run children over the mailbox bus (the unified actor+mailbox transport).
316 /// When set, local children dial this bus and are driven by mailbox id; when
317 /// unset they use the legacy direct-WS path. The server passes its in-process
318 /// broker here (`subagents.broker`); tests without a broker leave it unset.
319 pub fn with_bus(mut self, bus: Option<bamboo_subagent::BusEndpoint>) -> Self {
320 self.bus = bus.filter(|b| !b.endpoint.trim().is_empty());
321 self
322 }
323
324 /// Wire the host-side decider for child gated-tool approval requests
325 /// (Phase 2). Without this the host fail-closed DENYs every request.
326 pub fn with_approval_decider(mut self, decider: Arc<dyn ChildApprovalDecider>) -> Self {
327 self.approval_decider = Some(decider);
328 self
329 }
330
331 pub fn with_approval_reviewer(mut self, reviewer: Arc<dyn ChildApprovalReviewer>) -> Self {
332 self.approval_reviewer = Some(reviewer);
333 self
334 }
335
336 /// Pin specific sub-agent roles to remote resident workers (#193). The map
337 /// is keyed by role (`subagent_type`); a child whose role is present connects
338 /// over `wss://` to the resolved endpoint instead of spawning a local
339 /// subprocess. Default (empty) keeps every role on the local path — exactly
340 /// today's behavior.
341 pub fn with_remote_placements(
342 mut self,
343 placements: HashMap<String, ResolvedRemotePlacement>,
344 ) -> Self {
345 self.remote_placements = placements;
346 self
347 }
348
349 /// Route specific sub-agent roles to a registry-SCHEDULED worker (#181, P2b).
350 /// The map is keyed by role (`subagent_type`); a child whose role is present
351 /// (and NOT already pinned by `remote_placements`, which takes precedence) is
352 /// run on a live worker discovered from the registry instead of a local
353 /// subprocess. Default (empty) keeps every role on the local path.
354 pub fn with_schedulable_placements(
355 mut self,
356 placements: HashMap<String, ResolvedSchedulablePlacement>,
357 ) -> Self {
358 self.schedulable_placements = placements;
359 self
360 }
361
362 /// Reuse fingerprint: two children are interchangeable on one warm worker iff
363 /// they share role, provider, model, workspace, disabled-tool set, AND every
364 /// capability the worker BAKES at provision time (`BambooRuntimeExecutor`
365 /// stamps these once and reuses them across runs): nesting depth, nested-spawn
366 /// stack, bypass mode, permission enforcement, and the depth cap. Omitting any
367 /// of these lets the pool hand a run a worker baked for a DIFFERENT posture —
368 /// e.g. a depth-1 worker (with its own spawn stack) reused for a depth-4
369 /// child would re-stamp `spawn_depth=1` and pass the depth-cap check, breaking
370 /// the recursion bound; or a bypass worker reused for a non-bypass child. So
371 /// these MUST split the pool bucket. Everything else (assignment, history) is
372 /// shipped per-run in the `RunSpec` and does not affect the fingerprint.
373 /// Reuse fingerprint (role/provider/model/workspace/disabled-tools/baked
374 /// caps): two children with the same fingerprint are interchangeable on one
375 /// warm worker, so they share a pool bucket. Any axis the worker bakes ONCE
376 /// at provision time MUST be in here, else a worker baked for one posture
377 /// gets reused for another (see the `fingerprint_*` tests).
378 fn fingerprint(spec: &ProvisionSpec) -> String {
379 let role = spec.identity.role.as_str();
380 let (provider, model) = spec
381 .model
382 .as_ref()
383 .map(|m| (m.provider.as_str(), m.model.as_str()))
384 .unwrap_or(("", ""));
385 let workspace = spec.workspace.as_deref().unwrap_or("");
386 let mut tools = spec.disabled_tools.clone().unwrap_or_default();
387 tools.sort();
388 let caps = &spec.capabilities;
389 format!(
390 "{role}\u{1}{provider}\u{1}{model}\u{1}{workspace}\u{1}{}\u{1}d={}\u{1}ns={}\u{1}by={}\u{1}ep={}\u{1}md={}\u{1}nha={}\u{1}gro={}",
391 tools.join(","),
392 spec.identity.depth,
393 caps.nested_spawn,
394 caps.bypass,
395 caps.enforce_permissions,
396 caps.max_spawn_depth.unwrap_or(0),
397 // #73 review (P1): a worker bakes `no_human_review` ONCE from this flag
398 // at build() and never re-reads it per run, so the pool MUST NOT hand a
399 // worker baked for one approval posture to a run of the opposite one —
400 // else a scheduled-root worker reused for an interactive child would
401 // silently model-review instead of asking the human (and vice-versa,
402 // reintroducing the 300s-deny). Split the bucket on it.
403 caps.no_human_approver,
404 // #71: the read-only Bash checker is baked once at build() from this
405 // flag, so a guardian-reviewer worker must NOT be reused for an
406 // ordinary child (which expects unrestricted Bash), and vice-versa.
407 caps.guardian_read_only,
408 )
409 }
410
411 /// Check out a warm bus worker for `key`, reusing a live parked one if any,
412 /// else spawning a fresh one that dials the bus. The returned worker is OWNED
413 /// by the caller for the run's duration (checkout removes it from the pool, so
414 /// a concurrent sibling gets a different worker or spawns its own — one run per
415 /// worker at a time, matching the pre-bus pool semantics).
416 async fn acquire_bus_worker(
417 &self,
418 key: &str,
419 spec: &ProvisionSpec,
420 ) -> crate::runtime::runner::Result<PooledWorker> {
421 // Drain the bucket, skipping (and reaping) any worker whose process exited
422 // while parked. A live one is handed straight out for reuse.
423 loop {
424 let candidate = {
425 let mut pool = self.pool.lock().await;
426 pool.get_mut(key).and_then(|bucket| bucket.pop())
427 };
428 let Some(mut candidate) = candidate else {
429 break;
430 };
431 if candidate.worker.is_alive() {
432 return Ok(candidate);
433 }
434 candidate.worker.kill().await;
435 }
436
437 let spawned = spawn_worker_on_bus(&self.worker_bin, &self.worker_args, spec)
438 .await
439 .map_err(|e| AgentError::LLM(format!("actor spawn (bus) failed: {e}")))?;
440 let mailbox_id = spawned.record.agent_id.clone();
441 Ok(PooledWorker {
442 worker: spawned,
443 mailbox_id,
444 })
445 }
446
447 /// Park a warm bus worker for reuse after a clean run; if its bucket is full
448 /// (or it died), kill it instead. The worker stays dialed-in + subscribed
449 /// while parked, so a reusing child just delivers a new `Run` to its mailbox.
450 async fn release_bus_worker(&self, key: &str, mut worker: PooledWorker) {
451 if !worker.worker.is_alive() {
452 worker.worker.kill().await;
453 return;
454 }
455 let mut pool = self.pool.lock().await;
456 let bucket = pool.entry(key.to_string()).or_default();
457 if bucket.len() >= self.max_idle_per_key {
458 drop(pool);
459 worker.worker.kill().await;
460 return;
461 }
462 bucket.push(worker);
463 }
464
465 /// Assemble the parent-resolved provisioning document for this child.
466 fn build_spec(&self, session: &Session, job: &SpawnJob) -> ProvisionSpec {
467 let mut spec = ProvisionSpec::new(
468 ChildIdentity {
469 child_id: job.child_session_id.clone(),
470 parent_id: Some(job.parent_session_id.clone()),
471 project_key: None,
472 role: session
473 .metadata
474 .get("subagent_type")
475 .cloned()
476 .unwrap_or_else(|| "worker".to_string()),
477 // The child session already carries the correct depth
478 // (create_child_action's new_child_of did parent.spawn_depth+1);
479 // stamp it so the worker can re-establish it on its run session
480 // and enforce the max-depth cap across the actor boundary.
481 depth: session.spawn_depth,
482 },
483 self.executor.clone(),
484 self.fabric_dir.to_string_lossy().into_owned(),
485 );
486 spec.workspace = session.workspace.clone();
487 // Unified transport: when a bus is configured, the child dials it (no
488 // listen socket / file discovery) and the parent drives it by mailbox id.
489 spec.bus = self.bus.clone();
490 // Final model: the session's pinned model_ref (create.model / routing already applied),
491 // falling back to the job's bare model on the parent's default provider.
492 spec.model = session
493 .model_ref
494 .as_ref()
495 .map(|r| ModelRefSpec {
496 provider: r.provider.clone(),
497 model: r.model.clone(),
498 })
499 .or_else(|| {
500 let m = job.model.trim();
501 (!m.is_empty()).then(|| ModelRefSpec {
502 provider: self.default_provider.clone(),
503 model: m.to_string(),
504 })
505 });
506 spec.disabled_tools = job.disabled_tools.clone();
507 // Least-privilege secrets: only the credential for the child's provider.
508 let provider = spec
509 .model
510 .as_ref()
511 .map(|m| m.provider.as_str())
512 .filter(|p| !p.trim().is_empty())
513 .unwrap_or(&self.default_provider);
514 if let Some(cred) = self.credentials.iter().find(|c| c.provider == provider) {
515 spec.secrets.provider_credentials.push(cred.clone());
516 } else {
517 tracing::warn!(
518 "actor child {}: no credential found for provider '{}'",
519 job.child_session_id,
520 provider
521 );
522 }
523 // Phase 6 (direct nested execution): a worker BELOW the depth cap may
524 // orchestrate its OWN children — on startup it builds its own spawn
525 // stack and runs the real SubAgent tool (no host proxy). The cap (the
526 // SubAgent tool refuses to spawn at/over `max_spawn_depth`) bounds the
527 // recursion. Driven purely by the child's depth, so it auto-propagates
528 // down the tree without any extra config threading.
529 spec.capabilities.nested_spawn = session.spawn_depth < MAX_SPAWN_DEPTH;
530 spec.capabilities.max_spawn_depth = Some(MAX_SPAWN_DEPTH);
531 // #69: activate child-approval review. Sub-agents enforce permissions so
532 // their DANGEROUS actions (the worker uses a HIGH threshold) reach the
533 // parent for review — escalated to the human, or model-reviewed off-loop
534 // when the parent is in bypass. The worker installs no checker without
535 // this, so the whole review chain would otherwise stay dormant.
536 spec.capabilities.enforce_permissions = true;
537 // Propagate "bypass permissions" so a self-orchestrating worker knows it
538 // is a bypassed parent and installs the off-loop model-reviewer for its
539 // children's forced-ask actions (Phase 6, Part B). The child session
540 // already carries the inherited flag (create_child_action seeds it).
541 spec.capabilities.bypass = session
542 .agent_runtime_state
543 .as_ref()
544 .is_some_and(|s| s.bypass_permissions);
545 // #73: propagate "no interactive human approver" (headless / scheduled /
546 // deployed root, inherited by the child session). When set, the worker's
547 // per-run approval proxy model-reviews a gated action locally instead of
548 // escalating to a human who will never answer (which would 300s-deny).
549 spec.capabilities.no_human_approver = session
550 .agent_runtime_state
551 .as_ref()
552 .is_some_and(|s| s.no_human_approver);
553 // #71: mark a READ-ONLY Guardian reviewer so the worker installs the
554 // read-only Bash allowlist checker. The reviewer is spawned by
555 // `spawn_guardian_review` with `subagent_type == "guardian"` (the SAME
556 // marker the completion coordinator branches on to parse the verdict) AND
557 // the `guardian_read_only_disabled_tools` denylist. Keyed off that role
558 // marker (already read above to set `identity.role`), so it rides the same
559 // session-metadata path the denylist/subagent_type use — no new wire seam.
560 // Without this the worker keeps an UNRESTRICTED Bash, so the reviewer could
561 // still `rm -rf` / `git push` / `curl | sh`, defeating "read-only".
562 spec.capabilities.guardian_read_only =
563 session.metadata.get("subagent_type").map(String::as_str) == Some("guardian");
564 // #193: route this role to a REMOTE resident worker when one is pinned.
565 // `spec.identity.role` was just computed from `subagent_type` above; a
566 // match flips the placement to Remote and rides the worker's bearer on the
567 // scoped secrets envelope (TLS handshake / Authorization header only — the
568 // token is never logged). No match leaves the default `Placement::Local`,
569 // so the local path is byte-for-byte unchanged for every non-pinned role.
570 if let Some(placement) = self.remote_placements.get(spec.identity.role.as_str()) {
571 spec.placement = Placement::Remote {
572 endpoint: placement.endpoint.clone(),
573 };
574 spec.secrets.worker_auth_token = placement.token.clone();
575 } else if let Some(placement) = self.schedulable_placements.get(spec.identity.role.as_str())
576 {
577 // #181 (P2b): route this role to a SCHEDULED worker — ONLY when it is
578 // NOT already pinned to a fixed remote endpoint (the `else if` makes
579 // remote_placements take precedence for a role in both). The concrete
580 // worker is picked at run time in `execute_external_child` from the bus
581 // (a live connected worker of the pool role). No per-placement bearer
582 // now — the bus connection uses the bus token. No match in either map
583 // leaves the default `Placement::Local`.
584 spec.placement = Placement::Schedulable {
585 pool: placement.pool.clone(),
586 };
587 }
588 spec
589 }
590
591 /// The `metadata["placement"]` JSON to stamp on a child from its resolved
592 /// placement, preferring the matching cluster node's `host_label` (its
593 /// operator label/host) over the raw endpoint/pool. `None` for a Local child
594 /// (the DTO defaults it to the backend's own host). Split out of
595 /// `execute_external_child` so the role→placement→host resolution is unit-testable.
596 fn placement_stamp_for(&self, spec: &ProvisionSpec) -> Option<String> {
597 let host_label = match &spec.placement {
598 Placement::Remote { .. } => self
599 .remote_placements
600 .get(spec.identity.role.as_str())
601 .and_then(|p| p.host_label.as_deref()),
602 Placement::Schedulable { .. } => self
603 .schedulable_placements
604 .get(spec.identity.role.as_str())
605 .and_then(|p| p.host_label.as_deref()),
606 Placement::Local => None,
607 };
608 placement_metadata(&spec.placement, host_label)
609 }
610
611 /// Pick a live worker for a SCHEDULABLE role from the BUS (#181, Phase 3):
612 /// ask the broker which actors are connected serving the pool role (presence
613 /// is connection-truth — no HTTP registry, no leases, no connect-fail
614 /// failover), then round-robin one per resolve for spread. Returns the chosen
615 /// worker's mailbox id. An empty pool ⇒ a terminal `AgentError` — NEVER a
616 /// local-subprocess fallback (that would silently defeat the placement).
617 async fn resolve_schedulable_worker(
618 &self,
619 role: &str,
620 ) -> std::result::Result<String, AgentError> {
621 let pool = self
622 .schedulable_placements
623 .get(role)
624 .ok_or_else(|| {
625 AgentError::LLM(format!(
626 "schedulable placement for role '{role}' vanished before scheduling"
627 ))
628 })?
629 .pool
630 .clone();
631 let bus = self.bus.as_ref().ok_or_else(|| {
632 AgentError::LLM(format!(
633 "schedulable role '{role}': no mailbox bus configured (subagents.broker)"
634 ))
635 })?;
636
637 // Ask the BUS who is connected serving the pool role — presence is
638 // connection-truth (no HTTP registry, no leases, no stale-record failover).
639 let mut q = bamboo_broker::BrokerClient::connect(
640 &bus.endpoint,
641 bamboo_subagent::AgentRef {
642 session_id: format!("sched-q-{role}"),
643 role: None,
644 },
645 &bus.token,
646 )
647 .await
648 .map_err(|e| {
649 AgentError::LLM(format!(
650 "schedulable role '{role}': bus connect failed: {e}"
651 ))
652 })?;
653 let candidates = q.list_connected(&pool).await.map_err(|e| {
654 AgentError::LLM(format!(
655 "schedulable role '{role}': bus presence query failed: {e}"
656 ))
657 })?;
658
659 if candidates.is_empty() {
660 return Err(AgentError::LLM(format!(
661 "schedulable role '{role}': no live worker in pool '{pool}' on the bus \
662 (NOT spawning a local subprocess — a schedulable role has no local fallback)"
663 )));
664 }
665
666 // Round-robin: advance a per-pool cursor once per resolve so successive
667 // sibling spawns spread across the connected pool workers. No failover
668 // needed — a listed worker is connected NOW (the bus only lists live
669 // subscribers), so there is no stale-but-leased candidate to skip.
670 let idx = {
671 let mut cursors = self.schedule_cursor.lock().recover_poison();
672 let cursor = cursors.entry(pool.clone()).or_insert(0);
673 let i = *cursor % candidates.len();
674 *cursor = cursor.wrapping_add(1);
675 i
676 };
677 Ok(candidates[idx].clone())
678 }
679}
680
681#[async_trait]
682impl ExternalChildRunner for ActorChildRunner {
683 async fn should_handle(&self, session: &Session) -> bool {
684 session.metadata.get("runtime.kind") == Some(&"external".to_string())
685 && session.metadata.get("external.protocol") == Some(&"actor".to_string())
686 && session.metadata.get("external.agent_id") == Some(&self.agent_id)
687 }
688
689 fn set_escalation_bridge(&self, bridge: Option<bamboo_subagent::executor::HostBridge>) {
690 *self.escalation_bridge.lock().recover_poison() = bridge;
691 }
692
693 async fn execute_external_child(
694 &self,
695 session: &mut Session,
696 job: &SpawnJob,
697 event_tx: mpsc::Sender<AgentEvent>,
698 cancel_token: CancellationToken,
699 ) -> crate::runtime::runner::Result<()> {
700 // #68 CORRECTNESS CRUX: capture the per-run escalation bridge HERE, at the
701 // moment this grandchild is spawned — while the parent run's bridge is
702 // still in our slot — into an owned local handed to `drive()` for this
703 // grandchild's whole lifetime. A fire-and-forget grandchild that OUTLIVES
704 // the run that spawned it must NOT re-read `self.escalation_bridge` at
705 // approval time: by then `run()` may have cleared/overwritten it (a worker
706 // serves runs sequentially), and re-proxying through a closed bridge
707 // fail-closed denies. Capturing at spawn pins the right bridge per run.
708 let escalation = self.escalation_bridge.lock().recover_poison().clone();
709 let assignment = extract_assignment(session);
710 let mut spec = self.build_spec(session, job);
711 // Mark the worker reusable + give it an idle timeout so it self-reaps if
712 // orphaned. Warm bus workers are pooled per fingerprint and reused.
713 spec.reusable = true;
714 if spec.limits.idle_timeout_secs.is_none() {
715 spec.limits.idle_timeout_secs = Some(POOLED_IDLE_TIMEOUT_SECS);
716 }
717 let pool_key = Self::fingerprint(&spec);
718 // Rehydration: the child session in the parent's store is the actor's
719 // durable state. Ship the full conversation so a reactivation
720 // (send_message / update / rerun) carries its history. A reused worker is
721 // stateless between runs, so this is also what isolates each child's
722 // context on a shared process.
723 let messages: Vec<serde_json::Value> = session
724 .messages
725 .iter()
726 .filter_map(|m| serde_json::to_value(m).ok())
727 .collect();
728 // Policy is captured per activation (not only when a worker is
729 // provisioned), so reused local workers and resident remote/broker
730 // workers observe the latest durable revision and bypass flag at the
731 // next run boundary. Session grants are intentionally not inherited.
732 let permission_policy = self.permission_config.as_ref().and_then(|config| {
733 serde_json::to_value(config.to_serializable())
734 .ok()
735 .map(|policy| PermissionPolicyContext {
736 revision: config.policy_revision(),
737 bypass_permissions: session
738 .agent_runtime_state
739 .as_ref()
740 .is_some_and(|state| state.bypass_permissions),
741 session_id: session.id.clone(),
742 workspace_path: session.workspace.clone(),
743 inherit_session_grants: false,
744 policy,
745 })
746 });
747
748 // Backpressure: hold a concurrency slot for the lifetime of the *run*
749 // (cancellation still proceeds — the cancel branch in drive() runs while
750 // we hold the permit). Released when this fn returns, i.e. once the worker
751 // is parked back into the pool, so idle workers don't pin slots.
752 let _slot = self
753 .concurrency
754 .acquire()
755 .await
756 .map_err(|_| AgentError::LLM("actor concurrency limiter closed".to_string()))?;
757
758 // Split LOCAL (spawn + warm-pool) from the two process-less remote paths
759 // ONLY at the divergent spots — acquire/connect here and the park/retire at
760 // the end. Everything between (Run dispatch, live-actor registration,
761 // drive, the close) is identical for all three. `kind` is the single guard.
762 // - Local (#0): byte-for-byte the pre-#193 reuse-or-spawn path.
763 // - Remote (#194): connect to a FIXED resident endpoint, no spawn.
764 // - Schedulable (#181): resolve a live worker from the registry, connect.
765 let kind = match spec.placement {
766 Placement::Remote { .. } => PlacementKind::Remote,
767 Placement::Schedulable { .. } => PlacementKind::Schedulable,
768 Placement::Local => PlacementKind::Local,
769 };
770 let remote = !matches!(kind, PlacementKind::Local);
771
772 // Stamp WHICH machine this child runs on onto its session metadata, so the
773 // UI can show it (mirrored into the session index → SessionSummary.placement).
774 // Only remote/scheduled placements need a stamp — a Local child falls through
775 // to the DTO default (this backend's own host). Persisted by the caller with
776 // the rest of the child session after we return.
777 if let Some(placement_meta) = self.placement_stamp_for(&spec) {
778 session
779 .metadata
780 .insert("placement".to_string(), placement_meta);
781 }
782
783 // Retry-once loop: a pooled local worker can die between its liveness
784 // check and handling the Run (a tiny TOCTOU window) — its Run then sits
785 // queued with no server. The first-frame watchdog in `drive` surfaces that
786 // as `WorkerUnresponsive`; we reap the dead worker and re-acquire ONCE
787 // (which spawns fresh / reuses the next live one). Remote/schedulable have
788 // no spawn fallback, so they never retry.
789 let mut attempt = 0u8;
790 let (result, actor) = loop {
791 let (actor, mut client) = match kind {
792 PlacementKind::Remote => {
793 // REMOTE branch: connect to a resident worker. No spawn, no pool
794 // touch, no drain. We do not own the worker, so a connect failure
795 // has NO respawn fallback — it is a clear, terminal error.
796 let placement = self
797 .remote_placements
798 .get(spec.identity.role.as_str())
799 .ok_or_else(|| {
800 AgentError::LLM(format!(
801 "remote placement for role '{}' vanished before connect",
802 spec.identity.role
803 ))
804 })?;
805 let endpoint = placement.endpoint.clone();
806 // Build the TLS trust: a pinned CA pins a self-signed worker cert;
807 // otherwise default webpki roots (or plaintext for `ws://`).
808 let trust_cfg = match placement.ca_cert_file.as_deref() {
809 Some(path) => Some(client_config_trusting_cert(path).map_err(|e| {
810 AgentError::LLM(format!(
811 "remote worker CA cert '{}': {e}",
812 path.display()
813 ))
814 })?),
815 None => None,
816 };
817 let client = ChildClient::connect_with_auth_tls(
818 &endpoint,
819 placement.token.as_deref(),
820 trust_cfg,
821 )
822 .await
823 .map_err(|e| {
824 AgentError::LLM(format!("remote actor connect to '{endpoint}' failed: {e}"))
825 })?;
826 // Process-less handle so live-actor registration (in-band steering)
827 // works exactly as for a local worker; `kill()` is a no-op.
828 let record = AgentRecord {
829 agent_id: job.child_session_id.clone(),
830 role: spec.identity.role.clone(),
831 labels: Vec::new(),
832 endpoint: endpoint.clone(),
833 pid: 0,
834 version: String::new(),
835 started_at: chrono::Utc::now(),
836 lease_expires_at: chrono::Utc::now(),
837 };
838 let _ = endpoint;
839 let actor = PooledWorker {
840 worker: SpawnedChild::remote(record),
841 mailbox_id: job.child_session_id.clone(),
842 };
843 let client: Box<dyn bamboo_subagent::ChildLink> = Box::new(client);
844 (actor, client)
845 }
846 PlacementKind::Schedulable => {
847 // SCHEDULABLE branch (#181): pick a LIVE worker of the pool role
848 // from the BUS (presence = connection-truth; no HTTP registry, no
849 // leases, no failover) and drive it by mailbox id. The pool worker
850 // stays connected and is reused next time. No spawn, no kill, NO
851 // local fallback — an empty pool is a terminal error (raised in
852 // resolve_schedulable_worker).
853 let bus = self.bus.as_ref().ok_or_else(|| {
854 AgentError::LLM(
855 "schedulable sub-agents require a mailbox bus (subagents.broker)"
856 .to_string(),
857 )
858 })?;
859 let mailbox_id = self
860 .resolve_schedulable_worker(spec.identity.role.as_str())
861 .await?;
862 let parent = bamboo_subagent::AgentRef {
863 session_id: format!("p-{}", job.child_session_id),
864 role: None,
865 };
866 let link = bamboo_broker::BrokerChildLink::connect(
867 &bus.endpoint,
868 parent,
869 &bus.token,
870 mailbox_id.clone(),
871 )
872 .await
873 .map_err(|e| {
874 AgentError::LLM(format!(
875 "schedulable link connect to '{mailbox_id}' failed: {e}"
876 ))
877 })?;
878 // Process-less handle — a bus-resident pool worker is never ours to
879 // kill (remote ⇒ dropped, not pooled, after the run).
880 let actor = PooledWorker {
881 worker: SpawnedChild::remote(AgentRecord {
882 agent_id: mailbox_id.clone(),
883 role: spec.identity.role.clone(),
884 labels: Vec::new(),
885 endpoint: bus.endpoint.clone(),
886 pid: 0,
887 version: String::new(),
888 started_at: chrono::Utc::now(),
889 lease_expires_at: chrono::Utc::now(),
890 }),
891 mailbox_id,
892 };
893 let client: Box<dyn bamboo_subagent::ChildLink> = Box::new(link);
894 (actor, client)
895 }
896 PlacementKind::Local => {
897 // LOCAL = the mailbox bus (the unified transport): check out a warm
898 // pooled worker (reuse a live parked one, else spawn fresh) and
899 // drive it by mailbox id — no listen socket, no file discovery, no
900 // respawn-on-connect-miss (the broker queues the Run until the
901 // worker handles it). The legacy direct-WS path was retired; the bus
902 // is required.
903 let bus = self.bus.as_ref().ok_or_else(|| {
904 AgentError::LLM(
905 "local sub-agents require a mailbox bus (subagents.broker); none is \
906 configured and the bus could not be embedded"
907 .to_string(),
908 )
909 })?;
910 let actor = self.acquire_bus_worker(&pool_key, &spec).await?;
911 let parent = bamboo_subagent::AgentRef {
912 session_id: format!("p-{}", job.child_session_id),
913 role: None,
914 };
915 let link = bamboo_broker::BrokerChildLink::connect(
916 &bus.endpoint,
917 parent,
918 &bus.token,
919 actor.mailbox_id.clone(),
920 )
921 .await
922 .map_err(|e| {
923 AgentError::LLM(format!("broker child link connect failed: {e}"))
924 })?;
925 let client: Box<dyn bamboo_subagent::ChildLink> = Box::new(link);
926 (actor, client)
927 }
928 };
929
930 if let Err(e) = client
931 .send(ParentFrame::Run(RunSpec {
932 // Cloned (not moved) so a retry can re-dispatch to a fresh worker.
933 assignment: assignment.clone(),
934 reasoning_effort: None,
935 permission_policy: permission_policy.clone(),
936 messages: messages.clone(),
937 }))
938 .await
939 {
940 if !remote {
941 actor.worker.kill().await;
942 }
943 return Err(AgentError::LLM(format!("actor run dispatch failed: {e}")));
944 }
945
946 // Register as a live actor so send_message (running, no interrupt) can
947 // steer this child in-band over the existing WS connection. The guard
948 // unregisters on every exit path.
949 let (live_tx, mut live_rx) = mpsc::unbounded_channel::<ParentFrame>();
950 let live_guard = super::live::register(
951 &job.child_session_id,
952 live_tx,
953 attempt as u32,
954 self.approval_registry.clone(),
955 );
956
957 let result = drive(
958 &mut *client,
959 &job.parent_session_id,
960 &job.child_session_id,
961 attempt as u32,
962 self.approval_registry.as_ref(),
963 self.approval_decider.as_ref(),
964 self.approval_reviewer.as_ref(),
965 escalation.clone(),
966 &event_tx,
967 &cancel_token,
968 &mut live_rx,
969 // First-frame watchdog for EVERY placement: a wedged-but-connected
970 // worker (subscribed ≠ serving — e.g. stuck on a prior LLM call) emits
971 // no first frame; without a deadline drive() blocks forever. Bounding it
972 // turns the "running-but-unresponsive" hang into a recoverable
973 // WorkerUnresponsive (reap+respawn local / re-pick schedulable / error
974 // on a fixed remote endpoint).
975 Some(WORKER_FIRST_FRAME_TIMEOUT),
976 )
977 .await;
978 // Unregister IMMEDIATELY: after drive returns nobody consumes live_rx,
979 // so a send_message landing in the close/park window below must see
980 // "not live" and take the durable-queue fallback instead of vanishing.
981 // (Even if one slipped in earlier, send_message also appends it to the
982 // durable transcript, so the next activation still rehydrates it.)
983 drop(live_guard);
984 // Close the parent link (dropping it closes our broker connection; the
985 // worker stays dialed-in + subscribed, ready for its next Run).
986 drop(client);
987
988 // No first frame ⇒ the worker is wedged. Recover ONCE before giving up:
989 // - Local: reap the dead pooled worker + respawn.
990 // - Schedulable: not ours to kill — drop it and re-select a live pool
991 // member (a wedged worker must not fail the run when the pool has others).
992 // - Remote: a FIXED endpoint has no alternative — fall through to a bounded
993 // WorkerUnresponsive error (far better than the previous infinite hang).
994 if attempt == 0 && matches!(result, Err(AgentError::WorkerUnresponsive(_))) {
995 match kind {
996 PlacementKind::Local => {
997 tracing::warn!(
998 "actor child {} got no first frame; reaping the worker and respawning once",
999 job.child_session_id
1000 );
1001 actor.worker.kill().await;
1002 attempt += 1;
1003 continue;
1004 }
1005 PlacementKind::Schedulable => {
1006 tracing::warn!(
1007 "scheduled actor child {} got no first frame; re-selecting a pool worker",
1008 job.child_session_id
1009 );
1010 drop(actor);
1011 attempt += 1;
1012 continue;
1013 }
1014 PlacementKind::Remote => {}
1015 }
1016 }
1017 break (result, actor);
1018 };
1019
1020 // Park the warm worker for reuse on a clean run, or kill it on
1021 // error/cancel (a wedged worker must not be reused). Remote / schedulable
1022 // workers are registry-managed — never ours to pool/kill, just drop.
1023 if remote {
1024 drop(actor);
1025 } else {
1026 match &result {
1027 Ok(_) => self.release_bus_worker(&pool_key, actor).await,
1028 Err(_) => actor.worker.kill().await,
1029 }
1030 }
1031
1032 // Write-back: persist the actor's final reply onto the child session so
1033 // the transcript survives and the NEXT activation sees it as history.
1034 // (run_child_spawn saves the session right after we return.)
1035 match result {
1036 Ok(Some(text)) => {
1037 if !text.is_empty() {
1038 session.add_message(bamboo_agent_core::Message::assistant(text, None));
1039 }
1040 Ok(())
1041 }
1042 Ok(None) => Ok(()),
1043 Err(e) => Err(e),
1044 }
1045 }
1046}
1047
1048/// The `{kind,host}` placement descriptor stamped onto a child session's metadata
1049/// under `"placement"` — read back by the storage index → `SessionSummary.placement`
1050/// → the UI's machine badge. `None` for `Local` (those fall through to the DTO's
1051/// default of this backend's own host). The value is a JSON string matching
1052/// `bamboo_storage::SessionPlacement { kind, host }`.
1053fn placement_metadata(placement: &Placement, host_label: Option<&str>) -> Option<String> {
1054 // Prefer the cluster node's own label/host (its metadata) when the placement
1055 // maps to a node; else fall back to the raw endpoint host / pool name.
1056 let value = match placement {
1057 Placement::Local => return None,
1058 Placement::Remote { endpoint } => serde_json::json!({
1059 "kind": "remote",
1060 "host": host_label.map(str::to_string).unwrap_or_else(|| host_of_endpoint(endpoint)),
1061 }),
1062 Placement::Schedulable { pool } => serde_json::json!({
1063 "kind": "remote",
1064 "host": host_label.unwrap_or(pool),
1065 }),
1066 };
1067 serde_json::to_string(&value).ok()
1068}
1069
1070/// Extract the host from a `ws[s]://host:port[/path]` bus endpoint, for display.
1071fn host_of_endpoint(endpoint: &str) -> String {
1072 endpoint
1073 .trim()
1074 .trim_start_matches("wss://")
1075 .trim_start_matches("ws://")
1076 .split(['/', ':'])
1077 .next()
1078 .unwrap_or(endpoint)
1079 .to_string()
1080}
1081
1082/// Pump child frames -> parent events until a terminal frame (or cancellation).
1083/// On success, yields the actor's final result text (for session write-back).
1084/// `live_rx` carries in-band frames (steering messages) from the live registry.
1085///
1086/// `escalation_bridge` (#68) is the per-run escalation host bridge CAPTURED BY
1087/// VALUE at spawn time in `execute_external_child` (NOT read live here): when a
1088/// non-bypass child re-proxies an approval request, this owned bridge routes it
1089/// UP to the parent run. Owning it for the call's lifetime is what lets a
1090/// fire-and-forget grandchild that outlives its spawning run still escalate to
1091/// the correct (then-current) parent bridge rather than a stale/overwritten one.
1092async fn drive(
1093 client: &mut dyn bamboo_subagent::ChildLink,
1094 parent_session_id: &str,
1095 child_session_id: &str,
1096 child_attempt: u32,
1097 approval_registry: Option<&super::approval_registry::SharedApprovalRegistry>,
1098 approval_decider: Option<&Arc<dyn ChildApprovalDecider>>,
1099 approval_reviewer: Option<&Arc<dyn ChildApprovalReviewer>>,
1100 escalation_bridge: Option<bamboo_subagent::executor::HostBridge>,
1101 event_tx: &mpsc::Sender<AgentEvent>,
1102 cancel_token: &CancellationToken,
1103 live_rx: &mut mpsc::UnboundedReceiver<ParentFrame>,
1104 first_frame_timeout: Option<Duration>,
1105) -> crate::runtime::runner::Result<Option<String>> {
1106 // First-frame watchdog: a live worker emits its first frame (run-started /
1107 // first token) within seconds; total silence past the deadline means the
1108 // worker is dead (e.g. a pooled worker that exited right after checkout), so
1109 // its Run sits queued forever. We trip ONLY before the first frame — once any
1110 // frame arrives the worker is proven live and a legitimately long run (a slow
1111 // tool between tokens) never trips it.
1112 let mut got_first_frame = false;
1113 let mut first_frame_watch = first_frame_timeout.map(|d| Box::pin(tokio::time::sleep(d)));
1114 loop {
1115 tokio::select! {
1116 _ = cancel_token.cancelled() => {
1117 // fall through to the cancel handling below
1118 break;
1119 }
1120 _ = async {
1121 match first_frame_watch.as_mut() {
1122 Some(s) => s.as_mut().await,
1123 None => std::future::pending::<()>().await,
1124 }
1125 }, if !got_first_frame => {
1126 return Err(AgentError::WorkerUnresponsive(format!(
1127 "child {child_session_id} produced no frame within {:?}",
1128 first_frame_timeout.unwrap_or_default()
1129 )));
1130 }
1131 Some(frame) = live_rx.recv() => {
1132 // Forward in-band steering to the worker over the existing WS.
1133 if client.send(frame).await.is_err() {
1134 tracing::warn!("live steering frame could not be sent; connection failing");
1135 }
1136 }
1137 frame = client.next_frame() => {
1138 // Any frame (event / approval / terminal / close / error) proves
1139 // the worker responded — disarm the first-frame watchdog.
1140 got_first_frame = true;
1141 first_frame_watch = None;
1142 match frame {
1143 Ok(Some(ChildFrame::Event { event })) => {
1144 // AgentEvent is serialized verbatim on the wire (zero mapping).
1145 if let Ok(ev) = serde_json::from_value::<AgentEvent>(event) {
1146 let _ = event_tx.send(ev).await;
1147 }
1148 }
1149 Ok(Some(ChildFrame::ApprovalRequest { id, body })) => {
1150 // Phase 2: a worker proxied a gated-tool approval back to
1151 // the host. The WORKER side is live — its executor installs
1152 // a per-run task-local `ApprovalProxy` (subagent_worker.rs)
1153 // that calls `host.approval_call`, so this frame arrives
1154 // when a child hits `ConfirmationRequired`.
1155 if let Some(reviewer) = approval_reviewer
1156 .cloned()
1157 .or_else(child_approval_reviewer)
1158 {
1159 // Phase 6, Part B: a BYPASSED parent worker
1160 // model-reviews its children's forced-ask (dangerous)
1161 // actions. The review is an LLM call, so run it OFF
1162 // the frame pump in a spawned task and deliver the
1163 // verdict async via the live channel — the pump keeps
1164 // forwarding events and the agent loop never blocks. A
1165 // timeout denies a hung review so the child can't hang.
1166 let child = child_session_id.to_string();
1167 let parent = parent_session_id.to_string();
1168 let req_id = id.clone();
1169 let body = body.clone();
1170 let registry = approval_registry.cloned();
1171 tokio::spawn(async move {
1172 let approved = tokio::time::timeout(
1173 CHILD_APPROVAL_TIMEOUT,
1174 reviewer.review(&parent, &child, &body),
1175 )
1176 .await
1177 .unwrap_or(false);
1178 super::live::deliver_approval_scoped(
1179 registry.as_ref(),
1180 &child,
1181 child_attempt,
1182 &req_id,
1183 approved,
1184 );
1185 });
1186 } else if approval_decider.is_some() {
1187 // A decider is wired (policy / auto): decide promptly
1188 // and reply inline. (Must not block the pump — see the
1189 // `ChildApprovalDecider` doc.)
1190 let approved =
1191 decide_child_approval(approval_decider, child_session_id, &body)
1192 .await;
1193 if client
1194 .send(ParentFrame::ApprovalReply { id, approved })
1195 .await
1196 .is_err()
1197 {
1198 tracing::warn!(
1199 "failed to answer approval_request; connection failing"
1200 );
1201 }
1202 } else if let Some(host) = escalation_bridge.clone() {
1203 // Non-bypass WORKER: ESCALATE up our own actor link
1204 // (re-proxy) so the request chains to our parent — and
1205 // up every level until a bypass level (model-review) or
1206 // the top orchestrator (human) decides. Off-loop so the
1207 // pump never blocks; relay the reply down to the child.
1208 let child = child_session_id.to_string();
1209 let req_id = id.clone();
1210 let body = body.clone();
1211 let registry = approval_registry.cloned();
1212 tokio::spawn(async move {
1213 let approved = match tokio::time::timeout(
1214 CHILD_APPROVAL_TIMEOUT,
1215 host.approval_call(body),
1216 )
1217 .await
1218 {
1219 Ok(Ok(reply)) => reply
1220 .get("approved")
1221 .and_then(|v| v.as_bool())
1222 .unwrap_or(false),
1223 // Transport error or timeout ⇒ fail closed.
1224 _ => false,
1225 };
1226 super::live::deliver_approval_scoped(
1227 registry.as_ref(),
1228 &child,
1229 child_attempt,
1230 &req_id,
1231 approved,
1232 );
1233 });
1234 } else {
1235 // Top orchestrator (no escalation bridge): human-in-the-
1236 // loop. Surface the request on the parent's event stream
1237 // and DEFER — the decision arrives out-of-band via
1238 // `live::deliver_approval(child, request_id, approved)`
1239 // (→ this child's `live_rx` → forwarded to the worker
1240 // above). A timeout denies a never-answered request so
1241 // it can't hang the child forever.
1242 let (tool_name, permission, resource) =
1243 approval_request_fields(&body);
1244 // Register the pending request BEFORE surfacing it so
1245 // the external handler's `deliver_approval_checked` can
1246 // correlate an out-of-band POST against a genuine
1247 // human-loop request (and consume it one-shot).
1248 let (approval_version, approval_created_at) =
1249 super::live::register_pending_approval_observed(
1250 approval_registry,
1251 parent_session_id,
1252 child_session_id,
1253 child_attempt,
1254 &id,
1255 &tool_name,
1256 &permission,
1257 &resource,
1258 event_tx.clone(),
1259 );
1260 if approval_version == 0 {
1261 super::live::deliver_approval_scoped(
1262 approval_registry,
1263 child_session_id,
1264 child_attempt,
1265 &id,
1266 false,
1267 );
1268 let _ = event_tx
1269 .send(AgentEvent::ChildApprovalChanged {
1270 parent_session_id: parent_session_id.to_string(),
1271 child_session_id: child_session_id.to_string(),
1272 child_attempt,
1273 request_id: id,
1274 version: 1,
1275 status: "delivery_failed".to_string(),
1276 reason: Some("persistence_failed".to_string()),
1277 tool_name,
1278 permission,
1279 resource,
1280 created_at: approval_created_at,
1281 resolved_at: Some(chrono::Utc::now().to_rfc3339()),
1282 })
1283 .await;
1284 continue;
1285 }
1286 let _ = event_tx.send(AgentEvent::ChildApprovalChanged {
1287 parent_session_id: parent_session_id.to_string(),
1288 child_session_id: child_session_id.to_string(),
1289 child_attempt,
1290 request_id: id.clone(),
1291 version: approval_version,
1292 status: "pending".to_string(),
1293 reason: None,
1294 tool_name: tool_name.clone(),
1295 permission: permission.clone(),
1296 resource: resource.clone(),
1297 created_at: approval_created_at,
1298 resolved_at: None,
1299 }).await;
1300 let _ = event_tx
1301 .send(AgentEvent::ChildApprovalRequested {
1302 child_session_id: child_session_id.to_string(),
1303 request_id: id.clone(),
1304 tool_name,
1305 permission,
1306 resource,
1307 })
1308 .await;
1309 let child = child_session_id.to_string();
1310 let approval_registry = approval_registry.cloned();
1311 tokio::spawn(async move {
1312 tokio::time::sleep(CHILD_APPROVAL_TIMEOUT).await;
1313 // Deny only if still pending: a one-shot consume so
1314 // we don't double-deliver if the human already
1315 // answered (the POST took it), and so a late POST
1316 // after this fires finds nothing pending.
1317 if super::live::expire_pending_approval(approval_registry.as_ref(), &child, &id) {
1318 super::live::deliver_approval_scoped(
1319 approval_registry.as_ref(),
1320 &child,
1321 child_attempt,
1322 &id,
1323 false,
1324 );
1325 }
1326 });
1327 }
1328 }
1329 Ok(Some(ChildFrame::Terminal { status, result, error, .. })) => {
1330 return match status {
1331 TerminalStatus::Completed => Ok(result),
1332 TerminalStatus::Cancelled => Err(AgentError::Cancelled),
1333 TerminalStatus::Error => Err(AgentError::LLM(
1334 error.unwrap_or_else(|| "actor child errored".to_string()),
1335 )),
1336 // The suspend/resume round-trip (host re-dispatch of a
1337 // nested parent) is not wired here yet; a worker in
1338 // this build never emits Suspended, so this is
1339 // unreachable in practice.
1340 TerminalStatus::Suspended => Err(AgentError::LLM(
1341 "nested sub-agent suspend received but resume transport is not wired"
1342 .to_string(),
1343 )),
1344 };
1345 }
1346 Ok(None) => {
1347 return Err(AgentError::LLM(
1348 "actor child closed before terminal".to_string(),
1349 ));
1350 }
1351 Err(e) => {
1352 return Err(AgentError::LLM(format!("actor transport error: {e}")));
1353 }
1354 }
1355 }
1356 }
1357 }
1358
1359 // Only reached on cancellation: ask the child to stop (best-effort), then report cancelled.
1360 let _ = client.send(ParentFrame::Cancel).await;
1361 Err(AgentError::Cancelled)
1362}
1363
1364/// The assignment text = the child session's latest user message (falls back to its title).
1365fn extract_assignment(session: &Session) -> String {
1366 session
1367 .messages
1368 .iter()
1369 .rev()
1370 .find(|m| matches!(m.role, Role::User))
1371 .map(|m| m.content.clone())
1372 .unwrap_or_else(|| {
1373 session
1374 .metadata
1375 .get("title")
1376 .cloned()
1377 .unwrap_or_else(|| "Execute task".to_string())
1378 })
1379}
1380
1381#[cfg(test)]
1382mod tests {
1383 use super::*;
1384
1385 fn spec_with(
1386 role: &str,
1387 provider: &str,
1388 model: &str,
1389 workspace: Option<&str>,
1390 disabled: Option<Vec<&str>>,
1391 ) -> ProvisionSpec {
1392 let mut spec = ProvisionSpec::new(
1393 ChildIdentity {
1394 child_id: "c".into(),
1395 parent_id: None,
1396 project_key: None,
1397 role: role.into(),
1398 depth: 0,
1399 },
1400 ExecutorSpec::Echo,
1401 "/tmp/fab".into(),
1402 );
1403 spec.workspace = workspace.map(|w| w.to_string());
1404 spec.model = Some(ModelRefSpec {
1405 provider: provider.into(),
1406 model: model.into(),
1407 });
1408 spec.disabled_tools = disabled.map(|d| d.into_iter().map(String::from).collect());
1409 spec
1410 }
1411
1412 #[test]
1413 fn fingerprint_matches_interchangeable_children() {
1414 // Same role/provider/model/workspace and equal tool sets (order-insensitive)
1415 // are interchangeable on one warm worker — and differ only in child_id.
1416 let a = spec_with(
1417 "explorer",
1418 "p",
1419 "m",
1420 Some("/ws"),
1421 Some(vec!["Bash", "Edit"]),
1422 );
1423 let mut b = spec_with(
1424 "explorer",
1425 "p",
1426 "m",
1427 Some("/ws"),
1428 Some(vec!["Edit", "Bash"]),
1429 );
1430 b.identity.child_id = "other".into();
1431 assert_eq!(
1432 ActorChildRunner::fingerprint(&a),
1433 ActorChildRunner::fingerprint(&b)
1434 );
1435 }
1436
1437 #[test]
1438 fn fingerprint_separates_distinct_runtimes() {
1439 let base = spec_with("explorer", "p", "m", Some("/ws"), None);
1440 let base_fp = ActorChildRunner::fingerprint(&base);
1441 // Each axis that is baked into the worker must split the pool bucket.
1442 assert_ne!(
1443 base_fp,
1444 ActorChildRunner::fingerprint(&spec_with("writer", "p", "m", Some("/ws"), None))
1445 );
1446 assert_ne!(
1447 base_fp,
1448 ActorChildRunner::fingerprint(&spec_with("explorer", "p2", "m", Some("/ws"), None))
1449 );
1450 assert_ne!(
1451 base_fp,
1452 ActorChildRunner::fingerprint(&spec_with("explorer", "p", "m2", Some("/ws"), None))
1453 );
1454 assert_ne!(
1455 base_fp,
1456 ActorChildRunner::fingerprint(&spec_with("explorer", "p", "m", Some("/ws2"), None))
1457 );
1458 assert_ne!(
1459 base_fp,
1460 ActorChildRunner::fingerprint(&spec_with(
1461 "explorer",
1462 "p",
1463 "m",
1464 Some("/ws"),
1465 Some(vec!["Bash"])
1466 ))
1467 );
1468 }
1469
1470 #[test]
1471 fn fingerprint_splits_on_baked_capabilities() {
1472 // Every capability baked once at provision time must split the pool
1473 // bucket, else a worker baked for one posture gets reused for another
1474 // (e.g. a depth-1 worker re-stamping spawn_depth onto a depth-4 child,
1475 // breaking the depth cap; or a bypass worker reused for a non-bypass one).
1476 let base_fp =
1477 ActorChildRunner::fingerprint(&spec_with("explorer", "p", "m", Some("/ws"), None));
1478
1479 let mut depth = spec_with("explorer", "p", "m", Some("/ws"), None);
1480 depth.identity.depth = 2;
1481 assert_ne!(
1482 base_fp,
1483 ActorChildRunner::fingerprint(&depth),
1484 "depth must split"
1485 );
1486
1487 let mut nested = spec_with("explorer", "p", "m", Some("/ws"), None);
1488 nested.capabilities.nested_spawn = true;
1489 assert_ne!(
1490 base_fp,
1491 ActorChildRunner::fingerprint(&nested),
1492 "nested_spawn must split"
1493 );
1494
1495 let mut bypass = spec_with("explorer", "p", "m", Some("/ws"), None);
1496 bypass.capabilities.bypass = true;
1497 assert_ne!(
1498 base_fp,
1499 ActorChildRunner::fingerprint(&bypass),
1500 "bypass must split"
1501 );
1502
1503 let mut enforce = spec_with("explorer", "p", "m", Some("/ws"), None);
1504 enforce.capabilities.enforce_permissions = true;
1505 assert_ne!(
1506 base_fp,
1507 ActorChildRunner::fingerprint(&enforce),
1508 "enforce_permissions must split"
1509 );
1510
1511 let mut cap = spec_with("explorer", "p", "m", Some("/ws"), None);
1512 cap.capabilities.max_spawn_depth = Some(8);
1513 assert_ne!(
1514 base_fp,
1515 ActorChildRunner::fingerprint(&cap),
1516 "max_spawn_depth must split"
1517 );
1518
1519 // #73 (P1): the worker bakes `no_human_review` from this flag once at
1520 // build(), so it MUST split the pool or a worker baked for one approval
1521 // posture is reused for the opposite one.
1522 let mut nha = spec_with("explorer", "p", "m", Some("/ws"), None);
1523 nha.capabilities.no_human_approver = true;
1524 assert_ne!(
1525 base_fp,
1526 ActorChildRunner::fingerprint(&nha),
1527 "no_human_approver must split"
1528 );
1529
1530 // #71: the read-only Bash checker is baked once at build() from this flag,
1531 // so a guardian reviewer worker must not be reused for an ordinary child.
1532 let mut gro = spec_with("explorer", "p", "m", Some("/ws"), None);
1533 gro.capabilities.guardian_read_only = true;
1534 assert_ne!(
1535 base_fp,
1536 ActorChildRunner::fingerprint(&gro),
1537 "guardian_read_only must split"
1538 );
1539 }
1540
1541 struct StaticDecider(bool);
1542
1543 #[async_trait]
1544 impl ChildApprovalDecider for StaticDecider {
1545 async fn decide(&self, _child: &str, _req: &serde_json::Value) -> bool {
1546 self.0
1547 }
1548 }
1549
1550 struct RecordingReviewer {
1551 reviewed: mpsc::UnboundedSender<(String, String, serde_json::Value)>,
1552 }
1553
1554 #[async_trait]
1555 impl ChildApprovalReviewer for RecordingReviewer {
1556 async fn review(&self, parent: &str, child: &str, request: &serde_json::Value) -> bool {
1557 let _ = self
1558 .reviewed
1559 .send((parent.to_string(), child.to_string(), request.clone()));
1560 true
1561 }
1562 }
1563
1564 // ---- first-frame watchdog (dead-pooled-worker recovery) -----------------
1565
1566 /// A link that never yields a frame — models a worker that died (or never
1567 /// subscribed) so its Run sits queued with no server.
1568 struct SilentLink;
1569 #[async_trait]
1570 impl bamboo_subagent::ChildLink for SilentLink {
1571 async fn send(&mut self, _: ParentFrame) -> bamboo_subagent::TransportResult<()> {
1572 Ok(())
1573 }
1574 async fn next_frame(&mut self) -> bamboo_subagent::TransportResult<Option<ChildFrame>> {
1575 std::future::pending().await
1576 }
1577 }
1578
1579 /// A link that immediately yields one terminal frame (a healthy fast worker).
1580 struct InstantTerminalLink {
1581 done: bool,
1582 }
1583
1584 struct ApprovalThenTerminalLink {
1585 step: u8,
1586 }
1587
1588 #[async_trait]
1589 impl bamboo_subagent::ChildLink for ApprovalThenTerminalLink {
1590 async fn send(&mut self, _: ParentFrame) -> bamboo_subagent::TransportResult<()> {
1591 Ok(())
1592 }
1593
1594 async fn next_frame(&mut self) -> bamboo_subagent::TransportResult<Option<ChildFrame>> {
1595 self.step += 1;
1596 match self.step {
1597 1 => Ok(Some(ChildFrame::ApprovalRequest {
1598 id: "approval-1".into(),
1599 body: serde_json::json!({
1600 "tool_name": "Bash",
1601 "permission": "execute",
1602 "resource": "rm -rf target",
1603 "permission_request": {"reason_code": "hard_dangerous"}
1604 }),
1605 })),
1606 2 => Ok(Some(ChildFrame::Terminal {
1607 status: TerminalStatus::Completed,
1608 result: Some("done".into()),
1609 error: None,
1610 transcript: vec![],
1611 })),
1612 _ => std::future::pending().await,
1613 }
1614 }
1615 }
1616
1617 #[tokio::test]
1618 async fn drive_routes_forced_ask_to_parent_reviewer_without_human_event() {
1619 let (event_tx, mut event_rx) = mpsc::channel::<AgentEvent>(8);
1620 let (review_tx, mut review_rx) = mpsc::unbounded_channel();
1621 let reviewer: Arc<dyn ChildApprovalReviewer> = Arc::new(RecordingReviewer {
1622 reviewed: review_tx,
1623 });
1624 let cancel = CancellationToken::new();
1625 let (_live_tx, mut live_rx) = mpsc::unbounded_channel::<ParentFrame>();
1626 let mut link = ApprovalThenTerminalLink { step: 0 };
1627
1628 let result = drive(
1629 &mut link,
1630 "parent-reviewer",
1631 "child-reviewer",
1632 0,
1633 None,
1634 None,
1635 Some(&reviewer),
1636 None,
1637 &event_tx,
1638 &cancel,
1639 &mut live_rx,
1640 None,
1641 )
1642 .await;
1643
1644 assert_eq!(result.ok().flatten().as_deref(), Some("done"));
1645 let (parent, child, body) = tokio::time::timeout(Duration::from_secs(1), review_rx.recv())
1646 .await
1647 .expect("reviewer should be invoked off-loop")
1648 .expect("review channel should remain open");
1649 assert_eq!(parent, "parent-reviewer");
1650 assert_eq!(child, "child-reviewer");
1651 assert_eq!(
1652 body.pointer("/permission_request/reason_code")
1653 .and_then(serde_json::Value::as_str),
1654 Some("hard_dangerous")
1655 );
1656 assert!(
1657 event_rx.try_recv().is_err(),
1658 "must not emit a human-review event"
1659 );
1660 }
1661 #[async_trait]
1662 impl bamboo_subagent::ChildLink for InstantTerminalLink {
1663 async fn send(&mut self, _: ParentFrame) -> bamboo_subagent::TransportResult<()> {
1664 Ok(())
1665 }
1666 async fn next_frame(&mut self) -> bamboo_subagent::TransportResult<Option<ChildFrame>> {
1667 if self.done {
1668 std::future::pending().await
1669 } else {
1670 self.done = true;
1671 Ok(Some(ChildFrame::Terminal {
1672 status: TerminalStatus::Completed,
1673 result: Some("done".into()),
1674 error: None,
1675 transcript: vec![],
1676 }))
1677 }
1678 }
1679 }
1680
1681 #[tokio::test]
1682 async fn drive_trips_first_frame_watchdog_on_a_silent_worker() {
1683 let (event_tx, _rx) = mpsc::channel::<AgentEvent>(8);
1684 let cancel = CancellationToken::new();
1685 let (_live_tx, mut live_rx) = mpsc::unbounded_channel::<ParentFrame>();
1686 let mut link = SilentLink;
1687 let r = drive(
1688 &mut link,
1689 "parent-x",
1690 "child-x",
1691 0,
1692 None,
1693 None,
1694 None,
1695 None,
1696 &event_tx,
1697 &cancel,
1698 &mut live_rx,
1699 Some(Duration::from_millis(100)),
1700 )
1701 .await;
1702 assert!(
1703 matches!(r, Err(AgentError::WorkerUnresponsive(_))),
1704 "a silent worker must trip the first-frame watchdog, got {r:?}"
1705 );
1706 }
1707
1708 #[tokio::test]
1709 async fn drive_does_not_trip_when_a_frame_arrives() {
1710 let (event_tx, _rx) = mpsc::channel::<AgentEvent>(8);
1711 let cancel = CancellationToken::new();
1712 let (_live_tx, mut live_rx) = mpsc::unbounded_channel::<ParentFrame>();
1713 let mut link = InstantTerminalLink { done: false };
1714 // Even a tiny timeout must NOT trip: the terminal frame arrives first and
1715 // disarms the watchdog.
1716 let r = drive(
1717 &mut link,
1718 "parent-y",
1719 "child-y",
1720 0,
1721 None,
1722 None,
1723 None,
1724 None,
1725 &event_tx,
1726 &cancel,
1727 &mut live_rx,
1728 Some(Duration::from_millis(50)),
1729 )
1730 .await;
1731 assert_eq!(r.ok().flatten().as_deref(), Some("done"));
1732 }
1733
1734 #[tokio::test]
1735 async fn child_approval_fails_closed_without_decider() {
1736 // No decider wired ⇒ the host denies (safe default), unchanged behavior.
1737 let body = serde_json::json!({"tool_name":"Bash","permission":"run","resource":"rm -rf /"});
1738 assert!(!decide_child_approval(None, "child-1", &body).await);
1739 }
1740
1741 #[tokio::test]
1742 async fn child_approval_honors_wired_decider() {
1743 let body =
1744 serde_json::json!({"tool_name":"Write","permission":"write","resource":"/tmp/x"});
1745 let approve: Arc<dyn ChildApprovalDecider> = Arc::new(StaticDecider(true));
1746 let deny: Arc<dyn ChildApprovalDecider> = Arc::new(StaticDecider(false));
1747 assert!(decide_child_approval(Some(&approve), "child-1", &body).await);
1748 assert!(!decide_child_approval(Some(&deny), "child-1", &body).await);
1749 }
1750
1751 #[test]
1752 fn approval_request_fields_extracts_and_defaults() {
1753 let full = serde_json::json!({"tool_name":"Bash","permission":"run","resource":"ls"});
1754 assert_eq!(
1755 approval_request_fields(&full),
1756 ("Bash".to_string(), "run".to_string(), "ls".to_string())
1757 );
1758 // Missing fields default to empty strings.
1759 let partial = serde_json::json!({"tool_name":"Write"});
1760 assert_eq!(
1761 approval_request_fields(&partial),
1762 ("Write".to_string(), String::new(), String::new())
1763 );
1764 }
1765
1766 // ---- #193: remote placement routing -------------------------------------
1767
1768 use crate::runtime::execution::SpawnJob;
1769 use bamboo_agent_core::Session;
1770
1771 /// A runner with a BOGUS worker_bin (`/bin/false`): a local spawn here would
1772 /// FAIL, so a passing remote test proves the remote path never spawns.
1773 fn bogus_runner(placements: HashMap<String, ResolvedRemotePlacement>) -> ActorChildRunner {
1774 ActorChildRunner::new(
1775 "test-actor".into(),
1776 PathBuf::from("/bin/false"),
1777 vec![],
1778 std::env::temp_dir().join("bamboo-test-fab-193"),
1779 ExecutorSpec::Echo,
1780 vec![],
1781 "anthropic".into(),
1782 4,
1783 )
1784 .with_remote_placements(placements)
1785 }
1786
1787 /// A child session of the given role (the role rides `subagent_type`, the
1788 /// path build_spec + the remote lookup both read).
1789 fn session_of_role(role: &str, assignment: &str) -> Session {
1790 let mut s = Session::new("child-1", "test-model");
1791 s.metadata
1792 .insert("subagent_type".to_string(), role.to_string());
1793 s.add_message(bamboo_agent_core::Message::user(assignment));
1794 s
1795 }
1796
1797 fn job_for(child: &str) -> SpawnJob {
1798 SpawnJob {
1799 parent_session_id: "parent-1".into(),
1800 child_session_id: child.into(),
1801 model: String::new(),
1802 disabled_tools: None,
1803 }
1804 }
1805
1806 #[test]
1807 fn build_spec_sets_remote_placement_for_matching_role() {
1808 let mut placements = HashMap::new();
1809 placements.insert(
1810 "explorer".to_string(),
1811 ResolvedRemotePlacement {
1812 endpoint: "wss://gpu-host:8443".into(),
1813 token: Some("T-secret".into()),
1814 ca_cert_file: None,
1815 host_label: None,
1816 },
1817 );
1818 let runner = bogus_runner(placements);
1819
1820 // Matching role -> Placement::Remote + the bearer on the secrets envelope.
1821 let s = session_of_role("explorer", "do the thing");
1822 let spec = runner.build_spec(&s, &job_for("child-1"));
1823 match &spec.placement {
1824 Placement::Remote { endpoint } => assert_eq!(endpoint, "wss://gpu-host:8443"),
1825 other => panic!("expected Remote, got {other:?}"),
1826 }
1827 assert_eq!(spec.secrets.worker_auth_token.as_deref(), Some("T-secret"));
1828 }
1829
1830 #[test]
1831 fn build_spec_leaves_local_for_unmatched_role() {
1832 let mut placements = HashMap::new();
1833 placements.insert(
1834 "explorer".to_string(),
1835 ResolvedRemotePlacement {
1836 endpoint: "wss://gpu-host:8443".into(),
1837 token: Some("T".into()),
1838 ca_cert_file: None,
1839 host_label: None,
1840 },
1841 );
1842 let runner = bogus_runner(placements);
1843
1844 // A DIFFERENT role keeps the default Local placement + no bearer.
1845 let s = session_of_role("writer", "do the thing");
1846 let spec = runner.build_spec(&s, &job_for("child-1"));
1847 assert_eq!(spec.placement, Placement::Local);
1848 assert!(spec.secrets.worker_auth_token.is_none());
1849 }
1850
1851 #[test]
1852 fn build_spec_local_when_no_placements() {
1853 let runner = bogus_runner(HashMap::new());
1854 let s = session_of_role("explorer", "do the thing");
1855 let spec = runner.build_spec(&s, &job_for("child-1"));
1856 assert_eq!(spec.placement, Placement::Local);
1857 assert!(spec.secrets.worker_auth_token.is_none());
1858 }
1859
1860 #[test]
1861 fn placement_metadata_stamps_remote_and_schedulable_not_local() {
1862 // Local children carry no stamp — the DTO defaults them to the backend host.
1863 assert_eq!(placement_metadata(&Placement::Local, None), None);
1864
1865 // Remote, no node label → host derived from the endpoint.
1866 let r = placement_metadata(
1867 &Placement::Remote {
1868 endpoint: "wss://10.0.0.5:8443/stream".into(),
1869 },
1870 None,
1871 )
1872 .unwrap();
1873 assert!(r.contains(r#""kind":"remote""#), "{r}");
1874 assert!(r.contains(r#""host":"10.0.0.5""#), "{r}");
1875
1876 // A cluster node's label (its metadata) OVERRIDES the raw endpoint host.
1877 let labeled = placement_metadata(
1878 &Placement::Remote {
1879 endpoint: "ws://169.254.230.101:8899".into(),
1880 },
1881 Some("mini"),
1882 )
1883 .unwrap();
1884 assert!(labeled.contains(r#""host":"mini""#), "{labeled}");
1885
1886 // Schedulable → {kind:"remote", host:<node label, else pool>}.
1887 let s = placement_metadata(
1888 &Placement::Schedulable {
1889 pool: "explorers".into(),
1890 },
1891 Some("mini"),
1892 )
1893 .unwrap();
1894 assert!(s.contains(r#""kind":"remote""#), "{s}");
1895 assert!(s.contains(r#""host":"mini""#), "{s}");
1896
1897 // The stamp round-trips through the storage placement type.
1898 let p: bamboo_storage::SessionPlacement = serde_json::from_str(&labeled).unwrap();
1899 assert_eq!(p.kind, "remote");
1900 assert_eq!(p.host, "mini");
1901 }
1902
1903 /// End-to-end remote run through `execute_external_child`: a resident worker
1904 /// (Bearer-gated `WsServer` + `EchoExecutor`) serves the role; the runner is
1905 /// built with a `remote_placements` entry pointing at it AND a BOGUS
1906 /// worker_bin (`/bin/false`). A passing test proves the remote path CONNECTS
1907 /// to the resident worker and NEVER spawns (a spawn would fail on /bin/false),
1908 /// and that a terminal/echo result flows back.
1909 #[tokio::test]
1910 async fn execute_external_child_routes_role_to_remote_worker_without_spawning() {
1911 // 1. Stand up the resident worker on loopback with a required bearer.
1912 let token = "remote-test-token";
1913 let server = bamboo_subagent::transport::WsServer::bind_with_token(
1914 (std::net::Ipv4Addr::LOCALHOST, 0).into(),
1915 Some(token.to_string()),
1916 )
1917 .await
1918 .expect("bind resident worker");
1919 let endpoint = server.ws_endpoint(); // ws://127.0.0.1:<port>
1920 let srv = tokio::spawn(async move {
1921 // serve() loops connection-after-connection; the test exits, dropping it.
1922 let _ = server
1923 .serve(Arc::new(bamboo_subagent::executor::EchoExecutor))
1924 .await;
1925 });
1926
1927 // 2. Build the runner: role "explorer" pinned remote, bogus worker_bin.
1928 let mut placements = HashMap::new();
1929 placements.insert(
1930 "explorer".to_string(),
1931 ResolvedRemotePlacement {
1932 endpoint: endpoint.clone(),
1933 token: Some(token.to_string()),
1934 ca_cert_file: None,
1935 host_label: Some("mini-e2e".into()), // node label, surfaced on the badge
1936 },
1937 );
1938 let runner = bogus_runner(placements);
1939
1940 // 3. Drive a real run for that role.
1941 let mut session = session_of_role("explorer", "hello remote");
1942 let job = job_for("child-1");
1943 let (event_tx, mut event_rx) = mpsc::channel::<AgentEvent>(64);
1944 let cancel = CancellationToken::new();
1945
1946 let result = tokio::time::timeout(
1947 Duration::from_secs(10),
1948 runner.execute_external_child(&mut session, &job, event_tx, cancel),
1949 )
1950 .await
1951 .expect("run did not hang")
1952 .expect("remote run succeeded (connected to resident worker, did not spawn)");
1953
1954 let _ = result;
1955 // The EchoExecutor's reply is written back onto the child session as an
1956 // assistant message — proof a terminal result flowed back over the link.
1957 let last = session
1958 .messages
1959 .iter()
1960 .rev()
1961 .find(|m| matches!(m.role, Role::Assistant))
1962 .expect("an assistant reply was written back");
1963 assert!(
1964 last.content.contains("echo:"),
1965 "expected echo reply, got {:?}",
1966 last.content
1967 );
1968
1969 // A remote run must stamp WHICH machine it ran on onto the child session
1970 // (mirrored to the UI badge) using the placement's node label.
1971 let placement = session
1972 .metadata
1973 .get("placement")
1974 .expect("remote child session stamped with a placement");
1975 assert!(placement.contains(r#""kind":"remote""#), "{placement}");
1976 assert!(placement.contains(r#""host":"mini-e2e""#), "{placement}");
1977
1978 // Drain a couple of streamed events to confirm the event pipe carried the
1979 // worker's tokens too (best-effort; the reply assertion above is primary).
1980 let mut saw_event = false;
1981 while let Ok(Some(_ev)) =
1982 tokio::time::timeout(Duration::from_millis(50), event_rx.recv()).await
1983 {
1984 saw_event = true;
1985 }
1986 let _ = saw_event;
1987
1988 srv.abort();
1989 }
1990
1991 // ---- #181 (P2b): schedulable placement routing --------------------------
1992
1993 /// A bogus-worker_bin runner carrying SCHEDULABLE placements (and optionally
1994 /// remote ones, to test precedence). A local spawn here would fail on
1995 /// `/bin/false`, so a passing schedulable test proves no subprocess spawned.
1996 fn bogus_sched_runner(
1997 remote: HashMap<String, ResolvedRemotePlacement>,
1998 sched: HashMap<String, ResolvedSchedulablePlacement>,
1999 ) -> ActorChildRunner {
2000 ActorChildRunner::new(
2001 "test-actor".into(),
2002 PathBuf::from("/bin/false"),
2003 vec![],
2004 std::env::temp_dir().join("bamboo-test-fab-181"),
2005 ExecutorSpec::Echo,
2006 vec![],
2007 "anthropic".into(),
2008 4,
2009 )
2010 .with_remote_placements(remote)
2011 .with_schedulable_placements(sched)
2012 }
2013
2014 fn sched_placement(
2015 pool: &str,
2016 _registry_url: impl Into<String>,
2017 ) -> ResolvedSchedulablePlacement {
2018 ResolvedSchedulablePlacement {
2019 pool: pool.into(),
2020 host_label: None,
2021 }
2022 }
2023
2024 #[test]
2025 fn build_spec_sets_schedulable_placement_for_matching_role() {
2026 let mut sched = HashMap::new();
2027 sched.insert(
2028 "explorer".to_string(),
2029 sched_placement("gpu-pool", "unused"),
2030 );
2031 let runner = bogus_sched_runner(HashMap::new(), sched);
2032
2033 let s = session_of_role("explorer", "do the thing");
2034 let spec = runner.build_spec(&s, &job_for("child-1"));
2035 match &spec.placement {
2036 Placement::Schedulable { pool } => assert_eq!(pool, "gpu-pool"),
2037 other => panic!("expected Schedulable, got {other:?}"),
2038 }
2039 // No per-placement bearer now — the bus connection carries the bus token.
2040 assert!(spec.secrets.worker_auth_token.is_none());
2041 }
2042
2043 #[test]
2044 fn build_spec_remote_wins_when_role_in_both_maps() {
2045 // A role present in BOTH remote_placements and schedulable_placements must
2046 // resolve to the FIXED remote placement (documented precedence).
2047 let mut remote = HashMap::new();
2048 remote.insert(
2049 "explorer".to_string(),
2050 ResolvedRemotePlacement {
2051 endpoint: "wss://fixed-host:8443".into(),
2052 token: Some("T-remote".into()),
2053 ca_cert_file: None,
2054 host_label: None,
2055 },
2056 );
2057 let mut sched = HashMap::new();
2058 sched.insert(
2059 "explorer".to_string(),
2060 sched_placement("gpu-pool", "https://control-plane:9562"),
2061 );
2062 let runner = bogus_sched_runner(remote, sched);
2063
2064 let s = session_of_role("explorer", "do the thing");
2065 let spec = runner.build_spec(&s, &job_for("child-1"));
2066 match &spec.placement {
2067 Placement::Remote { endpoint } => assert_eq!(endpoint, "wss://fixed-host:8443"),
2068 other => panic!("expected Remote (precedence), got {other:?}"),
2069 }
2070 assert_eq!(spec.secrets.worker_auth_token.as_deref(), Some("T-remote"));
2071 }
2072
2073 #[test]
2074 fn build_spec_local_for_unmatched_schedulable_role() {
2075 let mut sched = HashMap::new();
2076 sched.insert(
2077 "explorer".to_string(),
2078 sched_placement("gpu-pool", "https://control-plane:9562"),
2079 );
2080 let runner = bogus_sched_runner(HashMap::new(), sched);
2081 let s = session_of_role("writer", "do the thing");
2082 let spec = runner.build_spec(&s, &job_for("child-1"));
2083 assert_eq!(spec.placement, Placement::Local);
2084 assert!(spec.secrets.worker_auth_token.is_none());
2085 }
2086
2087 /// The full role → resolved-placement → badge-host chain: a child routed to a
2088 /// remote/schedulable placement carrying a cluster node's `host_label` stamps
2089 /// that label; without a label it falls back to the endpoint host / pool; a
2090 /// Local child gets no stamp (the DTO defaults it to the backend host).
2091 #[test]
2092 fn placement_stamp_uses_node_label_for_remote_and_schedulable() {
2093 // Remote WITH a node label → {remote, <label>}, overriding the raw IP.
2094 let mut remote = HashMap::new();
2095 remote.insert(
2096 "explorer".to_string(),
2097 ResolvedRemotePlacement {
2098 endpoint: "ws://169.254.230.101:8899".into(),
2099 token: None,
2100 ca_cert_file: None,
2101 host_label: Some("mini".into()),
2102 },
2103 );
2104 let runner = bogus_runner(remote);
2105 let spec = runner.build_spec(&session_of_role("explorer", "go"), &job_for("c1"));
2106 let stamp = runner
2107 .placement_stamp_for(&spec)
2108 .expect("remote child is stamped");
2109 assert!(stamp.contains(r#""kind":"remote""#), "{stamp}");
2110 assert!(stamp.contains(r#""host":"mini""#), "{stamp}");
2111
2112 // Remote WITHOUT a node label → falls back to the endpoint host.
2113 let mut remote_nolabel = HashMap::new();
2114 remote_nolabel.insert(
2115 "explorer".to_string(),
2116 ResolvedRemotePlacement {
2117 endpoint: "ws://169.254.230.101:8899".into(),
2118 token: None,
2119 ca_cert_file: None,
2120 host_label: None,
2121 },
2122 );
2123 let r2 = bogus_runner(remote_nolabel);
2124 let spec2 = r2.build_spec(&session_of_role("explorer", "go"), &job_for("c1"));
2125 assert!(r2
2126 .placement_stamp_for(&spec2)
2127 .unwrap()
2128 .contains(r#""host":"169.254.230.101""#));
2129
2130 // Schedulable WITH a node label → {remote, <label>} (a node, not a pool name).
2131 let mut sched = HashMap::new();
2132 sched.insert(
2133 "mac-mini-monitor".to_string(),
2134 ResolvedSchedulablePlacement {
2135 pool: "mac-mini-monitor".into(),
2136 host_label: Some("mini".into()),
2137 },
2138 );
2139 let sr = bogus_sched_runner(HashMap::new(), sched);
2140 let spec3 = sr.build_spec(&session_of_role("mac-mini-monitor", "go"), &job_for("c1"));
2141 let stamp3 = sr
2142 .placement_stamp_for(&spec3)
2143 .expect("scheduled child is stamped");
2144 assert!(stamp3.contains(r#""kind":"remote""#), "{stamp3}");
2145 assert!(stamp3.contains(r#""host":"mini""#), "{stamp3}");
2146
2147 // A Local (unmatched) child gets NO stamp.
2148 let local = bogus_runner(HashMap::new());
2149 let spec4 = local.build_spec(&session_of_role("writer", "go"), &job_for("c1"));
2150 assert_eq!(local.placement_stamp_for(&spec4), None);
2151 }
2152
2153 // ---- #181: schedulable selection over the BUS (Phase 3 cutover) ----------
2154
2155 async fn start_bus() -> (String, tempfile::TempDir) {
2156 let dir = tempfile::tempdir().unwrap();
2157 let core = std::sync::Arc::new(bamboo_broker::BrokerCore::new(dir.path()));
2158 let server = std::sync::Arc::new(bamboo_broker::BrokerServer::new(core, "t"));
2159 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
2160 let addr = listener.local_addr().unwrap();
2161 tokio::spawn(async move {
2162 let _ = server.serve(listener).await;
2163 });
2164 (format!("ws://{addr}"), dir)
2165 }
2166
2167 async fn join_pool(endpoint: &str, id: &str, pool: &str) -> bamboo_broker::BrokerClient {
2168 let mut c = bamboo_broker::BrokerClient::connect(
2169 endpoint,
2170 bamboo_subagent::AgentRef {
2171 session_id: id.into(),
2172 role: Some(pool.into()),
2173 },
2174 "t",
2175 )
2176 .await
2177 .unwrap();
2178 c.subscribe().await.unwrap();
2179 c
2180 }
2181
2182 fn sched_runner_on_bus(endpoint: &str, child_role: &str, pool: &str) -> ActorChildRunner {
2183 let mut sched = HashMap::new();
2184 sched.insert(child_role.to_string(), sched_placement(pool, "unused"));
2185 bogus_sched_runner(HashMap::new(), sched).with_bus(Some(bamboo_subagent::BusEndpoint {
2186 endpoint: endpoint.into(),
2187 token: "t".into(),
2188 }))
2189 }
2190
2191 #[tokio::test]
2192 async fn resolve_schedulable_picks_a_live_bus_worker() {
2193 let (endpoint, _dir) = start_bus().await;
2194 let _w = join_pool(&endpoint, "w-gpu", "gpu-pool").await;
2195 let runner = sched_runner_on_bus(&endpoint, "explorer", "gpu-pool");
2196
2197 let mailbox = runner
2198 .resolve_schedulable_worker("explorer")
2199 .await
2200 .expect("a live pool worker is found on the bus");
2201 assert_eq!(mailbox, "w-gpu");
2202 }
2203
2204 #[tokio::test]
2205 async fn resolve_schedulable_round_robins_over_pool_workers() {
2206 let (endpoint, _dir) = start_bus().await;
2207 let _a = join_pool(&endpoint, "w-a", "gpu-pool").await;
2208 let _b = join_pool(&endpoint, "w-b", "gpu-pool").await;
2209 let runner = sched_runner_on_bus(&endpoint, "explorer", "gpu-pool");
2210
2211 // Successive resolves spread across both connected workers.
2212 let mut picked = std::collections::HashSet::new();
2213 for _ in 0..6 {
2214 picked.insert(runner.resolve_schedulable_worker("explorer").await.unwrap());
2215 }
2216 assert_eq!(
2217 picked,
2218 ["w-a".to_string(), "w-b".to_string()].into_iter().collect(),
2219 "round-robin must cover every connected pool worker"
2220 );
2221 }
2222
2223 #[tokio::test]
2224 async fn resolve_schedulable_errors_on_empty_pool() {
2225 let (endpoint, _dir) = start_bus().await;
2226 // No worker subscribes to "gpu-pool".
2227 let runner = sched_runner_on_bus(&endpoint, "explorer", "gpu-pool");
2228
2229 let err = runner
2230 .resolve_schedulable_worker("explorer")
2231 .await
2232 .expect_err("an empty pool is terminal — no local fallback")
2233 .to_string();
2234 assert!(err.contains("no live worker in pool"), "got: {err}");
2235 assert!(err.contains("NOT spawning"), "got: {err}");
2236 }
2237
2238 /// FULL schedulable run over the bus: a worker SERVING `EchoExecutor` joins the
2239 /// pool by role; `execute_external_child` with a Schedulable placement resolves
2240 /// it from the bus (no local subprocess — the worker_bin is `/bin/false`),
2241 /// drives the run, gets the echo back, AND stamps the child session with the
2242 /// pool's cluster-node label — `{kind:remote, host:"mini"}`. The end-to-end
2243 /// analogue of the live `mac-mini-monitor`→mini run.
2244 #[tokio::test]
2245 async fn execute_external_child_runs_schedulable_over_bus_and_stamps_node_label() {
2246 let (endpoint, _dir) = start_bus().await;
2247
2248 // A bus worker SERVING runs (not just presence), joined to the pool by role.
2249 let ep = endpoint.clone();
2250 let worker = tokio::spawn(async move {
2251 let _ = bamboo_broker::serve_executor(
2252 &ep,
2253 bamboo_subagent::AgentRef {
2254 session_id: "mmm-worker".into(),
2255 role: Some("mac-mini-monitor".into()),
2256 },
2257 "t",
2258 std::sync::Arc::new(bamboo_subagent::executor::EchoExecutor),
2259 )
2260 .await;
2261 });
2262
2263 // Wait until the worker is visible on the bus so the pool is non-empty
2264 // when execute_external_child resolves it (serve_executor connects async).
2265 let mut probe = bamboo_broker::BrokerClient::connect(
2266 &endpoint,
2267 bamboo_subagent::AgentRef {
2268 session_id: "probe".into(),
2269 role: None,
2270 },
2271 "t",
2272 )
2273 .await
2274 .unwrap();
2275 let mut ready = false;
2276 for _ in 0..100 {
2277 if probe
2278 .list_connected("mac-mini-monitor")
2279 .await
2280 .unwrap()
2281 .iter()
2282 .any(|id| id == "mmm-worker")
2283 {
2284 ready = true;
2285 break;
2286 }
2287 tokio::time::sleep(Duration::from_millis(30)).await;
2288 }
2289 assert!(ready, "worker never joined the pool");
2290
2291 // Runner: child role → schedulable pool "mac-mini-monitor" carrying the
2292 // cluster node's label "mini"; bogus worker_bin so any local spawn fails.
2293 let mut sched = HashMap::new();
2294 sched.insert(
2295 "mac-mini-monitor".to_string(),
2296 ResolvedSchedulablePlacement {
2297 pool: "mac-mini-monitor".into(),
2298 host_label: Some("mini".into()),
2299 },
2300 );
2301 let runner = bogus_sched_runner(HashMap::new(), sched).with_bus(Some(
2302 bamboo_subagent::BusEndpoint {
2303 endpoint: endpoint.clone(),
2304 token: "t".into(),
2305 },
2306 ));
2307
2308 let mut session = session_of_role("mac-mini-monitor", "hello scheduled");
2309 let job = job_for("child-1");
2310 let (event_tx, _rx) = mpsc::channel::<AgentEvent>(64);
2311 let cancel = CancellationToken::new();
2312
2313 tokio::time::timeout(
2314 Duration::from_secs(10),
2315 runner.execute_external_child(&mut session, &job, event_tx, cancel),
2316 )
2317 .await
2318 .expect("run did not hang")
2319 .expect("schedulable run succeeded over the bus (no local spawn)");
2320
2321 // Echo reply flowed back — proves it routed to the bus worker, not local.
2322 let last = session
2323 .messages
2324 .iter()
2325 .rev()
2326 .find(|m| matches!(m.role, Role::Assistant))
2327 .expect("an assistant reply was written back");
2328 assert!(last.content.contains("echo:"), "got {:?}", last.content);
2329
2330 // ...and the child is stamped with the pool's cluster-node label.
2331 let placement = session
2332 .metadata
2333 .get("placement")
2334 .expect("scheduled child session stamped with a placement");
2335 assert!(placement.contains(r#""kind":"remote""#), "{placement}");
2336 assert!(placement.contains(r#""host":"mini""#), "{placement}");
2337
2338 worker.abort();
2339 }
2340}