1use std::collections::{HashMap, VecDeque};
14use std::path::{Path, PathBuf};
15use std::sync::Arc;
16use std::time::Duration;
17
18use async_trait::async_trait;
19use bamboo_agent_core::{AgentError, AgentEvent, Role, Session};
20use bamboo_domain::poison::PoisonRecover;
21use bamboo_domain::SessionInboxClaim;
22use tokio::sync::mpsc;
23use tokio_util::sync::CancellationToken;
24
25use bamboo_subagent::fleet::{spawn_worker_on_bus, SpawnedChild};
26use bamboo_subagent::proto::{
27 AgentRecord, ChildFrame, LogicalSessionIdentity, ParentFrame, PermissionPolicyContext, RunSpec,
28 SessionMessageDelivery, TerminalStatus,
29};
30use bamboo_subagent::provision::{
31 ChildIdentity, ExecutorSpec, ModelRefSpec, Placement, ProvisionSpec, ScopedCredential,
32};
33use bamboo_subagent::transport::{client_config_trusting_cert, ChildClient};
34
35use crate::runtime::execution::{ExternalChildRunner, SessionInboxRuntimeBinding, SpawnJob};
36
37pub const DEFAULT_MAX_CONCURRENT_ACTORS: usize = 8;
39
40pub const MAX_SPAWN_DEPTH: u32 = 4;
46
47const DEFAULT_MAX_IDLE_PER_KEY: usize = 4;
49
50const POOLED_IDLE_TIMEOUT_SECS: u64 = 300;
53
54const WORKER_FIRST_FRAME_TIMEOUT: Duration = Duration::from_secs(60);
61
62pub struct IssuedCodexRunToken {
65 pub token_id: String,
66 pub token: String,
67}
68
69pub trait CodexRunTokenAuthority: Send + Sync + 'static {
72 fn issue(&self, session_id: &str) -> Result<IssuedCodexRunToken, String>;
73 fn revoke(&self, token_id: &str);
74}
75
76struct CodexRunTokenGuard {
77 authority: Arc<dyn CodexRunTokenAuthority>,
78 token_id: String,
79}
80
81impl Drop for CodexRunTokenGuard {
82 fn drop(&mut self) {
83 self.authority.revoke(&self.token_id);
84 }
85}
86
87fn executor_uses_bamboo_codex(executor: &ExecutorSpec) -> bool {
88 matches!(
89 executor,
90 ExecutorSpec::Codex {
91 auth_mode: Some(mode),
92 ..
93 } if mode == "bamboo"
94 ) || matches!(
95 executor,
96 ExecutorSpec::Codex {
97 auth_mode: None,
98 inherit_user_config,
99 ..
100 } if !inherit_user_config.unwrap_or(false)
101 )
102}
103
104fn workspace_is_bamboo_owned(raw: &str) -> bool {
105 let workspace = std::fs::canonicalize(raw).unwrap_or_else(|_| PathBuf::from(raw));
106 let configured_root = bamboo_config::paths::resolve_workspace_root();
107 let configured_root = std::fs::canonicalize(&configured_root).unwrap_or(configured_root);
108 if workspace.starts_with(&configured_root) {
109 return true;
110 }
111
112 workspace.ancestors().any(|candidate| {
117 let Some(name) = candidate.file_name().and_then(|name| name.to_str()) else {
118 return false;
119 };
120 let Some(worktree_root) = candidate.parent() else {
121 return false;
122 };
123 if worktree_root.file_name() != Some(std::ffi::OsStr::new("worktree"))
124 || worktree_root.parent().and_then(Path::file_name)
125 != Some(std::ffi::OsStr::new(".bamboo"))
126 {
127 return false;
128 }
129 let marker = worktree_root.join(".bamboo-owned").join(name);
130 std::fs::read_to_string(marker).is_ok_and(|branch| branch == format!("bamboo/{name}"))
131 })
132}
133
134fn build_codex_run_secrets(
135 executor: &ExecutorSpec,
136 authority: Option<Arc<dyn CodexRunTokenAuthority>>,
137 child_session_id: &str,
138) -> Result<
139 (
140 bamboo_subagent::proto::RunSecrets,
141 Option<CodexRunTokenGuard>,
142 ),
143 AgentError,
144> {
145 if !executor_uses_bamboo_codex(executor) {
146 return Ok((bamboo_subagent::proto::RunSecrets::default(), None));
147 }
148
149 let authority = authority.ok_or_else(|| {
150 AgentError::LLM(
151 "Codex auth mode 'bamboo' requires the server per-run token authority".to_string(),
152 )
153 })?;
154 let issued = authority
155 .issue(child_session_id)
156 .map_err(|error| AgentError::LLM(format!("mint Codex per-run provider token: {error}")))?;
157 let guard = CodexRunTokenGuard {
158 authority,
159 token_id: issued.token_id,
160 };
161 Ok((
162 bamboo_subagent::proto::RunSecrets {
163 codex_provider_token: Some(bamboo_subagent::proto::SecretValue::new(issued.token)),
164 },
165 Some(guard),
166 ))
167}
168
169struct PooledWorker {
175 worker: SpawnedChild,
176 mailbox_id: String,
178}
179
180#[derive(Debug, Clone)]
186pub struct ResolvedRemotePlacement {
187 pub endpoint: String,
188 pub token: Option<String>,
189 pub ca_cert_file: Option<PathBuf>,
190 pub host_label: Option<String>,
194}
195
196#[derive(Debug, Clone)]
203pub struct ResolvedSchedulablePlacement {
204 pub pool: String,
205 pub host_label: Option<String>,
209}
210
211enum PlacementKind {
217 Local,
218 Remote,
219 Schedulable,
220}
221
222pub struct ActorChildRunner {
224 approval_registry: Option<super::approval_registry::SharedApprovalRegistry>,
225 permission_config: Option<Arc<bamboo_tools::permission::PermissionConfig>>,
226 agent_id: String,
227 worker_bin: PathBuf,
228 worker_args: Vec<String>,
229 fabric_dir: PathBuf,
230 executor: ExecutorSpec,
231 credentials: Vec<ScopedCredential>,
234 default_provider: String,
236 bus: Option<bamboo_subagent::BusEndpoint>,
239 concurrency: std::sync::Arc<tokio::sync::Semaphore>,
243 pool: Arc<tokio::sync::Mutex<HashMap<String, Vec<PooledWorker>>>>,
249 max_idle_per_key: usize,
250 approval_decider: Option<Arc<dyn ChildApprovalDecider>>,
254 approval_reviewer: Option<Arc<dyn ChildApprovalReviewer>>,
258 escalation_bridge: Arc<std::sync::Mutex<Option<bamboo_subagent::executor::HostBridge>>>,
269 remote_placements: HashMap<String, ResolvedRemotePlacement>,
275 schedulable_placements: HashMap<String, ResolvedSchedulablePlacement>,
283 schedule_cursor: Arc<std::sync::Mutex<HashMap<String, usize>>>,
288 codex_run_tokens: Option<Arc<dyn CodexRunTokenAuthority>>,
290 session_inbox_runtime: Arc<std::sync::Mutex<Option<SessionInboxRuntimeBinding>>>,
293}
294
295#[async_trait]
305pub trait ChildApprovalDecider: Send + Sync {
306 async fn decide(&self, child_session_id: &str, request: &serde_json::Value) -> bool;
309}
310
311async fn decide_child_approval(
314 decider: Option<&Arc<dyn ChildApprovalDecider>>,
315 child_session_id: &str,
316 request: &serde_json::Value,
317) -> bool {
318 match decider {
319 Some(decider) => decider.decide(child_session_id, request).await,
320 None => false,
321 }
322}
323
324const CHILD_APPROVAL_TIMEOUT: Duration = Duration::from_secs(300);
328
329#[async_trait]
338pub trait ChildApprovalReviewer: Send + Sync {
339 async fn review(
342 &self,
343 parent_session_id: &str,
344 child_session_id: &str,
345 request: &serde_json::Value,
346 ) -> bool;
347}
348
349fn child_approval_reviewer_slot() -> &'static std::sync::OnceLock<Arc<dyn ChildApprovalReviewer>> {
350 static SLOT: std::sync::OnceLock<Arc<dyn ChildApprovalReviewer>> = std::sync::OnceLock::new();
351 &SLOT
352}
353
354pub fn set_child_approval_reviewer(reviewer: Arc<dyn ChildApprovalReviewer>) {
356 let _ = child_approval_reviewer_slot().set(reviewer);
357}
358
359pub fn child_approval_reviewer() -> Option<Arc<dyn ChildApprovalReviewer>> {
361 child_approval_reviewer_slot().get().cloned()
362}
363
364impl ActorChildRunner {
365 #[allow(clippy::too_many_arguments)]
366 pub fn new(
367 agent_id: String,
368 worker_bin: PathBuf,
369 worker_args: Vec<String>,
370 fabric_dir: PathBuf,
371 executor: ExecutorSpec,
372 credentials: Vec<ScopedCredential>,
373 default_provider: String,
374 max_concurrent: usize,
375 ) -> Self {
376 Self {
377 approval_registry: None,
378 permission_config: None,
379 agent_id,
380 worker_bin,
381 worker_args,
382 fabric_dir,
383 executor,
384 credentials,
385 default_provider,
386 bus: None,
387 concurrency: std::sync::Arc::new(tokio::sync::Semaphore::new(max_concurrent.max(1))),
388 pool: Arc::new(tokio::sync::Mutex::new(HashMap::new())),
389 max_idle_per_key: DEFAULT_MAX_IDLE_PER_KEY,
390 approval_decider: None,
391 approval_reviewer: None,
392 escalation_bridge: Arc::new(std::sync::Mutex::new(None)),
393 remote_placements: HashMap::new(),
394 schedulable_placements: HashMap::new(),
395 schedule_cursor: Arc::new(std::sync::Mutex::new(HashMap::new())),
396 codex_run_tokens: None,
397 session_inbox_runtime: Arc::new(std::sync::Mutex::new(None)),
398 }
399 }
400
401 pub fn with_approval_registry(
402 mut self,
403 registry: super::approval_registry::SharedApprovalRegistry,
404 ) -> Self {
405 self.approval_registry = Some(registry);
406 self
407 }
408
409 pub fn with_permission_config(
410 mut self,
411 config: Arc<bamboo_tools::permission::PermissionConfig>,
412 ) -> Self {
413 self.permission_config = Some(config);
414 self
415 }
416
417 pub fn with_bus(mut self, bus: Option<bamboo_subagent::BusEndpoint>) -> Self {
422 self.bus = bus.filter(|b| !b.endpoint.trim().is_empty());
423 self
424 }
425
426 pub fn with_approval_decider(mut self, decider: Arc<dyn ChildApprovalDecider>) -> Self {
429 self.approval_decider = Some(decider);
430 self
431 }
432
433 pub fn with_approval_reviewer(mut self, reviewer: Arc<dyn ChildApprovalReviewer>) -> Self {
434 self.approval_reviewer = Some(reviewer);
435 self
436 }
437
438 pub fn with_codex_run_tokens(
439 mut self,
440 authority: Option<Arc<dyn CodexRunTokenAuthority>>,
441 ) -> Self {
442 self.codex_run_tokens = authority;
443 self
444 }
445
446 pub fn with_remote_placements(
452 mut self,
453 placements: HashMap<String, ResolvedRemotePlacement>,
454 ) -> Self {
455 self.remote_placements = placements;
456 self
457 }
458
459 pub fn with_schedulable_placements(
465 mut self,
466 placements: HashMap<String, ResolvedSchedulablePlacement>,
467 ) -> Self {
468 self.schedulable_placements = placements;
469 self
470 }
471
472 fn fingerprint(spec: &ProvisionSpec) -> String {
489 let role = spec.identity.role.as_str();
490 let (provider, model) = spec
491 .model
492 .as_ref()
493 .map(|m| (m.provider.as_str(), m.model.as_str()))
494 .unwrap_or(("", ""));
495 let workspace = spec.workspace.as_deref().unwrap_or("");
496 let mut tools = spec.disabled_tools.clone().unwrap_or_default();
497 tools.sort();
498 let caps = &spec.capabilities;
499 let executor = serde_json::to_string(&spec.executor).unwrap_or_default();
502 format!(
503 "{role}\u{1}{provider}\u{1}{model}\u{1}{workspace}\u{1}{}\u{1}d={}\u{1}ns={}\u{1}by={}\u{1}ep={}\u{1}md={}\u{1}nha={}\u{1}gro={}\u{1}executor={executor}",
504 tools.join(","),
505 spec.identity.depth,
506 caps.nested_spawn,
507 caps.bypass,
508 caps.enforce_permissions,
509 caps.max_spawn_depth.unwrap_or(0),
510 caps.no_human_approver,
517 caps.guardian_read_only,
521 )
522 }
523
524 async fn acquire_bus_worker(
530 &self,
531 key: &str,
532 spec: &ProvisionSpec,
533 ) -> crate::runtime::runner::Result<PooledWorker> {
534 loop {
537 let candidate = {
538 let mut pool = self.pool.lock().await;
539 pool.get_mut(key).and_then(|bucket| bucket.pop())
540 };
541 let Some(mut candidate) = candidate else {
542 break;
543 };
544 if candidate.worker.is_alive() {
545 return Ok(candidate);
546 }
547 candidate.worker.kill().await;
548 }
549
550 let spawned = spawn_worker_on_bus(&self.worker_bin, &self.worker_args, spec)
551 .await
552 .map_err(|e| AgentError::LLM(format!("actor spawn (bus) failed: {e}")))?;
553 let mailbox_id = spawned.record.agent_id.clone();
554 Ok(PooledWorker {
555 worker: spawned,
556 mailbox_id,
557 })
558 }
559
560 async fn release_bus_worker(&self, key: &str, mut worker: PooledWorker) {
564 if !worker.worker.is_alive() {
565 worker.worker.kill().await;
566 return;
567 }
568 let mut pool = self.pool.lock().await;
569 let bucket = pool.entry(key.to_string()).or_default();
570 if bucket.len() >= self.max_idle_per_key {
571 drop(pool);
572 worker.worker.kill().await;
573 return;
574 }
575 bucket.push(worker);
576 }
577
578 fn build_spec(&self, session: &Session, job: &SpawnJob) -> ProvisionSpec {
580 let mut spec = ProvisionSpec::new(
581 ChildIdentity {
582 child_id: job.child_session_id.clone(),
583 parent_id: Some(job.parent_session_id.clone()),
584 project_key: None,
585 role: session
586 .metadata
587 .get("subagent_type")
588 .cloned()
589 .unwrap_or_else(|| "worker".to_string()),
590 depth: session.spawn_depth,
595 },
596 self.executor.clone(),
597 self.fabric_dir.to_string_lossy().into_owned(),
598 );
599 spec.workspace = session.workspace.clone();
600 if let ExecutorSpec::Codex {
601 workspace_owned, ..
602 } = &mut spec.executor
603 {
604 *workspace_owned = Some(
605 spec.workspace
606 .as_deref()
607 .is_some_and(workspace_is_bamboo_owned),
608 );
609 }
610 spec.bus = self.bus.clone();
613 spec.model = session
616 .model_ref
617 .as_ref()
618 .map(|r| ModelRefSpec {
619 provider: r.provider.clone(),
620 model: r.model.clone(),
621 })
622 .or_else(|| {
623 let m = job.model.trim();
624 (!m.is_empty()).then(|| ModelRefSpec {
625 provider: self.default_provider.clone(),
626 model: m.to_string(),
627 })
628 });
629 spec.disabled_tools = job.disabled_tools.clone();
630 match &spec.executor {
631 ExecutorSpec::Codex {
637 auth_mode,
638 provider_key_ref,
639 ..
640 } => {
641 if auth_mode.as_deref() == Some("custom") {
642 if let Some(reference) = provider_key_ref {
643 if let Some(credential) = self.credentials.iter().find(|credential| {
644 credential.credential_ref.as_deref() == Some(reference)
645 }) {
646 spec.secrets.provider_credentials.push(credential.clone());
647 } else {
648 tracing::warn!(
649 "actor child {}: custom Codex credential reference '{}' did not resolve",
650 job.child_session_id,
651 reference
652 );
653 }
654 } else {
655 tracing::warn!(
656 "actor child {}: custom Codex executor has no credential reference",
657 job.child_session_id
658 );
659 }
660 }
661 }
662 _ => {
665 let provider = spec
666 .model
667 .as_ref()
668 .map(|model| model.provider.as_str())
669 .filter(|provider| !provider.trim().is_empty())
670 .unwrap_or(&self.default_provider);
671 if let Some(credential) = self
672 .credentials
673 .iter()
674 .find(|credential| credential.provider == provider)
675 {
676 spec.secrets.provider_credentials.push(credential.clone());
677 } else {
678 tracing::warn!(
679 "actor child {}: no credential found for provider '{}'",
680 job.child_session_id,
681 provider
682 );
683 }
684 }
685 }
686 spec.capabilities.nested_spawn = session.spawn_depth < MAX_SPAWN_DEPTH;
693 spec.capabilities.max_spawn_depth = Some(MAX_SPAWN_DEPTH);
694 spec.capabilities.enforce_permissions = true;
700 spec.capabilities.bypass = session
705 .agent_runtime_state
706 .as_ref()
707 .is_some_and(|s| s.bypass_permissions);
708 spec.capabilities.no_human_approver = session
713 .agent_runtime_state
714 .as_ref()
715 .is_some_and(|s| s.no_human_approver);
716 spec.capabilities.guardian_read_only =
726 session.metadata.get("subagent_type").map(String::as_str) == Some("guardian");
727 if spec.capabilities.guardian_read_only {
728 if let ExecutorSpec::Codex {
729 permission_profile, ..
730 } = &mut spec.executor
731 {
732 *permission_profile = Some("read-only".to_string());
733 }
734 }
735 if let Some(placement) = self.remote_placements.get(spec.identity.role.as_str()) {
742 spec.placement = Placement::Remote {
743 endpoint: placement.endpoint.clone(),
744 };
745 spec.secrets.worker_auth_token = placement.token.clone();
746 } else if let Some(placement) = self.schedulable_placements.get(spec.identity.role.as_str())
747 {
748 spec.placement = Placement::Schedulable {
756 pool: placement.pool.clone(),
757 };
758 }
759 spec
760 }
761
762 fn placement_stamp_for(&self, spec: &ProvisionSpec) -> Option<String> {
768 let host_label = match &spec.placement {
769 Placement::Remote { .. } => self
770 .remote_placements
771 .get(spec.identity.role.as_str())
772 .and_then(|p| p.host_label.as_deref()),
773 Placement::Schedulable { .. } => self
774 .schedulable_placements
775 .get(spec.identity.role.as_str())
776 .and_then(|p| p.host_label.as_deref()),
777 Placement::Local => None,
778 };
779 placement_metadata(&spec.placement, host_label)
780 }
781
782 async fn resolve_schedulable_worker(
789 &self,
790 role: &str,
791 ) -> std::result::Result<String, AgentError> {
792 let pool = self
793 .schedulable_placements
794 .get(role)
795 .ok_or_else(|| {
796 AgentError::LLM(format!(
797 "schedulable placement for role '{role}' vanished before scheduling"
798 ))
799 })?
800 .pool
801 .clone();
802 let bus = self.bus.as_ref().ok_or_else(|| {
803 AgentError::LLM(format!(
804 "schedulable role '{role}': no mailbox bus configured (subagents.broker)"
805 ))
806 })?;
807
808 let mut q = bamboo_broker::BrokerClient::connect(
811 &bus.endpoint,
812 bamboo_subagent::AgentRef {
813 session_id: format!("sched-q-{role}"),
814 role: None,
815 },
816 &bus.token,
817 )
818 .await
819 .map_err(|e| {
820 AgentError::LLM(format!(
821 "schedulable role '{role}': bus connect failed: {e}"
822 ))
823 })?;
824 let candidates = q.list_connected(&pool).await.map_err(|e| {
825 AgentError::LLM(format!(
826 "schedulable role '{role}': bus presence query failed: {e}"
827 ))
828 })?;
829
830 if candidates.is_empty() {
831 return Err(AgentError::LLM(format!(
832 "schedulable role '{role}': no live worker in pool '{pool}' on the bus \
833 (NOT spawning a local subprocess — a schedulable role has no local fallback)"
834 )));
835 }
836
837 let idx = {
842 let mut cursors = self.schedule_cursor.lock().recover_poison();
843 let cursor = cursors.entry(pool.clone()).or_insert(0);
844 let i = *cursor % candidates.len();
845 *cursor = cursor.wrapping_add(1);
846 i
847 };
848 Ok(candidates[idx].clone())
849 }
850}
851
852#[async_trait]
853impl ExternalChildRunner for ActorChildRunner {
854 async fn should_handle(&self, session: &Session) -> bool {
855 session.metadata.get("runtime.kind") == Some(&"external".to_string())
856 && session.metadata.get("external.protocol") == Some(&"actor".to_string())
857 && session.metadata.get("external.agent_id") == Some(&self.agent_id)
858 }
859
860 fn set_escalation_bridge(&self, bridge: Option<bamboo_subagent::executor::HostBridge>) {
861 *self.escalation_bridge.lock().recover_poison() = bridge;
862 }
863
864 fn set_session_inbox_runtime(&self, binding: Option<SessionInboxRuntimeBinding>) {
865 *self.session_inbox_runtime.lock().recover_poison() = binding;
866 }
867
868 async fn execute_external_child(
869 &self,
870 session: &mut Session,
871 job: &SpawnJob,
872 event_tx: mpsc::Sender<AgentEvent>,
873 cancel_token: CancellationToken,
874 ) -> crate::runtime::runner::Result<()> {
875 let escalation = self.escalation_bridge.lock().recover_poison().clone();
884 let session_inbox_runtime = self.session_inbox_runtime.lock().recover_poison().clone();
885 let assignment = extract_assignment(session);
886 let mut spec = self.build_spec(session, job);
887 spec.reusable = true;
890 if spec.limits.idle_timeout_secs.is_none() {
891 spec.limits.idle_timeout_secs = Some(POOLED_IDLE_TIMEOUT_SECS);
892 }
893 let pool_key = Self::fingerprint(&spec);
894
895 if executor_uses_bamboo_codex(&spec.executor) && !matches!(spec.placement, Placement::Local)
900 {
901 return Err(AgentError::LLM(
902 "Codex auth mode 'bamboo' requires local actor placement; use custom mode with a reachable URL for remote workers"
903 .to_string(),
904 ));
905 }
906 let project_id = project_id_for_actor_run(session)?;
907 let permission_policy = self.permission_config.as_ref().and_then(|config| {
912 serde_json::to_value(config.to_serializable())
913 .ok()
914 .map(|policy| PermissionPolicyContext {
915 revision: config.policy_revision(),
916 bypass_permissions: session
917 .agent_runtime_state
918 .as_ref()
919 .is_some_and(|state| state.bypass_permissions),
920 session_id: session.id.clone(),
921 workspace_path: session.workspace.clone(),
922 inherit_session_grants: false,
923 policy,
924 })
925 });
926
927 let _slot = self
932 .concurrency
933 .acquire()
934 .await
935 .map_err(|_| AgentError::LLM("actor concurrency limiter closed".to_string()))?;
936
937 let (run_secrets, _codex_token_guard) = build_codex_run_secrets(
944 &spec.executor,
945 self.codex_run_tokens.clone(),
946 &job.child_session_id,
947 )?;
948
949 let kind = match spec.placement {
957 Placement::Remote { .. } => PlacementKind::Remote,
958 Placement::Schedulable { .. } => PlacementKind::Schedulable,
959 Placement::Local => PlacementKind::Local,
960 };
961 let remote = !matches!(kind, PlacementKind::Local);
962
963 if let Some(placement_meta) = self.placement_stamp_for(&spec) {
969 session
970 .metadata
971 .insert("placement".to_string(), placement_meta);
972 }
973
974 let mut attempt = 0u8;
981 let (result, actor) = loop {
982 let (actor, mut client) = match kind {
983 PlacementKind::Remote => {
984 let placement = self
988 .remote_placements
989 .get(spec.identity.role.as_str())
990 .ok_or_else(|| {
991 AgentError::LLM(format!(
992 "remote placement for role '{}' vanished before connect",
993 spec.identity.role
994 ))
995 })?;
996 let endpoint = placement.endpoint.clone();
997 let trust_cfg = match placement.ca_cert_file.as_deref() {
1000 Some(path) => Some(client_config_trusting_cert(path).map_err(|e| {
1001 AgentError::LLM(format!(
1002 "remote worker CA cert '{}': {e}",
1003 path.display()
1004 ))
1005 })?),
1006 None => None,
1007 };
1008 let client = ChildClient::connect_with_auth_tls(
1009 &endpoint,
1010 placement.token.as_deref(),
1011 trust_cfg,
1012 )
1013 .await
1014 .map_err(|e| {
1015 AgentError::LLM(format!("remote actor connect to '{endpoint}' failed: {e}"))
1016 })?;
1017 let record = AgentRecord {
1020 agent_id: job.child_session_id.clone(),
1021 role: spec.identity.role.clone(),
1022 labels: Vec::new(),
1023 endpoint: endpoint.clone(),
1024 pid: 0,
1025 version: String::new(),
1026 started_at: chrono::Utc::now(),
1027 lease_expires_at: chrono::Utc::now(),
1028 };
1029 let _ = endpoint;
1030 let actor = PooledWorker {
1031 worker: SpawnedChild::remote(record),
1032 mailbox_id: job.child_session_id.clone(),
1033 };
1034 let client: Box<dyn bamboo_subagent::ChildLink> = Box::new(client);
1035 (actor, client)
1036 }
1037 PlacementKind::Schedulable => {
1038 let bus = self.bus.as_ref().ok_or_else(|| {
1045 AgentError::LLM(
1046 "schedulable sub-agents require a mailbox bus (subagents.broker)"
1047 .to_string(),
1048 )
1049 })?;
1050 let mailbox_id = self
1051 .resolve_schedulable_worker(spec.identity.role.as_str())
1052 .await?;
1053 let parent = bamboo_subagent::AgentRef {
1054 session_id: format!("p-{}", job.child_session_id),
1055 role: None,
1056 };
1057 let link = bamboo_broker::BrokerChildLink::connect(
1058 &bus.endpoint,
1059 parent,
1060 &bus.token,
1061 mailbox_id.clone(),
1062 )
1063 .await
1064 .map_err(|e| {
1065 AgentError::LLM(format!(
1066 "schedulable link connect to '{mailbox_id}' failed: {e}"
1067 ))
1068 })?;
1069 let actor = PooledWorker {
1072 worker: SpawnedChild::remote(AgentRecord {
1073 agent_id: mailbox_id.clone(),
1074 role: spec.identity.role.clone(),
1075 labels: Vec::new(),
1076 endpoint: bus.endpoint.clone(),
1077 pid: 0,
1078 version: String::new(),
1079 started_at: chrono::Utc::now(),
1080 lease_expires_at: chrono::Utc::now(),
1081 }),
1082 mailbox_id,
1083 };
1084 let client: Box<dyn bamboo_subagent::ChildLink> = Box::new(link);
1085 (actor, client)
1086 }
1087 PlacementKind::Local => {
1088 let bus = self.bus.as_ref().ok_or_else(|| {
1095 AgentError::LLM(
1096 "local sub-agents require a mailbox bus (subagents.broker); none is \
1097 configured and the bus could not be embedded"
1098 .to_string(),
1099 )
1100 })?;
1101 let actor = self.acquire_bus_worker(&pool_key, &spec).await?;
1102 let parent = bamboo_subagent::AgentRef {
1103 session_id: format!("p-{}", job.child_session_id),
1104 role: None,
1105 };
1106 let link = bamboo_broker::BrokerChildLink::connect(
1107 &bus.endpoint,
1108 parent,
1109 &bus.token,
1110 actor.mailbox_id.clone(),
1111 )
1112 .await
1113 .map_err(|e| {
1114 AgentError::LLM(format!("broker child link connect failed: {e}"))
1115 })?;
1116 let client: Box<dyn bamboo_subagent::ChildLink> = Box::new(link);
1117 (actor, client)
1118 }
1119 };
1120
1121 let (delivery_tx, mut delivery_rx) = mpsc::unbounded_channel::<u64>();
1126 let bound_activation_run_id = match session_inbox_runtime.as_ref() {
1127 Some(binding) => {
1128 let run_id = binding
1129 .router
1130 .attach_delivery_sink(&job.child_session_id, delivery_tx.clone())
1131 .await;
1132 if run_id.is_none() {
1133 tracing::debug!(
1134 session_id = %job.child_session_id,
1135 "actor driver had no current SessionInbox activation owner to bind"
1136 );
1137 }
1138 run_id
1139 }
1140 None => None,
1141 };
1142 drop(delivery_tx);
1143 let initial_pairs = match (
1144 session_inbox_runtime.as_ref(),
1145 bound_activation_run_id.as_deref(),
1146 ) {
1147 (Some(binding), Some(run_id)) => {
1148 match claim_canonical_deliveries(binding, session, run_id, usize::MAX).await {
1149 Ok(deliveries) => deliveries,
1150 Err(error) => {
1151 binding
1152 .router
1153 .detach_delivery_sink(&job.child_session_id, run_id)
1154 .await;
1155 if !remote {
1156 actor.worker.kill().await;
1157 }
1158 return Err(error);
1159 }
1160 }
1161 }
1162 _ => Vec::new(),
1163 };
1164 let initial_session_messages = initial_pairs
1165 .iter()
1166 .map(|(_, delivery)| delivery.clone())
1167 .collect::<Vec<_>>();
1168 let initial_inflight_claims = initial_pairs
1169 .into_iter()
1170 .map(|(claim, _)| claim)
1171 .collect::<VecDeque<_>>();
1172 let messages = session
1175 .messages
1176 .iter()
1177 .filter_map(|message| serde_json::to_value(message).ok())
1178 .collect();
1179
1180 if let Err(e) = client
1181 .send(ParentFrame::Run(RunSpec {
1182 assignment: assignment.clone(),
1184 logical_session: Some(logical_identity_for_actor_run(session, job)),
1185 project_id: project_id.clone(),
1186 reasoning_effort: None,
1187 permission_policy: permission_policy.clone(),
1188 messages,
1189 activation_run_id: bound_activation_run_id.clone(),
1190 initial_session_messages,
1191 secrets: run_secrets.clone(),
1192 }))
1193 .await
1194 {
1195 if let (Some(binding), Some(run_id)) = (
1196 session_inbox_runtime.as_ref(),
1197 bound_activation_run_id.as_deref(),
1198 ) {
1199 binding
1200 .router
1201 .detach_delivery_sink(&job.child_session_id, run_id)
1202 .await;
1203 }
1204 if !remote {
1205 actor.worker.kill().await;
1206 }
1207 return Err(AgentError::LLM(format!("actor run dispatch failed: {e}")));
1208 }
1209
1210 let (live_tx, mut live_rx) = mpsc::unbounded_channel::<ParentFrame>();
1214 let live_guard = super::live::register(
1215 &job.child_session_id,
1216 live_tx,
1217 attempt as u32,
1218 self.approval_registry.clone(),
1219 );
1220
1221 let result = drive(
1222 &mut *client,
1223 &job.parent_session_id,
1224 &job.child_session_id,
1225 attempt as u32,
1226 self.approval_registry.as_ref(),
1227 self.approval_decider.as_ref(),
1228 self.approval_reviewer.as_ref(),
1229 escalation.clone(),
1230 &event_tx,
1231 &cancel_token,
1232 &mut live_rx,
1233 &mut delivery_rx,
1234 session,
1235 session_inbox_runtime.as_ref(),
1236 bound_activation_run_id.as_deref(),
1237 initial_inflight_claims,
1238 Some(WORKER_FIRST_FRAME_TIMEOUT),
1245 )
1246 .await;
1247 if let (Some(binding), Some(run_id)) = (
1248 session_inbox_runtime.as_ref(),
1249 bound_activation_run_id.as_deref(),
1250 ) {
1251 binding
1252 .router
1253 .detach_delivery_sink(&job.child_session_id, run_id)
1254 .await;
1255 }
1256 drop(live_guard);
1262 drop(client);
1265
1266 if attempt == 0 && matches!(result, Err(AgentError::WorkerUnresponsive(_))) {
1273 match kind {
1274 PlacementKind::Local => {
1275 tracing::warn!(
1276 "actor child {} got no first frame; reaping the worker and respawning once",
1277 job.child_session_id
1278 );
1279 actor.worker.kill().await;
1280 attempt += 1;
1281 continue;
1282 }
1283 PlacementKind::Schedulable => {
1284 tracing::warn!(
1285 "scheduled actor child {} got no first frame; re-selecting a pool worker",
1286 job.child_session_id
1287 );
1288 drop(actor);
1289 attempt += 1;
1290 continue;
1291 }
1292 PlacementKind::Remote => {}
1293 }
1294 }
1295 break (result, actor);
1296 };
1297
1298 if remote {
1302 drop(actor);
1303 } else {
1304 match &result {
1305 Ok(_) => self.release_bus_worker(&pool_key, actor).await,
1306 Err(_) => actor.worker.kill().await,
1307 }
1308 }
1309
1310 match result {
1314 Ok(Some(text)) => {
1315 if !text.is_empty() {
1316 session.add_message(bamboo_agent_core::Message::assistant(text, None));
1317 }
1318 Ok(())
1319 }
1320 Ok(None) => Ok(()),
1321 Err(e) => Err(e),
1322 }
1323 }
1324}
1325
1326fn placement_metadata(placement: &Placement, host_label: Option<&str>) -> Option<String> {
1332 let value = match placement {
1335 Placement::Local => return None,
1336 Placement::Remote { endpoint } => serde_json::json!({
1337 "kind": "remote",
1338 "host": host_label.map(str::to_string).unwrap_or_else(|| host_of_endpoint(endpoint)),
1339 }),
1340 Placement::Schedulable { pool } => serde_json::json!({
1341 "kind": "remote",
1342 "host": host_label.unwrap_or(pool),
1343 }),
1344 };
1345 serde_json::to_string(&value).ok()
1346}
1347
1348fn host_of_endpoint(endpoint: &str) -> String {
1350 endpoint
1351 .trim()
1352 .trim_start_matches("wss://")
1353 .trim_start_matches("ws://")
1354 .split(['/', ':'])
1355 .next()
1356 .unwrap_or(endpoint)
1357 .to_string()
1358}
1359
1360async fn reconcile_already_admitted_claim(
1361 binding: &SessionInboxRuntimeBinding,
1362 session: &mut Session,
1363 claim: &SessionInboxClaim,
1364) -> crate::runtime::runner::Result<()> {
1365 let latest = binding
1366 .storage
1367 .load_session(&session.id)
1368 .await
1369 .map_err(|error| {
1370 AgentError::LLM(format!(
1371 "load canonical SessionInbox checkpoint for {}: {error}",
1372 session.id
1373 ))
1374 })?
1375 .ok_or_else(|| {
1376 AgentError::LLM(format!(
1377 "canonical SessionInbox target disappeared: {}",
1378 session.id
1379 ))
1380 })?;
1381 let Some(message) = latest
1382 .messages
1383 .iter()
1384 .find(|message| bamboo_domain::is_matching_session_message(message, &claim.envelope))
1385 .cloned()
1386 else {
1387 return Err(AgentError::LLM(format!(
1388 "canonical admitted receipt for {} exists without transcript message {}",
1389 session.id, claim.envelope.id
1390 )));
1391 };
1392 if let Some(existing) = session
1393 .messages
1394 .iter_mut()
1395 .find(|existing| existing.id == message.id)
1396 {
1397 *existing = message;
1398 } else {
1399 session.add_message(message);
1400 }
1401 bamboo_domain::merge_session_inbox_admission(session, &latest);
1402 binding
1403 .inbox
1404 .ack(&session.id, claim)
1405 .await
1406 .map_err(|error| {
1407 AgentError::LLM(format!(
1408 "ack recovered canonical SessionInbox claim {}: {error}",
1409 claim.envelope.id
1410 ))
1411 })
1412}
1413
1414async fn checkpoint_and_ack_canonical_claim(
1418 binding: &SessionInboxRuntimeBinding,
1419 session: &mut Session,
1420 claim: &SessionInboxClaim,
1421) -> crate::runtime::runner::Result<()> {
1422 if claim.envelope.target_session_id != session.id {
1423 return Err(AgentError::LLM(format!(
1424 "canonical SessionInbox claim target {} does not match active logical session {}",
1425 claim.envelope.target_session_id, session.id
1426 )));
1427 }
1428 if binding
1429 .inbox
1430 .was_admitted(&session.id, &claim.envelope.id)
1431 .await
1432 .map_err(|error| {
1433 AgentError::LLM(format!(
1434 "inspect canonical admitted receipt {}: {error}",
1435 claim.envelope.id
1436 ))
1437 })?
1438 {
1439 return reconcile_already_admitted_claim(binding, session, claim).await;
1440 }
1441
1442 let transcript_has_id = session
1443 .messages
1444 .iter()
1445 .any(|message| bamboo_domain::is_matching_session_message(message, &claim.envelope));
1446 if session
1447 .messages
1448 .iter()
1449 .any(|message| message.id == claim.envelope.id.as_str())
1450 && !transcript_has_id
1451 {
1452 return Err(AgentError::LLM(format!(
1453 "canonical SessionInbox id {} collides with a non-matching transcript message",
1454 claim.envelope.id
1455 )));
1456 }
1457 let cursor_has_id = session
1458 .session_inbox_admission()
1459 .is_some_and(|state| state.contains(&claim.envelope.id));
1460 if cursor_has_id && !transcript_has_id {
1461 return Err(AgentError::LLM(format!(
1462 "canonical SessionInbox cursor exists without transcript message {}",
1463 claim.envelope.id
1464 )));
1465 }
1466 let before = session.clone();
1467 if !transcript_has_id {
1468 let message = claim.envelope.to_provider_message().map_err(|error| {
1469 AgentError::LLM(format!(
1470 "translate canonical SessionInbox envelope {}: {error}",
1471 claim.envelope.id
1472 ))
1473 })?;
1474 session.add_message(message);
1475 }
1476 session
1477 .session_inbox_admission_mut()
1478 .record(claim.envelope.id.clone(), claim.generation);
1479 session.updated_at = chrono::Utc::now();
1480
1481 if let Err(error) = binding
1482 .persistence
1483 .checkpoint_runtime_session(session)
1484 .await
1485 {
1486 *session = before;
1487 return Err(AgentError::LLM(format!(
1488 "checkpoint canonical SessionInbox claim {}: {error}",
1489 claim.envelope.id
1490 )));
1491 }
1492 if !session
1493 .messages
1494 .iter()
1495 .any(|message| bamboo_domain::is_matching_session_message(message, &claim.envelope))
1496 {
1497 *session = before;
1498 return Err(AgentError::LLM(format!(
1499 "canonical SessionInbox checkpoint lost typed transcript proof for {}",
1500 claim.envelope.id
1501 )));
1502 }
1503 binding
1504 .inbox
1505 .ack(&session.id, claim)
1506 .await
1507 .map_err(|error| {
1508 AgentError::LLM(format!(
1509 "ack canonical SessionInbox claim {} after checkpoint: {error}",
1510 claim.envelope.id
1511 ))
1512 })
1513}
1514
1515async fn checkpoint_claim_context_before_dispatch(
1526 binding: &SessionInboxRuntimeBinding,
1527 session: &mut Session,
1528 claims: &[SessionInboxClaim],
1529) -> crate::runtime::runner::Result<()> {
1530 if claims.is_empty() {
1531 return Ok(());
1532 }
1533 let before = session.clone();
1534 for claim in claims {
1535 if claim.envelope.target_session_id != session.id {
1536 return Err(AgentError::LLM(format!(
1537 "canonical SessionInbox claim target {} does not match actor session {}",
1538 claim.envelope.target_session_id, session.id
1539 )));
1540 }
1541 let matching = session
1542 .messages
1543 .iter()
1544 .any(|message| bamboo_domain::is_matching_session_message(message, &claim.envelope));
1545 if session
1546 .messages
1547 .iter()
1548 .any(|message| message.id == claim.envelope.id.as_str())
1549 && !matching
1550 {
1551 return Err(AgentError::LLM(format!(
1552 "canonical SessionInbox id {} collides before actor dispatch",
1553 claim.envelope.id
1554 )));
1555 }
1556 if session
1557 .session_inbox_admission()
1558 .is_some_and(|state| state.contains(&claim.envelope.id))
1559 && !matching
1560 {
1561 return Err(AgentError::LLM(format!(
1562 "canonical SessionInbox cursor exists without transcript proof for {}",
1563 claim.envelope.id
1564 )));
1565 }
1566 if !matching {
1567 let message = claim.envelope.to_provider_message().map_err(|error| {
1568 AgentError::LLM(format!(
1569 "translate canonical SessionInbox envelope {} before actor dispatch: {error}",
1570 claim.envelope.id
1571 ))
1572 })?;
1573 session.add_message(message);
1574 }
1575 }
1576 session.updated_at = chrono::Utc::now();
1577 if let Err(error) = binding
1578 .persistence
1579 .checkpoint_runtime_session(session)
1580 .await
1581 {
1582 *session = before;
1583 return Err(AgentError::LLM(format!(
1584 "checkpoint canonical SessionInbox actor context: {error}"
1585 )));
1586 }
1587 for claim in claims {
1588 if !session
1589 .messages
1590 .iter()
1591 .any(|message| bamboo_domain::is_matching_session_message(message, &claim.envelope))
1592 {
1593 *session = before;
1594 return Err(AgentError::LLM(format!(
1595 "actor context checkpoint lost typed transcript proof for {}",
1596 claim.envelope.id
1597 )));
1598 }
1599 }
1600 Ok(())
1601}
1602
1603async fn claim_canonical_deliveries(
1604 binding: &SessionInboxRuntimeBinding,
1605 session: &mut Session,
1606 activation_run_id: &str,
1607 limit: usize,
1608) -> crate::runtime::runner::Result<Vec<(SessionInboxClaim, SessionMessageDelivery)>> {
1609 let claims = binding
1610 .inbox
1611 .claim(&session.id, limit)
1612 .await
1613 .map_err(|error| {
1614 AgentError::LLM(format!(
1615 "claim canonical SessionInbox for active actor {}: {error}",
1616 session.id
1617 ))
1618 })?;
1619 if claims.is_empty() {
1620 return Ok(Vec::new());
1621 }
1622 let interrupt_generation = binding
1623 .inbox
1624 .inspect(&session.id)
1625 .await
1626 .map_err(|error| {
1627 AgentError::LLM(format!(
1628 "inspect canonical SessionInbox activation policy for {}: {error}",
1629 session.id
1630 ))
1631 })?
1632 .interrupt_generation;
1633 let mut unconfirmed = Vec::with_capacity(claims.len());
1634 for claim in claims {
1635 if binding
1636 .inbox
1637 .was_admitted(&session.id, &claim.envelope.id)
1638 .await
1639 .map_err(|error| {
1640 AgentError::LLM(format!(
1641 "inspect canonical SessionInbox claim {}: {error}",
1642 claim.envelope.id
1643 ))
1644 })?
1645 {
1646 reconcile_already_admitted_claim(binding, session, &claim).await?;
1647 continue;
1648 }
1649 unconfirmed.push(claim);
1650 }
1651 checkpoint_claim_context_before_dispatch(binding, session, &unconfirmed).await?;
1652
1653 let mut deliveries = Vec::with_capacity(unconfirmed.len());
1654 for claim in unconfirmed {
1655 if session
1660 .session_inbox_admission()
1661 .is_some_and(|state| state.contains(&claim.envelope.id))
1662 {
1663 binding
1664 .inbox
1665 .ack(&session.id, &claim)
1666 .await
1667 .map_err(|error| {
1668 AgentError::LLM(format!(
1669 "finish confirmed canonical SessionInbox ack {}: {error}",
1670 claim.envelope.id
1671 ))
1672 })?;
1673 continue;
1674 }
1675 let activation_policy = if claim.generation <= interrupt_generation {
1676 bamboo_domain::SessionActivationPolicy::InterruptSpecificWait
1677 } else {
1678 bamboo_domain::SessionActivationPolicy::RespectSpecificWait
1679 };
1680 let delivery = SessionMessageDelivery {
1681 target_session_id: session.id.clone(),
1682 envelope: claim.envelope.clone(),
1683 canonical_claim_generation: claim.generation,
1684 activation_run_id: activation_run_id.to_string(),
1685 activation_policy,
1686 };
1687 deliveries.push((claim, delivery));
1688 }
1689 Ok(deliveries)
1690}
1691
1692async fn forward_next_canonical_claim(
1693 client: &mut dyn bamboo_subagent::ChildLink,
1694 binding: &SessionInboxRuntimeBinding,
1695 session: &mut Session,
1696 activation_run_id: &str,
1697 inflight: &mut VecDeque<SessionInboxClaim>,
1698) -> crate::runtime::runner::Result<()> {
1699 if !inflight.is_empty() {
1700 return Ok(());
1701 }
1702 let Some((claim, delivery)) =
1703 claim_canonical_deliveries(binding, session, activation_run_id, 1)
1704 .await?
1705 .pop()
1706 else {
1707 return Ok(());
1708 };
1709 client
1710 .send(ParentFrame::SessionMessage { delivery })
1711 .await
1712 .map_err(|error| {
1713 AgentError::LLM(format!(
1714 "forward canonical SessionInbox claim {} to active actor: {error}",
1715 claim.envelope.id
1716 ))
1717 })?;
1718 inflight.push_back(claim);
1719 Ok(())
1720}
1721
1722async fn drive(
1733 client: &mut dyn bamboo_subagent::ChildLink,
1734 parent_session_id: &str,
1735 child_session_id: &str,
1736 child_attempt: u32,
1737 approval_registry: Option<&super::approval_registry::SharedApprovalRegistry>,
1738 approval_decider: Option<&Arc<dyn ChildApprovalDecider>>,
1739 approval_reviewer: Option<&Arc<dyn ChildApprovalReviewer>>,
1740 escalation_bridge: Option<bamboo_subagent::executor::HostBridge>,
1741 event_tx: &mpsc::Sender<AgentEvent>,
1742 cancel_token: &CancellationToken,
1743 live_rx: &mut mpsc::UnboundedReceiver<ParentFrame>,
1744 delivery_rx: &mut mpsc::UnboundedReceiver<u64>,
1745 logical_session: &mut Session,
1746 session_inbox_runtime: Option<&SessionInboxRuntimeBinding>,
1747 activation_run_id: Option<&str>,
1748 initial_inflight_claims: VecDeque<SessionInboxClaim>,
1749 first_frame_timeout: Option<Duration>,
1750) -> crate::runtime::runner::Result<Option<String>> {
1751 let mut got_first_frame = false;
1758 let mut first_frame_watch = first_frame_timeout.map(|d| Box::pin(tokio::time::sleep(d)));
1759 let mut inflight_claims = initial_inflight_claims;
1760 loop {
1761 tokio::select! {
1762 _ = cancel_token.cancelled() => {
1763 break;
1765 }
1766 _ = async {
1767 match first_frame_watch.as_mut() {
1768 Some(s) => s.as_mut().await,
1769 None => std::future::pending::<()>().await,
1770 }
1771 }, if !got_first_frame => {
1772 return Err(AgentError::WorkerUnresponsive(format!(
1773 "child {child_session_id} produced no frame within {:?}",
1774 first_frame_timeout.unwrap_or_default()
1775 )));
1776 }
1777 Some(_generation) = delivery_rx.recv(),
1778 if session_inbox_runtime.is_some() && activation_run_id.is_some() =>
1779 {
1780 forward_next_canonical_claim(
1781 client,
1782 session_inbox_runtime.expect("guarded"),
1783 logical_session,
1784 activation_run_id.expect("guarded"),
1785 &mut inflight_claims,
1786 )
1787 .await?;
1788 }
1789 Some(frame) = live_rx.recv() => {
1790 if client.send(frame).await.is_err() {
1792 tracing::warn!("live steering frame could not be sent; connection failing");
1793 }
1794 }
1795 frame = client.next_frame() => {
1796 got_first_frame = true;
1799 first_frame_watch = None;
1800 match frame {
1801 Ok(Some(ChildFrame::Event { event })) => {
1802 if let Ok(ev) = serde_json::from_value::<AgentEvent>(event) {
1804 let _ = event_tx.send(ev).await;
1805 }
1806 }
1807 Ok(Some(ChildFrame::ApprovalRequest { id, body })) => {
1808 if let Some(reviewer) = approval_reviewer
1814 .cloned()
1815 .or_else(child_approval_reviewer)
1816 {
1817 let child = child_session_id.to_string();
1825 let parent = parent_session_id.to_string();
1826 let req_id = id.clone();
1827 let body = body.clone();
1828 let registry = approval_registry.cloned();
1829 tokio::spawn(async move {
1830 let approved = tokio::time::timeout(
1831 CHILD_APPROVAL_TIMEOUT,
1832 reviewer.review(&parent, &child, &body),
1833 )
1834 .await
1835 .unwrap_or(false);
1836 super::live::deliver_approval_scoped(
1837 registry.as_ref(),
1838 &child,
1839 child_attempt,
1840 &req_id,
1841 approved,
1842 );
1843 });
1844 } else if approval_decider.is_some() {
1845 let approved =
1849 decide_child_approval(approval_decider, child_session_id, &body)
1850 .await;
1851 if client
1852 .send(ParentFrame::ApprovalReply { id, approved })
1853 .await
1854 .is_err()
1855 {
1856 tracing::warn!(
1857 "failed to answer approval_request; connection failing"
1858 );
1859 }
1860 } else if let Some(host) = escalation_bridge.clone() {
1861 let child = child_session_id.to_string();
1868 let req_id = id.clone();
1869 let body = body.clone();
1870 let registry = approval_registry.cloned();
1871 tokio::spawn(async move {
1872 let approved = match tokio::time::timeout(
1873 CHILD_APPROVAL_TIMEOUT,
1874 host.approval_call(body),
1875 )
1876 .await
1877 {
1878 Ok(Ok(reply)) => reply
1879 .get("approved")
1880 .and_then(|v| v.as_bool())
1881 .unwrap_or(false),
1882 _ => false,
1884 };
1885 super::live::deliver_approval_scoped(
1886 registry.as_ref(),
1887 &child,
1888 child_attempt,
1889 &req_id,
1890 approved,
1891 );
1892 });
1893 } else {
1894 tracing::warn!(
1898 parent_session_id,
1899 child_session_id,
1900 request_id = %id,
1901 "forced-ask request has no parent-agent reviewer; denying"
1902 );
1903 if client
1904 .send(ParentFrame::ApprovalReply {
1905 id,
1906 approved: false,
1907 })
1908 .await
1909 .is_err()
1910 {
1911 tracing::warn!(
1912 "failed to send fail-closed approval reply; connection failing"
1913 );
1914 }
1915 }
1916 }
1917 Ok(Some(ChildFrame::SessionMessageAdmitted { confirmation })) => {
1918 let Some(binding) = session_inbox_runtime else {
1919 tracing::warn!(
1920 child_session_id,
1921 "ignoring SessionInbox confirmation without a runtime binding"
1922 );
1923 continue;
1924 };
1925 let Some(bound_run_id) = activation_run_id else {
1926 tracing::warn!(
1927 child_session_id,
1928 "ignoring SessionInbox confirmation without an activation owner"
1929 );
1930 continue;
1931 };
1932 let Some(claim) = inflight_claims.front() else {
1933 tracing::warn!(
1934 child_session_id,
1935 envelope_id = %confirmation.envelope_id,
1936 "rejecting stale SessionInbox confirmation with no in-flight canonical claim"
1937 );
1938 continue;
1939 };
1940 let exact = confirmation.target_session_id == logical_session.id
1941 && confirmation.envelope_id == claim.envelope.id.as_str()
1942 && confirmation.canonical_claim_generation == claim.generation
1943 && confirmation.activation_run_id == bound_run_id;
1944 if !exact
1945 || !binding
1946 .router
1947 .owns_run(&logical_session.id, bound_run_id)
1948 .await
1949 {
1950 tracing::warn!(
1951 child_session_id,
1952 expected_target = %logical_session.id,
1953 received_target = %confirmation.target_session_id,
1954 expected_envelope_id = %claim.envelope.id,
1955 received_envelope_id = %confirmation.envelope_id,
1956 expected_generation = claim.generation,
1957 received_generation = confirmation.canonical_claim_generation,
1958 expected_run_id = bound_run_id,
1959 received_run_id = %confirmation.activation_run_id,
1960 "rejecting stale or mismatched SessionInbox admission confirmation"
1961 );
1962 continue;
1963 }
1964 let claim = inflight_claims
1965 .pop_front()
1966 .expect("validated in-flight canonical claim");
1967 checkpoint_and_ack_canonical_claim(binding, logical_session, &claim)
1970 .await?;
1971 if inflight_claims.is_empty() {
1975 forward_next_canonical_claim(
1976 client,
1977 binding,
1978 logical_session,
1979 bound_run_id,
1980 &mut inflight_claims,
1981 )
1982 .await?;
1983 }
1984 }
1985 Ok(Some(ChildFrame::Terminal { status, result, error, .. })) => {
1986 if let Some(claim) = inflight_claims.front() {
1987 return Err(AgentError::LLM(format!(
1988 "actor terminated before durably admitting SessionInbox message {}; canonical claim remains recoverable",
1989 claim.envelope.id
1990 )));
1991 }
1992 return match status {
1993 TerminalStatus::Completed => Ok(result),
1994 TerminalStatus::Cancelled => Err(AgentError::Cancelled),
1995 TerminalStatus::Error => Err(AgentError::LLM(
1996 error.unwrap_or_else(|| "actor child errored".to_string()),
1997 )),
1998 TerminalStatus::Suspended => Err(AgentError::LLM(
2003 "nested sub-agent suspend received but resume transport is not wired"
2004 .to_string(),
2005 )),
2006 };
2007 }
2008 Ok(None) => {
2009 return Err(AgentError::LLM(
2010 "actor child closed before terminal".to_string(),
2011 ));
2012 }
2013 Err(e) => {
2014 return Err(AgentError::LLM(format!("actor transport error: {e}")));
2015 }
2016 }
2017 }
2018 }
2019 }
2020
2021 let _ = client.send(ParentFrame::Cancel).await;
2023 Err(AgentError::Cancelled)
2024}
2025
2026fn project_id_for_actor_run(
2028 session: &Session,
2029) -> Result<Option<bamboo_domain::ProjectId>, AgentError> {
2030 match crate::project_context::ProjectContextResolver::session_project_identity(session) {
2031 crate::project_context::SessionProjectIdentity::Assigned(project_id) => {
2032 Ok(Some(project_id))
2033 }
2034 crate::project_context::SessionProjectIdentity::Unassigned => Ok(None),
2035 crate::project_context::SessionProjectIdentity::Invalid { raw, message } => {
2036 Err(AgentError::LLM(format!(
2037 "child session carries an invalid Project identity '{raw}': {message}"
2038 )))
2039 }
2040 }
2041}
2042
2043fn logical_identity_for_actor_run(session: &Session, job: &SpawnJob) -> LogicalSessionIdentity {
2044 LogicalSessionIdentity {
2045 session_id: session.id.clone(),
2046 parent_session_id: session
2047 .parent_session_id
2048 .clone()
2049 .or_else(|| Some(job.parent_session_id.clone())),
2050 root_session_id: if session.root_session_id.trim().is_empty() {
2051 job.parent_session_id.clone()
2052 } else {
2053 session.root_session_id.clone()
2054 },
2055 }
2056}
2057
2058fn extract_assignment(session: &Session) -> String {
2059 session
2060 .messages
2061 .iter()
2062 .rev()
2063 .find(|m| matches!(m.role, Role::User))
2064 .map(|m| m.content.clone())
2065 .unwrap_or_else(|| {
2066 session
2067 .metadata
2068 .get("title")
2069 .cloned()
2070 .unwrap_or_else(|| "Execute task".to_string())
2071 })
2072}
2073
2074#[cfg(test)]
2075mod tests {
2076 use super::*;
2077 use crate::SessionActivationRouter;
2078 use bamboo_domain::{RuntimeSessionPersistence, SessionInboxPort, Storage};
2079
2080 struct ActorFaultingPersistence {
2081 inner: Arc<bamboo_storage::LockedSessionStore>,
2082 fail_checkpoint_once: std::sync::atomic::AtomicBool,
2083 }
2084
2085 #[async_trait]
2086 impl RuntimeSessionPersistence for ActorFaultingPersistence {
2087 async fn save_runtime_session(&self, session: &mut Session) -> std::io::Result<()> {
2088 self.inner.merge_save_runtime(session).await
2089 }
2090
2091 async fn checkpoint_runtime_session(&self, session: &mut Session) -> std::io::Result<()> {
2092 if self
2093 .fail_checkpoint_once
2094 .swap(false, std::sync::atomic::Ordering::SeqCst)
2095 {
2096 return Err(std::io::Error::other("injected actor checkpoint failure"));
2097 }
2098 self.inner.checkpoint_runtime_session(session).await
2099 }
2100
2101 async fn load_runtime_session(&self, session_id: &str) -> std::io::Result<Option<Session>> {
2102 self.inner.storage().load_session(session_id).await
2103 }
2104 }
2105
2106 struct ActorFailBeforeAckInbox {
2107 inner: Arc<dyn SessionInboxPort>,
2108 fail_once: std::sync::atomic::AtomicBool,
2109 }
2110
2111 #[async_trait]
2112 impl SessionInboxPort for ActorFailBeforeAckInbox {
2113 async fn deliver(
2114 &self,
2115 envelope: &bamboo_domain::SessionMessageEnvelope,
2116 ) -> Result<bamboo_domain::SessionInboxReceipt, bamboo_domain::SessionInboxError> {
2117 self.inner.deliver(envelope).await
2118 }
2119
2120 async fn mark_activation_eligible(
2121 &self,
2122 target_session_id: &str,
2123 generation: u64,
2124 policy: bamboo_domain::SessionActivationPolicy,
2125 ) -> Result<(), bamboo_domain::SessionInboxError> {
2126 self.inner
2127 .mark_activation_eligible(target_session_id, generation, policy)
2128 .await
2129 }
2130
2131 async fn claim(
2132 &self,
2133 target_session_id: &str,
2134 limit: usize,
2135 ) -> Result<Vec<SessionInboxClaim>, bamboo_domain::SessionInboxError> {
2136 self.inner.claim(target_session_id, limit).await
2137 }
2138
2139 async fn was_admitted(
2140 &self,
2141 target_session_id: &str,
2142 id: &bamboo_domain::SessionMessageId,
2143 ) -> Result<bool, bamboo_domain::SessionInboxError> {
2144 self.inner.was_admitted(target_session_id, id).await
2145 }
2146
2147 async fn ack(
2148 &self,
2149 target_session_id: &str,
2150 claim: &SessionInboxClaim,
2151 ) -> Result<(), bamboo_domain::SessionInboxError> {
2152 if self
2153 .fail_once
2154 .swap(false, std::sync::atomic::Ordering::SeqCst)
2155 {
2156 return Err(bamboo_domain::SessionInboxError::Storage(
2157 "injected actor pre-ack failure".to_string(),
2158 ));
2159 }
2160 self.inner.ack(target_session_id, claim).await
2161 }
2162
2163 async fn inspect(
2164 &self,
2165 target_session_id: &str,
2166 ) -> Result<bamboo_domain::SessionInboxBacklog, bamboo_domain::SessionInboxError> {
2167 self.inner.inspect(target_session_id).await
2168 }
2169 }
2170
2171 async fn actor_inbox_fixture(
2172 session_id: &str,
2173 ) -> (
2174 tempfile::TempDir,
2175 Arc<bamboo_storage::SessionStoreV2>,
2176 Arc<bamboo_storage::LockedSessionStore>,
2177 Arc<dyn SessionInboxPort>,
2178 Session,
2179 SessionInboxClaim,
2180 ) {
2181 let temp = tempfile::tempdir().unwrap();
2182 let store = Arc::new(
2183 bamboo_storage::SessionStoreV2::new(temp.path().to_path_buf())
2184 .await
2185 .unwrap(),
2186 );
2187 let storage: Arc<dyn Storage> = store.clone();
2188 let locked = Arc::new(bamboo_storage::LockedSessionStore::new(storage));
2189 let inbox: Arc<dyn SessionInboxPort> = Arc::new(bamboo_storage::FileSessionInbox::new(
2190 store.clone(),
2191 bamboo_domain::SessionInboxLimits::default(),
2192 ));
2193 let session = Session::new(session_id, "model");
2194 store.save_session(&session).await.unwrap();
2195 let mut envelope =
2196 bamboo_domain::SessionMessageEnvelope::user_input(session_id, "actor follow-up");
2197 envelope.id =
2198 bamboo_domain::SessionMessageId::parse(format!("{session_id}-message")).unwrap();
2199 let receipt = inbox.deliver(&envelope).await.unwrap();
2200 inbox
2201 .mark_activation_eligible(
2202 session_id,
2203 receipt.generation,
2204 bamboo_domain::SessionActivationPolicy::InterruptSpecificWait,
2205 )
2206 .await
2207 .unwrap();
2208 let claim = inbox.claim(session_id, 1).await.unwrap().remove(0);
2209 (temp, store, locked, inbox, session, claim)
2210 }
2211
2212 fn actor_binding(
2213 store: Arc<bamboo_storage::SessionStoreV2>,
2214 inbox: Arc<dyn SessionInboxPort>,
2215 persistence: Arc<dyn RuntimeSessionPersistence>,
2216 ) -> SessionInboxRuntimeBinding {
2217 let storage: Arc<dyn Storage> = store;
2218 SessionInboxRuntimeBinding {
2219 router: SessionActivationRouter::new(),
2220 inbox,
2221 storage,
2222 persistence,
2223 }
2224 }
2225
2226 #[tokio::test]
2227 async fn actor_mismatched_typed_marker_id_collision_never_acks() {
2228 let (_temp, store, locked, inbox, mut session, claim) =
2229 actor_inbox_fixture("actor-live-id-collision").await;
2230 let mut forged = claim.envelope.to_provider_message().unwrap();
2231 forged.metadata = Some(serde_json::json!({
2232 "session_message": {
2233 "id": claim.envelope.id,
2234 "target_session_id": "different-session"
2235 }
2236 }));
2237 session.add_message(forged);
2238 let persistence: Arc<dyn RuntimeSessionPersistence> = locked;
2239 let binding = actor_binding(store, inbox.clone(), persistence);
2240
2241 assert!(
2242 checkpoint_and_ack_canonical_claim(&binding, &mut session, &claim)
2243 .await
2244 .is_err()
2245 );
2246 assert_eq!(inbox.inspect(&session.id).await.unwrap().claimed, 1);
2247 assert!(!inbox
2248 .was_admitted(&session.id, &claim.envelope.id)
2249 .await
2250 .unwrap());
2251 }
2252
2253 #[tokio::test]
2254 async fn actor_concurrent_durable_id_collision_after_claim_never_acks() {
2255 let (_temp, store, locked, inbox, mut session, claim) =
2256 actor_inbox_fixture("actor-durable-id-collision").await;
2257 let mut concurrent = store.load_session(&session.id).await.unwrap().unwrap();
2258 let mut forged = bamboo_agent_core::Message::user("concurrent actor collision");
2259 forged.id = claim.envelope.id.to_string();
2260 concurrent.add_message(forged);
2261 store.save_session(&concurrent).await.unwrap();
2262 let persistence: Arc<dyn RuntimeSessionPersistence> = locked;
2263 let binding = actor_binding(store.clone(), inbox.clone(), persistence);
2264
2265 assert!(
2266 checkpoint_and_ack_canonical_claim(&binding, &mut session, &claim)
2267 .await
2268 .is_err()
2269 );
2270 assert_eq!(inbox.inspect(&session.id).await.unwrap().claimed, 1);
2271 assert!(!inbox
2272 .was_admitted(&session.id, &claim.envelope.id)
2273 .await
2274 .unwrap());
2275 let durable = store.load_session(&session.id).await.unwrap().unwrap();
2276 assert!(!durable.messages.iter().any(|message| {
2277 bamboo_domain::is_matching_session_message(message, &claim.envelope)
2278 }));
2279 }
2280
2281 #[tokio::test]
2282 async fn actor_concurrent_durable_typed_body_mismatch_never_acks() {
2283 let (_temp, store, locked, inbox, mut session, claim) =
2284 actor_inbox_fixture("actor-durable-typed-body-collision").await;
2285 let mut different = claim.envelope.clone();
2286 different.body = bamboo_domain::SessionMessageBody::Content(
2287 bamboo_domain::SessionMessageContent::text("forged actor body"),
2288 );
2289 let mut concurrent = store.load_session(&session.id).await.unwrap().unwrap();
2290 concurrent.add_message(different.to_provider_message().unwrap());
2291 store.save_session(&concurrent).await.unwrap();
2292 let persistence: Arc<dyn RuntimeSessionPersistence> = locked;
2293 let binding = actor_binding(store.clone(), inbox.clone(), persistence);
2294
2295 assert!(
2296 checkpoint_and_ack_canonical_claim(&binding, &mut session, &claim)
2297 .await
2298 .is_err()
2299 );
2300 assert_eq!(inbox.inspect(&session.id).await.unwrap().claimed, 1);
2301 assert!(!inbox
2302 .was_admitted(&session.id, &claim.envelope.id)
2303 .await
2304 .unwrap());
2305 let durable = store.load_session(&session.id).await.unwrap().unwrap();
2306 assert!(!durable
2307 .messages
2308 .iter()
2309 .any(|message| bamboo_domain::is_matching_session_message(message, &claim.envelope)));
2310 }
2311
2312 #[tokio::test]
2313 async fn actor_checkpoint_failure_rolls_back_and_restart_admits_once() {
2314 let (_temp, store, locked, inbox, mut session, claim) =
2315 actor_inbox_fixture("actor-checkpoint-failure").await;
2316 let envelope_id = claim.envelope.id.clone();
2317 let fault: Arc<dyn RuntimeSessionPersistence> = Arc::new(ActorFaultingPersistence {
2318 inner: locked.clone(),
2319 fail_checkpoint_once: std::sync::atomic::AtomicBool::new(true),
2320 });
2321 let binding = actor_binding(store.clone(), inbox.clone(), fault);
2322
2323 assert!(
2324 checkpoint_and_ack_canonical_claim(&binding, &mut session, &claim)
2325 .await
2326 .is_err()
2327 );
2328 assert!(!session
2329 .messages
2330 .iter()
2331 .any(|message| message.id == envelope_id.as_str()));
2332 assert_eq!(inbox.inspect(&session.id).await.unwrap().claimed, 1);
2333 assert!(!inbox.was_admitted(&session.id, &envelope_id).await.unwrap());
2334
2335 let reopened: Arc<dyn SessionInboxPort> = Arc::new(bamboo_storage::FileSessionInbox::new(
2336 store.clone(),
2337 bamboo_domain::SessionInboxLimits::default(),
2338 ));
2339 let recovered = reopened.claim(&session.id, 1).await.unwrap().remove(0);
2340 let persistence: Arc<dyn RuntimeSessionPersistence> = locked;
2341 let binding = actor_binding(store.clone(), reopened.clone(), persistence);
2342 let mut restarted = store.load_session(&session.id).await.unwrap().unwrap();
2343 checkpoint_and_ack_canonical_claim(&binding, &mut restarted, &recovered)
2344 .await
2345 .unwrap();
2346 assert_eq!(
2347 restarted
2348 .messages
2349 .iter()
2350 .filter(|message| message.id == envelope_id.as_str())
2351 .count(),
2352 1
2353 );
2354 assert!(reopened
2355 .was_admitted(&session.id, &envelope_id)
2356 .await
2357 .unwrap());
2358 let backlog = reopened.inspect(&session.id).await.unwrap();
2359 assert_eq!(backlog.pending + backlog.claimed, 0);
2360 }
2361
2362 #[tokio::test]
2363 async fn actor_checkpoint_success_pre_ack_failure_recovers_without_duplicate() {
2364 let (_temp, store, locked, real_inbox, mut session, claim) =
2365 actor_inbox_fixture("actor-pre-ack-failure").await;
2366 let envelope_id = claim.envelope.id.clone();
2367 let faulted: Arc<dyn SessionInboxPort> = Arc::new(ActorFailBeforeAckInbox {
2368 inner: real_inbox.clone(),
2369 fail_once: std::sync::atomic::AtomicBool::new(true),
2370 });
2371 let persistence: Arc<dyn RuntimeSessionPersistence> = locked.clone();
2372 let binding = actor_binding(store.clone(), faulted, persistence);
2373
2374 assert!(
2375 checkpoint_and_ack_canonical_claim(&binding, &mut session, &claim)
2376 .await
2377 .is_err()
2378 );
2379 let durable = store.load_session(&session.id).await.unwrap().unwrap();
2380 assert_eq!(
2381 durable
2382 .messages
2383 .iter()
2384 .filter(|message| message.id == envelope_id.as_str())
2385 .count(),
2386 1
2387 );
2388 assert_eq!(real_inbox.inspect(&session.id).await.unwrap().claimed, 1);
2389 assert!(!real_inbox
2390 .was_admitted(&session.id, &envelope_id)
2391 .await
2392 .unwrap());
2393
2394 let reopened: Arc<dyn SessionInboxPort> = Arc::new(bamboo_storage::FileSessionInbox::new(
2395 store.clone(),
2396 bamboo_domain::SessionInboxLimits::default(),
2397 ));
2398 let recovered = reopened.claim(&session.id, 1).await.unwrap().remove(0);
2399 let persistence: Arc<dyn RuntimeSessionPersistence> = locked;
2400 let binding = actor_binding(store.clone(), reopened.clone(), persistence);
2401 let mut restarted = durable;
2402 checkpoint_and_ack_canonical_claim(&binding, &mut restarted, &recovered)
2403 .await
2404 .unwrap();
2405 assert_eq!(
2406 restarted
2407 .messages
2408 .iter()
2409 .filter(|message| message.id == envelope_id.as_str())
2410 .count(),
2411 1
2412 );
2413 assert!(reopened
2414 .was_admitted(&session.id, &envelope_id)
2415 .await
2416 .unwrap());
2417 let backlog = reopened.inspect(&session.id).await.unwrap();
2418 assert_eq!(backlog.pending + backlog.claimed, 0);
2419 }
2420
2421 struct ConfirmationSequenceLink {
2422 frames: VecDeque<ChildFrame>,
2423 sent: Vec<ParentFrame>,
2424 }
2425
2426 #[async_trait]
2427 impl bamboo_subagent::ChildLink for ConfirmationSequenceLink {
2428 async fn send(&mut self, frame: ParentFrame) -> bamboo_subagent::TransportResult<()> {
2429 self.sent.push(frame);
2430 Ok(())
2431 }
2432
2433 async fn next_frame(&mut self) -> bamboo_subagent::TransportResult<Option<ChildFrame>> {
2434 match self.frames.pop_front() {
2435 Some(frame) => Ok(Some(frame)),
2436 None => std::future::pending().await,
2437 }
2438 }
2439 }
2440
2441 fn admission_confirmation(
2442 session_id: &str,
2443 claim: &SessionInboxClaim,
2444 run_id: &str,
2445 ) -> bamboo_subagent::proto::SessionMessageAdmissionConfirmation {
2446 bamboo_subagent::proto::SessionMessageAdmissionConfirmation {
2447 target_session_id: session_id.to_string(),
2448 envelope_id: claim.envelope.id.to_string(),
2449 canonical_claim_generation: claim.generation,
2450 activation_run_id: run_id.to_string(),
2451 }
2452 }
2453
2454 #[tokio::test]
2455 async fn actor_initial_batch_acks_in_order_and_rejects_stale_confirmation() {
2456 let temp = tempfile::tempdir().unwrap();
2457 let store = Arc::new(
2458 bamboo_storage::SessionStoreV2::new(temp.path().to_path_buf())
2459 .await
2460 .unwrap(),
2461 );
2462 let storage: Arc<dyn Storage> = store.clone();
2463 let locked = Arc::new(bamboo_storage::LockedSessionStore::new(storage.clone()));
2464 let inbox: Arc<dyn SessionInboxPort> = Arc::new(bamboo_storage::FileSessionInbox::new(
2465 store.clone(),
2466 bamboo_domain::SessionInboxLimits::default(),
2467 ));
2468 let session_id = "actor-confirmation-order";
2469 let run_id = "actor-run-current";
2470 let mut session = Session::new(session_id, "model");
2471 store.save_session(&session).await.unwrap();
2472 for (id, text) in [("actor-first", "first"), ("actor-second", "second")] {
2473 let mut envelope = bamboo_domain::SessionMessageEnvelope::user_input(session_id, text);
2474 envelope.id = bamboo_domain::SessionMessageId::parse(id).unwrap();
2475 inbox.deliver(&envelope).await.unwrap();
2476 }
2477 inbox
2478 .mark_activation_eligible(
2479 session_id,
2480 2,
2481 bamboo_domain::SessionActivationPolicy::InterruptSpecificWait,
2482 )
2483 .await
2484 .unwrap();
2485 let router = SessionActivationRouter::new();
2486 let mut owner_registration = router.register_run(session_id, run_id).await.unwrap();
2487 let binding = SessionInboxRuntimeBinding {
2488 router,
2489 inbox: inbox.clone(),
2490 storage,
2491 persistence: locked,
2492 };
2493 let pairs = claim_canonical_deliveries(&binding, &mut session, run_id, usize::MAX)
2494 .await
2495 .unwrap();
2496 assert_eq!(
2497 pairs
2498 .iter()
2499 .map(|(claim, _)| claim.envelope.id.as_str())
2500 .collect::<Vec<_>>(),
2501 vec!["actor-first", "actor-second"]
2502 );
2503 let seeded = store.load_session(session_id).await.unwrap().unwrap();
2504 assert_eq!(
2505 seeded
2506 .messages
2507 .iter()
2508 .filter(|message| matches!(message.id.as_str(), "actor-first" | "actor-second"))
2509 .map(|message| message.id.as_str())
2510 .collect::<Vec<_>>(),
2511 vec!["actor-first", "actor-second"],
2512 "host context must be durable before actor dispatch"
2513 );
2514 for (claim, _) in &pairs {
2515 assert_eq!(
2516 seeded
2517 .messages
2518 .iter()
2519 .filter(|message| bamboo_domain::is_matching_session_message(
2520 message,
2521 &claim.envelope
2522 ))
2523 .count(),
2524 1,
2525 "pre-dispatch host checkpoint must contain exactly one canonical marker for {}",
2526 claim.envelope.id
2527 );
2528 }
2529 assert!(
2530 seeded.session_inbox_admission().is_none_or(|cursor| {
2531 !cursor.contains(&pairs[0].0.envelope.id)
2532 && !cursor.contains(&pairs[1].0.envelope.id)
2533 }),
2534 "pre-dispatch transcript seeding must not forge worker confirmation"
2535 );
2536 assert_eq!(inbox.inspect(session_id).await.unwrap().claimed, 2);
2537 let claims = pairs
2538 .into_iter()
2539 .map(|(claim, _)| claim)
2540 .collect::<VecDeque<_>>();
2541 let first = claims[0].clone();
2542 let second = claims[1].clone();
2543 let mut stale = admission_confirmation(session_id, &first, "stale-run");
2544 stale.canonical_claim_generation = second.generation;
2545 let mut link = ConfirmationSequenceLink {
2546 frames: VecDeque::from([
2547 ChildFrame::SessionMessageAdmitted {
2548 confirmation: stale,
2549 },
2550 ChildFrame::SessionMessageAdmitted {
2551 confirmation: admission_confirmation(session_id, &first, run_id),
2552 },
2553 ChildFrame::SessionMessageAdmitted {
2554 confirmation: admission_confirmation(session_id, &second, run_id),
2555 },
2556 ChildFrame::Terminal {
2557 status: TerminalStatus::Completed,
2558 result: Some("done".to_string()),
2559 error: None,
2560 transcript: Vec::new(),
2561 },
2562 ]),
2563 sent: Vec::new(),
2564 };
2565 let (event_tx, _event_rx) = mpsc::channel(8);
2566 let cancel = CancellationToken::new();
2567 let (_live_tx, mut live_rx) = mpsc::unbounded_channel();
2568 let (_delivery_tx, mut delivery_rx) = mpsc::unbounded_channel();
2569 let result = drive(
2570 &mut link,
2571 "parent",
2572 session_id,
2573 0,
2574 None,
2575 None,
2576 None,
2577 None,
2578 &event_tx,
2579 &cancel,
2580 &mut live_rx,
2581 &mut delivery_rx,
2582 &mut session,
2583 Some(&binding),
2584 Some(run_id),
2585 claims,
2586 Some(Duration::from_secs(1)),
2587 )
2588 .await
2589 .unwrap();
2590 assert_eq!(result.as_deref(), Some("done"));
2591 assert_eq!(
2592 session
2593 .messages
2594 .iter()
2595 .filter(|message| matches!(message.id.as_str(), "actor-first" | "actor-second"))
2596 .map(|message| message.id.as_str())
2597 .collect::<Vec<_>>(),
2598 vec!["actor-first", "actor-second"]
2599 );
2600 let backlog = inbox.inspect(session_id).await.unwrap();
2601 assert_eq!(backlog.pending + backlog.claimed, 0);
2602 let confirmed = store.load_session(session_id).await.unwrap().unwrap();
2603 let cursor = confirmed
2604 .session_inbox_admission()
2605 .expect("exact worker confirmation must checkpoint the admission cursor");
2606 assert!(cursor.contains(&first.envelope.id));
2607 assert!(cursor.contains(&second.envelope.id));
2608 for claim in [&first, &second] {
2609 assert_eq!(
2610 confirmed
2611 .messages
2612 .iter()
2613 .filter(|message| bamboo_domain::is_matching_session_message(
2614 message,
2615 &claim.envelope
2616 ))
2617 .count(),
2618 1,
2619 "confirmation must retain one exact canonical marker for {}",
2620 claim.envelope.id
2621 );
2622 }
2623 assert!(inbox
2624 .was_admitted(session_id, &first.envelope.id)
2625 .await
2626 .unwrap());
2627 assert!(inbox
2628 .was_admitted(session_id, &second.envelope.id)
2629 .await
2630 .unwrap());
2631 owner_registration.begin_finalization().await;
2632 owner_registration.finish(2).await.unwrap();
2633 }
2634
2635 #[derive(Default)]
2636 struct RecordingCodexTokenAuthority {
2637 issued_for: std::sync::Mutex<Vec<String>>,
2638 revoked: std::sync::Mutex<Vec<String>>,
2639 }
2640
2641 impl CodexRunTokenAuthority for RecordingCodexTokenAuthority {
2642 fn issue(&self, session_id: &str) -> Result<IssuedCodexRunToken, String> {
2643 self.issued_for
2644 .lock()
2645 .expect("issued fixture lock")
2646 .push(session_id.to_string());
2647 Ok(IssuedCodexRunToken {
2648 token_id: format!("id-{session_id}"),
2649 token: format!("bcx1_secret-{session_id}"),
2650 })
2651 }
2652
2653 fn revoke(&self, token_id: &str) {
2654 self.revoked
2655 .lock()
2656 .expect("revoked fixture lock")
2657 .push(token_id.to_string());
2658 }
2659 }
2660
2661 fn codex_executor(auth_mode: Option<&str>, inherit_user_config: Option<bool>) -> ExecutorSpec {
2662 let bamboo_mode = auth_mode == Some("bamboo")
2663 || (auth_mode.is_none() && !inherit_user_config.unwrap_or(false));
2664 ExecutorSpec::Codex {
2665 binary: None,
2666 model: None,
2667 mode: None,
2668 sandbox: None,
2669 inherit_user_config,
2670 auth_mode: auth_mode.map(str::to_string),
2671 base_url: bamboo_mode.then(|| "http://127.0.0.1:9562/openai/v1".to_string()),
2672 wire_api: Some("responses".to_string()),
2673 provider_key_ref: None,
2674 forward_env: None,
2675 approval_policy: None,
2676 network_access: None,
2677 allow_danger_bypass: None,
2678 permission_profile: None,
2679 workspace_owned: None,
2680 }
2681 }
2682
2683 #[test]
2684 fn only_bamboo_managed_non_git_workspaces_are_marked_owned() {
2685 let project = tempfile::tempdir().unwrap();
2686 let managed = project.path().join(".bamboo/worktree/child-571");
2687 std::fs::create_dir_all(&managed).unwrap();
2688 assert!(!workspace_is_bamboo_owned(managed.to_str().unwrap()));
2689 let marker = project
2690 .path()
2691 .join(".bamboo/worktree/.bamboo-owned/child-571");
2692 std::fs::create_dir_all(marker.parent().unwrap()).unwrap();
2693 std::fs::write(&marker, "bamboo/child-571").unwrap();
2694 assert!(workspace_is_bamboo_owned(managed.to_str().unwrap()));
2695 let nested = managed.join("nested/path");
2696 std::fs::create_dir_all(&nested).unwrap();
2697 assert!(workspace_is_bamboo_owned(nested.to_str().unwrap()));
2698
2699 let arbitrary = tempfile::tempdir().unwrap();
2700 assert!(!workspace_is_bamboo_owned(
2701 arbitrary.path().to_str().unwrap()
2702 ));
2703 }
2704
2705 #[test]
2706 fn bamboo_codex_token_is_per_run_redacted_and_revoked_on_guard_drop() {
2707 let authority = Arc::new(RecordingCodexTokenAuthority::default());
2708 let authority_dyn: Arc<dyn CodexRunTokenAuthority> = authority.clone();
2709
2710 let (secrets, guard) = build_codex_run_secrets(
2711 &codex_executor(Some("bamboo"), None),
2712 Some(authority_dyn),
2713 "child-570",
2714 )
2715 .unwrap();
2716
2717 let token = secrets
2718 .codex_provider_token
2719 .as_ref()
2720 .expect("bamboo mode mints a token");
2721 assert_eq!(token.expose(), "bcx1_secret-child-570");
2722 assert!(!format!("{token:?}").contains("secret-child-570"));
2723 assert_eq!(
2724 authority.issued_for.lock().unwrap().as_slice(),
2725 ["child-570"]
2726 );
2727 assert!(authority.revoked.lock().unwrap().is_empty());
2728
2729 drop(guard);
2730 assert_eq!(
2731 authority.revoked.lock().unwrap().as_slice(),
2732 ["id-child-570"]
2733 );
2734 }
2735
2736 #[test]
2737 fn non_bamboo_codex_never_mints_and_bamboo_fails_closed_without_authority() {
2738 let authority = Arc::new(RecordingCodexTokenAuthority::default());
2739 let authority_dyn: Arc<dyn CodexRunTokenAuthority> = authority.clone();
2740 let (secrets, guard) = build_codex_run_secrets(
2741 &codex_executor(Some("custom"), None),
2742 Some(authority_dyn),
2743 "child-custom",
2744 )
2745 .unwrap();
2746 assert!(secrets.codex_provider_token.is_none());
2747 assert!(guard.is_none());
2748 assert!(authority.issued_for.lock().unwrap().is_empty());
2749
2750 let error = build_codex_run_secrets(
2751 &codex_executor(Some("bamboo"), None),
2752 None,
2753 "child-no-authority",
2754 )
2755 .err()
2756 .expect("bamboo mode without an authority must fail closed");
2757 assert!(error.to_string().contains("per-run token authority"));
2758 }
2759
2760 #[test]
2761 fn codex_provisioning_never_leaks_the_session_provider_credential() {
2762 let credentials = vec![ScopedCredential {
2763 provider: "openai".to_string(),
2764 api_key: "upstream-secret-must-not-cross".to_string(),
2765 base_url: None,
2766 provider_type: Some("openai".to_string()),
2767 credential_ref: Some("provider.openai.api_key".to_string()),
2768 }];
2769
2770 for (mode, label) in [
2771 (Some("inherit"), "inherit"),
2772 (Some("api_key"), "api_key"),
2773 (Some("bamboo"), "bamboo"),
2774 (None, "default-bamboo"),
2775 ] {
2776 let runner = ActorChildRunner::new(
2777 format!("codex-{label}-test"),
2778 PathBuf::from("/bin/false"),
2779 Vec::new(),
2780 std::env::temp_dir().join(format!("bamboo-codex-{label}-570")),
2781 codex_executor(mode, None),
2782 credentials.clone(),
2783 "openai".to_string(),
2784 1,
2785 );
2786 let mut session = Session::new(format!("child-{label}"), "model");
2787 session.add_message(bamboo_agent_core::Message::user("test"));
2788 let spec = runner.build_spec(
2789 &session,
2790 &crate::runtime::execution::SpawnJob {
2791 parent_session_id: "parent".to_string(),
2792 child_session_id: format!("child-{label}"),
2793 model: "gpt-5.4".to_string(),
2794 disabled_tools: None,
2795 },
2796 );
2797 assert!(
2798 spec.secrets.provider_credentials.is_empty(),
2799 "{label} Codex must not receive the session provider key"
2800 );
2801 }
2802 }
2803
2804 #[test]
2805 fn non_codex_provisioning_still_receives_only_its_selected_provider_credential() {
2806 let credentials = vec![
2807 ScopedCredential {
2808 provider: "openai".to_string(),
2809 api_key: "selected-openai-secret".to_string(),
2810 base_url: None,
2811 provider_type: Some("openai".to_string()),
2812 credential_ref: Some("provider.openai.api_key".to_string()),
2813 },
2814 ScopedCredential {
2815 provider: "other".to_string(),
2816 api_key: "unrelated-secret".to_string(),
2817 base_url: None,
2818 provider_type: Some("openai".to_string()),
2819 credential_ref: Some("provider.other.api_key".to_string()),
2820 },
2821 ];
2822 let runner = ActorChildRunner::new(
2823 "echo-test".to_string(),
2824 PathBuf::from("/bin/false"),
2825 Vec::new(),
2826 std::env::temp_dir().join("bamboo-echo-provider-570"),
2827 ExecutorSpec::Echo,
2828 credentials,
2829 "openai".to_string(),
2830 1,
2831 );
2832 let spec = runner.build_spec(
2833 &Session::new("child-echo", "model"),
2834 &crate::runtime::execution::SpawnJob {
2835 parent_session_id: "parent".to_string(),
2836 child_session_id: "child-echo".to_string(),
2837 model: "gpt-5.4".to_string(),
2838 disabled_tools: None,
2839 },
2840 );
2841
2842 assert_eq!(spec.secrets.provider_credentials.len(), 1);
2843 assert_eq!(
2844 spec.secrets.provider_credentials[0].api_key,
2845 "selected-openai-secret"
2846 );
2847 }
2848
2849 #[test]
2850 fn custom_codex_provisioning_scopes_only_the_referenced_credential() {
2851 let mut executor = codex_executor(Some("custom"), None);
2852 if let ExecutorSpec::Codex {
2853 base_url,
2854 provider_key_ref,
2855 ..
2856 } = &mut executor
2857 {
2858 *base_url = Some("https://provider.example/v1".to_string());
2859 *provider_key_ref = Some("provider.custom.api_key".to_string());
2860 }
2861 let credentials = vec![
2862 ScopedCredential {
2863 provider: "openai".to_string(),
2864 api_key: "session-provider-secret".to_string(),
2865 base_url: None,
2866 provider_type: Some("openai".to_string()),
2867 credential_ref: Some("provider.openai.api_key".to_string()),
2868 },
2869 ScopedCredential {
2870 provider: "custom".to_string(),
2871 api_key: "selected-secret".to_string(),
2872 base_url: None,
2873 provider_type: Some("openai".to_string()),
2874 credential_ref: Some("provider.custom.api_key".to_string()),
2875 },
2876 ScopedCredential {
2877 provider: "other".to_string(),
2878 api_key: "unrelated-secret".to_string(),
2879 base_url: None,
2880 provider_type: Some("openai".to_string()),
2881 credential_ref: Some("provider.other.api_key".to_string()),
2882 },
2883 ];
2884 let runner = ActorChildRunner::new(
2885 "codex-test".to_string(),
2886 PathBuf::from("/bin/false"),
2887 Vec::new(),
2888 std::env::temp_dir().join("bamboo-codex-570"),
2889 executor,
2890 credentials,
2891 "openai".to_string(),
2892 1,
2893 );
2894 let mut session = Session::new("child-custom", "model");
2895 session.add_message(bamboo_agent_core::Message::user("test"));
2896 let spec = runner.build_spec(
2897 &session,
2898 &crate::runtime::execution::SpawnJob {
2899 parent_session_id: "parent".to_string(),
2900 child_session_id: "child-custom".to_string(),
2901 model: "gpt-5.4".to_string(),
2902 disabled_tools: None,
2903 },
2904 );
2905
2906 assert_eq!(spec.secrets.provider_credentials.len(), 1);
2907 assert_eq!(
2908 spec.secrets.provider_credentials[0]
2909 .credential_ref
2910 .as_deref(),
2911 Some("provider.custom.api_key")
2912 );
2913 assert_eq!(
2914 spec.secrets.provider_credentials[0].api_key,
2915 "selected-secret"
2916 );
2917 }
2918
2919 fn spec_with(
2920 role: &str,
2921 provider: &str,
2922 model: &str,
2923 workspace: Option<&str>,
2924 disabled: Option<Vec<&str>>,
2925 ) -> ProvisionSpec {
2926 let mut spec = ProvisionSpec::new(
2927 ChildIdentity {
2928 child_id: "c".into(),
2929 parent_id: None,
2930 project_key: None,
2931 role: role.into(),
2932 depth: 0,
2933 },
2934 ExecutorSpec::Echo,
2935 "/tmp/fab".into(),
2936 );
2937 spec.workspace = workspace.map(|w| w.to_string());
2938 spec.model = Some(ModelRefSpec {
2939 provider: provider.into(),
2940 model: model.into(),
2941 });
2942 spec.disabled_tools = disabled.map(|d| d.into_iter().map(String::from).collect());
2943 spec
2944 }
2945
2946 #[test]
2947 fn fingerprint_matches_interchangeable_children() {
2948 let a = spec_with(
2951 "explorer",
2952 "p",
2953 "m",
2954 Some("/ws"),
2955 Some(vec!["Bash", "Edit"]),
2956 );
2957 let mut b = spec_with(
2958 "explorer",
2959 "p",
2960 "m",
2961 Some("/ws"),
2962 Some(vec!["Edit", "Bash"]),
2963 );
2964 b.identity.child_id = "other".into();
2965 assert_eq!(
2966 ActorChildRunner::fingerprint(&a),
2967 ActorChildRunner::fingerprint(&b)
2968 );
2969 }
2970
2971 #[test]
2972 fn logical_identity_is_invariant_across_local_remote_scheduled_and_warm_reuse() {
2973 let mut session =
2974 Session::new_child("logical-child-681", "logical-parent-681", "model", "child");
2975 session.root_session_id = "logical-root-681".to_string();
2976 let job = SpawnJob {
2977 parent_session_id: "logical-parent-681".to_string(),
2978 child_session_id: "logical-child-681".to_string(),
2979 model: "model".to_string(),
2980 disabled_tools: None,
2981 };
2982 let expected = LogicalSessionIdentity {
2983 session_id: "logical-child-681".to_string(),
2984 parent_session_id: Some("logical-parent-681".to_string()),
2985 root_session_id: "logical-root-681".to_string(),
2986 };
2987
2988 let placements_and_transport_ids = [
2989 (Placement::Local, "local-mailbox-first"),
2990 (
2991 Placement::Remote {
2992 endpoint: "wss://remote.example/actor".to_string(),
2993 },
2994 "remote-process-44",
2995 ),
2996 (
2997 Placement::Schedulable {
2998 pool: "gpu-pool".to_string(),
2999 },
3000 "scheduled-mailbox-9",
3001 ),
3002 (Placement::Local, "warm-mailbox-reused-77"),
3004 ];
3005 for (placement, transport_id) in placements_and_transport_ids {
3006 let mut provision = spec_with("worker", "provider", "model", None, None);
3007 provision.placement = placement;
3008 provision.identity.child_id = transport_id.to_string();
3009 assert_eq!(logical_identity_for_actor_run(&session, &job), expected);
3010 assert_ne!(
3011 provision.identity.child_id, expected.session_id,
3012 "test fixture must prove transport identity is independent"
3013 );
3014 }
3015 }
3016
3017 #[test]
3018 fn fingerprint_separates_distinct_runtimes() {
3019 let base = spec_with("explorer", "p", "m", Some("/ws"), None);
3020 let base_fp = ActorChildRunner::fingerprint(&base);
3021 assert_ne!(
3023 base_fp,
3024 ActorChildRunner::fingerprint(&spec_with("writer", "p", "m", Some("/ws"), None))
3025 );
3026 assert_ne!(
3027 base_fp,
3028 ActorChildRunner::fingerprint(&spec_with("explorer", "p2", "m", Some("/ws"), None))
3029 );
3030 assert_ne!(
3031 base_fp,
3032 ActorChildRunner::fingerprint(&spec_with("explorer", "p", "m2", Some("/ws"), None))
3033 );
3034 assert_ne!(
3035 base_fp,
3036 ActorChildRunner::fingerprint(&spec_with("explorer", "p", "m", Some("/ws2"), None))
3037 );
3038 assert_ne!(
3039 base_fp,
3040 ActorChildRunner::fingerprint(&spec_with(
3041 "explorer",
3042 "p",
3043 "m",
3044 Some("/ws"),
3045 Some(vec!["Bash"])
3046 ))
3047 );
3048 }
3049
3050 #[test]
3051 fn fingerprint_splits_on_baked_capabilities() {
3052 let base_fp =
3057 ActorChildRunner::fingerprint(&spec_with("explorer", "p", "m", Some("/ws"), None));
3058
3059 let mut depth = spec_with("explorer", "p", "m", Some("/ws"), None);
3060 depth.identity.depth = 2;
3061 assert_ne!(
3062 base_fp,
3063 ActorChildRunner::fingerprint(&depth),
3064 "depth must split"
3065 );
3066
3067 let mut nested = spec_with("explorer", "p", "m", Some("/ws"), None);
3068 nested.capabilities.nested_spawn = true;
3069 assert_ne!(
3070 base_fp,
3071 ActorChildRunner::fingerprint(&nested),
3072 "nested_spawn must split"
3073 );
3074
3075 let mut bypass = spec_with("explorer", "p", "m", Some("/ws"), None);
3076 bypass.capabilities.bypass = true;
3077 assert_ne!(
3078 base_fp,
3079 ActorChildRunner::fingerprint(&bypass),
3080 "bypass must split"
3081 );
3082
3083 let mut enforce = spec_with("explorer", "p", "m", Some("/ws"), None);
3084 enforce.capabilities.enforce_permissions = true;
3085 assert_ne!(
3086 base_fp,
3087 ActorChildRunner::fingerprint(&enforce),
3088 "enforce_permissions must split"
3089 );
3090
3091 let mut cap = spec_with("explorer", "p", "m", Some("/ws"), None);
3092 cap.capabilities.max_spawn_depth = Some(8);
3093 assert_ne!(
3094 base_fp,
3095 ActorChildRunner::fingerprint(&cap),
3096 "max_spawn_depth must split"
3097 );
3098
3099 let mut nha = spec_with("explorer", "p", "m", Some("/ws"), None);
3103 nha.capabilities.no_human_approver = true;
3104 assert_ne!(
3105 base_fp,
3106 ActorChildRunner::fingerprint(&nha),
3107 "no_human_approver must split"
3108 );
3109
3110 let mut gro = spec_with("explorer", "p", "m", Some("/ws"), None);
3113 gro.capabilities.guardian_read_only = true;
3114 assert_ne!(
3115 base_fp,
3116 ActorChildRunner::fingerprint(&gro),
3117 "guardian_read_only must split"
3118 );
3119 }
3120
3121 #[test]
3122 fn fingerprint_splits_codex_exec_and_app_server_workers() {
3123 let mut exec = spec_with("explorer", "p", "m", Some("/ws"), None);
3124 exec.executor = codex_executor(Some("inherit"), None);
3125 let mut app_server = exec.clone();
3126 if let ExecutorSpec::Codex { mode, .. } = &mut app_server.executor {
3127 *mode = Some("app_server".to_string());
3128 }
3129 assert_ne!(
3130 ActorChildRunner::fingerprint(&exec),
3131 ActorChildRunner::fingerprint(&app_server)
3132 );
3133 }
3134
3135 struct StaticDecider(bool);
3136
3137 #[async_trait]
3138 impl ChildApprovalDecider for StaticDecider {
3139 async fn decide(&self, _child: &str, _req: &serde_json::Value) -> bool {
3140 self.0
3141 }
3142 }
3143
3144 struct RecordingReviewer {
3145 reviewed: mpsc::UnboundedSender<(String, String, serde_json::Value)>,
3146 }
3147
3148 #[async_trait]
3149 impl ChildApprovalReviewer for RecordingReviewer {
3150 async fn review(&self, parent: &str, child: &str, request: &serde_json::Value) -> bool {
3151 let _ = self
3152 .reviewed
3153 .send((parent.to_string(), child.to_string(), request.clone()));
3154 true
3155 }
3156 }
3157
3158 struct SilentLink;
3163 #[async_trait]
3164 impl bamboo_subagent::ChildLink for SilentLink {
3165 async fn send(&mut self, _: ParentFrame) -> bamboo_subagent::TransportResult<()> {
3166 Ok(())
3167 }
3168 async fn next_frame(&mut self) -> bamboo_subagent::TransportResult<Option<ChildFrame>> {
3169 std::future::pending().await
3170 }
3171 }
3172
3173 struct InstantTerminalLink {
3175 done: bool,
3176 }
3177
3178 struct ApprovalRoundTripLink {
3179 step: u8,
3180 approval_reply: Option<(String, bool)>,
3181 }
3182
3183 #[async_trait]
3184 impl bamboo_subagent::ChildLink for ApprovalRoundTripLink {
3185 async fn send(&mut self, frame: ParentFrame) -> bamboo_subagent::TransportResult<()> {
3186 if let ParentFrame::ApprovalReply { id, approved } = frame {
3187 self.approval_reply = Some((id, approved));
3188 self.step = 2;
3189 }
3190 Ok(())
3191 }
3192
3193 async fn next_frame(&mut self) -> bamboo_subagent::TransportResult<Option<ChildFrame>> {
3194 match self.step {
3195 0 => {
3196 self.step = 1;
3197 Ok(Some(ChildFrame::ApprovalRequest {
3198 id: "approval-1".into(),
3199 body: serde_json::json!({
3200 "tool_name": "Bash",
3201 "permission": "execute",
3202 "resource": "rm -rf target",
3203 "permission_request": {"reason_code": "hard_dangerous"}
3204 }),
3205 }))
3206 }
3207 1 => std::future::pending().await,
3208 2 => {
3209 self.step = 3;
3210 Ok(Some(ChildFrame::Terminal {
3211 status: TerminalStatus::Completed,
3212 result: Some("done".into()),
3213 error: None,
3214 transcript: vec![],
3215 }))
3216 }
3217 _ => std::future::pending().await,
3218 }
3219 }
3220 }
3221
3222 #[tokio::test]
3223 async fn drive_routes_forced_ask_to_parent_reviewer_without_human_event() {
3224 let (event_tx, mut event_rx) = mpsc::channel::<AgentEvent>(8);
3225 let (review_tx, mut review_rx) = mpsc::unbounded_channel();
3226 let reviewer: Arc<dyn ChildApprovalReviewer> = Arc::new(RecordingReviewer {
3227 reviewed: review_tx,
3228 });
3229 let cancel = CancellationToken::new();
3230 let (live_tx, mut live_rx) = mpsc::unbounded_channel::<ParentFrame>();
3231 let (_delivery_tx, mut delivery_rx) = mpsc::unbounded_channel();
3232 let mut logical_session = Session::new("child-reviewer", "model");
3233 let live_guard = crate::external_agents::live::register("child-reviewer", live_tx, 0, None);
3234 let mut link = ApprovalRoundTripLink {
3235 step: 0,
3236 approval_reply: None,
3237 };
3238
3239 let result = tokio::time::timeout(
3240 Duration::from_secs(1),
3241 drive(
3242 &mut link,
3243 "parent-reviewer",
3244 "child-reviewer",
3245 0,
3246 None,
3247 None,
3248 Some(&reviewer),
3249 None,
3250 &event_tx,
3251 &cancel,
3252 &mut live_rx,
3253 &mut delivery_rx,
3254 &mut logical_session,
3255 None,
3256 None,
3257 VecDeque::new(),
3258 None,
3259 ),
3260 )
3261 .await
3262 .expect("worker must receive the reviewer verdict before terminating");
3263
3264 assert_eq!(result.ok().flatten().as_deref(), Some("done"));
3265 assert_eq!(
3266 link.approval_reply,
3267 Some(("approval-1".to_string(), true)),
3268 "reviewer verdict must traverse the live route back to the worker"
3269 );
3270 let (parent, child, body) = tokio::time::timeout(Duration::from_secs(1), review_rx.recv())
3271 .await
3272 .expect("reviewer should be invoked off-loop")
3273 .expect("review channel should remain open");
3274 assert_eq!(parent, "parent-reviewer");
3275 assert_eq!(child, "child-reviewer");
3276 assert_eq!(
3277 body.pointer("/permission_request/reason_code")
3278 .and_then(serde_json::Value::as_str),
3279 Some("hard_dangerous")
3280 );
3281 assert!(
3282 event_rx.try_recv().is_err(),
3283 "must not emit a human-review event"
3284 );
3285 drop(live_guard);
3286 }
3287
3288 #[tokio::test]
3289 async fn drive_denies_forced_ask_without_parent_reviewer_or_manual_event() {
3290 let (event_tx, mut event_rx) = mpsc::channel::<AgentEvent>(8);
3291 let cancel = CancellationToken::new();
3292 let (_live_tx, mut live_rx) = mpsc::unbounded_channel::<ParentFrame>();
3293 let (_delivery_tx, mut delivery_rx) = mpsc::unbounded_channel();
3294 let mut logical_session = Session::new("child-no-reviewer", "model");
3295 let mut link = ApprovalRoundTripLink {
3296 step: 0,
3297 approval_reply: None,
3298 };
3299
3300 let result = tokio::time::timeout(
3301 Duration::from_secs(1),
3302 drive(
3303 &mut link,
3304 "parent-no-reviewer",
3305 "child-no-reviewer",
3306 0,
3307 None,
3308 None,
3309 None,
3310 None,
3311 &event_tx,
3312 &cancel,
3313 &mut live_rx,
3314 &mut delivery_rx,
3315 &mut logical_session,
3316 None,
3317 None,
3318 VecDeque::new(),
3319 None,
3320 ),
3321 )
3322 .await
3323 .expect("fail-closed reply must unblock the child immediately");
3324
3325 assert_eq!(result.ok().flatten().as_deref(), Some("done"));
3326 assert_eq!(link.approval_reply, Some(("approval-1".to_string(), false)));
3327 assert!(
3328 event_rx.try_recv().is_err(),
3329 "missing parent review must not surface a manual approval event"
3330 );
3331 }
3332 #[async_trait]
3333 impl bamboo_subagent::ChildLink for InstantTerminalLink {
3334 async fn send(&mut self, _: ParentFrame) -> bamboo_subagent::TransportResult<()> {
3335 Ok(())
3336 }
3337 async fn next_frame(&mut self) -> bamboo_subagent::TransportResult<Option<ChildFrame>> {
3338 if self.done {
3339 std::future::pending().await
3340 } else {
3341 self.done = true;
3342 Ok(Some(ChildFrame::Terminal {
3343 status: TerminalStatus::Completed,
3344 result: Some("done".into()),
3345 error: None,
3346 transcript: vec![],
3347 }))
3348 }
3349 }
3350 }
3351
3352 #[tokio::test]
3353 async fn drive_trips_first_frame_watchdog_on_a_silent_worker() {
3354 let (event_tx, _rx) = mpsc::channel::<AgentEvent>(8);
3355 let cancel = CancellationToken::new();
3356 let (_live_tx, mut live_rx) = mpsc::unbounded_channel::<ParentFrame>();
3357 let (_delivery_tx, mut delivery_rx) = mpsc::unbounded_channel();
3358 let mut logical_session = Session::new("child-x", "model");
3359 let mut link = SilentLink;
3360 let r = drive(
3361 &mut link,
3362 "parent-x",
3363 "child-x",
3364 0,
3365 None,
3366 None,
3367 None,
3368 None,
3369 &event_tx,
3370 &cancel,
3371 &mut live_rx,
3372 &mut delivery_rx,
3373 &mut logical_session,
3374 None,
3375 None,
3376 VecDeque::new(),
3377 Some(Duration::from_millis(100)),
3378 )
3379 .await;
3380 assert!(
3381 matches!(r, Err(AgentError::WorkerUnresponsive(_))),
3382 "a silent worker must trip the first-frame watchdog, got {r:?}"
3383 );
3384 }
3385
3386 #[tokio::test]
3387 async fn drive_does_not_trip_when_a_frame_arrives() {
3388 let (event_tx, _rx) = mpsc::channel::<AgentEvent>(8);
3389 let cancel = CancellationToken::new();
3390 let (_live_tx, mut live_rx) = mpsc::unbounded_channel::<ParentFrame>();
3391 let (_delivery_tx, mut delivery_rx) = mpsc::unbounded_channel();
3392 let mut logical_session = Session::new("child-y", "model");
3393 let mut link = InstantTerminalLink { done: false };
3394 let r = drive(
3397 &mut link,
3398 "parent-y",
3399 "child-y",
3400 0,
3401 None,
3402 None,
3403 None,
3404 None,
3405 &event_tx,
3406 &cancel,
3407 &mut live_rx,
3408 &mut delivery_rx,
3409 &mut logical_session,
3410 None,
3411 None,
3412 VecDeque::new(),
3413 Some(Duration::from_millis(50)),
3414 )
3415 .await;
3416 assert_eq!(r.ok().flatten().as_deref(), Some("done"));
3417 }
3418
3419 #[tokio::test]
3420 async fn child_approval_fails_closed_without_decider() {
3421 let body = serde_json::json!({"tool_name":"Bash","permission":"run","resource":"rm -rf /"});
3423 assert!(!decide_child_approval(None, "child-1", &body).await);
3424 }
3425
3426 #[tokio::test]
3427 async fn child_approval_honors_wired_decider() {
3428 let body =
3429 serde_json::json!({"tool_name":"Write","permission":"write","resource":"/tmp/x"});
3430 let approve: Arc<dyn ChildApprovalDecider> = Arc::new(StaticDecider(true));
3431 let deny: Arc<dyn ChildApprovalDecider> = Arc::new(StaticDecider(false));
3432 assert!(decide_child_approval(Some(&approve), "child-1", &body).await);
3433 assert!(!decide_child_approval(Some(&deny), "child-1", &body).await);
3434 }
3435
3436 use crate::runtime::execution::SpawnJob;
3439 use bamboo_agent_core::Session;
3440
3441 fn bogus_runner(placements: HashMap<String, ResolvedRemotePlacement>) -> ActorChildRunner {
3444 ActorChildRunner::new(
3445 "test-actor".into(),
3446 PathBuf::from("/bin/false"),
3447 vec![],
3448 std::env::temp_dir().join("bamboo-test-fab-193"),
3449 ExecutorSpec::Echo,
3450 vec![],
3451 "anthropic".into(),
3452 4,
3453 )
3454 .with_remote_placements(placements)
3455 }
3456
3457 fn session_of_role(role: &str, assignment: &str) -> Session {
3460 let mut s = Session::new("child-1", "test-model");
3461 s.metadata
3462 .insert("subagent_type".to_string(), role.to_string());
3463 s.add_message(bamboo_agent_core::Message::user(assignment));
3464 s
3465 }
3466
3467 fn job_for(child: &str) -> SpawnJob {
3468 SpawnJob {
3469 parent_session_id: "parent-1".into(),
3470 child_session_id: child.into(),
3471 model: String::new(),
3472 disabled_tools: None,
3473 }
3474 }
3475
3476 #[derive(Default)]
3477 struct RecordingChildSessionPort {
3478 saved: std::sync::Mutex<Option<Session>>,
3479 }
3480
3481 impl RecordingChildSessionPort {
3482 fn saved_child(&self) -> Session {
3483 self.saved
3484 .lock()
3485 .expect("saved-child fixture lock")
3486 .clone()
3487 .expect("create_child_action must save the child")
3488 }
3489 }
3490
3491 #[async_trait]
3492 impl crate::session_app::child_session::ChildSessionPort for RecordingChildSessionPort {
3493 async fn load_root_session(
3494 &self,
3495 _root_id: &str,
3496 ) -> Result<Session, crate::session_app::child_session::ChildSessionError> {
3497 unreachable!("create_child_action does not load the root")
3498 }
3499
3500 async fn load_child_for_parent(
3501 &self,
3502 _parent_id: &str,
3503 _child_id: &str,
3504 ) -> Result<Session, crate::session_app::child_session::ChildSessionError> {
3505 unreachable!("create_child_action does not reload the child")
3506 }
3507
3508 async fn save_child_session(
3509 &self,
3510 child: &mut Session,
3511 ) -> Result<(), crate::session_app::child_session::ChildSessionError> {
3512 *self.saved.lock().expect("saved-child fixture lock") = Some(child.clone());
3513 Ok(())
3514 }
3515
3516 async fn save_child_session_authoritative_flags(
3517 &self,
3518 _child: &mut Session,
3519 ) -> Result<(), crate::session_app::child_session::ChildSessionError> {
3520 unreachable!("new-child creation uses the ordinary save")
3521 }
3522
3523 async fn is_child_running(&self, _child_id: &str) -> bool {
3524 false
3525 }
3526
3527 async fn list_children(
3528 &self,
3529 _parent_id: &str,
3530 ) -> Vec<crate::session_app::child_session::ChildSessionEntry> {
3531 Vec::new()
3532 }
3533
3534 async fn enqueue_child_run(
3535 &self,
3536 _parent: &Session,
3537 _child: &Session,
3538 ) -> Result<(), crate::session_app::child_session::ChildSessionError> {
3539 unreachable!("fixture creates the child with auto_run=false")
3540 }
3541
3542 async fn cancel_child_run_and_wait(
3543 &self,
3544 _child_id: &str,
3545 ) -> Result<(), crate::session_app::child_session::ChildSessionError> {
3546 unreachable!("create_child_action does not cancel")
3547 }
3548
3549 async fn delete_child_session(
3550 &self,
3551 _parent_id: &str,
3552 _child_id: &str,
3553 ) -> Result<
3554 crate::session_app::child_session::DeleteChildResult,
3555 crate::session_app::child_session::ChildSessionError,
3556 > {
3557 unreachable!("create_child_action does not delete")
3558 }
3559
3560 async fn get_child_runner_info(
3561 &self,
3562 _child_id: &str,
3563 ) -> Option<crate::session_app::child_session::ChildRunnerInfo> {
3564 None
3565 }
3566
3567 async fn register_parent_wait_for_child(
3568 &self,
3569 _parent_session_id: &str,
3570 _child_session_id: &str,
3571 _tool_call_id: Option<&str>,
3572 ) -> Result<(), crate::session_app::child_session::ChildSessionError> {
3573 unreachable!("create_child_action does not register a wait")
3574 }
3575
3576 async fn register_parent_wait_for_children(
3577 &self,
3578 _parent_session_id: &str,
3579 _child_session_ids: &[String],
3580 _policy: bamboo_domain::session::runtime_state::ChildWaitPolicy,
3581 ) -> Result<usize, crate::session_app::child_session::ChildSessionError> {
3582 unreachable!("create_child_action does not register a wait")
3583 }
3584
3585 async fn active_child_ids(&self, _parent_session_id: &str) -> Vec<String> {
3586 Vec::new()
3587 }
3588
3589 async fn find_resident_child(
3590 &self,
3591 _root_session_id: &str,
3592 _resident_name: &str,
3593 ) -> Option<String> {
3594 None
3595 }
3596
3597 async fn ensure_child_indexed(&self, _child_session_id: &str) {}
3598 }
3599
3600 #[test]
3601 fn build_spec_sets_remote_placement_for_matching_role() {
3602 let mut placements = HashMap::new();
3603 placements.insert(
3604 "explorer".to_string(),
3605 ResolvedRemotePlacement {
3606 endpoint: "wss://gpu-host:8443".into(),
3607 token: Some("T-secret".into()),
3608 ca_cert_file: None,
3609 host_label: None,
3610 },
3611 );
3612 let runner = bogus_runner(placements);
3613
3614 let s = session_of_role("explorer", "do the thing");
3616 let spec = runner.build_spec(&s, &job_for("child-1"));
3617 match &spec.placement {
3618 Placement::Remote { endpoint } => assert_eq!(endpoint, "wss://gpu-host:8443"),
3619 other => panic!("expected Remote, got {other:?}"),
3620 }
3621 assert_eq!(spec.secrets.worker_auth_token.as_deref(), Some("T-secret"));
3622 }
3623
3624 #[test]
3625 fn build_spec_leaves_local_for_unmatched_role() {
3626 let mut placements = HashMap::new();
3627 placements.insert(
3628 "explorer".to_string(),
3629 ResolvedRemotePlacement {
3630 endpoint: "wss://gpu-host:8443".into(),
3631 token: Some("T".into()),
3632 ca_cert_file: None,
3633 host_label: None,
3634 },
3635 );
3636 let runner = bogus_runner(placements);
3637
3638 let s = session_of_role("writer", "do the thing");
3640 let spec = runner.build_spec(&s, &job_for("child-1"));
3641 assert_eq!(spec.placement, Placement::Local);
3642 assert!(spec.secrets.worker_auth_token.is_none());
3643 }
3644
3645 #[test]
3646 fn build_spec_local_when_no_placements() {
3647 let runner = bogus_runner(HashMap::new());
3648 let s = session_of_role("explorer", "do the thing");
3649 let spec = runner.build_spec(&s, &job_for("child-1"));
3650 assert_eq!(spec.placement, Placement::Local);
3651 assert!(spec.secrets.worker_auth_token.is_none());
3652 }
3653
3654 #[tokio::test]
3655 async fn build_spec_preserves_inherited_bypass_for_child_worker() {
3656 let runner = bogus_runner(HashMap::new());
3660 let mut parent = Session::new("parent-bypass", "test-model");
3661 parent
3662 .agent_runtime_state
3663 .get_or_insert_with(bamboo_domain::AgentRuntimeState::default)
3664 .bypass_permissions = true;
3665 let workspace = tempfile::tempdir().expect("workspace fixture");
3666 let port = RecordingChildSessionPort::default();
3667 let child_id = format!("child-bypass-{}", uuid::Uuid::new_v4());
3668 crate::session_app::child_session::create_child_action(
3669 &port,
3670 crate::session_app::child_session::CreateChildInput {
3671 parent_session: parent,
3672 child_id: child_id.clone(),
3673 title: "Bypassed child".to_string(),
3674 responsibility: "Run ordinary commands".to_string(),
3675 assignment_prompt: "run an ordinary command".to_string(),
3676 subagent_type: "explorer".to_string(),
3677 workspace: workspace.path().to_string_lossy().into_owned(),
3678 model_override: None,
3679 model_ref_override: None,
3680 runtime_metadata: HashMap::new(),
3681 auto_run: false,
3682 reasoning_effort: None,
3683 lifecycle: None,
3684 resident_name: None,
3685 resident_context: None,
3686 disabled_tools: None,
3687 context_fork: None,
3688 },
3689 )
3690 .await
3691 .expect("create inherited-bypass child");
3692 let child = port.saved_child();
3693
3694 assert!(
3695 child
3696 .agent_runtime_state
3697 .as_ref()
3698 .is_some_and(|state| state.bypass_permissions),
3699 "create_child_action must inherit bypass from the parent"
3700 );
3701
3702 let spec = runner.build_spec(&child, &job_for(&child_id));
3703
3704 assert!(spec.capabilities.bypass, "child worker must inherit bypass");
3705 assert!(
3706 spec.capabilities.enforce_permissions,
3707 "forced-ask evaluation must remain active under bypass"
3708 );
3709 }
3710
3711 #[tokio::test]
3712 async fn child_resident_and_guardian_inherit_project_through_actor_run_spec() {
3713 let project_id = bamboo_domain::ProjectId::parse("project-inherited").expect("Project id");
3714 let workspace = tempfile::tempdir().expect("workspace fixture");
3715
3716 for (role, lifecycle, resident_name) in [
3717 ("explorer", None, None),
3718 ("resident", Some("resident"), Some("stable-reviewer")),
3719 ("guardian", None, None),
3720 ] {
3721 let mut parent = Session::new(format!("parent-{role}"), "test-model");
3722 parent.set_project_id_meta(project_id.to_string());
3723 let port = RecordingChildSessionPort::default();
3724 let child_id = format!("child-{role}-{}", uuid::Uuid::new_v4());
3725 crate::session_app::child_session::create_child_action(
3726 &port,
3727 crate::session_app::child_session::CreateChildInput {
3728 parent_session: parent,
3729 child_id: child_id.clone(),
3730 title: format!("{role} child"),
3731 responsibility: "Review the assigned work".to_string(),
3732 assignment_prompt: "inspect the change".to_string(),
3733 subagent_type: role.to_string(),
3734 workspace: workspace.path().to_string_lossy().into_owned(),
3735 model_override: None,
3736 model_ref_override: None,
3737 runtime_metadata: HashMap::new(),
3738 auto_run: false,
3739 reasoning_effort: None,
3740 lifecycle: lifecycle.map(str::to_string),
3741 resident_name: resident_name.map(str::to_string),
3742 resident_context: None,
3743 disabled_tools: None,
3744 context_fork: None,
3745 },
3746 )
3747 .await
3748 .expect("create Project-inheriting child");
3749 let child = port.saved_child();
3750 assert_eq!(
3751 crate::project_context::ProjectContextResolver::project_id_from_session(&child),
3752 Some(project_id.clone()),
3753 "{role} child must inherit its parent's Project"
3754 );
3755
3756 assert_eq!(
3757 project_id_for_actor_run(&child).expect("valid actor Project identity"),
3758 Some(project_id.clone()),
3759 "{role} actor RunSpec must preserve inherited Project identity"
3760 );
3761 }
3762 }
3763
3764 #[test]
3765 fn placement_metadata_stamps_remote_and_schedulable_not_local() {
3766 assert_eq!(placement_metadata(&Placement::Local, None), None);
3768
3769 let r = placement_metadata(
3771 &Placement::Remote {
3772 endpoint: "wss://10.0.0.5:8443/stream".into(),
3773 },
3774 None,
3775 )
3776 .unwrap();
3777 assert!(r.contains(r#""kind":"remote""#), "{r}");
3778 assert!(r.contains(r#""host":"10.0.0.5""#), "{r}");
3779
3780 let labeled = placement_metadata(
3782 &Placement::Remote {
3783 endpoint: "ws://169.254.230.101:8899".into(),
3784 },
3785 Some("mini"),
3786 )
3787 .unwrap();
3788 assert!(labeled.contains(r#""host":"mini""#), "{labeled}");
3789
3790 let s = placement_metadata(
3792 &Placement::Schedulable {
3793 pool: "explorers".into(),
3794 },
3795 Some("mini"),
3796 )
3797 .unwrap();
3798 assert!(s.contains(r#""kind":"remote""#), "{s}");
3799 assert!(s.contains(r#""host":"mini""#), "{s}");
3800
3801 let p: bamboo_storage::SessionPlacement = serde_json::from_str(&labeled).unwrap();
3803 assert_eq!(p.kind, "remote");
3804 assert_eq!(p.host, "mini");
3805 }
3806
3807 #[tokio::test]
3814 async fn execute_external_child_routes_role_to_remote_worker_without_spawning() {
3815 let token = "remote-test-token";
3817 let server = bamboo_subagent::transport::WsServer::bind_with_token(
3818 (std::net::Ipv4Addr::LOCALHOST, 0).into(),
3819 Some(token.to_string()),
3820 )
3821 .await
3822 .expect("bind resident worker");
3823 let endpoint = server.ws_endpoint(); let srv = tokio::spawn(async move {
3825 let _ = server
3827 .serve(Arc::new(bamboo_subagent::executor::EchoExecutor))
3828 .await;
3829 });
3830
3831 let mut placements = HashMap::new();
3833 placements.insert(
3834 "explorer".to_string(),
3835 ResolvedRemotePlacement {
3836 endpoint: endpoint.clone(),
3837 token: Some(token.to_string()),
3838 ca_cert_file: None,
3839 host_label: Some("mini-e2e".into()), },
3841 );
3842 let runner = bogus_runner(placements);
3843
3844 let mut session = session_of_role("explorer", "hello remote");
3846 let job = job_for("child-1");
3847 let (event_tx, mut event_rx) = mpsc::channel::<AgentEvent>(64);
3848 let cancel = CancellationToken::new();
3849
3850 let result = tokio::time::timeout(
3851 Duration::from_secs(10),
3852 runner.execute_external_child(&mut session, &job, event_tx, cancel),
3853 )
3854 .await
3855 .expect("run did not hang")
3856 .expect("remote run succeeded (connected to resident worker, did not spawn)");
3857
3858 let _ = result;
3859 let last = session
3862 .messages
3863 .iter()
3864 .rev()
3865 .find(|m| matches!(m.role, Role::Assistant))
3866 .expect("an assistant reply was written back");
3867 assert!(
3868 last.content.contains("echo:"),
3869 "expected echo reply, got {:?}",
3870 last.content
3871 );
3872
3873 let placement = session
3876 .metadata
3877 .get("placement")
3878 .expect("remote child session stamped with a placement");
3879 assert!(placement.contains(r#""kind":"remote""#), "{placement}");
3880 assert!(placement.contains(r#""host":"mini-e2e""#), "{placement}");
3881
3882 let mut saw_event = false;
3885 while let Ok(Some(_ev)) =
3886 tokio::time::timeout(Duration::from_millis(50), event_rx.recv()).await
3887 {
3888 saw_event = true;
3889 }
3890 let _ = saw_event;
3891
3892 srv.abort();
3893 }
3894
3895 fn bogus_sched_runner(
3901 remote: HashMap<String, ResolvedRemotePlacement>,
3902 sched: HashMap<String, ResolvedSchedulablePlacement>,
3903 ) -> ActorChildRunner {
3904 ActorChildRunner::new(
3905 "test-actor".into(),
3906 PathBuf::from("/bin/false"),
3907 vec![],
3908 std::env::temp_dir().join("bamboo-test-fab-181"),
3909 ExecutorSpec::Echo,
3910 vec![],
3911 "anthropic".into(),
3912 4,
3913 )
3914 .with_remote_placements(remote)
3915 .with_schedulable_placements(sched)
3916 }
3917
3918 fn sched_placement(
3919 pool: &str,
3920 _registry_url: impl Into<String>,
3921 ) -> ResolvedSchedulablePlacement {
3922 ResolvedSchedulablePlacement {
3923 pool: pool.into(),
3924 host_label: None,
3925 }
3926 }
3927
3928 #[test]
3929 fn build_spec_sets_schedulable_placement_for_matching_role() {
3930 let mut sched = HashMap::new();
3931 sched.insert(
3932 "explorer".to_string(),
3933 sched_placement("gpu-pool", "unused"),
3934 );
3935 let runner = bogus_sched_runner(HashMap::new(), sched);
3936
3937 let s = session_of_role("explorer", "do the thing");
3938 let spec = runner.build_spec(&s, &job_for("child-1"));
3939 match &spec.placement {
3940 Placement::Schedulable { pool } => assert_eq!(pool, "gpu-pool"),
3941 other => panic!("expected Schedulable, got {other:?}"),
3942 }
3943 assert!(spec.secrets.worker_auth_token.is_none());
3945 }
3946
3947 #[test]
3948 fn build_spec_remote_wins_when_role_in_both_maps() {
3949 let mut remote = HashMap::new();
3952 remote.insert(
3953 "explorer".to_string(),
3954 ResolvedRemotePlacement {
3955 endpoint: "wss://fixed-host:8443".into(),
3956 token: Some("T-remote".into()),
3957 ca_cert_file: None,
3958 host_label: None,
3959 },
3960 );
3961 let mut sched = HashMap::new();
3962 sched.insert(
3963 "explorer".to_string(),
3964 sched_placement("gpu-pool", "https://control-plane:9562"),
3965 );
3966 let runner = bogus_sched_runner(remote, sched);
3967
3968 let s = session_of_role("explorer", "do the thing");
3969 let spec = runner.build_spec(&s, &job_for("child-1"));
3970 match &spec.placement {
3971 Placement::Remote { endpoint } => assert_eq!(endpoint, "wss://fixed-host:8443"),
3972 other => panic!("expected Remote (precedence), got {other:?}"),
3973 }
3974 assert_eq!(spec.secrets.worker_auth_token.as_deref(), Some("T-remote"));
3975 }
3976
3977 #[test]
3978 fn build_spec_local_for_unmatched_schedulable_role() {
3979 let mut sched = HashMap::new();
3980 sched.insert(
3981 "explorer".to_string(),
3982 sched_placement("gpu-pool", "https://control-plane:9562"),
3983 );
3984 let runner = bogus_sched_runner(HashMap::new(), sched);
3985 let s = session_of_role("writer", "do the thing");
3986 let spec = runner.build_spec(&s, &job_for("child-1"));
3987 assert_eq!(spec.placement, Placement::Local);
3988 assert!(spec.secrets.worker_auth_token.is_none());
3989 }
3990
3991 #[test]
3996 fn placement_stamp_uses_node_label_for_remote_and_schedulable() {
3997 let mut remote = HashMap::new();
3999 remote.insert(
4000 "explorer".to_string(),
4001 ResolvedRemotePlacement {
4002 endpoint: "ws://169.254.230.101:8899".into(),
4003 token: None,
4004 ca_cert_file: None,
4005 host_label: Some("mini".into()),
4006 },
4007 );
4008 let runner = bogus_runner(remote);
4009 let spec = runner.build_spec(&session_of_role("explorer", "go"), &job_for("c1"));
4010 let stamp = runner
4011 .placement_stamp_for(&spec)
4012 .expect("remote child is stamped");
4013 assert!(stamp.contains(r#""kind":"remote""#), "{stamp}");
4014 assert!(stamp.contains(r#""host":"mini""#), "{stamp}");
4015
4016 let mut remote_nolabel = HashMap::new();
4018 remote_nolabel.insert(
4019 "explorer".to_string(),
4020 ResolvedRemotePlacement {
4021 endpoint: "ws://169.254.230.101:8899".into(),
4022 token: None,
4023 ca_cert_file: None,
4024 host_label: None,
4025 },
4026 );
4027 let r2 = bogus_runner(remote_nolabel);
4028 let spec2 = r2.build_spec(&session_of_role("explorer", "go"), &job_for("c1"));
4029 assert!(r2
4030 .placement_stamp_for(&spec2)
4031 .unwrap()
4032 .contains(r#""host":"169.254.230.101""#));
4033
4034 let mut sched = HashMap::new();
4036 sched.insert(
4037 "mac-mini-monitor".to_string(),
4038 ResolvedSchedulablePlacement {
4039 pool: "mac-mini-monitor".into(),
4040 host_label: Some("mini".into()),
4041 },
4042 );
4043 let sr = bogus_sched_runner(HashMap::new(), sched);
4044 let spec3 = sr.build_spec(&session_of_role("mac-mini-monitor", "go"), &job_for("c1"));
4045 let stamp3 = sr
4046 .placement_stamp_for(&spec3)
4047 .expect("scheduled child is stamped");
4048 assert!(stamp3.contains(r#""kind":"remote""#), "{stamp3}");
4049 assert!(stamp3.contains(r#""host":"mini""#), "{stamp3}");
4050
4051 let local = bogus_runner(HashMap::new());
4053 let spec4 = local.build_spec(&session_of_role("writer", "go"), &job_for("c1"));
4054 assert_eq!(local.placement_stamp_for(&spec4), None);
4055 }
4056
4057 async fn start_bus() -> (String, tempfile::TempDir) {
4060 let dir = tempfile::tempdir().unwrap();
4061 let core = std::sync::Arc::new(bamboo_broker::BrokerCore::new(dir.path()));
4062 let server = std::sync::Arc::new(bamboo_broker::BrokerServer::new(core, "t"));
4063 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
4064 let addr = listener.local_addr().unwrap();
4065 tokio::spawn(async move {
4066 let _ = server.serve(listener).await;
4067 });
4068 (format!("ws://{addr}"), dir)
4069 }
4070
4071 async fn join_pool(endpoint: &str, id: &str, pool: &str) -> bamboo_broker::BrokerClient {
4072 let mut c = bamboo_broker::BrokerClient::connect(
4073 endpoint,
4074 bamboo_subagent::AgentRef {
4075 session_id: id.into(),
4076 role: Some(pool.into()),
4077 },
4078 "t",
4079 )
4080 .await
4081 .unwrap();
4082 c.subscribe().await.unwrap();
4083 c
4084 }
4085
4086 fn sched_runner_on_bus(endpoint: &str, child_role: &str, pool: &str) -> ActorChildRunner {
4087 let mut sched = HashMap::new();
4088 sched.insert(child_role.to_string(), sched_placement(pool, "unused"));
4089 bogus_sched_runner(HashMap::new(), sched).with_bus(Some(bamboo_subagent::BusEndpoint {
4090 endpoint: endpoint.into(),
4091 token: "t".into(),
4092 }))
4093 }
4094
4095 #[tokio::test]
4096 async fn resolve_schedulable_picks_a_live_bus_worker() {
4097 let (endpoint, _dir) = start_bus().await;
4098 let _w = join_pool(&endpoint, "w-gpu", "gpu-pool").await;
4099 let runner = sched_runner_on_bus(&endpoint, "explorer", "gpu-pool");
4100
4101 let mailbox = runner
4102 .resolve_schedulable_worker("explorer")
4103 .await
4104 .expect("a live pool worker is found on the bus");
4105 assert_eq!(mailbox, "w-gpu");
4106 }
4107
4108 #[tokio::test]
4109 async fn resolve_schedulable_round_robins_over_pool_workers() {
4110 let (endpoint, _dir) = start_bus().await;
4111 let _a = join_pool(&endpoint, "w-a", "gpu-pool").await;
4112 let _b = join_pool(&endpoint, "w-b", "gpu-pool").await;
4113 let runner = sched_runner_on_bus(&endpoint, "explorer", "gpu-pool");
4114
4115 let mut picked = std::collections::HashSet::new();
4117 for _ in 0..6 {
4118 picked.insert(runner.resolve_schedulable_worker("explorer").await.unwrap());
4119 }
4120 assert_eq!(
4121 picked,
4122 ["w-a".to_string(), "w-b".to_string()].into_iter().collect(),
4123 "round-robin must cover every connected pool worker"
4124 );
4125 }
4126
4127 #[tokio::test]
4128 async fn resolve_schedulable_errors_on_empty_pool() {
4129 let (endpoint, _dir) = start_bus().await;
4130 let runner = sched_runner_on_bus(&endpoint, "explorer", "gpu-pool");
4132
4133 let err = runner
4134 .resolve_schedulable_worker("explorer")
4135 .await
4136 .expect_err("an empty pool is terminal — no local fallback")
4137 .to_string();
4138 assert!(err.contains("no live worker in pool"), "got: {err}");
4139 assert!(err.contains("NOT spawning"), "got: {err}");
4140 }
4141
4142 #[tokio::test]
4149 async fn execute_external_child_runs_schedulable_over_bus_and_stamps_node_label() {
4150 let (endpoint, _dir) = start_bus().await;
4151
4152 let ep = endpoint.clone();
4154 let worker = tokio::spawn(async move {
4155 let _ = bamboo_broker::serve_executor(
4156 &ep,
4157 bamboo_subagent::AgentRef {
4158 session_id: "mmm-worker".into(),
4159 role: Some("mac-mini-monitor".into()),
4160 },
4161 "t",
4162 std::sync::Arc::new(bamboo_subagent::executor::EchoExecutor),
4163 )
4164 .await;
4165 });
4166
4167 let mut probe = bamboo_broker::BrokerClient::connect(
4170 &endpoint,
4171 bamboo_subagent::AgentRef {
4172 session_id: "probe".into(),
4173 role: None,
4174 },
4175 "t",
4176 )
4177 .await
4178 .unwrap();
4179 let mut ready = false;
4180 for _ in 0..100 {
4181 if probe
4182 .list_connected("mac-mini-monitor")
4183 .await
4184 .unwrap()
4185 .iter()
4186 .any(|id| id == "mmm-worker")
4187 {
4188 ready = true;
4189 break;
4190 }
4191 tokio::time::sleep(Duration::from_millis(30)).await;
4192 }
4193 assert!(ready, "worker never joined the pool");
4194
4195 let mut sched = HashMap::new();
4198 sched.insert(
4199 "mac-mini-monitor".to_string(),
4200 ResolvedSchedulablePlacement {
4201 pool: "mac-mini-monitor".into(),
4202 host_label: Some("mini".into()),
4203 },
4204 );
4205 let runner = bogus_sched_runner(HashMap::new(), sched).with_bus(Some(
4206 bamboo_subagent::BusEndpoint {
4207 endpoint: endpoint.clone(),
4208 token: "t".into(),
4209 },
4210 ));
4211
4212 let mut session = session_of_role("mac-mini-monitor", "hello scheduled");
4213 let job = job_for("child-1");
4214 let (event_tx, _rx) = mpsc::channel::<AgentEvent>(64);
4215 let cancel = CancellationToken::new();
4216
4217 tokio::time::timeout(
4218 Duration::from_secs(10),
4219 runner.execute_external_child(&mut session, &job, event_tx, cancel),
4220 )
4221 .await
4222 .expect("run did not hang")
4223 .expect("schedulable run succeeded over the bus (no local spawn)");
4224
4225 let last = session
4227 .messages
4228 .iter()
4229 .rev()
4230 .find(|m| matches!(m.role, Role::Assistant))
4231 .expect("an assistant reply was written back");
4232 assert!(last.content.contains("echo:"), "got {:?}", last.content);
4233
4234 let placement = session
4236 .metadata
4237 .get("placement")
4238 .expect("scheduled child session stamped with a placement");
4239 assert!(placement.contains(r#""kind":"remote""#), "{placement}");
4240 assert!(placement.contains(r#""host":"mini""#), "{placement}");
4241
4242 worker.abort();
4243 }
4244}