1use std::collections::HashMap;
8use std::sync::{Arc, OnceLock, RwLock as StdRwLock};
9use std::time::Duration;
10
11use bamboo_domain::poison::PoisonRecover;
12use bamboo_domain::{AgentHookPoint, HookPayload, HookToolOutcome};
13
14use crate::execution::{
15 create_event_forwarder, finalize_runner, spawn_session_execution, try_reserve_runner,
16 AgentRunner, AgentStatus, ChildCompletion, ChildCompletionHandler, RunnerReservation,
17 SessionExecutionArgs,
18};
19use crate::runtime::config::{BashResumeHook, GuardianSpawner, BASH_COMPLETION_RESUME_KIND};
20use crate::runtime::guardian_state::{
21 parse_guardian_verdict, read_guardian_config, read_guardian_state, write_guardian_state,
22 GuardianVerdict,
23};
24use crate::Agent;
25use async_trait::async_trait;
26use bamboo_agent_core::storage::Storage;
27use bamboo_agent_core::tools::ToolExecutor;
28use bamboo_agent_core::{
29 AgentEvent, BashCompletionInfo, BashCompletionSink, Message, Role, Session,
30};
31use bamboo_domain::session::runtime_state::{
32 AgentRuntimeState, AgentStatusState, ChildWaitPolicy, SuspensionState, WaitingForChildrenState,
33};
34use bamboo_llm::{Config, ProviderModelRouter, ProviderRegistry};
35use bamboo_storage::LockedSessionStore;
36use chrono::Utc;
37use tokio::sync::{broadcast, RwLock};
38
39use crate::model_areas::resolve_global_area_models;
40use crate::model_config_helper::{
41 resolve_fast_model, resolve_gold_config, GOLD_CONFIG_METADATA_KEY,
42};
43use crate::session_app::provider_model::session_effective_model_ref;
44use crate::session_app::resume::{
45 resume_session_execution, ResumeExecutionPort, ResumeSpawnRequest,
46};
47use crate::session_app::types::{ResumeConfigSnapshot, ResumeOutcome};
48
49const AGENT_RUNTIME_STATE_METADATA_KEY: &str = "agent.runtime.state";
50const RUNTIME_RESUME_MESSAGE_HIDDEN_KEY: &str = "hidden_from_ui";
51const RUNTIME_RESUME_MESSAGE_KIND_KEY: &str = "runtime_kind";
52
53fn read_runtime_state(session: &Session) -> AgentRuntimeState {
54 session
55 .agent_runtime_state
56 .clone()
57 .or_else(|| {
58 session
59 .metadata
60 .get(AGENT_RUNTIME_STATE_METADATA_KEY)
61 .and_then(|raw| serde_json::from_str::<AgentRuntimeState>(raw).ok())
62 })
63 .unwrap_or_else(|| AgentRuntimeState::new(format!("{}-child-wait", session.id)))
64}
65
66fn write_runtime_state(session: &mut Session, runtime_state: &AgentRuntimeState) {
67 session.agent_runtime_state = Some(runtime_state.clone());
68 if let Ok(serialized) = serde_json::to_string(runtime_state) {
69 session
70 .metadata
71 .insert(AGENT_RUNTIME_STATE_METADATA_KEY.to_string(), serialized);
72 }
73}
74
75fn is_error_like(status: &str) -> bool {
76 matches!(status, "error" | "timeout" | "cancelled")
77}
78
79fn is_terminal_child_status(status: &str) -> bool {
81 matches!(
82 status,
83 "completed" | "error" | "timeout" | "cancelled" | "skipped"
84 )
85}
86
87async fn derive_completed_child_ids(
92 storage: &Arc<dyn Storage>,
93 parent_session_id: &str,
94 just_completed_child_id: &str,
95) -> Vec<String> {
96 let mut completed: Vec<String> = storage
97 .list_child_run_statuses(parent_session_id)
98 .await
99 .unwrap_or_default()
100 .into_iter()
101 .filter(|(_, status)| status.as_deref().is_some_and(is_terminal_child_status))
102 .map(|(id, _)| id)
103 .collect();
104 if !completed.iter().any(|id| id == just_completed_child_id) {
105 completed.push(just_completed_child_id.to_string());
106 }
107 completed.sort();
108 completed.dedup();
109 completed
110}
111
112fn read_config_snapshot(config: &Arc<RwLock<Config>>, cached_config: &StdRwLock<Config>) -> Config {
113 if let Ok(config_guard) = config.try_read() {
114 let snapshot = config_guard.clone();
115
116 if let Ok(mut cached_guard) = cached_config.try_write() {
117 *cached_guard = snapshot.clone();
118 }
119
120 snapshot
121 } else {
122 cached_config
123 .try_read()
124 .map(|guard| guard.clone())
125 .unwrap_or_default()
126 }
127}
128
129fn parent_locks() -> &'static std::sync::Mutex<HashMap<String, Arc<tokio::sync::Mutex<()>>>> {
146 static LOCKS: OnceLock<std::sync::Mutex<HashMap<String, Arc<tokio::sync::Mutex<()>>>>> =
147 OnceLock::new();
148 LOCKS.get_or_init(|| std::sync::Mutex::new(HashMap::new()))
149}
150
151fn session_resume_lock(session_id: &str) -> Arc<tokio::sync::Mutex<()>> {
158 let mut map = parent_locks().lock().recover_poison();
159 map.entry(session_id.to_string())
160 .or_insert_with(|| Arc::new(tokio::sync::Mutex::new(())))
161 .clone()
162}
163
164fn wait_policy_satisfied(
165 policy: ChildWaitPolicy,
166 wait_child_ids: &[String],
167 completed_child_ids: &[String],
168 latest_child_id: &str,
169 latest_status: &str,
170) -> bool {
171 if wait_child_ids.is_empty() {
172 return false;
173 }
174
175 match policy {
176 ChildWaitPolicy::All => wait_child_ids
177 .iter()
178 .all(|id| completed_child_ids.iter().any(|completed| completed == id)),
179 ChildWaitPolicy::Any => completed_child_ids
180 .iter()
181 .any(|id| wait_child_ids.iter().any(|wait_id| wait_id == id)),
182 ChildWaitPolicy::FirstError => {
183 (is_error_like(latest_status) && wait_child_ids.iter().any(|id| id == latest_child_id))
189 || wait_child_ids
190 .iter()
191 .all(|id| completed_child_ids.iter().any(|completed| completed == id))
192 }
193 }
194}
195
196fn child_final_assistant_text(child: &Session) -> Option<String> {
200 child
201 .messages
202 .iter()
203 .rev()
204 .find(|message| matches!(message.role, Role::Assistant))
205 .map(|message| message.content.clone())
206 .filter(|content| !content.trim().is_empty())
207}
208
209fn runtime_resume_message(
210 completion: &ChildCompletion,
211 remaining_children: usize,
212 child_final_response: Option<&str>,
213) -> Message {
214 let mut body = format!(
215 "Runtime notification: child session `{}` finished with status `{}`. Remaining child sessions: {}.",
216 completion.child_session_id, completion.status, remaining_children
217 );
218
219 let final_response = child_final_response.map(str::to_string);
225 if let Some(response) = final_response.as_deref() {
226 body.push_str("\n\nChild final response:\n");
227 body.push_str(response);
228 } else if let Some(error) = completion.error.as_deref() {
229 if !error.is_empty() {
230 body.push_str("\n\nChild error:\n");
231 body.push_str(error);
232 }
233 }
234
235 body.push_str(
236 "\n\nResume the parent task using this child result and continue from the previous plan. \
237 If you need the full child transcript, call SubAgent.get(child_session_id).",
238 );
239
240 let mut message = Message::user(body);
241 message.metadata = Some(serde_json::json!({
242 RUNTIME_RESUME_MESSAGE_HIDDEN_KEY: true,
243 RUNTIME_RESUME_MESSAGE_KIND_KEY: "child_completion_resume",
244 "child_session_id": completion.child_session_id,
245 "child_status": completion.status,
246 "child_error": completion.error,
247 "completed_at": completion.completed_at,
248 "child_final_response_included": final_response.is_some(),
249 }));
250 message.never_compress = false;
254 message
255}
256
257fn guardian_resume_message(completion: &ChildCompletion, verdict: &GuardianVerdict) -> Message {
262 let mut body = if verdict.approve {
263 String::from(
264 "Guardian review APPROVED: an independent reviewer verified the work and found no blocking issues. You may finalize the task.",
265 )
266 } else {
267 String::from(
268 "Guardian review REJECTED: an independent reviewer found issues. Address every finding below before completing — do NOT declare the task complete until they are resolved.",
269 )
270 };
271 if let Some(summary) = verdict.summary.as_deref().filter(|s| !s.trim().is_empty()) {
272 body.push_str("\n\nReviewer summary: ");
273 body.push_str(summary);
274 }
275 if !verdict.findings.is_empty() {
276 body.push_str("\n\nFindings:");
277 for (idx, finding) in verdict.findings.iter().enumerate() {
278 body.push_str(&format!("\n{}. {}", idx + 1, finding));
279 }
280 }
281 body.push_str(
282 "\n\nIf you need the full guardian transcript, call SubAgent.get(child_session_id).",
283 );
284
285 let mut message = Message::user(body);
286 message.metadata = Some(serde_json::json!({
287 RUNTIME_RESUME_MESSAGE_HIDDEN_KEY: true,
288 RUNTIME_RESUME_MESSAGE_KIND_KEY: "guardian_review_resume",
289 "child_session_id": completion.child_session_id,
290 "child_status": completion.status,
291 "guardian_approved": verdict.approve,
292 "completed_at": completion.completed_at,
293 }));
294 message.never_compress = false;
295 message
296}
297
298#[derive(Clone)]
299pub struct ChildCompletionCoordinator {
300 storage: Arc<dyn Storage>,
301 persistence: Arc<bamboo_storage::LockedSessionStore>,
302 sessions: crate::SessionCache,
303 agent_runners: Arc<RwLock<HashMap<String, AgentRunner>>>,
304 session_event_senders: Arc<RwLock<HashMap<String, broadcast::Sender<AgentEvent>>>>,
305 agent: Arc<Agent>,
306 config: Arc<RwLock<Config>>,
307 provider_registry: Arc<ProviderRegistry>,
308 provider_router: Arc<ProviderModelRouter>,
309 app_data_dir: std::path::PathBuf,
310 account_feed_inbox: Option<crate::execution::AccountFeedInbox>,
311 root_tools: Arc<RwLock<Option<Arc<dyn ToolExecutor>>>>,
312 guardian_spawner: Arc<RwLock<Option<Arc<dyn GuardianSpawner>>>>,
316}
317
318impl ChildCompletionCoordinator {
319 #[allow(clippy::too_many_arguments)]
320 pub fn new(
321 storage: Arc<dyn Storage>,
322 persistence: Arc<LockedSessionStore>,
323 sessions: crate::SessionCache,
324 agent_runners: Arc<RwLock<HashMap<String, AgentRunner>>>,
325 session_event_senders: Arc<RwLock<HashMap<String, broadcast::Sender<AgentEvent>>>>,
326 agent: Arc<Agent>,
327 config: Arc<RwLock<Config>>,
328 provider_registry: Arc<ProviderRegistry>,
329 provider_router: Arc<ProviderModelRouter>,
330 app_data_dir: std::path::PathBuf,
331 account_feed_inbox: Option<crate::execution::AccountFeedInbox>,
332 ) -> Self {
333 Self {
334 storage,
335 persistence,
336 sessions,
337 agent_runners,
338 session_event_senders,
339 agent,
340 config,
341 provider_registry,
342 provider_router,
343 app_data_dir,
344 account_feed_inbox,
345 root_tools: Arc::new(RwLock::new(None)),
346 guardian_spawner: Arc::new(RwLock::new(None)),
347 }
348 }
349
350 pub async fn set_root_tools(&self, tools: Arc<dyn ToolExecutor>) {
351 *self.root_tools.write().await = Some(tools);
352 }
353
354 pub async fn set_guardian_spawner(&self, spawner: Arc<dyn GuardianSpawner>) {
357 *self.guardian_spawner.write().await = Some(spawner);
358 }
359
360 fn build_resume_config(
361 &self,
362 session: &Session,
363 config_snapshot: &Config,
364 ) -> ResumeConfigSnapshot {
365 crate::session_app::resolution::resolve_resume_config_snapshot(
366 config_snapshot,
367 &self.provider_registry,
368 session,
369 None,
370 )
371 }
372
373 async fn resume_parent(&self, parent_session_id: String) -> ResumeOutcome {
381 for attempt in 0..=5u8 {
382 if attempt > 0 {
383 tokio::time::sleep(Duration::from_millis(250 * attempt as u64)).await;
384 }
385
386 let Some(session) = self.load_session(&parent_session_id).await else {
387 tracing::warn!(%parent_session_id, "cannot resume parent after child completion: session not found");
388 return ResumeOutcome::NotFound;
389 };
390 let config_snapshot = self.config.read().await.clone();
391 let resume_config = self.build_resume_config(&session, &config_snapshot);
392 let outcome = resume_session_execution(self, &parent_session_id, resume_config).await;
393 tracing::info!(
394 %parent_session_id,
395 attempt,
396 outcome = outcome.as_str(),
397 "child completion requested parent resume"
398 );
399
400 if !matches!(outcome, ResumeOutcome::AlreadyRunning { .. }) {
401 return outcome;
402 }
403 }
404 tracing::error!(
410 %parent_session_id,
411 "parent resume gave up after AlreadyRunning retry budget; \
412 relying on the child-wait watchdog backstop"
413 );
414 ResumeOutcome::AlreadyRunning {
415 run_id: String::new(),
416 }
417 }
418
419 async fn save_and_cache(&self, session: &mut Session) {
420 if let Err(error) = self.persistence.merge_save_runtime(session).await {
421 tracing::warn!(session_id = %session.id, %error, "failed to persist session");
422 }
423 self.sessions.insert(
424 session.id.clone(),
425 Arc::new(parking_lot::RwLock::new(session.clone())),
426 );
427 }
428}
429
430#[async_trait]
431impl ChildCompletionHandler for ChildCompletionCoordinator {
432 async fn on_child_completed(&self, completion: ChildCompletion) {
433 if !is_terminal_child_status(&completion.status) {
442 tracing::info!(
443 parent_session_id = %completion.parent_session_id,
444 child_session_id = %completion.child_session_id,
445 status = %completion.status,
446 "non-terminal child status; leaving the parent wait armed"
447 );
448 return;
449 }
450
451 let per_parent = session_resume_lock(&completion.parent_session_id);
456 let _per_parent_guard = per_parent.lock().await;
457
458 let Some(mut parent) = self.load_session(&completion.parent_session_id).await else {
459 tracing::warn!(
460 parent_session_id = %completion.parent_session_id,
461 child_session_id = %completion.child_session_id,
462 "child completion received for missing parent"
463 );
464 return;
465 };
466
467 let mut runtime_state = read_runtime_state(&parent);
473
474 let completed_child_ids = derive_completed_child_ids(
477 &self.storage,
478 &completion.parent_session_id,
479 &completion.child_session_id,
480 )
481 .await;
482
483 let mut should_resume = false;
484 let mut remaining_children = 0usize;
485 if let Some(wait) = runtime_state.waiting_for_children.clone() {
486 remaining_children = wait
487 .child_session_ids
488 .iter()
489 .filter(|id| !completed_child_ids.iter().any(|completed| completed == *id))
490 .count();
491 should_resume = wait_policy_satisfied(
492 wait.wait_for,
493 &wait.child_session_ids,
494 &completed_child_ids,
495 &completion.child_session_id,
496 &completion.status,
497 );
498 if should_resume {
499 runtime_state.waiting_for_children = None;
500 runtime_state.status = AgentStatusState::Idle;
501 runtime_state.suspension = None;
502 }
503 }
504
505 if should_resume {
506 parent.metadata.remove("runtime.suspend_reason");
507
508 let reported_child_owned = match self
520 .storage
521 .load_runtime_control_plane(&completion.child_session_id)
522 .await
523 {
524 Ok(Some(control_plane)) => completion_child_is_owned(
525 &completion.parent_session_id,
526 control_plane.parent_session_id.as_deref(),
527 ),
528 _ => false,
529 };
530
531 let loaded_child = if reported_child_owned {
536 match self
537 .storage
538 .load_session(&completion.child_session_id)
539 .await
540 {
541 Ok(child) => child,
542 Err(error) => {
543 tracing::warn!(
544 child_session_id = %completion.child_session_id,
545 %error,
546 "failed to load child session for runtime resume message"
547 );
548 None
549 }
550 }
551 } else {
552 tracing::warn!(
553 parent_session_id = %completion.parent_session_id,
554 child_session_id = %completion.child_session_id,
555 "completion child is not a child of this parent; resuming with a neutral \
556 message and NOT folding its content"
557 );
558 None
559 };
560
561 let reviewed_round = runtime_state.round.current_round;
567 let guardian_resume = loaded_child.as_ref().and_then(|child| {
568 if child.subagent_type().as_deref() != Some("guardian") {
569 return None;
570 }
571 let mut guardian_state = read_guardian_state(&parent)?;
572 if guardian_state.guardian_child_id.as_deref()
573 != Some(completion.child_session_id.as_str())
574 {
575 tracing::warn!(
578 parent_session_id = %completion.parent_session_id,
579 child_session_id = %completion.child_session_id,
580 expected = ?guardian_state.guardian_child_id,
581 "guardian completion does not match recorded guardian_child_id; using generic resume"
582 );
583 return None;
584 }
585 let verdict = child_final_assistant_text(child)
593 .and_then(|text| match parse_guardian_verdict(&text) {
594 Ok(verdict) => Some(verdict),
595 Err(error) => {
596 tracing::warn!(
597 child_session_id = %completion.child_session_id,
598 %error,
599 "guardian verdict unparseable; recording a synthetic reject"
600 );
601 None
602 }
603 })
604 .unwrap_or_else(|| {
605 GuardianVerdict::rejected(vec![
606 "The guardian reviewer did not return a usable verdict (it errored or \
607 emitted unparseable output); the work has NOT been independently \
608 verified."
609 .to_string(),
610 ])
611 });
612 let approved = verdict.approve;
613 let message = guardian_resume_message(&completion, &verdict);
614 guardian_state.record_verdict(verdict, reviewed_round);
615 write_guardian_state(&mut parent, guardian_state);
616 tracing::info!(
617 parent_session_id = %completion.parent_session_id,
618 child_session_id = %completion.child_session_id,
619 approved,
620 "guardian verdict recorded; resuming parent"
621 );
622 Some(message)
623 });
624
625 let resume_message = guardian_resume.unwrap_or_else(|| {
626 runtime_resume_message(
627 &completion,
628 remaining_children,
629 loaded_child
630 .as_ref()
631 .and_then(child_final_assistant_text)
632 .as_deref(),
633 )
634 });
635 parent.add_message(resume_message);
636 } else if runtime_state.waiting_for_children.is_some() {
637 runtime_state.status = AgentStatusState::Suspended;
638 runtime_state.suspension = Some(SuspensionState {
639 reason: "waiting_for_children".to_string(),
640 suspended_at: Utc::now(),
641 resumable: true,
642 hook_point: Some("ChildCompletion".to_string()),
643 });
644 }
645
646 parent.updated_at = Utc::now();
647 write_runtime_state(&mut parent, &runtime_state);
648 self.save_and_cache(&mut parent).await;
649
650 let resume_parent_id = parent.id.clone();
655 drop(_per_parent_guard);
656
657 if should_resume {
658 self.resume_parent(resume_parent_id).await;
659 }
660 }
661}
662
663#[async_trait]
664impl ResumeExecutionPort for ChildCompletionCoordinator {
665 async fn load_session(&self, session_id: &str) -> Option<Session> {
666 match self.storage.load_session(session_id).await {
667 Ok(Some(session)) => Some(session),
668 Ok(None) => self
669 .sessions
670 .get(session_id)
671 .map(|e| e.value().clone())
672 .map(|arc| arc.read().clone()),
673 Err(error) => {
674 tracing::warn!(%session_id, %error, "failed to load session from storage");
675 self.sessions
676 .get(session_id)
677 .map(|e| e.value().clone())
678 .map(|arc| arc.read().clone())
679 }
680 }
681 }
682
683 async fn save_and_cache_session(&self, session: &mut Session) {
684 self.save_and_cache(session).await;
685 }
686
687 async fn try_reserve_runner(
688 &self,
689 session_id: &str,
690 event_sender: &broadcast::Sender<AgentEvent>,
691 ) -> Option<RunnerReservation> {
692 try_reserve_runner(
693 &self.agent_runners,
694 &self.session_event_senders,
695 session_id,
696 event_sender,
697 )
698 .await
699 }
700
701 async fn get_existing_runner_run_id(&self, session_id: &str) -> Option<String> {
702 let runners = self.agent_runners.read().await;
703 runners.get(session_id).map(|r| r.run_id.clone())
704 }
705
706 async fn get_or_create_event_sender(&self, session_id: &str) -> broadcast::Sender<AgentEvent> {
707 crate::execution::session_events::get_or_create_event_sender(
708 &self.session_event_senders,
709 session_id,
710 )
711 .await
712 }
713
714 async fn spawn_resume_execution(&self, request: ResumeSpawnRequest) {
715 let ResumeSpawnRequest {
716 session_id,
717 session,
718 cancel_token,
719 run_id: _,
720 event_sender,
721 config,
722 } = request;
723
724 let Some(root_tools) = self.root_tools.read().await.clone() else {
725 tracing::error!(%session_id, "cannot resume parent after child completion: root tool surface is not initialized");
726 return;
727 };
728
729 let model = session.model.clone();
730 let resolved_provider_name = session_effective_model_ref(&session)
731 .map(|model_ref| model_ref.provider)
732 .unwrap_or(config.provider_name);
733 let provider_override = session_effective_model_ref(&session)
734 .and_then(|model_ref| match self.provider_router.route(&model_ref) {
735 Ok(provider) => Some(provider),
736 Err(error) => {
737 tracing::warn!(
738 session_id = %session_id,
739 provider = %model_ref.provider,
740 model = %model_ref.model,
741 error = %error,
742 "failed to resolve provider override for child-completion parent resume; falling back to runtime provider"
743 );
744 None
745 }
746 });
747 let config_snapshot = self.config.read().await.clone();
748 let resolved_fast_provider = resolve_fast_model(
749 &config_snapshot,
750 &resolved_provider_name,
751 &self.provider_registry,
752 )
753 .map(|model| model.provider);
754 let reasoning_effort = session.reasoning_effort;
755 let reasoning_effort_source = session
756 .metadata
757 .get("reasoning_effort_source")
758 .cloned()
759 .unwrap_or_default();
760 let gold_config = resolve_gold_config(
761 &config_snapshot,
762 session
763 .metadata
764 .get(GOLD_CONFIG_METADATA_KEY)
765 .map(String::as_str),
766 )
767 .or(config.gold_config.clone());
768
769 let (mpsc_tx, _forwarder) = create_event_forwarder(
770 session_id.clone(),
771 event_sender,
772 self.agent_runners.clone(),
773 self.account_feed_inbox.clone(),
774 );
775
776 let config_handle = self.config.clone();
777 let cached_config = Arc::new(StdRwLock::new(config_snapshot.clone()));
778 let provider_registry = self.provider_registry.clone();
779 let provider_name_for_aux = resolved_provider_name.clone();
780 let auxiliary_model_resolver = std::sync::Arc::new(move || {
781 let config_snapshot = read_config_snapshot(&config_handle, cached_config.as_ref());
782 let areas = resolve_global_area_models(
784 &config_snapshot,
785 &provider_name_for_aux,
786 &provider_registry,
787 );
788 crate::AuxiliaryModelConfig {
789 fast_model_name: areas.fast.as_ref().map(|m| m.model_name.clone()),
790 fast_model_provider: areas.fast.map(|m| m.provider),
791 background_model_name: areas.background.as_ref().map(|m| m.model_name.clone()),
792 planning_model_name: None,
793 search_model_name: None,
794 summarization_model_name: areas
795 .summarization
796 .as_ref()
797 .map(|m| m.model_name.clone()),
798 background_model_provider: areas.background.map(|m| m.provider),
799 summarization_model_provider: areas.summarization.map(|m| m.provider),
800 }
801 });
802 let model_roster = crate::ModelRoster {
803 model: Some(model),
804 provider_name: Some(resolved_provider_name),
805 provider_type: config.provider_type.clone(),
806 fast: crate::RoleModel::from_parts(config.fast_model, resolved_fast_provider),
807 background: crate::RoleModel::from_parts(
808 config.background_model,
809 config.background_model_provider,
810 ),
811 summarization: crate::RoleModel::from_parts(
812 config.summarization_model,
813 config.summarization_model_provider,
814 ),
815 };
816
817 let guardian_config = read_guardian_config(&session);
822 let guardian_spawner = self.guardian_spawner.read().await.clone();
823
824 spawn_session_execution(SessionExecutionArgs {
825 agent: self.agent.clone(),
826 session_id,
827 session,
828 tools_override: Some(root_tools),
829 provider_override,
830 model_roster,
831 reasoning_effort,
832 reasoning_effort_source,
833 auxiliary_model_resolver: Some(auxiliary_model_resolver),
834 disabled_filter_resolver: None,
837 disabled_tools: Some(config.disabled_tools),
838 disabled_skill_ids: Some(config.disabled_skill_ids),
839 selected_skill_ids: None,
840 selected_skill_mode: None,
841 cancel_token,
842 mpsc_tx,
843 image_fallback: config.image_fallback,
844 gold_config,
845 guardian_config,
846 guardian_spawner,
847 bash_resume_hook: {
848 let hook: Arc<dyn BashResumeHook> = Arc::new(self.clone());
849 Some(hook)
850 },
851 bash_completion_sink: {
852 let sink: Arc<dyn BashCompletionSink> = Arc::new(self.clone());
855 Some(sink)
856 },
857 app_data_dir: Some(self.app_data_dir.clone()),
858 run_budget: None,
861 runners: self.agent_runners.clone(),
862 sessions_cache: self.sessions.clone(),
863 on_complete: None,
864 child_completion_handler: Some(Arc::new(self.clone())),
868 });
869 }
870}
871
872fn bash_completion_resume_message(bash_ids: &[String], timed_out: bool) -> Message {
882 let body = if timed_out {
883 format!(
884 "Runtime notification: the background-Bash wait ceiling was reached while one or more \
885 shell(s) ({}) may still be running. The session is being resumed so it is not \
886 stranded; verify their actual status with BashOutput before assuming completion.",
887 bash_ids.join(", ")
888 )
889 } else {
890 format!(
891 "Runtime notification: all background Bash shell(s) ({}) have completed. \
892 Review their output with BashOutput and resume the task from where you left off.",
893 bash_ids.join(", ")
894 )
895 };
896 let mut message = Message::user(body);
897 message.metadata = Some(serde_json::json!({
898 RUNTIME_RESUME_MESSAGE_HIDDEN_KEY: true,
899 RUNTIME_RESUME_MESSAGE_KIND_KEY: BASH_COMPLETION_RESUME_KIND,
900 }));
901 message.never_compress = false;
902 message
903}
904
905fn bash_resume_should_retry(outcome: &ResumeOutcome, persisted_waiting_for_bash: bool) -> bool {
918 match outcome {
919 ResumeOutcome::Started { .. } | ResumeOutcome::NotFound => false,
920 ResumeOutcome::Completed | ResumeOutcome::AlreadyRunning { .. } => {
921 persisted_waiting_for_bash
922 }
923 }
924}
925
926fn bash_completion_should_resume(
934 loop_suspended_on_bash: bool,
935 all_waited_shells_done: bool,
936) -> bool {
937 loop_suspended_on_bash && all_waited_shells_done
938}
939
940fn background_bash_post_tool_payload(info: &BashCompletionInfo) -> HookPayload {
941 let success = info.status == "completed" && info.exit_code == Some(0);
942 let response = serde_json::json!({
943 "bash_id": info.bash_id,
944 "command": info.command,
945 "exit_code": info.exit_code,
946 "status": info.status,
947 "output_tail": info.output_tail,
948 });
949 HookPayload::ToolResult {
950 tool_name: "Bash".to_string(),
951 tool_call_id: info.bash_id.clone(),
952 outcome: HookToolOutcome {
953 success,
954 result: success.then(|| response.to_string()),
955 error: (!success).then(|| response.to_string()),
956 needs_human: false,
957 duration_ms: 0,
958 },
959 }
960}
961
962fn append_background_bash_hook_feedback(info: &mut BashCompletionInfo, feedback: Vec<String>) {
963 let feedback = feedback
964 .into_iter()
965 .map(|text| text.trim().to_string())
966 .filter(|text| !text.is_empty())
967 .collect::<Vec<_>>();
968 if feedback.is_empty() {
969 return;
970 }
971 if !info.output_tail.is_empty() {
972 info.output_tail.push_str("\n\n");
973 }
974 info.output_tail.push_str("<post_tool_use_feedback>\n");
975 info.output_tail.push_str(&feedback.join("\n"));
976 info.output_tail.push_str("\n</post_tool_use_feedback>");
977}
978
979async fn run_background_bash_post_tool_hooks(
980 config: &bamboo_config::LifecycleHooksConfig,
981 fallback_cwd: Option<std::path::PathBuf>,
982 session: &Session,
983 info: &mut BashCompletionInfo,
984) -> bool {
985 let runner = crate::HookRunner::new().with_lifecycle_config(config, fallback_cwd);
986 if !runner.has_hooks_for(AgentHookPoint::AfterToolExecution) {
987 return false;
988 }
989
990 let mut runtime_state = session
991 .agent_runtime_state
992 .clone()
993 .unwrap_or_else(|| AgentRuntimeState::new(&session.id));
994 let outcome = runner
995 .run_observer_hooks(
996 AgentHookPoint::AfterToolExecution,
997 &background_bash_post_tool_payload(info),
998 session,
999 &mut runtime_state,
1000 None,
1001 )
1002 .await;
1003 append_background_bash_hook_feedback(info, outcome.injected_contexts);
1004 true
1005}
1006
1007fn apply_bash_resume_transition(session: &mut Session, resume_message: &Message) -> bool {
1015 let mut runtime_state = read_runtime_state(session);
1016 if runtime_state.waiting_for_bash.is_none() {
1017 return false;
1018 }
1019 runtime_state.waiting_for_bash = None;
1020 runtime_state.status = AgentStatusState::Idle;
1021 runtime_state.suspension = None;
1022 write_runtime_state(session, &runtime_state);
1023 session.metadata.remove("runtime.suspend_reason");
1024 session.add_message(resume_message.clone());
1025 true
1026}
1027
1028impl ChildCompletionCoordinator {
1030 async fn bash_self_resume(&self, session_id: String, bash_ids: Vec<String>) {
1046 let mut delay = Duration::from_secs(1);
1047 let max_delay = Duration::from_secs(30);
1048 let max_poll = Duration::from_secs(6 * 3600 + 600);
1052 let deadline = tokio::time::Instant::now() + max_poll;
1053
1054 loop {
1055 tokio::time::sleep(delay).await;
1056
1057 let Some(session) = self.load_session(&session_id).await else {
1058 tracing::info!(%session_id, "bash self-resume backstop: session gone; nothing to do");
1059 return;
1060 };
1061 if read_runtime_state(&session).waiting_for_bash.is_none() {
1062 return;
1065 }
1066
1067 let still_running =
1068 bamboo_tools::tools::bash_runtime::running_shells_for_session(&session_id);
1069 let timed_out = tokio::time::Instant::now() >= deadline;
1070 if still_running.is_empty() || timed_out {
1071 let guard = session_resume_lock(&session_id);
1076 let _held = guard.lock().await;
1077 tracing::warn!(
1078 %session_id,
1079 shell_count = bash_ids.len(),
1080 timed_out,
1081 "bash self-resume backstop engaged (push lost or wait ceiling reached)"
1082 );
1083 self.perform_bash_resume(
1084 &session_id,
1085 bash_completion_resume_message(&bash_ids, timed_out),
1086 )
1087 .await;
1088 return;
1089 }
1090
1091 delay = (delay * 2).min(max_delay);
1092 }
1093 }
1094
1095 async fn perform_bash_resume(&self, session_id: &str, resume_message: Message) {
1116 let retry_backoff = Duration::from_millis(200);
1117 const MAX_RESUME_ATTEMPTS: u8 = 5;
1118 for attempt in 0..MAX_RESUME_ATTEMPTS {
1119 if attempt > 0 {
1120 tokio::time::sleep(retry_backoff).await;
1121 }
1122
1123 let Some(mut session) = self.load_session(session_id).await else {
1124 tracing::warn!(%session_id, "bash resume: session not found; nothing to resume");
1125 return;
1126 };
1127
1128 if !apply_bash_resume_transition(&mut session, &resume_message) {
1129 tracing::info!(
1133 %session_id, attempt,
1134 "bash resume: persisted bash wait already cleared; nothing to resume"
1135 );
1136 return;
1137 }
1138 session.updated_at = Utc::now();
1139 self.save_and_cache(&mut session).await;
1140 tracing::info!(
1141 %session_id, attempt,
1142 "bash resume: cleared bash wait and appended resume message"
1143 );
1144
1145 let outcome = self.resume_parent(session_id.to_string()).await;
1146 match outcome {
1147 ResumeOutcome::Started { .. } => {
1148 tracing::info!(%session_id, attempt, "bash resume: resume fired");
1149 return;
1150 }
1151 ResumeOutcome::NotFound => {
1152 tracing::warn!(%session_id, "bash resume: session vanished during resume");
1153 return;
1154 }
1155 _ => {
1156 let clobbered = match self.load_session(session_id).await {
1162 Some(reloaded) => read_runtime_state(&reloaded).waiting_for_bash.is_some(),
1163 None => {
1164 tracing::warn!(%session_id, "bash resume: session vanished after resume");
1165 return;
1166 }
1167 };
1168 if bash_resume_should_retry(&outcome, clobbered) {
1169 tracing::warn!(
1170 %session_id, attempt,
1171 outcome = outcome.as_str(),
1172 "bash resume: persisted wait still set after resume (finalize-clobber); retrying"
1173 );
1174 continue;
1175 }
1176 tracing::info!(
1177 %session_id, attempt,
1178 outcome = outcome.as_str(),
1179 "bash resume: wait cleared and resume handled; stopping"
1180 );
1181 return;
1182 }
1183 }
1184 }
1185
1186 tracing::warn!(
1187 %session_id,
1188 attempts = MAX_RESUME_ATTEMPTS,
1189 "bash resume: exhausted clobber-retry budget without confirming resume; giving up"
1190 );
1191 }
1192}
1193
1194impl BashResumeHook for ChildCompletionCoordinator {
1195 fn arrange_bash_self_resume(&self, session_id: String, bash_ids: Vec<String>) {
1196 let coordinator = Arc::new(self.clone());
1197 tokio::spawn(async move {
1198 coordinator.bash_self_resume(session_id, bash_ids).await;
1199 });
1200 }
1201}
1202
1203fn bash_completion_injection_body(info: &BashCompletionInfo) -> String {
1208 let exit = match info.exit_code {
1209 Some(code) => code.to_string(),
1210 None => "none (signal/killed)".to_string(),
1211 };
1212 let mut body = format!(
1213 "Runtime notification: background shell `{}` (`{}`) finished — status {}, exit code {}.",
1214 info.bash_id, info.command, info.status, exit
1215 );
1216 if info.output_tail.trim().is_empty() {
1217 body.push_str(" It produced no captured output.");
1218 } else {
1219 body.push_str("\n\nOutput tail:\n");
1220 body.push_str(&info.output_tail);
1221 }
1222 body.push_str(&format!(
1223 "\n\nUse BashOutput with bash_id=\"{}\" for the full output, then continue the task.",
1224 info.bash_id
1225 ));
1226 body
1227}
1228
1229async fn enqueue_bash_completion_injection(
1252 persistence: &LockedSessionStore,
1253 info: &BashCompletionInfo,
1254) -> std::io::Result<Option<Session>> {
1255 let body = bash_completion_injection_body(info);
1256 let queued = serde_json::json!({
1257 "content": body,
1258 "created_at": Utc::now(),
1259 });
1260 persistence
1261 .update_runtime_config(&info.session_id, move |session| {
1262 let mut pending = session.pending_injected_messages().unwrap_or_default();
1263 pending.push(queued);
1264 session.set_pending_injected_messages(pending);
1265 })
1266 .await
1267}
1268
1269fn bash_resume_message_from_info(info: &BashCompletionInfo) -> Message {
1277 let mut message = Message::user(bash_completion_injection_body(info));
1278 message.metadata = Some(serde_json::json!({
1279 RUNTIME_RESUME_MESSAGE_HIDDEN_KEY: true,
1280 RUNTIME_RESUME_MESSAGE_KIND_KEY: BASH_COMPLETION_RESUME_KIND,
1281 }));
1282 message.never_compress = false;
1283 message
1284}
1285
1286impl ChildCompletionCoordinator {
1287 async fn deliver_bash_completion(&self, mut info: BashCompletionInfo) {
1300 if let Some(hook_session) = self.load_session(&info.session_id).await {
1306 let config_snapshot = self.config.read().await.clone();
1307 let _ = run_background_bash_post_tool_hooks(
1308 &config_snapshot.lifecycle_hooks,
1309 Some(self.app_data_dir.clone()),
1310 &hook_session,
1311 &mut info,
1312 )
1313 .await;
1314 }
1315
1316 let guard = session_resume_lock(&info.session_id);
1317 let _held = guard.lock().await;
1318
1319 let Some(session) = self.load_session(&info.session_id).await else {
1320 tracing::warn!(
1321 session_id = %info.session_id,
1322 bash_id = %info.bash_id,
1323 "background bash completion: owning session not found; nothing to notify"
1324 );
1325 return;
1326 };
1327
1328 let waiting = read_runtime_state(&session).waiting_for_bash.is_some();
1329 let all_shells_done =
1335 bamboo_tools::tools::bash_runtime::running_shells_for_session(&info.session_id)
1336 .is_empty();
1337
1338 if bash_completion_should_resume(waiting, all_shells_done) {
1339 tracing::info!(
1340 session_id = %info.session_id,
1341 bash_id = %info.bash_id,
1342 status = %info.status,
1343 "background bash completion: push-resuming suspended loop (event-driven)"
1344 );
1345 self.perform_bash_resume(&info.session_id, bash_resume_message_from_info(&info))
1346 .await;
1347 return;
1348 }
1349
1350 match enqueue_bash_completion_injection(&self.persistence, &info).await {
1351 Ok(Some(_)) => tracing::info!(
1352 session_id = %info.session_id,
1353 bash_id = %info.bash_id,
1354 status = %info.status,
1355 waiting,
1356 "background bash completion queued for injection at the next round boundary"
1357 ),
1358 Ok(None) => tracing::warn!(
1359 session_id = %info.session_id,
1360 bash_id = %info.bash_id,
1361 "background bash completion: owning session not found; nothing to notify"
1362 ),
1363 Err(error) => tracing::warn!(
1364 session_id = %info.session_id,
1365 bash_id = %info.bash_id,
1366 %error,
1367 "background bash completion: failed to queue injection"
1368 ),
1369 }
1370 }
1371}
1372
1373impl BashCompletionSink for ChildCompletionCoordinator {
1374 fn on_bash_completed(&self, info: BashCompletionInfo) {
1375 let coordinator = Arc::new(self.clone());
1379 tokio::spawn(async move {
1380 coordinator.deliver_bash_completion(info).await;
1381 });
1382 }
1383}
1384
1385const CHILD_WAIT_SWEEP_INTERVAL_SECS: u64 = 30;
1391const CHILD_WAIT_REGISTRATION_GRACE_SECS: i64 = 60;
1395const DEAD_CHILD_GRACE_SECS: i64 = 120;
1398const STALE_RUNNER_SLACK_SECS: i64 = 600;
1404
1405fn is_dead_child_candidate_status(status: Option<&str>) -> bool {
1410 match status {
1411 Some(status) => !is_terminal_child_status(status) && status != "suspended",
1414 None => true,
1415 }
1416}
1417
1418fn completion_child_is_owned(reported_parent: &str, child_parent_linkage: Option<&str>) -> bool {
1426 child_parent_linkage == Some(reported_parent)
1427}
1428
1429fn select_replay_child(terminal: &[(String, String)]) -> Option<&(String, String)> {
1433 terminal
1434 .iter()
1435 .find(|(_, status)| is_error_like(status))
1436 .or_else(|| terminal.last())
1437}
1438
1439fn child_wait_watchdog_resume_message(body: String) -> Message {
1440 let mut message = Message::user(body);
1441 message.metadata = Some(serde_json::json!({
1442 RUNTIME_RESUME_MESSAGE_HIDDEN_KEY: true,
1443 RUNTIME_RESUME_MESSAGE_KIND_KEY: "child_wait_watchdog_resume",
1444 }));
1445 message.never_compress = false;
1446 message
1447}
1448
1449fn empty_child_wait_message() -> Message {
1450 child_wait_watchdog_resume_message(
1451 "Runtime notification: this session was suspended waiting for child sessions, but the \
1452 wait tracked no children (internal inconsistency). The session has been resumed; use \
1453 SubAgent.list to inspect child state and continue the task."
1454 .to_string(),
1455 )
1456}
1457
1458fn child_wait_lease_expired_message(child_ids: &[String]) -> Message {
1459 child_wait_watchdog_resume_message(format!(
1460 "Runtime notification: the wait lease for child session(s) [{}] expired before they all \
1461 reported completion. They were NOT cancelled and may still be running or already \
1462 finished — verify their actual status with SubAgent.list / SubAgent.get before assuming \
1463 anything, then continue the task.",
1464 child_ids.join(", ")
1465 ))
1466}
1467
1468impl ChildCompletionCoordinator {
1484 pub fn spawn_child_wait_watchdog(self: &Arc<Self>) {
1487 let coordinator = Arc::clone(self);
1488 tokio::spawn(async move {
1489 use futures::FutureExt;
1490 if std::panic::AssertUnwindSafe(coordinator.reconcile_orphans_at_boot())
1491 .catch_unwind()
1492 .await
1493 .is_err()
1494 {
1495 tracing::error!("child-wait watchdog: boot reconciliation panicked");
1496 }
1497 let mut ticker = tokio::time::interval(std::time::Duration::from_secs(
1498 CHILD_WAIT_SWEEP_INTERVAL_SECS,
1499 ));
1500 ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
1501 ticker.tick().await;
1503 loop {
1504 ticker.tick().await;
1505 if std::panic::AssertUnwindSafe(coordinator.sweep_child_waits())
1506 .catch_unwind()
1507 .await
1508 .is_err()
1509 {
1510 tracing::error!("child-wait watchdog: sweep panicked; continuing");
1511 }
1512 }
1513 });
1514 }
1515
1516 async fn reconcile_orphans_at_boot(&self) {
1521 let cutoff = Utc::now();
1522
1523 let running = self
1529 .storage
1530 .list_sessions_by_run_status("running")
1531 .await
1532 .unwrap_or_default();
1533 for (child_id, parent_id) in running {
1534 let Some(parent_id) = parent_id else { continue };
1535 if self.runner_is_running(&child_id).await {
1536 continue;
1537 }
1538 let Some(control_plane) = self.load_control_plane(&child_id).await else {
1539 continue;
1540 };
1541 if control_plane.updated_at >= cutoff {
1544 continue;
1545 }
1546 tracing::warn!(
1547 child_session_id = %child_id,
1548 parent_session_id = %parent_id,
1549 "boot reconciliation: child was running when the process died; \
1550 marking it error and waking the parent"
1551 );
1552 self.synthesize_child_completion(
1553 &parent_id,
1554 &child_id,
1555 "error",
1556 Some(
1557 "orphaned by server restart: the process died while this child session \
1558 was running"
1559 .to_string(),
1560 ),
1561 )
1562 .await;
1563 }
1564
1565 let suspended = self
1569 .storage
1570 .list_sessions_by_run_status("suspended")
1571 .await
1572 .unwrap_or_default();
1573 for (session_id, _) in suspended {
1574 let Some(control_plane) = self.load_control_plane(&session_id).await else {
1575 continue;
1576 };
1577 if control_plane
1578 .metadata
1579 .get("runtime.suspend_reason")
1580 .map(String::as_str)
1581 != Some("waiting_for_bash")
1582 {
1583 continue;
1584 }
1585 if let Some(wait) = read_runtime_state(&control_plane).waiting_for_bash {
1586 tracing::warn!(
1587 %session_id,
1588 "boot reconciliation: re-arming bash self-resume backstop lost in restart"
1589 );
1590 let coordinator = self.clone();
1591 tokio::spawn(async move {
1592 coordinator
1593 .bash_self_resume(session_id, wait.bash_ids)
1594 .await;
1595 });
1596 }
1597 }
1598 }
1599
1600 async fn runner_is_running(&self, session_id: &str) -> bool {
1601 let runners = self.agent_runners.read().await;
1602 runners
1603 .get(session_id)
1604 .is_some_and(|runner| matches!(runner.status, AgentStatus::Running))
1605 }
1606
1607 async fn load_control_plane(&self, session_id: &str) -> Option<Session> {
1608 match self.storage.load_runtime_control_plane(session_id).await {
1609 Ok(session) => session,
1610 Err(error) => {
1611 tracing::warn!(
1612 %session_id,
1613 %error,
1614 "child-wait watchdog: failed to load session control plane"
1615 );
1616 None
1617 }
1618 }
1619 }
1620
1621 async fn sweep_child_waits(&self) {
1625 let suspended = match self.storage.list_sessions_by_run_status("suspended").await {
1626 Ok(entries) => entries,
1627 Err(error) => {
1628 tracing::warn!(%error, "child-wait watchdog: failed to list suspended sessions");
1629 return;
1630 }
1631 };
1632 for (session_id, _) in suspended {
1633 self.sweep_one_suspended_session(&session_id).await;
1634 }
1635 }
1636
1637 async fn sweep_one_suspended_session(&self, session_id: &str) {
1638 if self.runner_is_running(session_id).await {
1641 return;
1642 }
1643 let Some(session) = self.load_control_plane(session_id).await else {
1644 return;
1645 };
1646 let runtime_state = read_runtime_state(&session);
1647 let suspend_reason = session
1648 .metadata
1649 .get("runtime.suspend_reason")
1650 .map(String::as_str)
1651 .unwrap_or_default()
1652 .to_string();
1653 match (
1654 suspend_reason.as_str(),
1655 runtime_state.waiting_for_children.clone(),
1656 ) {
1657 ("waiting_for_bash", _)
1659 | ("awaiting_clarification", _)
1660 | ("awaiting_parent_approval", _) => {}
1661 (_, Some(wait)) => self.sweep_child_wait(session_id, wait).await,
1662 ("waiting_for_children", None) | ("", None) => {
1665 self.rescue_stranded_resume(session_id).await;
1666 }
1667 _ => {}
1668 }
1669 }
1670
1671 async fn sweep_child_wait(&self, parent_session_id: &str, wait: WaitingForChildrenState) {
1676 let now = Utc::now();
1677
1678 if wait.child_session_ids.is_empty() {
1679 tracing::warn!(
1680 %parent_session_id,
1681 "child-wait watchdog: wait armed over an empty child set; force-resuming"
1682 );
1683 self.force_resume_child_wait(parent_session_id, empty_child_wait_message())
1684 .await;
1685 return;
1686 }
1687
1688 if wait.timeout_at.is_some_and(|deadline| now >= deadline) {
1692 tracing::warn!(
1693 %parent_session_id,
1694 "child-wait watchdog: wait lease expired; force-resuming parent"
1695 );
1696 self.force_resume_child_wait(
1697 parent_session_id,
1698 child_wait_lease_expired_message(&wait.child_session_ids),
1699 )
1700 .await;
1701 return;
1702 }
1703
1704 if now.signed_duration_since(wait.registered_at).num_seconds()
1706 < CHILD_WAIT_REGISTRATION_GRACE_SECS
1707 {
1708 return;
1709 }
1710
1711 let statuses: HashMap<String, Option<String>> = self
1712 .storage
1713 .list_child_run_statuses(parent_session_id)
1714 .await
1715 .unwrap_or_default()
1716 .into_iter()
1717 .collect();
1718
1719 struct DeadChild {
1720 child_id: String,
1721 status: String,
1722 reason: String,
1723 owned: bool,
1727 }
1728
1729 let mut terminal: Vec<(String, String)> = Vec::new();
1730 let mut dead: Vec<DeadChild> = Vec::new();
1731 for child_id in &wait.child_session_ids {
1732 let status = statuses.get(child_id).and_then(|status| status.as_deref());
1733 if let Some(status) = status {
1734 if is_terminal_child_status(status) {
1735 terminal.push((child_id.clone(), status.to_string()));
1736 continue;
1737 }
1738 }
1739 if !is_dead_child_candidate_status(status) {
1740 continue;
1741 }
1742
1743 let control_plane = self.load_control_plane(child_id).await;
1750 let owned = control_plane
1751 .as_ref()
1752 .is_some_and(|cp| cp.parent_session_id.as_deref() == Some(parent_session_id));
1753 if !owned {
1754 dead.push(DeadChild {
1755 child_id: child_id.clone(),
1756 status: "error".to_string(),
1757 reason: if control_plane.is_some() {
1758 "waited-on session id is not a child of this session; clearing it \
1759 from the wait without touching that session"
1760 .to_string()
1761 } else {
1762 "waited-on child session does not exist".to_string()
1763 },
1764 owned: false,
1765 });
1766 continue;
1767 }
1768
1769 let runner = { self.agent_runners.read().await.get(child_id).cloned() };
1770 match runner {
1771 Some(runner) if matches!(runner.status, AgentStatus::Running) => {
1772 let last_activity = runner.last_event_at.unwrap_or(runner.started_at);
1778 let idle_secs = now.signed_duration_since(last_activity).num_seconds();
1779 let total_secs = now.signed_duration_since(runner.started_at).num_seconds();
1780 let policy = match &control_plane {
1781 Some(child) => {
1782 crate::runtime::execution::spawn::watchdog_policy_for_session(child)
1783 }
1784 None => Default::default(),
1785 };
1786 let idle_limit = policy.max_idle_secs.saturating_add(STALE_RUNNER_SLACK_SECS);
1787 let total_limit = policy
1788 .max_total_secs
1789 .saturating_add(STALE_RUNNER_SLACK_SECS);
1790 if idle_secs >= idle_limit || total_secs >= total_limit {
1791 runner.cancel_token.cancel();
1792 dead.push(DeadChild {
1793 child_id: child_id.clone(),
1794 status: "timeout".to_string(),
1795 reason: format!(
1796 "child runner stalled: no events for {idle_secs}s \
1797 (limit {idle_limit}s including watchdog slack); \
1798 force-finalized by the child-wait watchdog"
1799 ),
1800 owned: true,
1801 });
1802 }
1803 }
1804 _ => {
1805 let quiet_secs = control_plane
1809 .as_ref()
1810 .map(|child| now.signed_duration_since(child.updated_at).num_seconds())
1811 .unwrap_or(i64::MAX);
1812 if quiet_secs >= DEAD_CHILD_GRACE_SECS {
1813 dead.push(DeadChild {
1814 child_id: child_id.clone(),
1815 status: "error".to_string(),
1816 reason: format!(
1817 "child runner lost (crashed task, dropped spawn job, or \
1818 process restart): index status {status:?} with no live \
1819 runner driving it"
1820 ),
1821 owned: true,
1822 });
1823 }
1824 }
1825 }
1826 }
1827
1828 if !dead.is_empty() {
1829 for entry in dead {
1830 tracing::warn!(
1831 %parent_session_id,
1832 child_session_id = %entry.child_id,
1833 status = %entry.status,
1834 reason = %entry.reason,
1835 owned = entry.owned,
1836 "child-wait watchdog: synthesizing terminal completion for dead child"
1837 );
1838 if entry.owned {
1839 self.synthesize_child_completion(
1840 parent_session_id,
1841 &entry.child_id,
1842 &entry.status,
1843 Some(entry.reason),
1844 )
1845 .await;
1846 } else {
1847 self.publish_synthetic_completion(
1849 parent_session_id,
1850 &entry.child_id,
1851 &entry.status,
1852 Some(entry.reason),
1853 )
1854 .await;
1855 }
1856 }
1857 return;
1859 }
1860
1861 let terminal_ids: Vec<String> = terminal.iter().map(|(id, _)| id.clone()).collect();
1867 if let Some((child_id, status)) = select_replay_child(&terminal) {
1868 if wait_policy_satisfied(
1869 wait.wait_for,
1870 &wait.child_session_ids,
1871 &terminal_ids,
1872 child_id,
1873 status,
1874 ) {
1875 tracing::warn!(
1876 %parent_session_id,
1877 child_session_id = %child_id,
1878 "child-wait watchdog: wait already satisfied but parent still suspended \
1879 (lost wake); replaying the completion"
1880 );
1881 let error = self
1882 .load_control_plane(child_id)
1883 .await
1884 .and_then(|child| child.last_run_error());
1885 self.publish_synthetic_completion(parent_session_id, child_id, status, error)
1886 .await;
1887 }
1888 }
1889 }
1890
1891 async fn synthesize_child_completion(
1897 &self,
1898 parent_session_id: &str,
1899 child_session_id: &str,
1900 status: &str,
1901 error: Option<String>,
1902 ) {
1903 match self.storage.load_session(child_session_id).await {
1904 Ok(Some(mut child)) => {
1905 if child.parent_session_id.as_deref() != Some(parent_session_id) {
1910 tracing::warn!(
1911 %parent_session_id,
1912 child_session_id = %child.id,
1913 "child-wait watchdog: refusing to synthesize status onto a session \
1914 that is not a child of this parent"
1915 );
1916 self.publish_synthetic_completion(
1917 parent_session_id,
1918 child_session_id,
1919 status,
1920 error,
1921 )
1922 .await;
1923 return;
1924 }
1925 child.set_last_run_status(status);
1926 match &error {
1927 Some(message) => child.set_last_run_error(message.clone()),
1928 None => child.clear_last_run_error(),
1929 }
1930 child.updated_at = Utc::now();
1931 if let Err(save_error) = self.persistence.merge_save_runtime(&mut child).await {
1932 tracing::warn!(
1933 child_session_id = %child.id,
1934 %save_error,
1935 "child-wait watchdog: failed to persist synthesized terminal status"
1936 );
1937 }
1938 self.sessions
1939 .insert(child.id.clone(), Arc::new(parking_lot::RwLock::new(child)));
1940 }
1941 Ok(None) => {}
1942 Err(load_error) => {
1943 tracing::warn!(
1944 %child_session_id,
1945 %load_error,
1946 "child-wait watchdog: failed to load child for synthesized terminal status"
1947 );
1948 }
1949 }
1950 finalize_runner(
1951 &self.agent_runners,
1952 child_session_id,
1953 &Err(bamboo_agent_core::AgentError::LLM(
1954 error
1955 .clone()
1956 .unwrap_or_else(|| format!("synthesized {status}")),
1957 )),
1958 )
1959 .await;
1960 self.publish_synthetic_completion(parent_session_id, child_session_id, status, error)
1961 .await;
1962 }
1963
1964 async fn publish_synthetic_completion(
1965 &self,
1966 parent_session_id: &str,
1967 child_session_id: &str,
1968 status: &str,
1969 error: Option<String>,
1970 ) {
1971 let parent_tx = crate::execution::session_events::get_or_create_event_sender(
1972 &self.session_event_senders,
1973 parent_session_id,
1974 )
1975 .await;
1976 let handler: Arc<dyn ChildCompletionHandler> = Arc::new(self.clone());
1977 crate::runtime::execution::spawn::publish_child_completion_parts(
1978 &parent_tx,
1979 Some(handler),
1980 parent_session_id.to_string(),
1981 child_session_id.to_string(),
1982 status.to_string(),
1983 error,
1984 )
1985 .await;
1986 }
1987
1988 async fn rescue_stranded_resume(&self, session_id: &str) {
1994 let Some(session) = self.load_session(session_id).await else {
1995 return;
1996 };
1997 let pending_runtime_resume = session.messages.last().is_some_and(|message| {
1998 matches!(message.role, Role::User)
1999 && message
2000 .metadata
2001 .as_ref()
2002 .is_some_and(|meta| meta.get(RUNTIME_RESUME_MESSAGE_KIND_KEY).is_some())
2003 });
2004 if !pending_runtime_resume {
2005 return;
2006 }
2007 tracing::warn!(
2008 %session_id,
2009 "child-wait watchdog: stranded resume detected (wait cleared, resume never \
2010 spawned); resuming"
2011 );
2012 self.resume_parent(session_id.to_string()).await;
2013 }
2014
2015 async fn force_resume_child_wait(&self, session_id: &str, resume_message: Message) {
2021 const MAX_ATTEMPTS: u8 = 5;
2022 let lock = session_resume_lock(session_id);
2023 for attempt in 0..MAX_ATTEMPTS {
2024 if attempt > 0 {
2025 tokio::time::sleep(Duration::from_millis(200)).await;
2026 }
2027 {
2028 let _held = lock.lock().await;
2029 let Some(mut session) = self.load_session(session_id).await else {
2030 return;
2031 };
2032 let mut runtime_state = read_runtime_state(&session);
2033 if runtime_state.waiting_for_children.is_none() {
2034 return;
2036 }
2037 runtime_state.waiting_for_children = None;
2038 runtime_state.status = AgentStatusState::Idle;
2039 runtime_state.suspension = None;
2040 write_runtime_state(&mut session, &runtime_state);
2041 session.metadata.remove("runtime.suspend_reason");
2042 session.add_message(resume_message.clone());
2043 session.updated_at = Utc::now();
2044 self.save_and_cache(&mut session).await;
2045 }
2046 let outcome = self.resume_parent(session_id.to_string()).await;
2047 match outcome {
2048 ResumeOutcome::Started { .. } | ResumeOutcome::NotFound => return,
2049 ResumeOutcome::Completed | ResumeOutcome::AlreadyRunning { .. } => {
2050 let clobbered = self
2054 .load_session(session_id)
2055 .await
2056 .map(|session| read_runtime_state(&session).waiting_for_children.is_some())
2057 .unwrap_or(false);
2058 if !clobbered {
2059 return;
2060 }
2061 }
2062 }
2063 }
2064 tracing::error!(
2065 %session_id,
2066 "child-wait watchdog: force-resume exhausted its clobber-retry budget"
2067 );
2068 }
2069}
2070
2071#[cfg(test)]
2072mod tests {
2073 use super::*;
2074 use bamboo_agent_core::Message;
2075
2076 #[test]
2079 fn dead_child_candidate_status_matrix() {
2080 assert!(is_dead_child_candidate_status(None));
2082 assert!(is_dead_child_candidate_status(Some("running")));
2085 assert!(is_dead_child_candidate_status(Some("pending")));
2086 assert!(!is_dead_child_candidate_status(Some("suspended")));
2088 for status in ["completed", "error", "timeout", "cancelled", "skipped"] {
2090 assert!(!is_dead_child_candidate_status(Some(status)), "{status}");
2091 }
2092 }
2093
2094 #[test]
2095 fn completion_child_ownership_gates_content_fold() {
2096 assert!(completion_child_is_owned("parent-1", Some("parent-1")));
2098 assert!(!completion_child_is_owned("parent-1", Some("parent-2")));
2101 assert!(!completion_child_is_owned("parent-1", None));
2103 }
2104
2105 #[test]
2106 fn replay_child_prefers_error_like_for_first_error_policy() {
2107 let terminal = vec![
2108 ("c-ok".to_string(), "completed".to_string()),
2109 ("c-err".to_string(), "timeout".to_string()),
2110 ("c-late".to_string(), "completed".to_string()),
2111 ];
2112 let (id, status) = select_replay_child(&terminal).expect("non-empty");
2113 assert_eq!(id, "c-err");
2114 assert_eq!(status, "timeout");
2115
2116 let all_ok = vec![
2117 ("c-1".to_string(), "completed".to_string()),
2118 ("c-2".to_string(), "completed".to_string()),
2119 ];
2120 let (id, _) = select_replay_child(&all_ok).expect("non-empty");
2121 assert_eq!(id, "c-2");
2122
2123 assert!(select_replay_child(&[]).is_none());
2124 }
2125
2126 #[test]
2127 fn watchdog_resume_messages_are_hidden_runtime_messages() {
2128 for message in [
2129 empty_child_wait_message(),
2130 child_wait_lease_expired_message(&["c-1".to_string(), "c-2".to_string()]),
2131 ] {
2132 assert!(matches!(message.role, Role::User));
2133 let meta = message.metadata.expect("hidden runtime metadata");
2134 assert_eq!(meta[RUNTIME_RESUME_MESSAGE_HIDDEN_KEY], true);
2135 assert_eq!(
2136 meta[RUNTIME_RESUME_MESSAGE_KIND_KEY],
2137 "child_wait_watchdog_resume"
2138 );
2139 }
2140 let lease = child_wait_lease_expired_message(&["c-1".to_string()]);
2141 assert!(lease.content.contains("NOT cancelled"));
2143 assert!(lease.content.contains("c-1"));
2144 }
2145
2146 #[test]
2149 fn non_terminal_statuses_never_satisfy_wait_policies() {
2150 assert!(!is_terminal_child_status("suspended"));
2153 assert!(!is_terminal_child_status("running"));
2154 assert!(!is_terminal_child_status("pending"));
2155 }
2156
2157 fn make_completion(status: &str) -> ChildCompletion {
2158 ChildCompletion {
2159 parent_session_id: "parent-1".to_string(),
2160 child_session_id: "child-1".to_string(),
2161 status: status.to_string(),
2162 error: None,
2163 completed_at: Utc::now(),
2164 }
2165 }
2166
2167 struct StubChildIndex {
2170 children: Vec<(String, Option<String>)>,
2171 }
2172
2173 #[async_trait]
2174 impl Storage for StubChildIndex {
2175 async fn save_session(&self, _session: &Session) -> std::io::Result<()> {
2176 Ok(())
2177 }
2178 async fn load_session(&self, _id: &str) -> std::io::Result<Option<Session>> {
2179 Ok(None)
2180 }
2181 async fn delete_session(&self, _id: &str) -> std::io::Result<bool> {
2182 Ok(false)
2183 }
2184 async fn list_child_run_statuses(
2185 &self,
2186 _parent_session_id: &str,
2187 ) -> std::io::Result<Vec<(String, Option<String>)>> {
2188 Ok(self.children.clone())
2189 }
2190 }
2191
2192 #[tokio::test]
2193 async fn derive_completed_only_includes_terminal_children() {
2194 let storage: Arc<dyn Storage> = Arc::new(StubChildIndex {
2195 children: vec![
2196 ("a".into(), Some("completed".into())),
2197 ("b".into(), Some("running".into())),
2198 ("c".into(), Some("error".into())),
2199 ("d".into(), None),
2200 ],
2201 });
2202 let completed = derive_completed_child_ids(&storage, "parent-1", "b").await;
2203 assert_eq!(
2205 completed,
2206 vec!["a".to_string(), "b".to_string(), "c".to_string()]
2207 );
2208 }
2209
2210 #[tokio::test]
2211 async fn derive_completed_folds_in_just_completed_when_index_lags() {
2212 let storage: Arc<dyn Storage> = Arc::new(StubChildIndex {
2214 children: vec![("only".into(), Some("running".into()))],
2215 });
2216 let completed = derive_completed_child_ids(&storage, "parent-1", "only").await;
2217 assert_eq!(completed, vec!["only".to_string()]);
2218 }
2219
2220 #[test]
2221 fn wait_policy_all_uses_derived_completed_set() {
2222 let waited = vec!["a".to_string(), "b".to_string()];
2223 assert!(!wait_policy_satisfied(
2224 ChildWaitPolicy::All,
2225 &waited,
2226 &["a".to_string()],
2227 "a",
2228 "completed"
2229 ));
2230 assert!(wait_policy_satisfied(
2231 ChildWaitPolicy::All,
2232 &waited,
2233 &["a".to_string(), "b".to_string()],
2234 "b",
2235 "completed"
2236 ));
2237 }
2238
2239 #[test]
2240 fn wait_policy_first_error_requires_tracked_membership() {
2241 let waited = vec!["a".to_string(), "b".to_string()];
2242 assert!(wait_policy_satisfied(
2244 ChildWaitPolicy::FirstError,
2245 &waited,
2246 &["a".to_string()],
2247 "a",
2248 "error"
2249 ));
2250 assert!(!wait_policy_satisfied(
2253 ChildWaitPolicy::FirstError,
2254 &waited,
2255 &["a".to_string()],
2256 "stray-child",
2257 "timeout"
2258 ));
2259 assert!(wait_policy_satisfied(
2261 ChildWaitPolicy::FirstError,
2262 &waited,
2263 &["a".to_string(), "b".to_string()],
2264 "stray-child",
2265 "completed"
2266 ));
2267 }
2268
2269 #[test]
2270 fn child_final_assistant_text_returns_last_assistant() {
2271 let mut session = Session::new("child-1", "gpt-4");
2272 session.messages.push(Message::user("hi"));
2273 session
2274 .messages
2275 .push(Message::assistant("first answer", None));
2276 session.messages.push(Message::user("again"));
2277 session
2278 .messages
2279 .push(Message::assistant("final answer", None));
2280
2281 assert_eq!(
2282 child_final_assistant_text(&session).as_deref(),
2283 Some("final answer")
2284 );
2285 }
2286
2287 #[test]
2288 fn child_final_assistant_text_returns_none_when_blank() {
2289 let mut session = Session::new("child-1", "gpt-4");
2290 session.messages.push(Message::assistant(" ", None));
2291 assert!(child_final_assistant_text(&session).is_none());
2292 }
2293
2294 #[test]
2295 fn child_final_assistant_text_returns_none_when_no_assistant() {
2296 let mut session = Session::new("child-1", "gpt-4");
2297 session.messages.push(Message::user("hi"));
2298 assert!(child_final_assistant_text(&session).is_none());
2299 }
2300
2301 #[test]
2302 fn runtime_resume_message_folds_full_response_without_truncation() {
2303 let completion = make_completion("completed");
2306 let long: String = "a".repeat(10_000);
2307 let message = runtime_resume_message(&completion, 0, Some(&long));
2308 assert!(message.content.contains(&long));
2309 assert!(!message.content.contains("truncated"));
2310 }
2311
2312 #[test]
2313 fn runtime_resume_message_includes_child_response_when_provided() {
2314 let completion = make_completion("completed");
2315 let message = runtime_resume_message(&completion, 0, Some("the answer is 42"));
2316
2317 assert!(matches!(message.role, Role::User));
2318 assert!(!message.never_compress);
2321 assert!(message.content.contains("Child final response:"));
2322 assert!(message.content.contains("the answer is 42"));
2323
2324 let metadata = message.metadata.expect("metadata present");
2325 assert_eq!(
2326 metadata.get("hidden_from_ui").and_then(|v| v.as_bool()),
2327 Some(true)
2328 );
2329 assert_eq!(
2330 metadata.get("runtime_kind").and_then(|v| v.as_str()),
2331 Some("child_completion_resume")
2332 );
2333 assert_eq!(
2334 metadata
2335 .get("child_final_response_included")
2336 .and_then(|v| v.as_bool()),
2337 Some(true)
2338 );
2339 }
2340
2341 #[test]
2342 fn runtime_resume_message_falls_back_to_error_when_no_response() {
2343 let mut completion = make_completion("error");
2344 completion.error = Some("boom".to_string());
2345
2346 let message = runtime_resume_message(&completion, 1, None);
2347 assert!(message.content.contains("Child error:"));
2348 assert!(message.content.contains("boom"));
2349 let metadata = message.metadata.expect("metadata present");
2350 assert_eq!(
2351 metadata
2352 .get("child_final_response_included")
2353 .and_then(|v| v.as_bool()),
2354 Some(false)
2355 );
2356 }
2357
2358 #[test]
2359 fn runtime_resume_message_minimal_when_no_response_and_no_error() {
2360 let completion = make_completion("completed");
2361 let message = runtime_resume_message(&completion, 2, None);
2362 assert!(!message.content.contains("Child final response:"));
2363 assert!(!message.content.contains("Child error:"));
2364 assert!(message.content.contains("Resume the parent task"));
2365 }
2366
2367 #[test]
2368 fn read_config_snapshot_refreshes_cached_snapshot_from_live_config() {
2369 let runtime = tokio::runtime::Runtime::new().expect("runtime");
2370
2371 runtime.block_on(async {
2372 let config = Arc::new(RwLock::new(Config::default()));
2373 config.write().await.provider = "copilot".to_string();
2374 let cached_config = StdRwLock::new(Config::default());
2375
2376 let snapshot = read_config_snapshot(&config, &cached_config);
2377
2378 assert_eq!(snapshot.provider, "copilot");
2379 assert_eq!(
2380 cached_config.read().expect("cached snapshot lock").provider,
2381 "copilot"
2382 );
2383 });
2384 }
2385
2386 #[test]
2387 fn read_config_snapshot_uses_cached_snapshot_when_live_lock_is_busy() {
2388 let runtime = tokio::runtime::Runtime::new().expect("runtime");
2389
2390 runtime.block_on(async {
2391 let mut cached_snapshot = Config::default();
2392 cached_snapshot.provider = "cached-provider".to_string();
2393
2394 let config = Arc::new(RwLock::new(Config::default()));
2395 let cached_config = StdRwLock::new(cached_snapshot);
2396 let _write_guard = config.write().await;
2397
2398 let snapshot = read_config_snapshot(&config, &cached_config);
2399
2400 assert_eq!(snapshot.provider, "cached-provider");
2401 });
2402 }
2403
2404 #[test]
2407 fn bash_completion_resume_message_normal_announces_completion() {
2408 let ids = vec!["bg-1".to_string(), "bg-2".to_string()];
2409 let message = bash_completion_resume_message(&ids, false);
2410 assert!(
2412 message.content.contains("have completed"),
2413 "normal resume message must announce completion: {}",
2414 message.content
2415 );
2416 let metadata = message.metadata.expect("metadata present");
2418 assert_eq!(
2419 metadata
2420 .get(RUNTIME_RESUME_MESSAGE_HIDDEN_KEY)
2421 .and_then(|v| v.as_bool()),
2422 Some(true),
2423 "resume message must be hidden from the UI"
2424 );
2425 assert_eq!(
2426 metadata
2427 .get(RUNTIME_RESUME_MESSAGE_KIND_KEY)
2428 .and_then(|v| v.as_str()),
2429 Some(BASH_COMPLETION_RESUME_KIND),
2430 "resume message must carry the bash-completion kind discriminant"
2431 );
2432 }
2433
2434 #[test]
2435 fn bash_completion_resume_message_deadline_does_not_claim_completion() {
2436 let ids = vec!["bg-long".to_string()];
2440 let message = bash_completion_resume_message(&ids, true);
2441 assert!(
2442 !message.content.contains("have completed"),
2443 "deadline resume message must NOT claim the shells completed: {}",
2444 message.content
2445 );
2446 assert!(
2447 message.content.contains("may still be running"),
2448 "deadline resume message must warn shells may still be running: {}",
2449 message.content
2450 );
2451 assert!(
2452 message.content.contains("BashOutput"),
2453 "deadline resume message must direct verification via BashOutput: {}",
2454 message.content
2455 );
2456 let metadata = message.metadata.expect("metadata present");
2458 assert_eq!(
2459 metadata
2460 .get(RUNTIME_RESUME_MESSAGE_KIND_KEY)
2461 .and_then(|v| v.as_str()),
2462 Some(BASH_COMPLETION_RESUME_KIND)
2463 );
2464 }
2465
2466 #[test]
2467 fn bash_resume_should_retry_matrix() {
2468 assert!(!bash_resume_should_retry(
2474 &ResumeOutcome::Started { run_id: "r".into() },
2475 true
2476 ));
2477 assert!(!bash_resume_should_retry(
2478 &ResumeOutcome::Started { run_id: "r".into() },
2479 false
2480 ));
2481
2482 assert!(!bash_resume_should_retry(&ResumeOutcome::NotFound, true));
2484 assert!(!bash_resume_should_retry(&ResumeOutcome::NotFound, false));
2485
2486 assert!(bash_resume_should_retry(&ResumeOutcome::Completed, true));
2488 assert!(!bash_resume_should_retry(&ResumeOutcome::Completed, false));
2491
2492 assert!(bash_resume_should_retry(
2495 &ResumeOutcome::AlreadyRunning { run_id: "r".into() },
2496 true
2497 ));
2498 assert!(!bash_resume_should_retry(
2500 &ResumeOutcome::AlreadyRunning { run_id: "r".into() },
2501 false
2502 ));
2503 }
2504
2505 #[test]
2508 fn injection_body_includes_status_exit_command_and_tail() {
2509 let info = BashCompletionInfo {
2510 session_id: "s".into(),
2511 bash_id: "abc123".into(),
2512 command: "make build".into(),
2513 exit_code: Some(0),
2514 status: "completed".into(),
2515 output_tail: "BUILD OK".into(),
2516 };
2517 let body = bash_completion_injection_body(&info);
2518 assert!(body.contains("abc123"), "body: {body}");
2519 assert!(body.contains("make build"), "body: {body}");
2520 assert!(body.contains("completed"), "body: {body}");
2521 assert!(body.contains("exit code 0"), "body: {body}");
2522 assert!(body.contains("BUILD OK"), "body: {body}");
2523 assert!(body.contains("BashOutput"), "body: {body}");
2525 assert!(body.contains("bash_id=\"abc123\""), "body: {body}");
2526 }
2527
2528 #[test]
2529 fn injection_body_handles_no_output_and_signal_kill() {
2530 let info = BashCompletionInfo {
2531 session_id: "s".into(),
2532 bash_id: "xyz".into(),
2533 command: "sleep 99".into(),
2534 exit_code: None,
2535 status: "killed".into(),
2536 output_tail: String::new(),
2537 };
2538 let body = bash_completion_injection_body(&info);
2539 assert!(body.contains("killed"), "body: {body}");
2540 assert!(body.contains("none (signal/killed)"), "body: {body}");
2541 assert!(body.contains("no captured output"), "body: {body}");
2542 assert!(!body.contains("Output tail:"), "body: {body}");
2544 }
2545
2546 #[test]
2547 fn background_completion_builds_post_tool_use_payload_and_feedback() {
2548 let mut info = BashCompletionInfo {
2549 session_id: "s".into(),
2550 bash_id: "bg-7".into(),
2551 command: "cargo test".into(),
2552 exit_code: Some(0),
2553 status: "completed".into(),
2554 output_tail: "test result: ok".into(),
2555 };
2556
2557 let payload = background_bash_post_tool_payload(&info);
2558 match payload {
2559 HookPayload::ToolResult {
2560 tool_name,
2561 tool_call_id,
2562 outcome,
2563 } => {
2564 assert_eq!(tool_name, "Bash");
2565 assert_eq!(tool_call_id, "bg-7");
2566 assert!(outcome.success);
2567 let response: serde_json::Value =
2568 serde_json::from_str(outcome.result.as_deref().unwrap()).unwrap();
2569 assert_eq!(response["command"], "cargo test");
2570 assert_eq!(response["exit_code"], 0);
2571 assert_eq!(response["status"], "completed");
2572 assert_eq!(response["output_tail"], "test result: ok");
2573 }
2574 other => panic!("expected PostToolUse payload, got {other:?}"),
2575 }
2576
2577 append_background_bash_hook_feedback(
2578 &mut info,
2579 vec!["Run the formatter before continuing".to_string()],
2580 );
2581 assert!(info.output_tail.contains("<post_tool_use_feedback>"));
2582 assert!(info
2583 .output_tail
2584 .contains("Run the formatter before continuing"));
2585 }
2586
2587 #[cfg(unix)]
2588 #[tokio::test]
2589 async fn background_completion_fires_configured_post_tool_use_command() {
2590 use bamboo_config::{
2591 LifecycleHookCommand, LifecycleHookGroup, LifecycleHookType, LifecycleHooksConfig,
2592 DEFAULT_LIFECYCLE_HOOK_TIMEOUT_MS,
2593 };
2594
2595 let dir = tempfile::tempdir().unwrap();
2596 let output = dir.path().join("background-post-tool.json");
2597 let command = format!(
2598 "cat > '{}'; printf '%s' '{{\"additional_context\":\"inspect the completed build log\"}}'",
2599 output.display()
2600 );
2601 let config = LifecycleHooksConfig {
2602 enabled: true,
2603 post_tool_use: vec![LifecycleHookGroup {
2604 enabled: true,
2605 matcher: Some("^Bash$".to_string()),
2606 hooks: vec![LifecycleHookCommand {
2607 hook_type: LifecycleHookType::Command,
2608 command,
2609 timeout_ms: DEFAULT_LIFECYCLE_HOOK_TIMEOUT_MS,
2610 }],
2611 }],
2612 ..Default::default()
2613 };
2614 let mut session = Session::new("session-bg-hook", "test-model");
2615 session.workspace = Some(dir.path().to_string_lossy().into_owned());
2616 let mut info = BashCompletionInfo {
2617 session_id: session.id.clone(),
2618 bash_id: "bg-9".into(),
2619 command: "cargo test".into(),
2620 exit_code: Some(0),
2621 status: "completed".into(),
2622 output_tail: "test result: ok".into(),
2623 };
2624
2625 assert!(run_background_bash_post_tool_hooks(&config, None, &session, &mut info).await);
2626 let envelope: serde_json::Value =
2627 serde_json::from_str(&std::fs::read_to_string(output).unwrap()).unwrap();
2628 assert_eq!(envelope["hook_event_name"], "PostToolUse");
2629 assert_eq!(envelope["tool_name"], "Bash");
2630 assert_eq!(envelope["payload"]["tool_call_id"], "bg-9");
2631 let response = envelope["tool_response"]["result"]
2632 .as_str()
2633 .map(serde_json::from_str::<serde_json::Value>)
2634 .transpose()
2635 .unwrap()
2636 .unwrap();
2637 assert_eq!(response["command"], "cargo test");
2638 assert_eq!(response["status"], "completed");
2639 assert!(info.output_tail.contains("inspect the completed build log"));
2640 }
2641
2642 async fn temp_store() -> (tempfile::TempDir, Arc<dyn Storage>, LockedSessionStore) {
2643 let temp = tempfile::tempdir().unwrap();
2644 let storage: Arc<dyn Storage> = Arc::new(
2645 bamboo_storage::v2::SessionStoreV2::new(temp.path().to_path_buf())
2646 .await
2647 .expect("storage init"),
2648 );
2649 let persistence = LockedSessionStore::new(storage.clone());
2650 (temp, storage, persistence)
2651 }
2652
2653 #[tokio::test]
2654 async fn enqueue_writes_pending_injection_and_preserves_messages() {
2655 let (_temp, storage, persistence) = temp_store().await;
2656
2657 let mut session = Session::new("sess-enq", "test-model");
2658 session.add_message(Message::user("do the build"));
2659 storage.save_session(&session).await.unwrap();
2660
2661 let info = BashCompletionInfo {
2662 session_id: "sess-enq".into(),
2663 bash_id: "sh-1".into(),
2664 command: "make".into(),
2665 exit_code: Some(0),
2666 status: "completed".into(),
2667 output_tail: "done".into(),
2668 };
2669 let saved = enqueue_bash_completion_injection(&persistence, &info)
2670 .await
2671 .expect("enqueue io ok")
2672 .expect("session exists");
2673
2674 let pending = saved
2675 .pending_injected_messages()
2676 .expect("pending injection present");
2677 assert_eq!(pending.len(), 1);
2678 let content = pending[0].get("content").and_then(|v| v.as_str()).unwrap();
2679 assert!(content.contains("sh-1"), "content: {content}");
2680 assert!(content.contains("make"), "content: {content}");
2681 assert!(content.contains("done"), "content: {content}");
2682 assert_eq!(saved.messages.len(), 1);
2684 }
2685
2686 #[tokio::test]
2687 async fn enqueue_returns_none_for_missing_session() {
2688 let (_temp, _storage, persistence) = temp_store().await;
2689 let info = BashCompletionInfo {
2690 session_id: "does-not-exist".into(),
2691 bash_id: "x".into(),
2692 command: "true".into(),
2693 exit_code: Some(0),
2694 status: "completed".into(),
2695 output_tail: String::new(),
2696 };
2697 let result = enqueue_bash_completion_injection(&persistence, &info)
2698 .await
2699 .expect("io ok");
2700 assert!(result.is_none(), "no session → nothing enqueued");
2701 }
2702
2703 #[test]
2711 fn apply_bash_resume_transition_clears_wait_and_appends_message() {
2712 use bamboo_domain::session::runtime_state::WaitingForBashState;
2713
2714 let mut session = Session::new("sess-resume", "test-model");
2715 session.add_message(Message::user("kick off the build"));
2716 let mut rt = read_runtime_state(&session);
2717 rt.status = AgentStatusState::Running;
2718 rt.waiting_for_bash = Some(WaitingForBashState::for_bash(
2719 vec!["sh-1".into()],
2720 Utc::now(),
2721 ));
2722 write_runtime_state(&mut session, &rt);
2723 session.metadata.insert(
2724 "runtime.suspend_reason".to_string(),
2725 "waiting_for_bash".to_string(),
2726 );
2727
2728 let resume = bash_completion_resume_message(&["sh-1".to_string()], false);
2729 let did = apply_bash_resume_transition(&mut session, &resume);
2730
2731 assert!(did, "a suspended session must transition");
2732 let after = read_runtime_state(&session);
2733 assert!(
2734 after.waiting_for_bash.is_none(),
2735 "bash wait must be cleared"
2736 );
2737 assert_eq!(after.status, AgentStatusState::Idle, "runtime must be Idle");
2738 assert!(
2739 !session.metadata.contains_key("runtime.suspend_reason"),
2740 "suspend-reason marker must be removed"
2741 );
2742 assert_eq!(session.messages.len(), 2, "resume message must be appended");
2743 assert!(matches!(
2744 session.messages.last().map(|m| &m.role),
2745 Some(Role::User)
2746 ));
2747 }
2748
2749 #[test]
2753 fn apply_bash_resume_transition_noops_when_not_waiting() {
2754 let mut session = Session::new("sess-live", "test-model");
2755 session.add_message(Message::user("hi"));
2756
2757 let resume = bash_completion_resume_message(&["sh-1".to_string()], false);
2758 let did = apply_bash_resume_transition(&mut session, &resume);
2759
2760 assert!(!did, "a non-waiting session must not transition");
2761 assert_eq!(session.messages.len(), 1, "no resume message appended");
2762 }
2763
2764 #[test]
2768 fn bash_completion_should_resume_only_when_suspended_and_all_done() {
2769 assert!(bash_completion_should_resume(true, true));
2770 assert!(!bash_completion_should_resume(true, false)); assert!(!bash_completion_should_resume(false, true)); assert!(!bash_completion_should_resume(false, false));
2773 }
2774
2775 #[test]
2779 fn bash_resume_message_from_info_carries_bashid_tail_and_kind() {
2780 let info = BashCompletionInfo {
2781 session_id: "s".into(),
2782 bash_id: "sh-42".into(),
2783 command: "cargo test".into(),
2784 exit_code: Some(0),
2785 status: "completed".into(),
2786 output_tail: "test result: ok".into(),
2787 };
2788 let msg = bash_resume_message_from_info(&info);
2789
2790 assert!(matches!(msg.role, Role::User));
2791 assert!(msg.content.contains("sh-42"), "content: {}", msg.content);
2792 assert!(
2793 msg.content.contains("cargo test"),
2794 "content: {}",
2795 msg.content
2796 );
2797 assert!(
2798 msg.content.contains("test result: ok"),
2799 "content: {}",
2800 msg.content
2801 );
2802 assert!(
2803 msg.content.contains("BashOutput"),
2804 "content: {}",
2805 msg.content
2806 );
2807 let meta = serde_json::to_string(&msg.metadata).unwrap();
2808 assert!(
2809 meta.contains(BASH_COMPLETION_RESUME_KIND),
2810 "resume message must be tagged as a bash-completion resume: {meta}"
2811 );
2812 }
2813}