1use std::collections::HashMap;
8use std::sync::{Arc, Mutex as StdMutex, OnceLock, RwLock as StdRwLock, Weak};
9use std::time::Duration;
10
11use bamboo_domain::poison::PoisonRecover;
12use bamboo_domain::{
13 AgentHookPoint, HookPayload, HookToolOutcome, SessionChildOutcome, SessionMessageBody,
14 SessionMessageContent, SessionMessageEnvelope, SessionMessageId, SessionMessageKind,
15 SessionMessageSource, SessionProviderMessage,
16};
17
18use crate::execution::{
19 create_event_forwarder, finalize_runner, reserve_runner_core, reserve_session_execution,
20 spawn_session_execution, AgentRunner, AgentStatus, ChildCompletion, ChildCompletionHandler,
21 ReserveOutcome, SessionExecutionArgs, SessionExecutionReservation,
22 SessionExecutionReserveOutcome, SpawnJob, SpawnScheduler,
23};
24use crate::runtime::config::{BashResumeHook, GuardianSpawner, BASH_COMPLETION_RESUME_KIND};
25use crate::runtime::guardian_state::{
26 parse_guardian_verdict, read_guardian_config, read_guardian_state, write_guardian_state,
27 GuardianVerdict,
28};
29use crate::Agent;
30use async_trait::async_trait;
31use bamboo_agent_core::storage::Storage;
32use bamboo_agent_core::tools::ToolExecutor;
33use bamboo_agent_core::{
34 AgentEvent, BashCompletionInfo, BashCompletionSink, Message, Role, Session, SessionKind,
35};
36use bamboo_domain::session::runtime_state::{
37 AgentRuntimeState, AgentStatusState, ChildWaitPolicy, SuspensionState, WaitingForChildrenState,
38};
39use bamboo_llm::{Config, ProviderModelRouter, ProviderRegistry};
40use bamboo_storage::LockedSessionStore;
41use chrono::Utc;
42use sha2::{Digest, Sha256};
43use tokio::sync::{broadcast, RwLock};
44
45use crate::model_areas::resolve_global_area_models;
46use crate::model_config_helper::{
47 resolve_fast_model, resolve_gold_config, resolve_provider_routing_key, GOLD_CONFIG_METADATA_KEY,
48};
49use crate::session_activation::{
50 SessionActivationLaunch, SessionActivationReserveOutcome, SessionActivationSpawner,
51};
52use crate::session_app::execute::consume_pending_clarification_resume;
53use crate::session_app::provider_model::{persist_model_ref, session_effective_model_ref};
54use crate::session_app::resume::{
55 resume_session_execution, ResumeExecutionPort, ResumeSpawnRequest,
56};
57use crate::session_app::types::{ResumeConfigSnapshot, ResumeOutcome};
58
59const AGENT_RUNTIME_STATE_METADATA_KEY: &str = "agent.runtime.state";
60const RUNTIME_RESUME_MESSAGE_HIDDEN_KEY: &str = "hidden_from_ui";
61const RUNTIME_RESUME_MESSAGE_KIND_KEY: &str = "runtime_kind";
62const CHILD_COMPLETION_INLINE_FIELD_BYTES: usize = 48 * 1024;
63const CHILD_COMPLETION_OVERSIZE_TAIL_BYTES: usize = 8 * 1024;
64
65fn read_runtime_state(session: &Session) -> AgentRuntimeState {
66 session
67 .agent_runtime_state
68 .clone()
69 .or_else(|| {
70 session
71 .metadata
72 .get(AGENT_RUNTIME_STATE_METADATA_KEY)
73 .and_then(|raw| serde_json::from_str::<AgentRuntimeState>(raw).ok())
74 })
75 .unwrap_or_else(|| AgentRuntimeState::new(format!("{}-child-wait", session.id)))
76}
77
78fn write_runtime_state(session: &mut Session, runtime_state: &AgentRuntimeState) {
79 session.agent_runtime_state = Some(runtime_state.clone());
80 if let Ok(serialized) = serde_json::to_string(runtime_state) {
81 session
82 .metadata
83 .insert(AGENT_RUNTIME_STATE_METADATA_KEY.to_string(), serialized);
84 }
85}
86
87async fn prepare_session_inbox_activation(
93 persistence: &LockedSessionStore,
94 session_id: &str,
95 interrupt_specific_wait: bool,
96) -> std::io::Result<Option<(Session, bool)>> {
97 let ready = Arc::new(std::sync::atomic::AtomicBool::new(false));
98 let ready_for_mutation = ready.clone();
99 let saved = persistence
100 .update_runtime_config(session_id, move |latest| {
101 let mut runtime_state = read_runtime_state(latest);
102 let specifically_waiting = runtime_state.waiting_for_children.is_some()
103 || runtime_state.waiting_for_bash.is_some();
104 if specifically_waiting && !interrupt_specific_wait {
105 return;
106 }
107 runtime_state.status = AgentStatusState::Idle;
113 runtime_state.suspension = None;
114 write_runtime_state(latest, &runtime_state);
115 latest.metadata.remove("runtime.suspend_reason");
116 latest.updated_at = Utc::now();
117 ready_for_mutation.store(true, std::sync::atomic::Ordering::Release);
118 })
119 .await?;
120 Ok(saved.map(|session| (session, ready.load(std::sync::atomic::Ordering::Acquire))))
121}
122
123fn is_error_like(status: &str) -> bool {
124 matches!(status, "error" | "timeout" | "cancelled")
125}
126
127fn is_terminal_child_status(status: &str) -> bool {
129 matches!(
130 status,
131 "completed" | "error" | "timeout" | "cancelled" | "skipped"
132 )
133}
134
135fn child_completion_envelope(
136 completion: &ChildCompletion,
137 wait_registered_at: chrono::DateTime<Utc>,
138 result: Option<String>,
139 provider_message: &Message,
140) -> SessionMessageEnvelope {
141 fn bounded_terminal_field(
142 label: &str,
143 child_session_id: &str,
144 value: Option<String>,
145 ) -> (Option<String>, serde_json::Value, bool) {
146 let Some(value) = value else {
147 return (None, serde_json::Value::Null, false);
148 };
149 if value.len() <= CHILD_COMPLETION_INLINE_FIELD_BYTES {
150 return (Some(value.clone()), serde_json::Value::String(value), false);
151 }
152 let digest = hex::encode(Sha256::digest(value.as_bytes()));
153 let mut tail_start = value
154 .len()
155 .saturating_sub(CHILD_COMPLETION_OVERSIZE_TAIL_BYTES);
156 while tail_start < value.len() && !value.is_char_boundary(tail_start) {
157 tail_start += 1;
158 }
159 let tail = &value[tail_start..];
160 let summary = format!(
161 "Child {label} exceeded the durable inline limit ({} UTF-8 bytes, sha256={digest}). \
162 Retrieve the full child transcript with SubAgent.get(child_session_id=\"{child_session_id}\").\
163 \n\nBounded tail:\n{tail}",
164 value.len()
165 );
166 (
167 Some(summary),
168 serde_json::json!({
169 "oversized": true,
170 "utf8_bytes": value.len(),
171 "sha256": digest,
172 }),
173 true,
174 )
175 }
176
177 let (stored_result, result_identity, _result_oversized) =
178 bounded_terminal_field("result", &completion.child_session_id, result);
179 let (stored_error, error_identity, _error_oversized) = bounded_terminal_field(
180 "error",
181 &completion.child_session_id,
182 completion.error.clone(),
183 );
184 let mut bounded_provider_message = provider_message.clone();
185 let provider_oversized = serde_json::to_vec(provider_message)
186 .map(|bytes| bytes.len() > CHILD_COMPLETION_INLINE_FIELD_BYTES)
187 .unwrap_or(true);
188 if provider_oversized {
189 let mut content = format!(
190 "Runtime notification: child session `{}` finished with status `{}`.",
191 completion.child_session_id, completion.status
192 );
193 if let Some(result) = stored_result.as_deref() {
194 content.push_str("\n\n");
195 content.push_str(result);
196 }
197 if let Some(error) = stored_error.as_deref() {
198 content.push_str("\n\n");
199 content.push_str(error);
200 }
201 bounded_provider_message.content = content;
202 bounded_provider_message.content_parts = None;
203 }
204 let body = SessionMessageBody::ChildOutcome(SessionChildOutcome {
205 child_session_id: completion.child_session_id.clone(),
206 status: completion.status.clone(),
207 result: stored_result,
208 error: stored_error,
209 provider_message: Some(session_provider_message(&bounded_provider_message)),
210 });
211 let semantic = serde_json::json!({
212 "parent_session_id": completion.parent_session_id,
213 "child_session_id": completion.child_session_id,
214 "status": completion.status,
215 "error": error_identity,
216 "result": result_identity,
217 "wait_registered_at": wait_registered_at,
218 });
219 SessionMessageEnvelope {
220 id: SessionMessageId::stable("session_child_completion", &semantic),
221 source: SessionMessageSource::Runtime {
222 subsystem: "child_completion_coordinator".to_string(),
223 },
224 target_session_id: completion.parent_session_id.clone(),
225 kind: SessionMessageKind::ChildOutcome,
226 body,
227 created_at: completion.completed_at,
228 thread_id: None,
229 in_reply_to: None,
230 attempt: None,
231 correlation_id: Some(format!("child_completion:{}", completion.child_session_id)),
232 }
233}
234
235fn session_provider_message(message: &Message) -> SessionProviderMessage {
236 SessionProviderMessage {
237 content: SessionMessageContent {
238 text: message.content.clone(),
239 parts: message.content_parts.clone().unwrap_or_default(),
240 },
241 metadata: message
242 .metadata
243 .as_ref()
244 .and_then(serde_json::Value::as_object)
245 .cloned()
246 .unwrap_or_default(),
247 never_compress: message.never_compress,
248 }
249}
250
251async fn derive_completed_child_ids(
256 storage: &Arc<dyn Storage>,
257 parent_session_id: &str,
258 just_completed_child_id: &str,
259) -> Vec<String> {
260 let mut completed: Vec<String> = storage
261 .list_child_run_statuses(parent_session_id)
262 .await
263 .unwrap_or_default()
264 .into_iter()
265 .filter(|(_, status)| status.as_deref().is_some_and(is_terminal_child_status))
266 .map(|(id, _)| id)
267 .collect();
268 if !completed.iter().any(|id| id == just_completed_child_id) {
269 completed.push(just_completed_child_id.to_string());
270 }
271 completed.sort();
272 completed.dedup();
273 completed
274}
275
276fn read_config_snapshot(config: &Arc<RwLock<Config>>, cached_config: &StdRwLock<Config>) -> Config {
277 if let Ok(config_guard) = config.try_read() {
278 let snapshot = config_guard.clone();
279
280 if let Ok(mut cached_guard) = cached_config.try_write() {
281 *cached_guard = snapshot.clone();
282 }
283
284 snapshot
285 } else {
286 cached_config
287 .try_read()
288 .map(|guard| guard.clone())
289 .unwrap_or_default()
290 }
291}
292
293fn parent_locks() -> &'static std::sync::Mutex<HashMap<String, Arc<tokio::sync::Mutex<()>>>> {
310 static LOCKS: OnceLock<std::sync::Mutex<HashMap<String, Arc<tokio::sync::Mutex<()>>>>> =
311 OnceLock::new();
312 LOCKS.get_or_init(|| std::sync::Mutex::new(HashMap::new()))
313}
314
315fn session_resume_lock(session_id: &str) -> Arc<tokio::sync::Mutex<()>> {
322 let mut map = parent_locks().lock().recover_poison();
323 map.entry(session_id.to_string())
324 .or_insert_with(|| Arc::new(tokio::sync::Mutex::new(())))
325 .clone()
326}
327
328fn wait_policy_satisfied(
329 policy: ChildWaitPolicy,
330 wait_child_ids: &[String],
331 completed_child_ids: &[String],
332 latest_child_id: &str,
333 latest_status: &str,
334) -> bool {
335 if wait_child_ids.is_empty() {
336 return false;
337 }
338
339 match policy {
340 ChildWaitPolicy::All => wait_child_ids
341 .iter()
342 .all(|id| completed_child_ids.iter().any(|completed| completed == id)),
343 ChildWaitPolicy::Any => completed_child_ids
344 .iter()
345 .any(|id| wait_child_ids.iter().any(|wait_id| wait_id == id)),
346 ChildWaitPolicy::FirstError => {
347 (is_error_like(latest_status) && wait_child_ids.iter().any(|id| id == latest_child_id))
353 || wait_child_ids
354 .iter()
355 .all(|id| completed_child_ids.iter().any(|completed| completed == id))
356 }
357 }
358}
359
360fn child_final_assistant_text(child: &Session) -> Option<String> {
364 child
365 .messages
366 .iter()
367 .rev()
368 .find(|message| matches!(message.role, Role::Assistant))
369 .map(|message| message.content.clone())
370 .filter(|content| !content.trim().is_empty())
371}
372
373fn runtime_resume_message(
374 completion: &ChildCompletion,
375 remaining_children: usize,
376 child_final_response: Option<&str>,
377) -> Message {
378 let mut body = format!(
379 "Runtime notification: child session `{}` finished with status `{}`. Remaining child sessions: {}.",
380 completion.child_session_id, completion.status, remaining_children
381 );
382
383 let final_response = child_final_response.map(str::to_string);
389 if let Some(response) = final_response.as_deref() {
390 body.push_str("\n\nChild final response:\n");
391 body.push_str(response);
392 } else if let Some(error) = completion.error.as_deref() {
393 if !error.is_empty() {
394 body.push_str("\n\nChild error:\n");
395 body.push_str(error);
396 }
397 }
398
399 body.push_str(
400 "\n\nResume the parent task using this child result and continue from the previous plan. \
401 If you need the full child transcript, call SubAgent.get(child_session_id).",
402 );
403
404 let mut message = Message::user(body);
405 message.metadata = Some(serde_json::json!({
406 RUNTIME_RESUME_MESSAGE_HIDDEN_KEY: true,
407 RUNTIME_RESUME_MESSAGE_KIND_KEY: "child_completion_resume",
408 "child_session_id": completion.child_session_id,
409 "child_status": completion.status,
410 "child_error": completion.error,
411 "child_final_response_included": final_response.is_some(),
412 }));
413 message.never_compress = false;
417 message
418}
419
420fn guardian_resume_message(completion: &ChildCompletion, verdict: &GuardianVerdict) -> Message {
425 let mut body = if verdict.approve {
426 String::from(
427 "Guardian review APPROVED: an independent reviewer verified the work and found no blocking issues. You may finalize the task.",
428 )
429 } else {
430 String::from(
431 "Guardian review REJECTED: an independent reviewer found issues. Address every finding below before completing — do NOT declare the task complete until they are resolved.",
432 )
433 };
434 if let Some(summary) = verdict.summary.as_deref().filter(|s| !s.trim().is_empty()) {
435 body.push_str("\n\nReviewer summary: ");
436 body.push_str(summary);
437 }
438 if !verdict.findings.is_empty() {
439 body.push_str("\n\nFindings:");
440 for (idx, finding) in verdict.findings.iter().enumerate() {
441 body.push_str(&format!("\n{}. {}", idx + 1, finding));
442 }
443 }
444 body.push_str(
445 "\n\nIf you need the full guardian transcript, call SubAgent.get(child_session_id).",
446 );
447
448 let mut message = Message::user(body);
449 message.metadata = Some(serde_json::json!({
450 RUNTIME_RESUME_MESSAGE_HIDDEN_KEY: true,
451 RUNTIME_RESUME_MESSAGE_KIND_KEY: "guardian_review_resume",
452 "child_session_id": completion.child_session_id,
453 "child_status": completion.status,
454 "guardian_approved": verdict.approve,
455 }));
456 message.never_compress = false;
457 message
458}
459
460#[derive(Clone)]
461pub struct ChildCompletionCoordinator {
462 storage: Arc<dyn Storage>,
463 persistence: Arc<bamboo_storage::LockedSessionStore>,
464 sessions: crate::SessionCache,
465 agent_runners: Arc<RwLock<HashMap<String, AgentRunner>>>,
466 session_event_senders: Arc<RwLock<HashMap<String, broadcast::Sender<AgentEvent>>>>,
467 agent: Arc<Agent>,
468 config: Arc<RwLock<Config>>,
469 provider_registry: Arc<ProviderRegistry>,
470 provider_router: Arc<ProviderModelRouter>,
471 app_data_dir: std::path::PathBuf,
472 account_feed_inbox: Option<crate::execution::AccountFeedInbox>,
473 root_tools: Arc<RwLock<Option<Arc<dyn ToolExecutor>>>>,
474 guardian_spawner: Arc<RwLock<Option<Arc<dyn GuardianSpawner>>>>,
478 spawn_scheduler: Arc<RwLock<Weak<SpawnScheduler>>>,
482}
483
484impl ChildCompletionCoordinator {
485 #[allow(clippy::too_many_arguments)]
486 pub fn new(
487 storage: Arc<dyn Storage>,
488 persistence: Arc<LockedSessionStore>,
489 sessions: crate::SessionCache,
490 agent_runners: Arc<RwLock<HashMap<String, AgentRunner>>>,
491 session_event_senders: Arc<RwLock<HashMap<String, broadcast::Sender<AgentEvent>>>>,
492 agent: Arc<Agent>,
493 config: Arc<RwLock<Config>>,
494 provider_registry: Arc<ProviderRegistry>,
495 provider_router: Arc<ProviderModelRouter>,
496 app_data_dir: std::path::PathBuf,
497 account_feed_inbox: Option<crate::execution::AccountFeedInbox>,
498 ) -> Self {
499 Self {
500 storage,
501 persistence,
502 sessions,
503 agent_runners,
504 session_event_senders,
505 agent,
506 config,
507 provider_registry,
508 provider_router,
509 app_data_dir,
510 account_feed_inbox,
511 root_tools: Arc::new(RwLock::new(None)),
512 guardian_spawner: Arc::new(RwLock::new(None)),
513 spawn_scheduler: Arc::new(RwLock::new(Weak::new())),
514 }
515 }
516
517 pub async fn set_root_tools(&self, tools: Arc<dyn ToolExecutor>) {
518 *self.root_tools.write().await = Some(tools);
519 }
520
521 pub async fn set_spawn_scheduler(&self, scheduler: &Arc<SpawnScheduler>) {
522 *self.spawn_scheduler.write().await = Arc::downgrade(scheduler);
523 }
524
525 pub async fn set_guardian_spawner(&self, spawner: Arc<dyn GuardianSpawner>) {
528 *self.guardian_spawner.write().await = Some(spawner);
529 }
530
531 fn build_resume_config(
532 &self,
533 session: &Session,
534 config_snapshot: &Config,
535 ) -> ResumeConfigSnapshot {
536 crate::session_app::resolution::resolve_resume_config_snapshot(
537 config_snapshot,
538 &self.provider_registry,
539 session,
540 None,
541 )
542 }
543
544 async fn resume_parent(&self, parent_session_id: String) -> ResumeOutcome {
552 for attempt in 0..=5u8 {
553 if attempt > 0 {
554 tokio::time::sleep(Duration::from_millis(250 * attempt as u64)).await;
555 }
556
557 let Some(session) = self.load_session(&parent_session_id).await else {
558 tracing::warn!(%parent_session_id, "cannot resume parent after child completion: session not found");
559 return ResumeOutcome::NotFound;
560 };
561 let config_snapshot = self.config.read().await.clone();
562 let resume_config = self.build_resume_config(&session, &config_snapshot);
563 let outcome = resume_session_execution(self, &parent_session_id, resume_config).await;
564 tracing::info!(
565 %parent_session_id,
566 attempt,
567 outcome = outcome.as_str(),
568 "child completion requested parent resume"
569 );
570
571 if !matches!(outcome, ResumeOutcome::AlreadyRunning { .. }) {
572 return outcome;
573 }
574 }
575 tracing::error!(
581 %parent_session_id,
582 "parent resume gave up after AlreadyRunning retry budget; \
583 relying on the child-wait watchdog backstop"
584 );
585 ResumeOutcome::AlreadyRunning {
586 run_id: String::new(),
587 }
588 }
589
590 async fn save_and_cache(&self, session: &mut Session) {
591 if let Err(error) = self.persistence.merge_save_runtime(session).await {
592 tracing::warn!(session_id = %session.id, %error, "failed to persist session");
593 }
594 self.sessions.insert(
595 session.id.clone(),
596 Arc::new(parking_lot::RwLock::new(session.clone())),
597 );
598 }
599}
600
601#[async_trait]
602impl ChildCompletionHandler for ChildCompletionCoordinator {
603 async fn on_child_completed(&self, completion: ChildCompletion) {
604 if !is_terminal_child_status(&completion.status) {
613 tracing::info!(
614 parent_session_id = %completion.parent_session_id,
615 child_session_id = %completion.child_session_id,
616 status = %completion.status,
617 "non-terminal child status; leaving the parent wait armed"
618 );
619 return;
620 }
621
622 let per_parent = session_resume_lock(&completion.parent_session_id);
627 let _per_parent_guard = per_parent.lock().await;
628
629 let Some(mut parent) = self.load_session(&completion.parent_session_id).await else {
630 tracing::warn!(
631 parent_session_id = %completion.parent_session_id,
632 child_session_id = %completion.child_session_id,
633 "child completion received for missing parent"
634 );
635 return;
636 };
637
638 let mut runtime_state = read_runtime_state(&parent);
644
645 let completed_child_ids = derive_completed_child_ids(
648 &self.storage,
649 &completion.parent_session_id,
650 &completion.child_session_id,
651 )
652 .await;
653
654 let mut should_resume = false;
655 let mut remaining_children = 0usize;
656 let active_wait = runtime_state.waiting_for_children.clone();
657 if let Some(wait) = active_wait.as_ref() {
658 remaining_children = wait
659 .child_session_ids
660 .iter()
661 .filter(|id| !completed_child_ids.iter().any(|completed| completed == *id))
662 .count();
663 should_resume = wait_policy_satisfied(
664 wait.wait_for,
665 &wait.child_session_ids,
666 &completed_child_ids,
667 &completion.child_session_id,
668 &completion.status,
669 );
670 }
671
672 let reported_child_owned = match self
684 .storage
685 .load_runtime_control_plane(&completion.child_session_id)
686 .await
687 {
688 Ok(Some(control_plane)) => completion_child_is_owned(
689 &completion.parent_session_id,
690 control_plane.parent_session_id.as_deref(),
691 ),
692 _ => false,
693 };
694
695 let loaded_child = if reported_child_owned {
700 match self
701 .storage
702 .load_session(&completion.child_session_id)
703 .await
704 {
705 Ok(child) => child,
706 Err(error) => {
707 tracing::warn!(
708 child_session_id = %completion.child_session_id,
709 %error,
710 "failed to load child session for runtime resume message"
711 );
712 None
713 }
714 }
715 } else {
716 tracing::warn!(
717 parent_session_id = %completion.parent_session_id,
718 child_session_id = %completion.child_session_id,
719 "completion child is not a child of this parent; resuming with a neutral \
720 message and NOT folding its content"
721 );
722 None
723 };
724
725 let child_final_response = loaded_child.as_ref().and_then(child_final_assistant_text);
726 let guardian_resume = if should_resume {
731 let reviewed_round = runtime_state.round.current_round;
732 loaded_child.as_ref().and_then(|child| {
733 if child.subagent_type().as_deref() != Some("guardian") {
734 return None;
735 }
736 let mut guardian_state = read_guardian_state(&parent)?;
737 if guardian_state.guardian_child_id.as_deref()
738 != Some(completion.child_session_id.as_str())
739 {
740 tracing::warn!(
743 parent_session_id = %completion.parent_session_id,
744 child_session_id = %completion.child_session_id,
745 expected = ?guardian_state.guardian_child_id,
746 "guardian completion does not match recorded guardian_child_id; using generic resume"
747 );
748 return None;
749 }
750 let verdict = child_final_assistant_text(child)
758 .and_then(|text| match parse_guardian_verdict(&text) {
759 Ok(verdict) => Some(verdict),
760 Err(error) => {
761 tracing::warn!(
762 child_session_id = %completion.child_session_id,
763 %error,
764 "guardian verdict unparseable; recording a synthetic reject"
765 );
766 None
767 }
768 })
769 .unwrap_or_else(|| {
770 GuardianVerdict::rejected(vec![
771 "The guardian reviewer did not return a usable verdict (it errored or \
772 emitted unparseable output); the work has NOT been independently \
773 verified."
774 .to_string(),
775 ])
776 });
777 let approved = verdict.approve;
778 let message = guardian_resume_message(&completion, &verdict);
779 guardian_state.record_verdict(verdict, reviewed_round);
780 write_guardian_state(&mut parent, guardian_state);
781 tracing::info!(
782 parent_session_id = %completion.parent_session_id,
783 child_session_id = %completion.child_session_id,
784 approved,
785 "guardian verdict recorded; resuming parent"
786 );
787 Some(message)
788 })
789 } else {
790 None
791 };
792 let resume_message = guardian_resume.unwrap_or_else(|| {
793 runtime_resume_message(
794 &completion,
795 remaining_children,
796 child_final_response.as_deref(),
797 )
798 });
799
800 let messenger = self.agent.session_messenger().cloned();
805 let child_admission =
806 if let (Some(wait), Some(messenger)) = (active_wait.as_ref(), messenger.as_ref()) {
807 let envelope = child_completion_envelope(
808 &completion,
809 wait.registered_at,
810 child_final_response,
811 &resume_message,
812 );
813 match messenger.admit(envelope).await {
814 Ok(admission) => Some(admission),
815 Err(error) => {
816 tracing::warn!(
817 parent_session_id = %completion.parent_session_id,
818 child_session_id = %completion.child_session_id,
819 %error,
820 "child outcome SessionInbox admission failed; leaving parent wait armed"
821 );
822 return;
823 }
824 }
825 } else {
826 None
827 };
828
829 if should_resume {
830 if let (Some(messenger), Some(admission)) =
831 (messenger.as_ref(), child_admission.as_ref())
832 {
833 if let Err(error) = messenger.prepare_activation(admission).await {
834 tracing::warn!(
835 parent_session_id = %completion.parent_session_id,
836 child_session_id = %completion.child_session_id,
837 %error,
838 "child outcome activation watermark failed; leaving parent wait armed"
839 );
840 return;
841 }
842 }
843 }
844
845 if should_resume {
846 runtime_state.waiting_for_children = None;
847 runtime_state.status = AgentStatusState::Idle;
848 runtime_state.suspension = None;
849 parent.metadata.remove("runtime.suspend_reason");
850
851 if child_admission.is_none() {
852 parent.add_message(resume_message);
856 }
857 } else if runtime_state.waiting_for_children.is_some() {
858 runtime_state.status = AgentStatusState::Suspended;
859 runtime_state.suspension = Some(SuspensionState {
860 reason: "waiting_for_children".to_string(),
861 suspended_at: Utc::now(),
862 resumable: true,
863 hook_point: Some("ChildCompletion".to_string()),
864 });
865 }
866
867 parent.updated_at = Utc::now();
868 write_runtime_state(&mut parent, &runtime_state);
869 if let Err(error) = self
870 .persistence
871 .checkpoint_runtime_session(&mut parent)
872 .await
873 {
874 tracing::warn!(
875 parent_session_id = %completion.parent_session_id,
876 child_session_id = %completion.child_session_id,
877 %error,
878 "child outcome is durable but parent wait transition failed; leaving activation deferred"
879 );
880 return;
881 }
882 self.sessions.insert(
883 parent.id.clone(),
884 Arc::new(parking_lot::RwLock::new(parent.clone())),
885 );
886
887 let resume_parent_id = parent.id.clone();
892 drop(_per_parent_guard);
893
894 if should_resume {
895 if let (Some(messenger), Some(admission)) = (messenger, child_admission) {
896 if let Err(error) = messenger.activate_prepared(&admission).await {
897 tracing::warn!(
898 parent_session_id = %resume_parent_id,
899 %error,
900 "child outcome and wait transition are durable but activation failed"
901 );
902 }
903 } else {
904 self.resume_parent(resume_parent_id).await;
905 }
906 }
907 }
908}
909
910#[async_trait]
911impl ResumeExecutionPort for ChildCompletionCoordinator {
912 async fn load_session(&self, session_id: &str) -> Option<Session> {
913 match self.storage.load_session(session_id).await {
914 Ok(Some(session)) => Some(session),
915 Ok(None) => self
916 .sessions
917 .get(session_id)
918 .map(|e| e.value().clone())
919 .map(|arc| arc.read().clone()),
920 Err(error) => {
921 tracing::warn!(%session_id, %error, "failed to load session from storage");
922 self.sessions
923 .get(session_id)
924 .map(|e| e.value().clone())
925 .map(|arc| arc.read().clone())
926 }
927 }
928 }
929
930 async fn save_and_cache_session(&self, session: &mut Session) {
931 self.save_and_cache(session).await;
932 }
933
934 async fn reserve_session_execution(
935 &self,
936 session_id: &str,
937 event_sender: &broadcast::Sender<AgentEvent>,
938 ) -> SessionExecutionReserveOutcome {
939 reserve_session_execution(
940 &self.agent,
941 &self.agent_runners,
942 &self.session_event_senders,
943 session_id,
944 event_sender,
945 )
946 .await
947 }
948
949 async fn get_or_create_event_sender(&self, session_id: &str) -> broadcast::Sender<AgentEvent> {
950 crate::execution::session_events::get_or_create_event_sender(
951 &self.session_event_senders,
952 session_id,
953 )
954 .await
955 }
956
957 fn dispatch_resume_execution(
958 &self,
959 request: ResumeSpawnRequest,
960 ) -> Result<(), ResumeSpawnRequest> {
961 let owner = self.clone();
962 tokio::spawn(async move {
963 ResumeExecutionPort::spawn_resume_execution(&owner, request).await;
964 });
965 Ok(())
966 }
967
968 async fn spawn_resume_execution(&self, request: ResumeSpawnRequest) {
969 let ResumeSpawnRequest {
970 session_id,
971 mut session,
972 mut execution_reservation,
973 event_sender,
974 config,
975 } = request;
976 if let Err(error) = execution_reservation.ensure_registered().await {
977 tracing::warn!(
978 %session_id,
979 run_id = %execution_reservation.run_id(),
980 %error,
981 "cannot resume after child completion without exact router ownership"
982 );
983 return;
984 }
985
986 let Some(root_tools) = self.root_tools.read().await.clone() else {
987 tracing::error!(%session_id, "cannot resume parent after child completion: root tool surface is not initialized");
988 return;
989 };
990
991 let config_snapshot = self.config.read().await.clone();
992 let model = session.model.clone();
993 let session_model_ref = session_effective_model_ref(&session);
994 let requested_provider = session_model_ref
995 .as_ref()
996 .map(|model_ref| model_ref.provider.as_str())
997 .unwrap_or(config.provider_name.as_str());
998 let resolved_provider_name = match resolve_provider_routing_key(
999 &config_snapshot,
1000 requested_provider,
1001 &self.provider_registry,
1002 ) {
1003 Ok(provider) => provider,
1004 Err(error) => {
1005 tracing::error!(
1006 session_id = %session_id,
1007 provider = requested_provider,
1008 %error,
1009 "child-completion resume provider is unavailable; refusing to fall back"
1010 );
1011 execution_reservation.abandon().await;
1012 return;
1013 }
1014 };
1015 let provider_override = if let Some(mut model_ref) = session_model_ref {
1016 model_ref.provider = resolved_provider_name.clone();
1017 persist_model_ref(&mut session, &model_ref);
1018 match self.provider_router.route(&model_ref) {
1019 Ok(provider) => Some(provider),
1020 Err(error) => {
1021 tracing::error!(
1022 session_id = %session_id,
1023 provider = %model_ref.provider,
1024 model = %model_ref.model,
1025 %error,
1026 "child-completion resume provider routing failed closed"
1027 );
1028 execution_reservation.abandon().await;
1029 return;
1030 }
1031 }
1032 } else {
1033 match self.provider_registry.get(&resolved_provider_name) {
1034 Some(provider) => Some(provider),
1035 None => {
1036 tracing::error!(
1037 session_id = %session_id,
1038 provider = %resolved_provider_name,
1039 "child-completion resume provider disappeared after resolution; refusing to fall back"
1040 );
1041 execution_reservation.abandon().await;
1042 return;
1043 }
1044 }
1045 };
1046 let resolved_fast_provider = resolve_fast_model(
1047 &config_snapshot,
1048 &resolved_provider_name,
1049 &self.provider_registry,
1050 )
1051 .map(|model| model.provider);
1052 let reasoning_effort = session.reasoning_effort;
1053 let reasoning_effort_source = session
1054 .metadata
1055 .get("reasoning_effort_source")
1056 .cloned()
1057 .unwrap_or_default();
1058 let gold_config = resolve_gold_config(
1059 &config_snapshot,
1060 session
1061 .metadata
1062 .get(GOLD_CONFIG_METADATA_KEY)
1063 .map(String::as_str),
1064 )
1065 .or(config.gold_config.clone());
1066
1067 let (mpsc_tx, _forwarder) = create_event_forwarder(
1068 session_id.clone(),
1069 execution_reservation.run_id().to_string(),
1070 event_sender,
1071 self.agent_runners.clone(),
1072 self.account_feed_inbox.clone(),
1073 );
1074
1075 let config_handle = self.config.clone();
1076 let cached_config = Arc::new(StdRwLock::new(config_snapshot.clone()));
1077 let provider_registry = self.provider_registry.clone();
1078 let provider_name_for_aux = resolved_provider_name.clone();
1079 let auxiliary_model_resolver = std::sync::Arc::new(move || {
1080 let config_snapshot = read_config_snapshot(&config_handle, cached_config.as_ref());
1081 let areas = resolve_global_area_models(
1083 &config_snapshot,
1084 &provider_name_for_aux,
1085 &provider_registry,
1086 );
1087 crate::AuxiliaryModelConfig {
1088 fast_model_name: areas.fast.as_ref().map(|m| m.model_name.clone()),
1089 fast_model_provider: areas.fast.map(|m| m.provider),
1090 background_model_name: areas.background.as_ref().map(|m| m.model_name.clone()),
1091 planning_model_name: None,
1092 search_model_name: None,
1093 summarization_model_name: areas
1094 .summarization
1095 .as_ref()
1096 .map(|m| m.model_name.clone()),
1097 background_model_provider: areas.background.map(|m| m.provider),
1098 summarization_model_provider: areas.summarization.map(|m| m.provider),
1099 }
1100 });
1101 let model_roster = crate::ModelRoster {
1102 model: Some(model),
1103 provider_name: Some(resolved_provider_name),
1104 provider_type: config.provider_type.clone(),
1105 fast: crate::RoleModel::from_parts(config.fast_model, resolved_fast_provider),
1106 background: crate::RoleModel::from_parts(
1107 config.background_model,
1108 config.background_model_provider,
1109 ),
1110 summarization: crate::RoleModel::from_parts(
1111 config.summarization_model,
1112 config.summarization_model_provider,
1113 ),
1114 };
1115
1116 let guardian_config = read_guardian_config(&session);
1121 let guardian_spawner = self.guardian_spawner.read().await.clone();
1122
1123 consume_pending_clarification_resume(&mut session);
1124 spawn_session_execution(SessionExecutionArgs {
1125 agent: self.agent.clone(),
1126 session_id,
1127 session,
1128 execution_reservation,
1129 tools_override: Some(root_tools),
1130 provider_override,
1131 model_roster,
1132 reasoning_effort,
1133 reasoning_effort_source,
1134 auxiliary_model_resolver: Some(auxiliary_model_resolver),
1135 disabled_filter_resolver: None,
1138 disabled_tools: Some(config.disabled_tools),
1139 disabled_skill_ids: Some(config.disabled_skill_ids),
1140 selected_skill_ids: None,
1141 selected_skill_mode: None,
1142 mpsc_tx,
1143 image_fallback: config.image_fallback,
1144 gold_config,
1145 guardian_config,
1146 guardian_spawner,
1147 bash_resume_hook: {
1148 let hook: Arc<dyn BashResumeHook> = Arc::new(self.clone());
1149 Some(hook)
1150 },
1151 bash_completion_sink: {
1152 let sink: Arc<dyn BashCompletionSink> = Arc::new(self.clone());
1155 Some(sink)
1156 },
1157 app_data_dir: Some(self.app_data_dir.clone()),
1158 run_budget: None,
1161 runners: self.agent_runners.clone(),
1162 sessions_cache: self.sessions.clone(),
1163 on_complete: None,
1164 child_completion_handler: Some(Arc::new(self.clone())),
1168 });
1169 }
1170}
1171
1172#[async_trait]
1177impl SessionActivationSpawner for ChildCompletionCoordinator {
1178 async fn reserve_activation(
1179 &self,
1180 target_session_id: &str,
1181 _inbox_generation: u64,
1182 ) -> Result<SessionActivationReserveOutcome, bamboo_domain::SessionActivationError> {
1183 let Some(inbox) = self.agent.session_inbox() else {
1184 return Err(bamboo_domain::SessionActivationError::Internal(
1185 "agent runtime has no SessionInbox".to_string(),
1186 ));
1187 };
1188 let backlog = inbox
1189 .inspect(target_session_id)
1190 .await
1191 .map_err(|error| bamboo_domain::SessionActivationError::Internal(error.to_string()))?;
1192 if !backlog.activation_pending() {
1193 return Ok(SessionActivationReserveOutcome::NoWork);
1194 }
1195 let prepared = prepare_session_inbox_activation(
1198 &self.persistence,
1199 target_session_id,
1200 backlog.interrupt_pending(),
1201 )
1202 .await
1203 .map_err(|error| {
1204 bamboo_domain::SessionActivationError::Internal(format!(
1205 "persist resumable SessionInbox target: {error}"
1206 ))
1207 })?;
1208 let Some((session, ready)) = prepared else {
1209 return Ok(SessionActivationReserveOutcome::NotFound);
1210 };
1211 if !ready {
1212 tracing::info!(
1213 session_id = target_session_id,
1214 "SessionInbox backlog is activation-eligible but a specific durable wait remains armed"
1215 );
1216 return Ok(SessionActivationReserveOutcome::NoWork);
1217 }
1218
1219 enum LaunchPlan {
1220 Root(Box<ResumeConfigSnapshot>),
1221 Child {
1222 scheduler: Arc<SpawnScheduler>,
1223 parent_session_id: String,
1224 model: String,
1225 disabled_tools: Option<Vec<String>>,
1226 },
1227 }
1228
1229 let launch_plan = match session.kind {
1232 SessionKind::Root => {
1233 if self.root_tools.read().await.is_none() {
1234 return Err(bamboo_domain::SessionActivationError::Internal(
1235 "root tool surface is not initialized".to_string(),
1236 ));
1237 }
1238 let config_snapshot = self.config.read().await.clone();
1239 LaunchPlan::Root(Box::new(
1240 self.build_resume_config(&session, &config_snapshot),
1241 ))
1242 }
1243 SessionKind::Child => {
1244 let scheduler = self.spawn_scheduler.read().await.upgrade().ok_or_else(|| {
1245 bamboo_domain::SessionActivationError::Internal(
1246 "child spawn scheduler is not initialized".to_string(),
1247 )
1248 })?;
1249 let parent_session_id = session
1250 .parent_session_id
1251 .as_deref()
1252 .map(str::trim)
1253 .filter(|id| !id.is_empty())
1254 .map(ToOwned::to_owned)
1255 .ok_or_else(|| {
1256 bamboo_domain::SessionActivationError::Internal(format!(
1257 "child SessionInbox target {target_session_id} has no parent owner"
1258 ))
1259 })?;
1260 let parent = self
1261 .storage
1262 .load_session(&parent_session_id)
1263 .await
1264 .map_err(|error| {
1265 bamboo_domain::SessionActivationError::Internal(format!(
1266 "load parent owner {parent_session_id}: {error}"
1267 ))
1268 })?
1269 .ok_or_else(|| {
1270 bamboo_domain::SessionActivationError::Internal(format!(
1271 "child SessionInbox parent owner {parent_session_id} disappeared"
1272 ))
1273 })?;
1274 let child_root = if session.root_session_id.trim().is_empty() {
1275 parent_session_id.as_str()
1276 } else {
1277 session.root_session_id.as_str()
1278 };
1279 let parent_root = if parent.root_session_id.trim().is_empty() {
1280 parent.id.as_str()
1281 } else {
1282 parent.root_session_id.as_str()
1283 };
1284 if child_root != parent_root {
1285 return Err(bamboo_domain::SessionActivationError::Internal(format!(
1286 "child SessionInbox target {target_session_id} does not share its parent owner's root"
1287 )));
1288 }
1289 let model = if session.model.trim().is_empty() {
1290 parent.model.clone()
1291 } else {
1292 session.model.clone()
1293 };
1294 if model.trim().is_empty() {
1295 return Err(bamboo_domain::SessionActivationError::Internal(format!(
1296 "child SessionInbox target {target_session_id} has no executable model"
1297 )));
1298 }
1299 let disabled_tools = match session.metadata.get("disabled_tools") {
1300 None => None,
1301 Some(raw) => {
1302 let tools = serde_json::from_str::<std::collections::BTreeSet<String>>(raw)
1303 .map_err(|error| {
1304 bamboo_domain::SessionActivationError::Internal(format!(
1305 "child SessionInbox target {target_session_id} has malformed disabled_tools: {error}"
1306 ))
1307 })?;
1308 (!tools.is_empty()).then(|| tools.into_iter().collect())
1309 }
1310 };
1311 LaunchPlan::Child {
1312 scheduler,
1313 parent_session_id,
1314 model,
1315 disabled_tools,
1316 }
1317 }
1318 };
1319 let event_sender =
1320 ResumeExecutionPort::get_or_create_event_sender(self, target_session_id).await;
1321
1322 let reservation = match reserve_runner_core(
1323 &self.agent_runners,
1324 &self.session_event_senders,
1325 target_session_id,
1326 &event_sender,
1327 )
1328 .await
1329 {
1330 ReserveOutcome::Reserved(reservation) => reservation,
1331 ReserveOutcome::AlreadyRunning(run_id) => {
1332 return Ok(SessionActivationReserveOutcome::AlreadyRunning { run_id });
1333 }
1334 };
1335 let run_id = reservation.run_id.clone();
1336 let launch = match launch_plan {
1337 LaunchPlan::Root(config) => {
1338 let execution_reservation =
1339 SessionExecutionReservation::from_activation_placeholder(
1340 target_session_id,
1341 reservation,
1342 self.agent
1343 .activation_router()
1344 .expect("SessionInbox activation requires an activation router")
1345 .clone(),
1346 self.agent_runners.clone(),
1347 );
1348 let reservation_cell = Arc::new(StdMutex::new(Some(execution_reservation)));
1352 let launch_reservation = reservation_cell.clone();
1353 let rollback_reservation = reservation_cell;
1354 let coordinator = self.clone();
1355 let launch_sessions = self.sessions.clone();
1356 let launch_session_id = session.id.clone();
1357 let launch_session = session.clone();
1358 let request_session_id = target_session_id.to_string();
1359 SessionActivationLaunch::new_with_async_rollback(
1360 run_id,
1361 move || {
1362 let mut execution_reservation = launch_reservation
1363 .lock()
1364 .unwrap_or_else(std::sync::PoisonError::into_inner)
1365 .take()
1366 .expect("activation reservation launches or rolls back exactly once");
1367 execution_reservation.mark_activation_published();
1368 launch_sessions.insert(
1372 launch_session_id,
1373 Arc::new(parking_lot::RwLock::new(launch_session)),
1374 );
1375 let request = ResumeSpawnRequest {
1376 session_id: request_session_id,
1377 session,
1378 execution_reservation,
1379 event_sender,
1380 config: *config,
1381 };
1382 tokio::spawn(async move {
1383 ResumeExecutionPort::spawn_resume_execution(&coordinator, request)
1384 .await;
1385 });
1386 },
1387 move || async move {
1388 let reservation = rollback_reservation
1389 .lock()
1390 .unwrap_or_else(std::sync::PoisonError::into_inner)
1391 .take();
1392 if let Some(reservation) = reservation {
1393 reservation.rollback_unpublished_activation().await;
1394 }
1395 },
1396 )
1397 }
1398 LaunchPlan::Child {
1399 scheduler,
1400 parent_session_id,
1401 model,
1402 disabled_tools,
1403 } => {
1404 let execution_reservation =
1405 SessionExecutionReservation::from_activation_placeholder(
1406 target_session_id,
1407 reservation,
1408 self.agent
1409 .activation_router()
1410 .expect("SessionInbox activation requires an activation router")
1411 .clone(),
1412 self.agent_runners.clone(),
1413 );
1414 let reservation_cell = Arc::new(StdMutex::new(Some(execution_reservation)));
1415 let launch_reservation = reservation_cell.clone();
1416 let rollback_reservation = reservation_cell;
1417 let job = SpawnJob {
1418 parent_session_id,
1419 child_session_id: target_session_id.to_string(),
1420 model,
1421 disabled_tools,
1422 };
1423 let launch_sessions = self.sessions.clone();
1424 let launch_session_id = session.id.clone();
1425 let launch_session = session;
1426 SessionActivationLaunch::new_with_async_rollback(
1427 run_id,
1428 move || {
1429 let mut execution_reservation = launch_reservation
1430 .lock()
1431 .unwrap_or_else(std::sync::PoisonError::into_inner)
1432 .take()
1433 .expect("activation reservation launches or rolls back exactly once");
1434 execution_reservation.mark_activation_published();
1435 launch_sessions.insert(
1438 launch_session_id,
1439 Arc::new(parking_lot::RwLock::new(launch_session)),
1440 );
1441 drop(scheduler.launch_reserved(job, execution_reservation));
1445 },
1446 move || async move {
1447 let reservation = rollback_reservation
1448 .lock()
1449 .unwrap_or_else(std::sync::PoisonError::into_inner)
1450 .take();
1451 if let Some(reservation) = reservation {
1452 reservation.rollback_unpublished_activation().await;
1453 }
1454 },
1455 )
1456 }
1457 };
1458 Ok(SessionActivationReserveOutcome::Reserved(launch))
1459 }
1460}
1461
1462fn bash_completion_resume_message(bash_ids: &[String], timed_out: bool) -> Message {
1472 let body = if timed_out {
1473 format!(
1474 "Runtime notification: the background-Bash wait ceiling was reached while one or more \
1475 shell(s) ({}) may still be running. The session is being resumed so it is not \
1476 stranded; verify their actual status with BashOutput before assuming completion.",
1477 bash_ids.join(", ")
1478 )
1479 } else {
1480 format!(
1481 "Runtime notification: all background Bash shell(s) ({}) have completed. \
1482 Review their output with BashOutput and resume the task from where you left off.",
1483 bash_ids.join(", ")
1484 )
1485 };
1486 let mut message = Message::user(body);
1487 message.metadata = Some(serde_json::json!({
1488 RUNTIME_RESUME_MESSAGE_HIDDEN_KEY: true,
1489 RUNTIME_RESUME_MESSAGE_KIND_KEY: BASH_COMPLETION_RESUME_KIND,
1490 }));
1491 message.never_compress = false;
1492 message
1493}
1494
1495fn bash_resume_should_retry(outcome: &ResumeOutcome, persisted_waiting_for_bash: bool) -> bool {
1508 match outcome {
1509 ResumeOutcome::Started { .. } | ResumeOutcome::NotFound => false,
1510 ResumeOutcome::Completed | ResumeOutcome::AlreadyRunning { .. } => {
1511 persisted_waiting_for_bash
1512 }
1513 }
1514}
1515
1516fn bash_completion_should_resume(
1524 loop_suspended_on_bash: bool,
1525 all_waited_shells_done: bool,
1526) -> bool {
1527 loop_suspended_on_bash && all_waited_shells_done
1528}
1529
1530#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1531enum BashCompletionDeliveryPlan {
1532 DurableOnly,
1536 Activate,
1538 ClearWaitThenActivate,
1540}
1541
1542fn bash_completion_delivery_plan(
1543 loop_suspended_on_bash: bool,
1544 all_waited_shells_done: bool,
1545) -> BashCompletionDeliveryPlan {
1546 if bash_completion_should_resume(loop_suspended_on_bash, all_waited_shells_done) {
1547 BashCompletionDeliveryPlan::ClearWaitThenActivate
1548 } else if loop_suspended_on_bash {
1549 BashCompletionDeliveryPlan::DurableOnly
1550 } else {
1551 BashCompletionDeliveryPlan::Activate
1552 }
1553}
1554
1555fn background_bash_post_tool_payload(info: &BashCompletionInfo) -> HookPayload {
1556 let success = info.status == "completed" && info.exit_code == Some(0);
1557 let response = serde_json::json!({
1558 "bash_id": info.bash_id,
1559 "command": info.command,
1560 "exit_code": info.exit_code,
1561 "status": info.status,
1562 "output_tail": info.output_tail,
1563 });
1564 HookPayload::ToolResult {
1565 tool_name: "Bash".to_string(),
1566 tool_call_id: info.bash_id.clone(),
1567 outcome: HookToolOutcome {
1568 success,
1569 result: success.then(|| response.to_string()),
1570 error: (!success).then(|| response.to_string()),
1571 needs_human: false,
1572 duration_ms: 0,
1573 },
1574 }
1575}
1576
1577fn append_background_bash_hook_feedback(info: &mut BashCompletionInfo, feedback: Vec<String>) {
1578 let feedback = feedback
1579 .into_iter()
1580 .map(|text| text.trim().to_string())
1581 .filter(|text| !text.is_empty())
1582 .collect::<Vec<_>>();
1583 if feedback.is_empty() {
1584 return;
1585 }
1586 if !info.output_tail.is_empty() {
1587 info.output_tail.push_str("\n\n");
1588 }
1589 info.output_tail.push_str("<post_tool_use_feedback>\n");
1590 info.output_tail.push_str(&feedback.join("\n"));
1591 info.output_tail.push_str("\n</post_tool_use_feedback>");
1592}
1593
1594async fn run_background_bash_post_tool_hooks(
1595 config: &bamboo_config::LifecycleHooksConfig,
1596 fallback_cwd: Option<std::path::PathBuf>,
1597 session: &Session,
1598 info: &mut BashCompletionInfo,
1599) -> bool {
1600 let runner = crate::HookRunner::new().with_lifecycle_config(config, fallback_cwd);
1601 if !runner.has_hooks_for(AgentHookPoint::AfterToolExecution) {
1602 return false;
1603 }
1604
1605 let mut runtime_state = session
1606 .agent_runtime_state
1607 .clone()
1608 .unwrap_or_else(|| AgentRuntimeState::new(&session.id));
1609 let outcome = runner
1610 .run_observer_hooks(
1611 AgentHookPoint::AfterToolExecution,
1612 &background_bash_post_tool_payload(info),
1613 session,
1614 &mut runtime_state,
1615 None,
1616 )
1617 .await;
1618 append_background_bash_hook_feedback(info, outcome.injected_contexts);
1619 true
1620}
1621
1622fn apply_bash_resume_transition(session: &mut Session, resume_message: &Message) -> bool {
1630 let mut runtime_state = read_runtime_state(session);
1631 if runtime_state.waiting_for_bash.is_none() {
1632 return false;
1633 }
1634 runtime_state.waiting_for_bash = None;
1635 runtime_state.status = AgentStatusState::Idle;
1636 runtime_state.suspension = None;
1637 write_runtime_state(session, &runtime_state);
1638 session.metadata.remove("runtime.suspend_reason");
1639 session.add_message(resume_message.clone());
1640 true
1641}
1642
1643impl ChildCompletionCoordinator {
1645 async fn bash_self_resume(&self, session_id: String, bash_ids: Vec<String>) {
1661 let mut delay = Duration::from_secs(1);
1662 let max_delay = Duration::from_secs(30);
1663 let max_poll = Duration::from_secs(6 * 3600 + 600);
1667 let deadline = tokio::time::Instant::now() + max_poll;
1668
1669 loop {
1670 tokio::time::sleep(delay).await;
1671
1672 let Some(session) = self.load_session(&session_id).await else {
1673 tracing::info!(%session_id, "bash self-resume backstop: session gone; nothing to do");
1674 return;
1675 };
1676 if read_runtime_state(&session).waiting_for_bash.is_none() {
1677 return;
1680 }
1681
1682 let still_running =
1683 bamboo_tools::tools::bash_runtime::running_shells_for_session(&session_id);
1684 let timed_out = tokio::time::Instant::now() >= deadline;
1685 if still_running.is_empty() || timed_out {
1686 let guard = session_resume_lock(&session_id);
1691 let _held = guard.lock().await;
1692 tracing::warn!(
1693 %session_id,
1694 shell_count = bash_ids.len(),
1695 timed_out,
1696 "bash self-resume backstop engaged (push lost or wait ceiling reached)"
1697 );
1698 self.perform_bash_resume(
1699 &session_id,
1700 bash_completion_resume_message(&bash_ids, timed_out),
1701 )
1702 .await;
1703 return;
1704 }
1705
1706 delay = (delay * 2).min(max_delay);
1707 }
1708 }
1709
1710 async fn perform_bash_resume(&self, session_id: &str, resume_message: Message) {
1731 let retry_backoff = Duration::from_millis(200);
1732 const MAX_RESUME_ATTEMPTS: u8 = 5;
1733 for attempt in 0..MAX_RESUME_ATTEMPTS {
1734 if attempt > 0 {
1735 tokio::time::sleep(retry_backoff).await;
1736 }
1737
1738 let Some(mut session) = self.load_session(session_id).await else {
1739 tracing::warn!(%session_id, "bash resume: session not found; nothing to resume");
1740 return;
1741 };
1742
1743 if !apply_bash_resume_transition(&mut session, &resume_message) {
1744 tracing::info!(
1748 %session_id, attempt,
1749 "bash resume: persisted bash wait already cleared; nothing to resume"
1750 );
1751 return;
1752 }
1753 session.updated_at = Utc::now();
1754 self.save_and_cache(&mut session).await;
1755 tracing::info!(
1756 %session_id, attempt,
1757 "bash resume: cleared bash wait and appended resume message"
1758 );
1759
1760 let outcome = self.resume_parent(session_id.to_string()).await;
1761 match outcome {
1762 ResumeOutcome::Started { .. } => {
1763 tracing::info!(%session_id, attempt, "bash resume: resume fired");
1764 return;
1765 }
1766 ResumeOutcome::NotFound => {
1767 tracing::warn!(%session_id, "bash resume: session vanished during resume");
1768 return;
1769 }
1770 _ => {
1771 let clobbered = match self.load_session(session_id).await {
1777 Some(reloaded) => read_runtime_state(&reloaded).waiting_for_bash.is_some(),
1778 None => {
1779 tracing::warn!(%session_id, "bash resume: session vanished after resume");
1780 return;
1781 }
1782 };
1783 if bash_resume_should_retry(&outcome, clobbered) {
1784 tracing::warn!(
1785 %session_id, attempt,
1786 outcome = outcome.as_str(),
1787 "bash resume: persisted wait still set after resume (finalize-clobber); retrying"
1788 );
1789 continue;
1790 }
1791 tracing::info!(
1792 %session_id, attempt,
1793 outcome = outcome.as_str(),
1794 "bash resume: wait cleared and resume handled; stopping"
1795 );
1796 return;
1797 }
1798 }
1799 }
1800
1801 tracing::warn!(
1802 %session_id,
1803 attempts = MAX_RESUME_ATTEMPTS,
1804 "bash resume: exhausted clobber-retry budget without confirming resume; giving up"
1805 );
1806 }
1807}
1808
1809impl BashResumeHook for ChildCompletionCoordinator {
1810 fn arrange_bash_self_resume(&self, session_id: String, bash_ids: Vec<String>) {
1811 let coordinator = Arc::new(self.clone());
1812 tokio::spawn(async move {
1813 coordinator.bash_self_resume(session_id, bash_ids).await;
1814 });
1815 }
1816}
1817
1818fn bash_completion_injection_body(info: &BashCompletionInfo) -> String {
1823 let exit = match info.exit_code {
1824 Some(code) => code.to_string(),
1825 None => "none (signal/killed)".to_string(),
1826 };
1827 let mut body = format!(
1828 "Runtime notification: background shell `{}` (`{}`) finished — status {}, exit code {}.",
1829 info.bash_id, info.command, info.status, exit
1830 );
1831 if info.output_tail.trim().is_empty() {
1832 body.push_str(" It produced no captured output.");
1833 } else {
1834 body.push_str("\n\nOutput tail:\n");
1835 body.push_str(&info.output_tail);
1836 }
1837 body.push_str(&format!(
1838 "\n\nUse BashOutput with bash_id=\"{}\" for the full output, then continue the task.",
1839 info.bash_id
1840 ));
1841 body
1842}
1843
1844fn bash_completion_envelope(info: &BashCompletionInfo) -> bamboo_domain::SessionMessageEnvelope {
1845 let provider_message = bash_resume_message_from_info(info);
1846 let data = serde_json::json!({
1847 "session_id": info.session_id,
1848 "bash_id": info.bash_id,
1849 "command": info.command,
1850 "exit_code": info.exit_code,
1851 "status": info.status,
1852 "output_tail": info.output_tail,
1853 });
1854 let identity = data.clone();
1859 bamboo_domain::SessionMessageEnvelope {
1860 id: bamboo_domain::SessionMessageId::stable("background_bash_completion", &identity),
1861 source: bamboo_domain::SessionMessageSource::Runtime {
1862 subsystem: "background_bash".to_string(),
1863 },
1864 target_session_id: info.session_id.clone(),
1865 kind: bamboo_domain::SessionMessageKind::RuntimeInstruction,
1866 body: bamboo_domain::SessionMessageBody::RuntimeInstruction(
1867 bamboo_domain::SessionRuntimeInstruction {
1868 instruction: "background_bash_completed".to_string(),
1869 content: Some(bamboo_domain::SessionMessageContent::text(
1870 bash_completion_injection_body(info),
1871 )),
1872 data: Some(data),
1873 provider_message: Some(session_provider_message(&provider_message)),
1874 },
1875 ),
1876 created_at: Utc::now(),
1877 thread_id: None,
1878 in_reply_to: None,
1879 attempt: None,
1880 correlation_id: Some(info.bash_id.clone()),
1881 }
1882}
1883
1884async fn enqueue_bash_completion_injection(
1890 persistence: &LockedSessionStore,
1891 info: &BashCompletionInfo,
1892) -> std::io::Result<Option<Session>> {
1893 let body = bash_completion_injection_body(info);
1894 let queued = serde_json::json!({
1895 "content": body,
1896 "created_at": Utc::now(),
1897 });
1898 persistence
1899 .update_runtime_config(&info.session_id, move |session| {
1900 let mut pending = session.pending_injected_messages().unwrap_or_default();
1901 pending.push(queued);
1902 session.set_pending_injected_messages(pending);
1903 })
1904 .await
1905}
1906
1907fn bash_resume_message_from_info(info: &BashCompletionInfo) -> Message {
1915 let mut message = Message::user(bash_completion_injection_body(info));
1916 message.metadata = Some(serde_json::json!({
1917 RUNTIME_RESUME_MESSAGE_HIDDEN_KEY: true,
1918 RUNTIME_RESUME_MESSAGE_KIND_KEY: BASH_COMPLETION_RESUME_KIND,
1919 }));
1920 message.never_compress = false;
1921 message
1922}
1923
1924impl ChildCompletionCoordinator {
1925 async fn deliver_bash_completion(&self, mut info: BashCompletionInfo) {
1937 if let Some(hook_session) = self.load_session(&info.session_id).await {
1943 let config_snapshot = self.config.read().await.clone();
1944 let _ = run_background_bash_post_tool_hooks(
1945 &config_snapshot.lifecycle_hooks,
1946 Some(self.app_data_dir.clone()),
1947 &hook_session,
1948 &mut info,
1949 )
1950 .await;
1951 }
1952
1953 let guard = session_resume_lock(&info.session_id);
1954 let _held = guard.lock().await;
1955
1956 let Some(session) = self.load_session(&info.session_id).await else {
1957 tracing::warn!(
1958 session_id = %info.session_id,
1959 bash_id = %info.bash_id,
1960 "background bash completion: owning session not found; nothing to notify"
1961 );
1962 return;
1963 };
1964
1965 let waiting = read_runtime_state(&session).waiting_for_bash.is_some();
1966 let all_shells_done =
1972 bamboo_tools::tools::bash_runtime::running_shells_for_session(&info.session_id)
1973 .is_empty();
1974 let delivery_plan = bash_completion_delivery_plan(waiting, all_shells_done);
1975
1976 let messenger = self.agent.session_messenger().cloned();
1982 let admission = match messenger.as_ref() {
1983 Some(messenger) => match messenger.admit(bash_completion_envelope(&info)).await {
1984 Ok(admission) => Some(admission),
1985 Err(error) => {
1986 tracing::warn!(
1987 session_id = %info.session_id,
1988 bash_id = %info.bash_id,
1989 %error,
1990 "background bash completion: durable SessionInbox admission failed; leaving wait armed"
1991 );
1992 return;
1993 }
1994 },
1995 None => {
1996 tracing::warn!(
1997 session_id = %info.session_id,
1998 bash_id = %info.bash_id,
1999 "SessionMessenger unavailable; using compatibility injection before clearing wait"
2000 );
2001 match enqueue_bash_completion_injection(&self.persistence, &info).await {
2002 Ok(Some(_)) => None,
2003 Ok(None) => {
2004 tracing::warn!(
2005 session_id = %info.session_id,
2006 "background bash compatibility target disappeared"
2007 );
2008 return;
2009 }
2010 Err(error) => {
2011 tracing::warn!(
2012 session_id = %info.session_id,
2013 %error,
2014 "background bash compatibility admission failed; leaving wait armed"
2015 );
2016 return;
2017 }
2018 }
2019 }
2020 };
2021
2022 if delivery_plan == BashCompletionDeliveryPlan::DurableOnly {
2023 tracing::info!(
2024 session_id = %info.session_id,
2025 bash_id = %info.bash_id,
2026 "background bash completion is durable; sibling shells still run, so wait remains armed"
2027 );
2028 return;
2029 }
2030
2031 if let (Some(messenger), Some(admission)) = (messenger.as_ref(), admission.as_ref()) {
2032 if let Err(error) = messenger.prepare_activation(admission).await {
2033 tracing::warn!(
2034 session_id = %info.session_id,
2035 bash_id = %info.bash_id,
2036 %error,
2037 "background bash completion activation watermark failed; leaving wait armed"
2038 );
2039 return;
2040 }
2041 }
2042
2043 if delivery_plan == BashCompletionDeliveryPlan::ClearWaitThenActivate {
2044 tracing::info!(
2045 session_id = %info.session_id,
2046 bash_id = %info.bash_id,
2047 status = %info.status,
2048 "background bash completion: push-resuming suspended loop (event-driven)"
2049 );
2050 let mut resumable = session.clone();
2051 let mut runtime_state = read_runtime_state(&resumable);
2052 runtime_state.waiting_for_bash = None;
2053 runtime_state.status = AgentStatusState::Idle;
2054 runtime_state.suspension = None;
2055 write_runtime_state(&mut resumable, &runtime_state);
2056 resumable.metadata.remove("runtime.suspend_reason");
2057 resumable.updated_at = Utc::now();
2058 if let Err(error) = self.persistence.merge_save_runtime(&mut resumable).await {
2059 tracing::warn!(
2060 session_id = %info.session_id,
2061 %error,
2062 "background bash completion is durable but wait-state clear failed; leaving activation to the wait backstop"
2063 );
2064 return;
2065 }
2066 self.sessions.insert(
2067 resumable.id.clone(),
2068 Arc::new(parking_lot::RwLock::new(resumable)),
2069 );
2070 }
2071
2072 let (Some(messenger), Some(admission)) = (messenger, admission) else {
2076 if delivery_plan == BashCompletionDeliveryPlan::ClearWaitThenActivate {
2077 let _ = self.resume_parent(info.session_id.clone()).await;
2078 }
2079 return;
2080 };
2081 match messenger.activate_prepared(&admission).await {
2082 Ok(receipt) => tracing::info!(
2083 session_id = %info.session_id,
2084 bash_id = %info.bash_id,
2085 status = %info.status,
2086 waiting,
2087 generation = receipt.delivery.generation,
2088 activation = ?receipt.activation,
2089 "background bash completion delivered through SessionMessenger"
2090 ),
2091 Err(error) => tracing::warn!(
2092 session_id = %info.session_id,
2093 bash_id = %info.bash_id,
2094 %error,
2095 "background bash completion: SessionMessenger delivery failed"
2096 ),
2097 }
2098 }
2099}
2100
2101impl BashCompletionSink for ChildCompletionCoordinator {
2102 fn on_bash_completed(&self, info: BashCompletionInfo) {
2103 let coordinator = Arc::new(self.clone());
2107 tokio::spawn(async move {
2108 coordinator.deliver_bash_completion(info).await;
2109 });
2110 }
2111}
2112
2113const CHILD_WAIT_SWEEP_INTERVAL_SECS: u64 = 30;
2119const CHILD_WAIT_REGISTRATION_GRACE_SECS: i64 = 60;
2123const DEAD_CHILD_GRACE_SECS: i64 = 120;
2126const STALE_RUNNER_SLACK_SECS: i64 = 600;
2132
2133fn is_dead_child_candidate_status(status: Option<&str>) -> bool {
2138 match status {
2139 Some(status) => !is_terminal_child_status(status) && status != "suspended",
2142 None => true,
2143 }
2144}
2145
2146fn completion_child_is_owned(reported_parent: &str, child_parent_linkage: Option<&str>) -> bool {
2154 child_parent_linkage == Some(reported_parent)
2155}
2156
2157fn select_replay_child(terminal: &[(String, String)]) -> Option<&(String, String)> {
2161 terminal
2162 .iter()
2163 .find(|(_, status)| is_error_like(status))
2164 .or_else(|| terminal.last())
2165}
2166
2167fn child_wait_watchdog_resume_message(body: String) -> Message {
2168 let mut message = Message::user(body);
2169 message.metadata = Some(serde_json::json!({
2170 RUNTIME_RESUME_MESSAGE_HIDDEN_KEY: true,
2171 RUNTIME_RESUME_MESSAGE_KIND_KEY: "child_wait_watchdog_resume",
2172 }));
2173 message.never_compress = false;
2174 message
2175}
2176
2177fn empty_child_wait_message() -> Message {
2178 child_wait_watchdog_resume_message(
2179 "Runtime notification: this session was suspended waiting for child sessions, but the \
2180 wait tracked no children (internal inconsistency). The session has been resumed; use \
2181 SubAgent.list to inspect child state and continue the task."
2182 .to_string(),
2183 )
2184}
2185
2186fn child_wait_lease_expired_message(child_ids: &[String]) -> Message {
2187 child_wait_watchdog_resume_message(format!(
2188 "Runtime notification: the wait lease for child session(s) [{}] expired before they all \
2189 reported completion. They were NOT cancelled and may still be running or already \
2190 finished — verify their actual status with SubAgent.list / SubAgent.get before assuming \
2191 anything, then continue the task.",
2192 child_ids.join(", ")
2193 ))
2194}
2195
2196impl ChildCompletionCoordinator {
2212 pub fn spawn_child_wait_watchdog(self: &Arc<Self>) {
2215 let coordinator = Arc::clone(self);
2216 tokio::spawn(async move {
2217 use futures::FutureExt;
2218 if std::panic::AssertUnwindSafe(coordinator.reconcile_orphans_at_boot())
2219 .catch_unwind()
2220 .await
2221 .is_err()
2222 {
2223 tracing::error!("child-wait watchdog: boot reconciliation panicked");
2224 }
2225 let mut ticker = tokio::time::interval(std::time::Duration::from_secs(
2226 CHILD_WAIT_SWEEP_INTERVAL_SECS,
2227 ));
2228 ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
2229 ticker.tick().await;
2231 loop {
2232 ticker.tick().await;
2233 if std::panic::AssertUnwindSafe(coordinator.sweep_child_waits())
2234 .catch_unwind()
2235 .await
2236 .is_err()
2237 {
2238 tracing::error!("child-wait watchdog: sweep panicked; continuing");
2239 }
2240 }
2241 });
2242 }
2243
2244 async fn reconcile_orphans_at_boot(&self) {
2249 let cutoff = Utc::now();
2250
2251 let running = self
2257 .storage
2258 .list_sessions_by_run_status("running")
2259 .await
2260 .unwrap_or_default();
2261 for (child_id, parent_id) in running {
2262 let Some(parent_id) = parent_id else { continue };
2263 if self.runner_is_running(&child_id).await {
2264 continue;
2265 }
2266 let Some(control_plane) = self.load_control_plane(&child_id).await else {
2267 continue;
2268 };
2269 if control_plane.updated_at >= cutoff {
2272 continue;
2273 }
2274 tracing::warn!(
2275 child_session_id = %child_id,
2276 parent_session_id = %parent_id,
2277 "boot reconciliation: child was running when the process died; \
2278 marking it error and waking the parent"
2279 );
2280 self.synthesize_child_completion(
2281 &parent_id,
2282 &child_id,
2283 "error",
2284 Some(
2285 "orphaned by server restart: the process died while this child session \
2286 was running"
2287 .to_string(),
2288 ),
2289 )
2290 .await;
2291 }
2292
2293 let suspended = self
2297 .storage
2298 .list_sessions_by_run_status("suspended")
2299 .await
2300 .unwrap_or_default();
2301 for (session_id, _) in suspended {
2302 let Some(control_plane) = self.load_control_plane(&session_id).await else {
2303 continue;
2304 };
2305 if control_plane
2306 .metadata
2307 .get("runtime.suspend_reason")
2308 .map(String::as_str)
2309 != Some("waiting_for_bash")
2310 {
2311 continue;
2312 }
2313 if let Some(wait) = read_runtime_state(&control_plane).waiting_for_bash {
2314 tracing::warn!(
2315 %session_id,
2316 "boot reconciliation: re-arming bash self-resume backstop lost in restart"
2317 );
2318 let coordinator = self.clone();
2319 tokio::spawn(async move {
2320 coordinator
2321 .bash_self_resume(session_id, wait.bash_ids)
2322 .await;
2323 });
2324 }
2325 }
2326 }
2327
2328 async fn runner_is_running(&self, session_id: &str) -> bool {
2329 let runners = self.agent_runners.read().await;
2330 runners
2331 .get(session_id)
2332 .is_some_and(|runner| matches!(runner.status, AgentStatus::Running))
2333 }
2334
2335 async fn load_control_plane(&self, session_id: &str) -> Option<Session> {
2336 match self.storage.load_runtime_control_plane(session_id).await {
2337 Ok(session) => session,
2338 Err(error) => {
2339 tracing::warn!(
2340 %session_id,
2341 %error,
2342 "child-wait watchdog: failed to load session control plane"
2343 );
2344 None
2345 }
2346 }
2347 }
2348
2349 async fn sweep_child_waits(&self) {
2353 let suspended = match self.storage.list_sessions_by_run_status("suspended").await {
2354 Ok(entries) => entries,
2355 Err(error) => {
2356 tracing::warn!(%error, "child-wait watchdog: failed to list suspended sessions");
2357 return;
2358 }
2359 };
2360 for (session_id, _) in suspended {
2361 self.sweep_one_suspended_session(&session_id).await;
2362 }
2363 }
2364
2365 async fn sweep_one_suspended_session(&self, session_id: &str) {
2366 if self.runner_is_running(session_id).await {
2369 return;
2370 }
2371 let Some(session) = self.load_control_plane(session_id).await else {
2372 return;
2373 };
2374 let runtime_state = read_runtime_state(&session);
2375 let suspend_reason = session
2376 .metadata
2377 .get("runtime.suspend_reason")
2378 .map(String::as_str)
2379 .unwrap_or_default()
2380 .to_string();
2381 match (
2382 suspend_reason.as_str(),
2383 runtime_state.waiting_for_children.clone(),
2384 ) {
2385 ("waiting_for_bash", _)
2387 | ("awaiting_clarification", _)
2388 | ("awaiting_parent_approval", _) => {}
2389 (_, Some(wait)) => self.sweep_child_wait(session_id, wait).await,
2390 ("waiting_for_children", None) | ("", None) => {
2393 self.rescue_stranded_resume(session_id).await;
2394 }
2395 _ => {}
2396 }
2397 }
2398
2399 async fn sweep_child_wait(&self, parent_session_id: &str, wait: WaitingForChildrenState) {
2404 let now = Utc::now();
2405
2406 if wait.child_session_ids.is_empty() {
2407 tracing::warn!(
2408 %parent_session_id,
2409 "child-wait watchdog: wait armed over an empty child set; force-resuming"
2410 );
2411 self.force_resume_child_wait(parent_session_id, empty_child_wait_message())
2412 .await;
2413 return;
2414 }
2415
2416 if wait.timeout_at.is_some_and(|deadline| now >= deadline) {
2420 tracing::warn!(
2421 %parent_session_id,
2422 "child-wait watchdog: wait lease expired; force-resuming parent"
2423 );
2424 self.force_resume_child_wait(
2425 parent_session_id,
2426 child_wait_lease_expired_message(&wait.child_session_ids),
2427 )
2428 .await;
2429 return;
2430 }
2431
2432 if now.signed_duration_since(wait.registered_at).num_seconds()
2434 < CHILD_WAIT_REGISTRATION_GRACE_SECS
2435 {
2436 return;
2437 }
2438
2439 let statuses: HashMap<String, Option<String>> = self
2440 .storage
2441 .list_child_run_statuses(parent_session_id)
2442 .await
2443 .unwrap_or_default()
2444 .into_iter()
2445 .collect();
2446
2447 struct DeadChild {
2448 child_id: String,
2449 status: String,
2450 reason: String,
2451 owned: bool,
2455 }
2456
2457 let mut terminal: Vec<(String, String)> = Vec::new();
2458 let mut dead: Vec<DeadChild> = Vec::new();
2459 for child_id in &wait.child_session_ids {
2460 let status = statuses.get(child_id).and_then(|status| status.as_deref());
2461 if let Some(status) = status {
2462 if is_terminal_child_status(status) {
2463 terminal.push((child_id.clone(), status.to_string()));
2464 continue;
2465 }
2466 }
2467 if !is_dead_child_candidate_status(status) {
2468 continue;
2469 }
2470
2471 let control_plane = self.load_control_plane(child_id).await;
2478 let owned = control_plane
2479 .as_ref()
2480 .is_some_and(|cp| cp.parent_session_id.as_deref() == Some(parent_session_id));
2481 if !owned {
2482 dead.push(DeadChild {
2483 child_id: child_id.clone(),
2484 status: "error".to_string(),
2485 reason: if control_plane.is_some() {
2486 "waited-on session id is not a child of this session; clearing it \
2487 from the wait without touching that session"
2488 .to_string()
2489 } else {
2490 "waited-on child session does not exist".to_string()
2491 },
2492 owned: false,
2493 });
2494 continue;
2495 }
2496
2497 let runner = { self.agent_runners.read().await.get(child_id).cloned() };
2498 match runner {
2499 Some(runner) if matches!(runner.status, AgentStatus::Running) => {
2500 let last_activity = runner.last_event_at.unwrap_or(runner.started_at);
2506 let idle_secs = now.signed_duration_since(last_activity).num_seconds();
2507 let total_secs = now.signed_duration_since(runner.started_at).num_seconds();
2508 let policy = match &control_plane {
2509 Some(child) => {
2510 crate::runtime::execution::spawn::watchdog_policy_for_session(child)
2511 }
2512 None => Default::default(),
2513 };
2514 let idle_limit = policy.max_idle_secs.saturating_add(STALE_RUNNER_SLACK_SECS);
2515 let total_limit = policy
2516 .max_total_secs
2517 .saturating_add(STALE_RUNNER_SLACK_SECS);
2518 if idle_secs >= idle_limit || total_secs >= total_limit {
2519 runner.cancel_token.cancel();
2520 dead.push(DeadChild {
2521 child_id: child_id.clone(),
2522 status: "timeout".to_string(),
2523 reason: format!(
2524 "child runner stalled: no events for {idle_secs}s \
2525 (limit {idle_limit}s including watchdog slack); \
2526 force-finalized by the child-wait watchdog"
2527 ),
2528 owned: true,
2529 });
2530 }
2531 }
2532 _ => {
2533 let quiet_secs = control_plane
2537 .as_ref()
2538 .map(|child| now.signed_duration_since(child.updated_at).num_seconds())
2539 .unwrap_or(i64::MAX);
2540 if quiet_secs >= DEAD_CHILD_GRACE_SECS {
2541 dead.push(DeadChild {
2542 child_id: child_id.clone(),
2543 status: "error".to_string(),
2544 reason: format!(
2545 "child runner lost (crashed task, dropped spawn job, or \
2546 process restart): index status {status:?} with no live \
2547 runner driving it"
2548 ),
2549 owned: true,
2550 });
2551 }
2552 }
2553 }
2554 }
2555
2556 if !dead.is_empty() {
2557 for entry in dead {
2558 tracing::warn!(
2559 %parent_session_id,
2560 child_session_id = %entry.child_id,
2561 status = %entry.status,
2562 reason = %entry.reason,
2563 owned = entry.owned,
2564 "child-wait watchdog: synthesizing terminal completion for dead child"
2565 );
2566 if entry.owned {
2567 self.synthesize_child_completion(
2568 parent_session_id,
2569 &entry.child_id,
2570 &entry.status,
2571 Some(entry.reason),
2572 )
2573 .await;
2574 } else {
2575 self.publish_synthetic_completion(
2577 parent_session_id,
2578 &entry.child_id,
2579 &entry.status,
2580 Some(entry.reason),
2581 )
2582 .await;
2583 }
2584 }
2585 return;
2587 }
2588
2589 let terminal_ids: Vec<String> = terminal.iter().map(|(id, _)| id.clone()).collect();
2595 if let Some((child_id, status)) = select_replay_child(&terminal) {
2596 if wait_policy_satisfied(
2597 wait.wait_for,
2598 &wait.child_session_ids,
2599 &terminal_ids,
2600 child_id,
2601 status,
2602 ) {
2603 tracing::warn!(
2604 %parent_session_id,
2605 child_session_id = %child_id,
2606 "child-wait watchdog: wait already satisfied but parent still suspended \
2607 (lost wake); replaying the completion"
2608 );
2609 let error = self
2610 .load_control_plane(child_id)
2611 .await
2612 .and_then(|child| child.last_run_error());
2613 self.publish_synthetic_completion(parent_session_id, child_id, status, error)
2614 .await;
2615 }
2616 }
2617 }
2618
2619 async fn synthesize_child_completion(
2625 &self,
2626 parent_session_id: &str,
2627 child_session_id: &str,
2628 status: &str,
2629 error: Option<String>,
2630 ) {
2631 match self.storage.load_session(child_session_id).await {
2632 Ok(Some(mut child)) => {
2633 if child.parent_session_id.as_deref() != Some(parent_session_id) {
2638 tracing::warn!(
2639 %parent_session_id,
2640 child_session_id = %child.id,
2641 "child-wait watchdog: refusing to synthesize status onto a session \
2642 that is not a child of this parent"
2643 );
2644 self.publish_synthetic_completion(
2645 parent_session_id,
2646 child_session_id,
2647 status,
2648 error,
2649 )
2650 .await;
2651 return;
2652 }
2653 child.set_last_run_status(status);
2654 match &error {
2655 Some(message) => child.set_last_run_error(message.clone()),
2656 None => child.clear_last_run_error(),
2657 }
2658 child.updated_at = Utc::now();
2659 if let Err(save_error) = self.persistence.merge_save_runtime(&mut child).await {
2660 tracing::warn!(
2661 child_session_id = %child.id,
2662 %save_error,
2663 "child-wait watchdog: failed to persist synthesized terminal status"
2664 );
2665 }
2666 self.sessions
2667 .insert(child.id.clone(), Arc::new(parking_lot::RwLock::new(child)));
2668 }
2669 Ok(None) => {}
2670 Err(load_error) => {
2671 tracing::warn!(
2672 %child_session_id,
2673 %load_error,
2674 "child-wait watchdog: failed to load child for synthesized terminal status"
2675 );
2676 }
2677 }
2678 finalize_runner(
2679 &self.agent_runners,
2680 child_session_id,
2681 &Err(bamboo_agent_core::AgentError::LLM(
2682 error
2683 .clone()
2684 .unwrap_or_else(|| format!("synthesized {status}")),
2685 )),
2686 )
2687 .await;
2688 self.publish_synthetic_completion(parent_session_id, child_session_id, status, error)
2689 .await;
2690 }
2691
2692 async fn publish_synthetic_completion(
2693 &self,
2694 parent_session_id: &str,
2695 child_session_id: &str,
2696 status: &str,
2697 error: Option<String>,
2698 ) {
2699 let publisher =
2700 crate::runtime::execution::session_events::ReplayableSessionEventPublisher::new(
2701 self.agent_runners.clone(),
2702 self.session_event_senders.clone(),
2703 self.account_feed_inbox.clone(),
2704 );
2705 let handler: Arc<dyn ChildCompletionHandler> = Arc::new(self.clone());
2706 crate::runtime::execution::spawn::publish_child_completion_parts(
2707 &publisher,
2708 Some(handler),
2709 parent_session_id.to_string(),
2710 child_session_id.to_string(),
2711 status.to_string(),
2712 error,
2713 )
2714 .await;
2715 }
2716
2717 async fn rescue_stranded_resume(&self, session_id: &str) {
2723 let Some(session) = self.load_session(session_id).await else {
2724 return;
2725 };
2726 let pending_runtime_resume = session.messages.last().is_some_and(|message| {
2727 matches!(message.role, Role::User)
2728 && message
2729 .metadata
2730 .as_ref()
2731 .is_some_and(|meta| meta.get(RUNTIME_RESUME_MESSAGE_KIND_KEY).is_some())
2732 });
2733 if !pending_runtime_resume {
2734 return;
2735 }
2736 tracing::warn!(
2737 %session_id,
2738 "child-wait watchdog: stranded resume detected (wait cleared, resume never \
2739 spawned); resuming"
2740 );
2741 self.resume_parent(session_id.to_string()).await;
2742 }
2743
2744 async fn force_resume_child_wait(&self, session_id: &str, resume_message: Message) {
2750 const MAX_ATTEMPTS: u8 = 5;
2751 let lock = session_resume_lock(session_id);
2752 for attempt in 0..MAX_ATTEMPTS {
2753 if attempt > 0 {
2754 tokio::time::sleep(Duration::from_millis(200)).await;
2755 }
2756 {
2757 let _held = lock.lock().await;
2758 let Some(mut session) = self.load_session(session_id).await else {
2759 return;
2760 };
2761 let mut runtime_state = read_runtime_state(&session);
2762 if runtime_state.waiting_for_children.is_none() {
2763 return;
2765 }
2766 runtime_state.waiting_for_children = None;
2767 runtime_state.status = AgentStatusState::Idle;
2768 runtime_state.suspension = None;
2769 write_runtime_state(&mut session, &runtime_state);
2770 session.metadata.remove("runtime.suspend_reason");
2771 session.add_message(resume_message.clone());
2772 session.updated_at = Utc::now();
2773 self.save_and_cache(&mut session).await;
2774 }
2775 let outcome = self.resume_parent(session_id.to_string()).await;
2776 match outcome {
2777 ResumeOutcome::Started { .. } | ResumeOutcome::NotFound => return,
2778 ResumeOutcome::Completed | ResumeOutcome::AlreadyRunning { .. } => {
2779 let clobbered = self
2783 .load_session(session_id)
2784 .await
2785 .map(|session| read_runtime_state(&session).waiting_for_children.is_some())
2786 .unwrap_or(false);
2787 if !clobbered {
2788 return;
2789 }
2790 }
2791 }
2792 }
2793 tracing::error!(
2794 %session_id,
2795 "child-wait watchdog: force-resume exhausted its clobber-retry budget"
2796 );
2797 }
2798}
2799
2800#[cfg(test)]
2801mod tests {
2802 use super::*;
2803 use bamboo_agent_core::Message;
2804 use bamboo_domain::SessionInboxPort;
2805 use futures::stream;
2806 use std::sync::atomic::{AtomicUsize, Ordering};
2807
2808 struct EmptyTools;
2809
2810 #[async_trait]
2811 impl bamboo_agent_core::tools::ToolExecutor for EmptyTools {
2812 async fn execute(
2813 &self,
2814 _call: &bamboo_agent_core::tools::ToolCall,
2815 ) -> Result<bamboo_agent_core::tools::ToolResult, bamboo_agent_core::tools::ToolError>
2816 {
2817 Err(bamboo_agent_core::tools::ToolError::NotFound(
2818 "no tools".to_string(),
2819 ))
2820 }
2821
2822 fn list_tools(&self) -> Vec<bamboo_agent_core::tools::ToolSchema> {
2823 Vec::new()
2824 }
2825 }
2826
2827 struct CompletedTestProvider;
2828
2829 #[async_trait]
2830 impl bamboo_llm::LLMProvider for CompletedTestProvider {
2831 async fn chat_stream(
2832 &self,
2833 _messages: &[Message],
2834 _tools: &[bamboo_agent_core::tools::ToolSchema],
2835 _max_output_tokens: Option<u32>,
2836 _model: &str,
2837 ) -> Result<bamboo_llm::LLMStream, bamboo_llm::LLMError> {
2838 let chunks: Vec<bamboo_llm::provider::Result<bamboo_llm::LLMChunk>> = vec![
2839 Ok(bamboo_llm::LLMChunk::Token("done".to_string())),
2840 Ok(bamboo_llm::LLMChunk::Done),
2841 ];
2842 Ok(Box::pin(stream::iter(chunks)))
2843 }
2844 }
2845
2846 struct CountingActivationSpawner {
2847 reservations: Arc<AtomicUsize>,
2848 launches: Arc<AtomicUsize>,
2849 }
2850
2851 #[async_trait]
2852 impl SessionActivationSpawner for CountingActivationSpawner {
2853 async fn reserve_activation(
2854 &self,
2855 target_session_id: &str,
2856 inbox_generation: u64,
2857 ) -> Result<SessionActivationReserveOutcome, bamboo_domain::SessionActivationError>
2858 {
2859 self.reservations.fetch_add(1, Ordering::SeqCst);
2860 let launches = self.launches.clone();
2861 Ok(SessionActivationReserveOutcome::Reserved(
2862 SessionActivationLaunch::new(
2863 format!("{target_session_id}-{inbox_generation}"),
2864 move || {
2865 launches.fetch_add(1, Ordering::SeqCst);
2866 },
2867 ),
2868 ))
2869 }
2870 }
2871
2872 async fn completion_inbox_fixture() -> (
2873 tempfile::TempDir,
2874 Arc<bamboo_storage::SessionStoreV2>,
2875 Arc<dyn SessionInboxPort>,
2876 Arc<ChildCompletionCoordinator>,
2877 Arc<AtomicUsize>,
2878 Arc<AtomicUsize>,
2879 ) {
2880 let temp = tempfile::tempdir().unwrap();
2881 let store = Arc::new(
2882 bamboo_storage::SessionStoreV2::new(temp.path().to_path_buf())
2883 .await
2884 .unwrap(),
2885 );
2886 let storage: Arc<dyn Storage> = store.clone();
2887 let locked = Arc::new(LockedSessionStore::new(storage.clone()));
2888 let inbox: Arc<dyn SessionInboxPort> = Arc::new(bamboo_storage::FileSessionInbox::new(
2889 store.clone(),
2890 bamboo_domain::SessionInboxLimits::default(),
2891 ));
2892 let router = crate::SessionActivationRouter::new();
2893 let messenger = Arc::new(crate::SessionMessenger::new(
2894 storage.clone(),
2895 inbox.clone(),
2896 router.clone(),
2897 ));
2898 let reservations = Arc::new(AtomicUsize::new(0));
2899 let launches = Arc::new(AtomicUsize::new(0));
2900 router
2901 .set_spawner(Arc::new(CountingActivationSpawner {
2902 reservations: reservations.clone(),
2903 launches: launches.clone(),
2904 }))
2905 .await;
2906 let provider: Arc<dyn bamboo_llm::LLMProvider> = Arc::new(CompletedTestProvider);
2907 let config = Arc::new(RwLock::new(Config::default()));
2908 let metrics = bamboo_metrics::MetricsCollector::spawn(
2909 Arc::new(bamboo_metrics::SqliteMetricsStorage::new(
2910 temp.path().join("metrics.db"),
2911 )),
2912 7,
2913 );
2914 let tools: Arc<dyn ToolExecutor> = Arc::new(EmptyTools);
2915 let agent = Arc::new(
2916 Agent::builder()
2917 .storage(storage.clone())
2918 .persistence(locked.clone())
2919 .session_inbox(inbox.clone())
2920 .activation_router(router)
2921 .session_messenger(messenger)
2922 .attachment_reader(store.clone())
2923 .skill_manager(Arc::new(bamboo_skills::SkillManager::new()))
2924 .metrics_collector(metrics)
2925 .config(config.clone())
2926 .provider(provider.clone())
2927 .default_tools(tools)
2928 .build()
2929 .unwrap(),
2930 );
2931 let mut providers = HashMap::new();
2932 providers.insert("test".to_string(), provider);
2933 let registry = Arc::new(ProviderRegistry::new(providers, "test".to_string()));
2934 let provider_router = Arc::new(ProviderModelRouter::new(registry.clone()));
2935 let coordinator = Arc::new(ChildCompletionCoordinator::new(
2936 storage,
2937 locked,
2938 Arc::new(dashmap::DashMap::new()),
2939 Arc::new(RwLock::new(HashMap::new())),
2940 Arc::new(RwLock::new(HashMap::new())),
2941 agent,
2942 config,
2943 registry,
2944 provider_router,
2945 temp.path().to_path_buf(),
2946 None,
2947 ));
2948 (temp, store, inbox, coordinator, reservations, launches)
2949 }
2950
2951 #[test]
2954 fn dead_child_candidate_status_matrix() {
2955 assert!(is_dead_child_candidate_status(None));
2957 assert!(is_dead_child_candidate_status(Some("running")));
2960 assert!(is_dead_child_candidate_status(Some("pending")));
2961 assert!(!is_dead_child_candidate_status(Some("suspended")));
2963 for status in ["completed", "error", "timeout", "cancelled", "skipped"] {
2965 assert!(!is_dead_child_candidate_status(Some(status)), "{status}");
2966 }
2967 }
2968
2969 #[test]
2970 fn completion_child_ownership_gates_content_fold() {
2971 assert!(completion_child_is_owned("parent-1", Some("parent-1")));
2973 assert!(!completion_child_is_owned("parent-1", Some("parent-2")));
2976 assert!(!completion_child_is_owned("parent-1", None));
2978 }
2979
2980 #[test]
2981 fn replay_child_prefers_error_like_for_first_error_policy() {
2982 let terminal = vec![
2983 ("c-ok".to_string(), "completed".to_string()),
2984 ("c-err".to_string(), "timeout".to_string()),
2985 ("c-late".to_string(), "completed".to_string()),
2986 ];
2987 let (id, status) = select_replay_child(&terminal).expect("non-empty");
2988 assert_eq!(id, "c-err");
2989 assert_eq!(status, "timeout");
2990
2991 let all_ok = vec![
2992 ("c-1".to_string(), "completed".to_string()),
2993 ("c-2".to_string(), "completed".to_string()),
2994 ];
2995 let (id, _) = select_replay_child(&all_ok).expect("non-empty");
2996 assert_eq!(id, "c-2");
2997
2998 assert!(select_replay_child(&[]).is_none());
2999 }
3000
3001 #[test]
3002 fn watchdog_resume_messages_are_hidden_runtime_messages() {
3003 for message in [
3004 empty_child_wait_message(),
3005 child_wait_lease_expired_message(&["c-1".to_string(), "c-2".to_string()]),
3006 ] {
3007 assert!(matches!(message.role, Role::User));
3008 let meta = message.metadata.expect("hidden runtime metadata");
3009 assert_eq!(meta[RUNTIME_RESUME_MESSAGE_HIDDEN_KEY], true);
3010 assert_eq!(
3011 meta[RUNTIME_RESUME_MESSAGE_KIND_KEY],
3012 "child_wait_watchdog_resume"
3013 );
3014 }
3015 let lease = child_wait_lease_expired_message(&["c-1".to_string()]);
3016 assert!(lease.content.contains("NOT cancelled"));
3018 assert!(lease.content.contains("c-1"));
3019 }
3020
3021 #[test]
3024 fn non_terminal_statuses_never_satisfy_wait_policies() {
3025 assert!(!is_terminal_child_status("suspended"));
3028 assert!(!is_terminal_child_status("running"));
3029 assert!(!is_terminal_child_status("pending"));
3030 }
3031
3032 fn make_completion(status: &str) -> ChildCompletion {
3033 ChildCompletion {
3034 parent_session_id: "parent-1".to_string(),
3035 child_session_id: "child-1".to_string(),
3036 status: status.to_string(),
3037 error: None,
3038 completed_at: Utc::now(),
3039 }
3040 }
3041
3042 #[test]
3043 fn oversized_child_outcome_is_bounded_and_keeps_full_content_identity() {
3044 let mut completion = make_completion("completed");
3045 let wait_registered_at = Utc::now();
3046 let huge = format!("prefix-A-{}", "x".repeat(300 * 1024));
3047 let presentation = runtime_resume_message(&completion, 0, Some(&huge));
3048 let first = child_completion_envelope(
3049 &completion,
3050 wait_registered_at,
3051 Some(huge.clone()),
3052 &presentation,
3053 );
3054 assert!(
3055 serde_json::to_vec(&first).unwrap().len()
3056 < bamboo_domain::SessionInboxLimits::default().max_payload_bytes
3057 );
3058 let SessionMessageBody::ChildOutcome(outcome) = &first.body else {
3059 panic!("typed child outcome");
3060 };
3061 let stored = outcome.result.as_deref().unwrap();
3062 assert!(stored.contains("sha256="));
3063 assert!(stored.contains("SubAgent.get"));
3064 assert!(stored.len() < CHILD_COMPLETION_INLINE_FIELD_BYTES);
3065
3066 completion.completed_at += chrono::Duration::seconds(1);
3069 let exact_retry = child_completion_envelope(
3070 &completion,
3071 wait_registered_at,
3072 Some(huge.clone()),
3073 &runtime_resume_message(&completion, 9, Some(&huge)),
3074 );
3075 assert_eq!(exact_retry.id, first.id);
3076 let changed = format!("prefix-B-{}", "x".repeat(300 * 1024));
3077 let corrected = child_completion_envelope(
3078 &completion,
3079 wait_registered_at,
3080 Some(changed.clone()),
3081 &runtime_resume_message(&completion, 0, Some(&changed)),
3082 );
3083 assert_ne!(corrected.id, first.id);
3084 }
3085
3086 #[tokio::test]
3087 async fn oversized_child_completion_clears_wait_and_activates_exactly_once() {
3088 let (_temp, store, inbox, coordinator, reservations, launches) =
3089 completion_inbox_fixture().await;
3090 let parent_id = "oversized-parent";
3091 let child_id = "oversized-child";
3092 let now = Utc::now();
3093 let mut parent = Session::new(parent_id, "model");
3094 let mut parent_runtime = AgentRuntimeState::new("waiting-run");
3095 parent_runtime.status = AgentStatusState::Suspended;
3096 parent_runtime.waiting_for_children = Some(WaitingForChildrenState::for_children(
3097 vec![child_id.to_string()],
3098 ChildWaitPolicy::All,
3099 now,
3100 ));
3101 parent_runtime.suspension = Some(SuspensionState {
3102 reason: "waiting_for_children".to_string(),
3103 suspended_at: now,
3104 resumable: true,
3105 hook_point: Some("ChildCompletion".to_string()),
3106 });
3107 write_runtime_state(&mut parent, &parent_runtime);
3108 parent.metadata.insert(
3109 "runtime.suspend_reason".to_string(),
3110 "waiting_for_children".to_string(),
3111 );
3112 store.save_session(&parent).await.unwrap();
3113
3114 let mut child = Session::new_child(child_id, parent_id, "model", "Child");
3115 child.add_message(Message::assistant("z".repeat(300 * 1024), None));
3116 child.set_last_run_status("completed");
3117 store.save_session(&child).await.unwrap();
3118 let completion = ChildCompletion {
3119 parent_session_id: parent_id.to_string(),
3120 child_session_id: child_id.to_string(),
3121 status: "completed".to_string(),
3122 error: None,
3123 completed_at: Utc::now(),
3124 };
3125
3126 ChildCompletionHandler::on_child_completed(coordinator.as_ref(), completion.clone()).await;
3127 let durable_parent = store.load_session(parent_id).await.unwrap().unwrap();
3128 let durable_runtime = read_runtime_state(&durable_parent);
3129 assert!(durable_runtime.waiting_for_children.is_none());
3130 assert_eq!(durable_runtime.status, AgentStatusState::Idle);
3131 assert!(!durable_parent
3132 .metadata
3133 .contains_key("runtime.suspend_reason"));
3134 let backlog = inbox.inspect(parent_id).await.unwrap();
3135 assert_eq!(backlog.pending + backlog.claimed, 1);
3136 assert!(backlog.activation_pending());
3137 assert_eq!(reservations.load(Ordering::SeqCst), 1);
3138 assert_eq!(launches.load(Ordering::SeqCst), 1);
3139
3140 let claim = inbox.claim(parent_id, 1).await.unwrap().remove(0);
3141 assert!(
3142 serde_json::to_vec(&claim.envelope).unwrap().len()
3143 < bamboo_domain::SessionInboxLimits::default().max_payload_bytes
3144 );
3145 ChildCompletionHandler::on_child_completed(coordinator.as_ref(), completion).await;
3148 assert_eq!(reservations.load(Ordering::SeqCst), 1);
3149 assert_eq!(launches.load(Ordering::SeqCst), 1);
3150 assert_eq!(inbox.inspect(parent_id).await.unwrap().claimed, 1);
3151 }
3152
3153 #[tokio::test]
3154 async fn latest_locked_activation_mutation_preserves_wait_armed_after_stale_load() {
3155 let temp = tempfile::tempdir().unwrap();
3156 let store = Arc::new(
3157 bamboo_storage::SessionStoreV2::new(temp.path().to_path_buf())
3158 .await
3159 .unwrap(),
3160 );
3161 let storage: Arc<dyn Storage> = store.clone();
3162 let locked = LockedSessionStore::new(storage);
3163 let mut session = Session::new("activation-stale-wait", "model");
3164 session.agent_runtime_state = Some(AgentRuntimeState::new("old-run"));
3165 store.save_session(&session).await.unwrap();
3166
3167 let stale = store.load_session(&session.id).await.unwrap().unwrap();
3169 assert!(read_runtime_state(&stale).waiting_for_children.is_none());
3170
3171 locked
3172 .update_runtime_config(&session.id, |latest| {
3173 let mut state = read_runtime_state(latest);
3174 state.status = AgentStatusState::Suspended;
3175 state.waiting_for_children = Some(WaitingForChildrenState::for_children(
3176 vec!["child-new".to_string()],
3177 ChildWaitPolicy::All,
3178 Utc::now(),
3179 ));
3180 state.suspension = Some(SuspensionState {
3181 reason: "waiting_for_children".to_string(),
3182 suspended_at: Utc::now(),
3183 resumable: true,
3184 hook_point: None,
3185 });
3186 write_runtime_state(latest, &state);
3187 latest.metadata.insert(
3188 "runtime.suspend_reason".to_string(),
3189 "waiting_for_children".to_string(),
3190 );
3191 })
3192 .await
3193 .unwrap();
3194
3195 let (prepared, ready) = prepare_session_inbox_activation(&locked, &session.id, false)
3196 .await
3197 .unwrap()
3198 .unwrap();
3199 assert!(!ready, "latest durable specific wait must block activation");
3200 let state = read_runtime_state(&prepared);
3201 assert!(state.waiting_for_children.is_some());
3202 assert_eq!(state.status, AgentStatusState::Suspended);
3203 assert_eq!(
3204 prepared
3205 .metadata
3206 .get("runtime.suspend_reason")
3207 .map(String::as_str),
3208 Some("waiting_for_children")
3209 );
3210 }
3211
3212 #[tokio::test]
3213 async fn partial_child_and_bash_backlogs_remain_inert_after_inbox_reopen() {
3214 let temp = tempfile::tempdir().unwrap();
3215 let store = Arc::new(
3216 bamboo_storage::SessionStoreV2::new(temp.path().to_path_buf())
3217 .await
3218 .unwrap(),
3219 );
3220 store
3221 .save_session(&Session::new("restart-child-parent", "model"))
3222 .await
3223 .unwrap();
3224 store
3225 .save_session(&Session::new("restart-bash-parent", "model"))
3226 .await
3227 .unwrap();
3228 let inbox = bamboo_storage::FileSessionInbox::new(
3229 store.clone(),
3230 bamboo_domain::SessionInboxLimits::default(),
3231 );
3232
3233 let completion = ChildCompletion {
3234 parent_session_id: "restart-child-parent".to_string(),
3235 child_session_id: "child-a".to_string(),
3236 status: "completed".to_string(),
3237 error: None,
3238 completed_at: Utc::now(),
3239 };
3240 let child_resume = runtime_resume_message(&completion, 1, Some("first child"));
3241 inbox
3242 .deliver(&child_completion_envelope(
3243 &completion,
3244 Utc::now(),
3245 Some("first child".to_string()),
3246 &child_resume,
3247 ))
3248 .await
3249 .unwrap();
3250 let bash = BashCompletionInfo {
3251 session_id: "restart-bash-parent".to_string(),
3252 bash_id: "bash-a".to_string(),
3253 command: "true".to_string(),
3254 exit_code: Some(0),
3255 status: "completed".to_string(),
3256 output_tail: String::new(),
3257 };
3258 inbox
3259 .deliver(&bash_completion_envelope(&bash))
3260 .await
3261 .unwrap();
3262
3263 let reopened = bamboo_storage::FileSessionInbox::new(
3264 store,
3265 bamboo_domain::SessionInboxLimits::default(),
3266 );
3267 for session_id in ["restart-child-parent", "restart-bash-parent"] {
3268 let backlog = reopened.inspect(session_id).await.unwrap();
3269 assert_eq!(backlog.pending, 1);
3270 assert_eq!(backlog.activation_generation, 0);
3271 assert!(
3272 !backlog.activation_pending(),
3273 "startup must not run a partial wait backlog for {session_id}"
3274 );
3275 }
3276 }
3277
3278 struct StubChildIndex {
3281 children: Vec<(String, Option<String>)>,
3282 }
3283
3284 #[async_trait]
3285 impl Storage for StubChildIndex {
3286 async fn save_session(&self, _session: &Session) -> std::io::Result<()> {
3287 Ok(())
3288 }
3289 async fn load_session(&self, _id: &str) -> std::io::Result<Option<Session>> {
3290 Ok(None)
3291 }
3292 async fn delete_session(&self, _id: &str) -> std::io::Result<bool> {
3293 Ok(false)
3294 }
3295 async fn list_child_run_statuses(
3296 &self,
3297 _parent_session_id: &str,
3298 ) -> std::io::Result<Vec<(String, Option<String>)>> {
3299 Ok(self.children.clone())
3300 }
3301 }
3302
3303 #[tokio::test]
3304 async fn derive_completed_only_includes_terminal_children() {
3305 let storage: Arc<dyn Storage> = Arc::new(StubChildIndex {
3306 children: vec![
3307 ("a".into(), Some("completed".into())),
3308 ("b".into(), Some("running".into())),
3309 ("c".into(), Some("error".into())),
3310 ("d".into(), None),
3311 ],
3312 });
3313 let completed = derive_completed_child_ids(&storage, "parent-1", "b").await;
3314 assert_eq!(
3316 completed,
3317 vec!["a".to_string(), "b".to_string(), "c".to_string()]
3318 );
3319 }
3320
3321 #[tokio::test]
3322 async fn derive_completed_folds_in_just_completed_when_index_lags() {
3323 let storage: Arc<dyn Storage> = Arc::new(StubChildIndex {
3325 children: vec![("only".into(), Some("running".into()))],
3326 });
3327 let completed = derive_completed_child_ids(&storage, "parent-1", "only").await;
3328 assert_eq!(completed, vec!["only".to_string()]);
3329 }
3330
3331 #[test]
3332 fn wait_policy_all_uses_derived_completed_set() {
3333 let waited = vec!["a".to_string(), "b".to_string()];
3334 assert!(!wait_policy_satisfied(
3335 ChildWaitPolicy::All,
3336 &waited,
3337 &["a".to_string()],
3338 "a",
3339 "completed"
3340 ));
3341 assert!(wait_policy_satisfied(
3342 ChildWaitPolicy::All,
3343 &waited,
3344 &["a".to_string(), "b".to_string()],
3345 "b",
3346 "completed"
3347 ));
3348 }
3349
3350 #[test]
3351 fn wait_policy_first_error_requires_tracked_membership() {
3352 let waited = vec!["a".to_string(), "b".to_string()];
3353 assert!(wait_policy_satisfied(
3355 ChildWaitPolicy::FirstError,
3356 &waited,
3357 &["a".to_string()],
3358 "a",
3359 "error"
3360 ));
3361 assert!(!wait_policy_satisfied(
3364 ChildWaitPolicy::FirstError,
3365 &waited,
3366 &["a".to_string()],
3367 "stray-child",
3368 "timeout"
3369 ));
3370 assert!(wait_policy_satisfied(
3372 ChildWaitPolicy::FirstError,
3373 &waited,
3374 &["a".to_string(), "b".to_string()],
3375 "stray-child",
3376 "completed"
3377 ));
3378 }
3379
3380 #[test]
3381 fn child_final_assistant_text_returns_last_assistant() {
3382 let mut session = Session::new("child-1", "gpt-4");
3383 session.messages.push(Message::user("hi"));
3384 session
3385 .messages
3386 .push(Message::assistant("first answer", None));
3387 session.messages.push(Message::user("again"));
3388 session
3389 .messages
3390 .push(Message::assistant("final answer", None));
3391
3392 assert_eq!(
3393 child_final_assistant_text(&session).as_deref(),
3394 Some("final answer")
3395 );
3396 }
3397
3398 #[test]
3399 fn child_final_assistant_text_returns_none_when_blank() {
3400 let mut session = Session::new("child-1", "gpt-4");
3401 session.messages.push(Message::assistant(" ", None));
3402 assert!(child_final_assistant_text(&session).is_none());
3403 }
3404
3405 #[test]
3406 fn child_final_assistant_text_returns_none_when_no_assistant() {
3407 let mut session = Session::new("child-1", "gpt-4");
3408 session.messages.push(Message::user("hi"));
3409 assert!(child_final_assistant_text(&session).is_none());
3410 }
3411
3412 #[test]
3413 fn runtime_resume_message_folds_full_response_without_truncation() {
3414 let completion = make_completion("completed");
3417 let long: String = "a".repeat(10_000);
3418 let message = runtime_resume_message(&completion, 0, Some(&long));
3419 assert!(message.content.contains(&long));
3420 assert!(!message.content.contains("truncated"));
3421 }
3422
3423 #[test]
3424 fn runtime_resume_message_includes_child_response_when_provided() {
3425 let completion = make_completion("completed");
3426 let message = runtime_resume_message(&completion, 0, Some("the answer is 42"));
3427
3428 assert!(matches!(message.role, Role::User));
3429 assert!(!message.never_compress);
3432 assert!(message.content.contains("Child final response:"));
3433 assert!(message.content.contains("the answer is 42"));
3434
3435 let metadata = message.metadata.expect("metadata present");
3436 assert_eq!(
3437 metadata.get("hidden_from_ui").and_then(|v| v.as_bool()),
3438 Some(true)
3439 );
3440 assert_eq!(
3441 metadata.get("runtime_kind").and_then(|v| v.as_str()),
3442 Some("child_completion_resume")
3443 );
3444 assert_eq!(
3445 metadata
3446 .get("child_final_response_included")
3447 .and_then(|v| v.as_bool()),
3448 Some(true)
3449 );
3450 }
3451
3452 #[test]
3453 fn runtime_resume_message_falls_back_to_error_when_no_response() {
3454 let mut completion = make_completion("error");
3455 completion.error = Some("boom".to_string());
3456
3457 let message = runtime_resume_message(&completion, 1, None);
3458 assert!(message.content.contains("Child error:"));
3459 assert!(message.content.contains("boom"));
3460 let metadata = message.metadata.expect("metadata present");
3461 assert_eq!(
3462 metadata
3463 .get("child_final_response_included")
3464 .and_then(|v| v.as_bool()),
3465 Some(false)
3466 );
3467 }
3468
3469 #[test]
3470 fn runtime_resume_message_minimal_when_no_response_and_no_error() {
3471 let completion = make_completion("completed");
3472 let message = runtime_resume_message(&completion, 2, None);
3473 assert!(!message.content.contains("Child final response:"));
3474 assert!(!message.content.contains("Child error:"));
3475 assert!(message.content.contains("Resume the parent task"));
3476 }
3477
3478 #[test]
3479 fn read_config_snapshot_refreshes_cached_snapshot_from_live_config() {
3480 let runtime = tokio::runtime::Runtime::new().expect("runtime");
3481
3482 runtime.block_on(async {
3483 let config = Arc::new(RwLock::new(Config::default()));
3484 config.write().await.provider = "copilot".to_string();
3485 let cached_config = StdRwLock::new(Config::default());
3486
3487 let snapshot = read_config_snapshot(&config, &cached_config);
3488
3489 assert_eq!(snapshot.provider, "copilot");
3490 assert_eq!(
3491 cached_config.read().expect("cached snapshot lock").provider,
3492 "copilot"
3493 );
3494 });
3495 }
3496
3497 #[test]
3498 fn read_config_snapshot_uses_cached_snapshot_when_live_lock_is_busy() {
3499 let runtime = tokio::runtime::Runtime::new().expect("runtime");
3500
3501 runtime.block_on(async {
3502 let mut cached_snapshot = Config::default();
3503 cached_snapshot.provider = "cached-provider".to_string();
3504
3505 let config = Arc::new(RwLock::new(Config::default()));
3506 let cached_config = StdRwLock::new(cached_snapshot);
3507 let _write_guard = config.write().await;
3508
3509 let snapshot = read_config_snapshot(&config, &cached_config);
3510
3511 assert_eq!(snapshot.provider, "cached-provider");
3512 });
3513 }
3514
3515 #[test]
3518 fn bash_completion_resume_message_normal_announces_completion() {
3519 let ids = vec!["bg-1".to_string(), "bg-2".to_string()];
3520 let message = bash_completion_resume_message(&ids, false);
3521 assert!(
3523 message.content.contains("have completed"),
3524 "normal resume message must announce completion: {}",
3525 message.content
3526 );
3527 let metadata = message.metadata.expect("metadata present");
3529 assert_eq!(
3530 metadata
3531 .get(RUNTIME_RESUME_MESSAGE_HIDDEN_KEY)
3532 .and_then(|v| v.as_bool()),
3533 Some(true),
3534 "resume message must be hidden from the UI"
3535 );
3536 assert_eq!(
3537 metadata
3538 .get(RUNTIME_RESUME_MESSAGE_KIND_KEY)
3539 .and_then(|v| v.as_str()),
3540 Some(BASH_COMPLETION_RESUME_KIND),
3541 "resume message must carry the bash-completion kind discriminant"
3542 );
3543 }
3544
3545 #[test]
3546 fn bash_completion_resume_message_deadline_does_not_claim_completion() {
3547 let ids = vec!["bg-long".to_string()];
3551 let message = bash_completion_resume_message(&ids, true);
3552 assert!(
3553 !message.content.contains("have completed"),
3554 "deadline resume message must NOT claim the shells completed: {}",
3555 message.content
3556 );
3557 assert!(
3558 message.content.contains("may still be running"),
3559 "deadline resume message must warn shells may still be running: {}",
3560 message.content
3561 );
3562 assert!(
3563 message.content.contains("BashOutput"),
3564 "deadline resume message must direct verification via BashOutput: {}",
3565 message.content
3566 );
3567 let metadata = message.metadata.expect("metadata present");
3569 assert_eq!(
3570 metadata
3571 .get(RUNTIME_RESUME_MESSAGE_KIND_KEY)
3572 .and_then(|v| v.as_str()),
3573 Some(BASH_COMPLETION_RESUME_KIND)
3574 );
3575 }
3576
3577 #[test]
3578 fn bash_resume_should_retry_matrix() {
3579 assert!(!bash_resume_should_retry(
3585 &ResumeOutcome::Started { run_id: "r".into() },
3586 true
3587 ));
3588 assert!(!bash_resume_should_retry(
3589 &ResumeOutcome::Started { run_id: "r".into() },
3590 false
3591 ));
3592
3593 assert!(!bash_resume_should_retry(&ResumeOutcome::NotFound, true));
3595 assert!(!bash_resume_should_retry(&ResumeOutcome::NotFound, false));
3596
3597 assert!(bash_resume_should_retry(&ResumeOutcome::Completed, true));
3599 assert!(!bash_resume_should_retry(&ResumeOutcome::Completed, false));
3602
3603 assert!(bash_resume_should_retry(
3606 &ResumeOutcome::AlreadyRunning { run_id: "r".into() },
3607 true
3608 ));
3609 assert!(!bash_resume_should_retry(
3611 &ResumeOutcome::AlreadyRunning { run_id: "r".into() },
3612 false
3613 ));
3614 }
3615
3616 #[test]
3619 fn injection_body_includes_status_exit_command_and_tail() {
3620 let info = BashCompletionInfo {
3621 session_id: "s".into(),
3622 bash_id: "abc123".into(),
3623 command: "make build".into(),
3624 exit_code: Some(0),
3625 status: "completed".into(),
3626 output_tail: "BUILD OK".into(),
3627 };
3628 let body = bash_completion_injection_body(&info);
3629 assert!(body.contains("abc123"), "body: {body}");
3630 assert!(body.contains("make build"), "body: {body}");
3631 assert!(body.contains("completed"), "body: {body}");
3632 assert!(body.contains("exit code 0"), "body: {body}");
3633 assert!(body.contains("BUILD OK"), "body: {body}");
3634 assert!(body.contains("BashOutput"), "body: {body}");
3636 assert!(body.contains("bash_id=\"abc123\""), "body: {body}");
3637 }
3638
3639 #[test]
3640 fn injection_body_handles_no_output_and_signal_kill() {
3641 let info = BashCompletionInfo {
3642 session_id: "s".into(),
3643 bash_id: "xyz".into(),
3644 command: "sleep 99".into(),
3645 exit_code: None,
3646 status: "killed".into(),
3647 output_tail: String::new(),
3648 };
3649 let body = bash_completion_injection_body(&info);
3650 assert!(body.contains("killed"), "body: {body}");
3651 assert!(body.contains("none (signal/killed)"), "body: {body}");
3652 assert!(body.contains("no captured output"), "body: {body}");
3653 assert!(!body.contains("Output tail:"), "body: {body}");
3655 }
3656
3657 #[test]
3658 fn background_completion_builds_post_tool_use_payload_and_feedback() {
3659 let mut info = BashCompletionInfo {
3660 session_id: "s".into(),
3661 bash_id: "bg-7".into(),
3662 command: "cargo test".into(),
3663 exit_code: Some(0),
3664 status: "completed".into(),
3665 output_tail: "test result: ok".into(),
3666 };
3667
3668 let payload = background_bash_post_tool_payload(&info);
3669 match payload {
3670 HookPayload::ToolResult {
3671 tool_name,
3672 tool_call_id,
3673 outcome,
3674 } => {
3675 assert_eq!(tool_name, "Bash");
3676 assert_eq!(tool_call_id, "bg-7");
3677 assert!(outcome.success);
3678 let response: serde_json::Value =
3679 serde_json::from_str(outcome.result.as_deref().unwrap()).unwrap();
3680 assert_eq!(response["command"], "cargo test");
3681 assert_eq!(response["exit_code"], 0);
3682 assert_eq!(response["status"], "completed");
3683 assert_eq!(response["output_tail"], "test result: ok");
3684 }
3685 other => panic!("expected PostToolUse payload, got {other:?}"),
3686 }
3687
3688 append_background_bash_hook_feedback(
3689 &mut info,
3690 vec!["Run the formatter before continuing".to_string()],
3691 );
3692 assert!(info.output_tail.contains("<post_tool_use_feedback>"));
3693 assert!(info
3694 .output_tail
3695 .contains("Run the formatter before continuing"));
3696 }
3697
3698 #[tokio::test]
3699 async fn bash_completion_payload_identity_matches_file_inbox_idempotency() {
3700 let baseline = BashCompletionInfo {
3701 session_id: "session".into(),
3702 bash_id: "bg-7".into(),
3703 command: "cargo test".into(),
3704 exit_code: Some(0),
3705 status: "completed".into(),
3706 output_tail: "first tail".into(),
3707 };
3708 let mut retried = baseline.clone();
3709 retried.output_tail = "first tail\nlater bytes\n<hook feedback>".into();
3710 retried.status = "completed-after-hook".into();
3711 let baseline_envelope = bash_completion_envelope(&baseline);
3712 let exact_retry = bash_completion_envelope(&baseline);
3713 let corrected_envelope = bash_completion_envelope(&retried);
3714 assert_eq!(baseline_envelope.id, exact_retry.id);
3715 assert_ne!(
3716 baseline_envelope.id, corrected_envelope.id,
3717 "changed payload semantics must receive a distinct id"
3718 );
3719
3720 let mut other_shell = baseline.clone();
3721 other_shell.bash_id = "bg-8".into();
3722 assert_ne!(
3723 bash_completion_envelope(&baseline).id,
3724 bash_completion_envelope(&other_shell).id
3725 );
3726
3727 let temp = tempfile::tempdir().unwrap();
3728 let store = Arc::new(
3729 bamboo_storage::SessionStoreV2::new(temp.path().to_path_buf())
3730 .await
3731 .unwrap(),
3732 );
3733 store
3734 .save_session(&Session::new("session", "model"))
3735 .await
3736 .unwrap();
3737 let inbox = bamboo_storage::FileSessionInbox::new(
3738 store,
3739 bamboo_domain::SessionInboxLimits::default(),
3740 );
3741 let first = inbox.deliver(&baseline_envelope).await.unwrap();
3742 let duplicate = inbox.deliver(&exact_retry).await.unwrap();
3743 let corrected = inbox.deliver(&corrected_envelope).await.unwrap();
3744 assert_eq!(duplicate, first, "exact payload retry is idempotent");
3745 assert_ne!(corrected.id, first.id);
3746 assert_eq!(corrected.generation, first.generation + 1);
3747 assert_eq!(inbox.inspect("session").await.unwrap().pending, 2);
3748 }
3749
3750 #[cfg(unix)]
3751 #[tokio::test]
3752 async fn background_completion_fires_configured_post_tool_use_command() {
3753 use bamboo_config::{
3754 LifecycleHookGroup, LifecycleHookHandler, LifecycleHooksConfig,
3755 DEFAULT_LIFECYCLE_HOOK_TIMEOUT_MS,
3756 };
3757
3758 let dir = tempfile::tempdir().unwrap();
3759 let output = dir.path().join("background-post-tool.json");
3760 let command = format!(
3761 "cat > '{}'; printf '%s' '{{\"additional_context\":\"inspect the completed build log\"}}'",
3762 output.display()
3763 );
3764 let config = LifecycleHooksConfig {
3765 enabled: true,
3766 post_tool_use: vec![LifecycleHookGroup {
3767 enabled: true,
3768 matcher: Some("^Bash$".to_string()),
3769 hooks: vec![LifecycleHookHandler::command(
3770 command,
3771 DEFAULT_LIFECYCLE_HOOK_TIMEOUT_MS,
3772 )],
3773 }],
3774 ..Default::default()
3775 };
3776 let mut session = Session::new("session-bg-hook", "test-model");
3777 session.workspace = Some(dir.path().to_string_lossy().into_owned());
3778 let mut info = BashCompletionInfo {
3779 session_id: session.id.clone(),
3780 bash_id: "bg-9".into(),
3781 command: "cargo test".into(),
3782 exit_code: Some(0),
3783 status: "completed".into(),
3784 output_tail: "test result: ok".into(),
3785 };
3786
3787 assert!(run_background_bash_post_tool_hooks(&config, None, &session, &mut info).await);
3788 let envelope: serde_json::Value =
3789 serde_json::from_str(&std::fs::read_to_string(output).unwrap()).unwrap();
3790 assert_eq!(envelope["hook_event_name"], "PostToolUse");
3791 assert_eq!(envelope["tool_name"], "Bash");
3792 assert_eq!(envelope["payload"]["tool_call_id"], "bg-9");
3793 let response = envelope["tool_response"]["result"]
3794 .as_str()
3795 .map(serde_json::from_str::<serde_json::Value>)
3796 .transpose()
3797 .unwrap()
3798 .unwrap();
3799 assert_eq!(response["command"], "cargo test");
3800 assert_eq!(response["status"], "completed");
3801 assert!(info.output_tail.contains("inspect the completed build log"));
3802 }
3803
3804 async fn temp_store() -> (tempfile::TempDir, Arc<dyn Storage>, LockedSessionStore) {
3805 let temp = tempfile::tempdir().unwrap();
3806 let storage: Arc<dyn Storage> = Arc::new(
3807 bamboo_storage::v2::SessionStoreV2::new(temp.path().to_path_buf())
3808 .await
3809 .expect("storage init"),
3810 );
3811 let persistence = LockedSessionStore::new(storage.clone());
3812 (temp, storage, persistence)
3813 }
3814
3815 #[tokio::test]
3816 async fn enqueue_writes_pending_injection_and_preserves_messages() {
3817 let (_temp, storage, persistence) = temp_store().await;
3818
3819 let mut session = Session::new("sess-enq", "test-model");
3820 session.add_message(Message::user("do the build"));
3821 storage.save_session(&session).await.unwrap();
3822
3823 let info = BashCompletionInfo {
3824 session_id: "sess-enq".into(),
3825 bash_id: "sh-1".into(),
3826 command: "make".into(),
3827 exit_code: Some(0),
3828 status: "completed".into(),
3829 output_tail: "done".into(),
3830 };
3831 let saved = enqueue_bash_completion_injection(&persistence, &info)
3832 .await
3833 .expect("enqueue io ok")
3834 .expect("session exists");
3835
3836 let pending = saved
3837 .pending_injected_messages()
3838 .expect("pending injection present");
3839 assert_eq!(pending.len(), 1);
3840 let content = pending[0].get("content").and_then(|v| v.as_str()).unwrap();
3841 assert!(content.contains("sh-1"), "content: {content}");
3842 assert!(content.contains("make"), "content: {content}");
3843 assert!(content.contains("done"), "content: {content}");
3844 assert_eq!(saved.messages.len(), 1);
3846 }
3847
3848 #[tokio::test]
3849 async fn enqueue_returns_none_for_missing_session() {
3850 let (_temp, _storage, persistence) = temp_store().await;
3851 let info = BashCompletionInfo {
3852 session_id: "does-not-exist".into(),
3853 bash_id: "x".into(),
3854 command: "true".into(),
3855 exit_code: Some(0),
3856 status: "completed".into(),
3857 output_tail: String::new(),
3858 };
3859 let result = enqueue_bash_completion_injection(&persistence, &info)
3860 .await
3861 .expect("io ok");
3862 assert!(result.is_none(), "no session → nothing enqueued");
3863 }
3864
3865 #[test]
3873 fn apply_bash_resume_transition_clears_wait_and_appends_message() {
3874 use bamboo_domain::session::runtime_state::WaitingForBashState;
3875
3876 let mut session = Session::new("sess-resume", "test-model");
3877 session.add_message(Message::user("kick off the build"));
3878 let mut rt = read_runtime_state(&session);
3879 rt.status = AgentStatusState::Running;
3880 rt.waiting_for_bash = Some(WaitingForBashState::for_bash(
3881 vec!["sh-1".into()],
3882 Utc::now(),
3883 ));
3884 write_runtime_state(&mut session, &rt);
3885 session.metadata.insert(
3886 "runtime.suspend_reason".to_string(),
3887 "waiting_for_bash".to_string(),
3888 );
3889
3890 let resume = bash_completion_resume_message(&["sh-1".to_string()], false);
3891 let did = apply_bash_resume_transition(&mut session, &resume);
3892
3893 assert!(did, "a suspended session must transition");
3894 let after = read_runtime_state(&session);
3895 assert!(
3896 after.waiting_for_bash.is_none(),
3897 "bash wait must be cleared"
3898 );
3899 assert_eq!(after.status, AgentStatusState::Idle, "runtime must be Idle");
3900 assert!(
3901 !session.metadata.contains_key("runtime.suspend_reason"),
3902 "suspend-reason marker must be removed"
3903 );
3904 assert_eq!(session.messages.len(), 2, "resume message must be appended");
3905 assert!(matches!(
3906 session.messages.last().map(|m| &m.role),
3907 Some(Role::User)
3908 ));
3909 }
3910
3911 #[test]
3915 fn apply_bash_resume_transition_noops_when_not_waiting() {
3916 let mut session = Session::new("sess-live", "test-model");
3917 session.add_message(Message::user("hi"));
3918
3919 let resume = bash_completion_resume_message(&["sh-1".to_string()], false);
3920 let did = apply_bash_resume_transition(&mut session, &resume);
3921
3922 assert!(!did, "a non-waiting session must not transition");
3923 assert_eq!(session.messages.len(), 1, "no resume message appended");
3924 }
3925
3926 #[test]
3930 fn bash_completion_should_resume_only_when_suspended_and_all_done() {
3931 assert!(bash_completion_should_resume(true, true));
3932 assert!(!bash_completion_should_resume(true, false)); assert!(!bash_completion_should_resume(false, true)); assert!(!bash_completion_should_resume(false, false));
3935 }
3936
3937 #[test]
3938 fn two_shell_delivery_stages_backlog_before_one_final_activation() {
3939 let mut waiting = true;
3940 let mut durable_backlog = 0;
3941 let mut reservations = 0;
3942
3943 durable_backlog += 1;
3945 let first = bash_completion_delivery_plan(waiting, false);
3946 assert_eq!(first, BashCompletionDeliveryPlan::DurableOnly);
3947 assert!(waiting);
3948 assert_eq!(durable_backlog, 1);
3949 assert_eq!(reservations, 0);
3950
3951 durable_backlog += 1;
3954 let last = bash_completion_delivery_plan(waiting, true);
3955 assert_eq!(last, BashCompletionDeliveryPlan::ClearWaitThenActivate);
3956 waiting = false;
3957 reservations += 1;
3958 assert!(!waiting);
3959 assert_eq!(durable_backlog, 2);
3960 assert_eq!(reservations, 1);
3961 }
3962
3963 #[test]
3967 fn bash_resume_message_from_info_carries_bashid_tail_and_kind() {
3968 let info = BashCompletionInfo {
3969 session_id: "s".into(),
3970 bash_id: "sh-42".into(),
3971 command: "cargo test".into(),
3972 exit_code: Some(0),
3973 status: "completed".into(),
3974 output_tail: "test result: ok".into(),
3975 };
3976 let msg = bash_resume_message_from_info(&info);
3977
3978 assert!(matches!(msg.role, Role::User));
3979 assert!(msg.content.contains("sh-42"), "content: {}", msg.content);
3980 assert!(
3981 msg.content.contains("cargo test"),
3982 "content: {}",
3983 msg.content
3984 );
3985 assert!(
3986 msg.content.contains("test result: ok"),
3987 "content: {}",
3988 msg.content
3989 );
3990 assert!(
3991 msg.content.contains("BashOutput"),
3992 "content: {}",
3993 msg.content
3994 );
3995 let meta = serde_json::to_string(&msg.metadata).unwrap();
3996 assert!(
3997 meta.contains(BASH_COMPLETION_RESUME_KIND),
3998 "resume message must be tagged as a bash-completion resume: {meta}"
3999 );
4000 }
4001}