1use std::collections::HashMap;
8use std::sync::{Arc, Mutex as StdMutex, OnceLock, RwLock as StdRwLock, Weak};
9use std::time::Duration;
10
11use bamboo_domain::poison::PoisonRecover;
12use bamboo_domain::{
13 AgentHookPoint, HookPayload, HookToolOutcome, SessionChildOutcome, SessionMessageBody,
14 SessionMessageContent, SessionMessageEnvelope, SessionMessageId, SessionMessageKind,
15 SessionMessageSource, SessionProviderMessage,
16};
17
18use crate::execution::{
19 create_event_forwarder, finalize_runner, reserve_runner_core, reserve_session_execution,
20 spawn_session_execution, AgentRunner, AgentStatus, ChildCompletion, ChildCompletionHandler,
21 ReserveOutcome, SessionExecutionArgs, SessionExecutionReservation,
22 SessionExecutionReserveOutcome, SpawnJob, SpawnScheduler,
23};
24use crate::runtime::config::{BashResumeHook, GuardianSpawner, BASH_COMPLETION_RESUME_KIND};
25use crate::runtime::guardian_state::{
26 parse_guardian_verdict, read_guardian_config, read_guardian_state, write_guardian_state,
27 GuardianVerdict,
28};
29use crate::Agent;
30use async_trait::async_trait;
31use bamboo_agent_core::storage::Storage;
32use bamboo_agent_core::tools::ToolExecutor;
33use bamboo_agent_core::{
34 AgentEvent, BashCompletionInfo, BashCompletionSink, Message, Role, Session, SessionKind,
35};
36use bamboo_domain::session::runtime_state::{
37 AgentRuntimeState, AgentStatusState, ChildWaitPolicy, SuspensionState, WaitingForChildrenState,
38};
39use bamboo_llm::{Config, ProviderModelRouter, ProviderRegistry};
40use bamboo_storage::LockedSessionStore;
41use chrono::Utc;
42use sha2::{Digest, Sha256};
43use tokio::sync::{broadcast, RwLock};
44
45use crate::model_areas::resolve_global_area_models;
46use crate::model_config_helper::{
47 resolve_fast_model, resolve_gold_config, GOLD_CONFIG_METADATA_KEY,
48};
49use crate::session_activation::{
50 SessionActivationLaunch, SessionActivationReserveOutcome, SessionActivationSpawner,
51};
52use crate::session_app::provider_model::session_effective_model_ref;
53use crate::session_app::resume::{
54 resume_session_execution, ResumeExecutionPort, ResumeSpawnRequest,
55};
56use crate::session_app::types::{ResumeConfigSnapshot, ResumeOutcome};
57
58const AGENT_RUNTIME_STATE_METADATA_KEY: &str = "agent.runtime.state";
59const RUNTIME_RESUME_MESSAGE_HIDDEN_KEY: &str = "hidden_from_ui";
60const RUNTIME_RESUME_MESSAGE_KIND_KEY: &str = "runtime_kind";
61const CHILD_COMPLETION_INLINE_FIELD_BYTES: usize = 48 * 1024;
62const CHILD_COMPLETION_OVERSIZE_TAIL_BYTES: usize = 8 * 1024;
63
64fn read_runtime_state(session: &Session) -> AgentRuntimeState {
65 session
66 .agent_runtime_state
67 .clone()
68 .or_else(|| {
69 session
70 .metadata
71 .get(AGENT_RUNTIME_STATE_METADATA_KEY)
72 .and_then(|raw| serde_json::from_str::<AgentRuntimeState>(raw).ok())
73 })
74 .unwrap_or_else(|| AgentRuntimeState::new(format!("{}-child-wait", session.id)))
75}
76
77fn write_runtime_state(session: &mut Session, runtime_state: &AgentRuntimeState) {
78 session.agent_runtime_state = Some(runtime_state.clone());
79 if let Ok(serialized) = serde_json::to_string(runtime_state) {
80 session
81 .metadata
82 .insert(AGENT_RUNTIME_STATE_METADATA_KEY.to_string(), serialized);
83 }
84}
85
86async fn prepare_session_inbox_activation(
92 persistence: &LockedSessionStore,
93 session_id: &str,
94 interrupt_specific_wait: bool,
95) -> std::io::Result<Option<(Session, bool)>> {
96 let ready = Arc::new(std::sync::atomic::AtomicBool::new(false));
97 let ready_for_mutation = ready.clone();
98 let saved = persistence
99 .update_runtime_config(session_id, move |latest| {
100 let mut runtime_state = read_runtime_state(latest);
101 let specifically_waiting = runtime_state.waiting_for_children.is_some()
102 || runtime_state.waiting_for_bash.is_some();
103 if specifically_waiting && !interrupt_specific_wait {
104 return;
105 }
106 runtime_state.status = AgentStatusState::Idle;
112 runtime_state.suspension = None;
113 write_runtime_state(latest, &runtime_state);
114 latest.metadata.remove("runtime.suspend_reason");
115 latest.updated_at = Utc::now();
116 ready_for_mutation.store(true, std::sync::atomic::Ordering::Release);
117 })
118 .await?;
119 Ok(saved.map(|session| (session, ready.load(std::sync::atomic::Ordering::Acquire))))
120}
121
122fn is_error_like(status: &str) -> bool {
123 matches!(status, "error" | "timeout" | "cancelled")
124}
125
126fn is_terminal_child_status(status: &str) -> bool {
128 matches!(
129 status,
130 "completed" | "error" | "timeout" | "cancelled" | "skipped"
131 )
132}
133
134fn child_completion_envelope(
135 completion: &ChildCompletion,
136 wait_registered_at: chrono::DateTime<Utc>,
137 result: Option<String>,
138 provider_message: &Message,
139) -> SessionMessageEnvelope {
140 fn bounded_terminal_field(
141 label: &str,
142 child_session_id: &str,
143 value: Option<String>,
144 ) -> (Option<String>, serde_json::Value, bool) {
145 let Some(value) = value else {
146 return (None, serde_json::Value::Null, false);
147 };
148 if value.len() <= CHILD_COMPLETION_INLINE_FIELD_BYTES {
149 return (Some(value.clone()), serde_json::Value::String(value), false);
150 }
151 let digest = hex::encode(Sha256::digest(value.as_bytes()));
152 let mut tail_start = value
153 .len()
154 .saturating_sub(CHILD_COMPLETION_OVERSIZE_TAIL_BYTES);
155 while tail_start < value.len() && !value.is_char_boundary(tail_start) {
156 tail_start += 1;
157 }
158 let tail = &value[tail_start..];
159 let summary = format!(
160 "Child {label} exceeded the durable inline limit ({} UTF-8 bytes, sha256={digest}). \
161 Retrieve the full child transcript with SubAgent.get(child_session_id=\"{child_session_id}\").\
162 \n\nBounded tail:\n{tail}",
163 value.len()
164 );
165 (
166 Some(summary),
167 serde_json::json!({
168 "oversized": true,
169 "utf8_bytes": value.len(),
170 "sha256": digest,
171 }),
172 true,
173 )
174 }
175
176 let (stored_result, result_identity, _result_oversized) =
177 bounded_terminal_field("result", &completion.child_session_id, result);
178 let (stored_error, error_identity, _error_oversized) = bounded_terminal_field(
179 "error",
180 &completion.child_session_id,
181 completion.error.clone(),
182 );
183 let mut bounded_provider_message = provider_message.clone();
184 let provider_oversized = serde_json::to_vec(provider_message)
185 .map(|bytes| bytes.len() > CHILD_COMPLETION_INLINE_FIELD_BYTES)
186 .unwrap_or(true);
187 if provider_oversized {
188 let mut content = format!(
189 "Runtime notification: child session `{}` finished with status `{}`.",
190 completion.child_session_id, completion.status
191 );
192 if let Some(result) = stored_result.as_deref() {
193 content.push_str("\n\n");
194 content.push_str(result);
195 }
196 if let Some(error) = stored_error.as_deref() {
197 content.push_str("\n\n");
198 content.push_str(error);
199 }
200 bounded_provider_message.content = content;
201 bounded_provider_message.content_parts = None;
202 }
203 let body = SessionMessageBody::ChildOutcome(SessionChildOutcome {
204 child_session_id: completion.child_session_id.clone(),
205 status: completion.status.clone(),
206 result: stored_result,
207 error: stored_error,
208 provider_message: Some(session_provider_message(&bounded_provider_message)),
209 });
210 let semantic = serde_json::json!({
211 "parent_session_id": completion.parent_session_id,
212 "child_session_id": completion.child_session_id,
213 "status": completion.status,
214 "error": error_identity,
215 "result": result_identity,
216 "wait_registered_at": wait_registered_at,
217 });
218 SessionMessageEnvelope {
219 id: SessionMessageId::stable("session_child_completion", &semantic),
220 source: SessionMessageSource::Runtime {
221 subsystem: "child_completion_coordinator".to_string(),
222 },
223 target_session_id: completion.parent_session_id.clone(),
224 kind: SessionMessageKind::ChildOutcome,
225 body,
226 created_at: completion.completed_at,
227 thread_id: None,
228 in_reply_to: None,
229 attempt: None,
230 correlation_id: Some(format!("child_completion:{}", completion.child_session_id)),
231 }
232}
233
234fn session_provider_message(message: &Message) -> SessionProviderMessage {
235 SessionProviderMessage {
236 content: SessionMessageContent {
237 text: message.content.clone(),
238 parts: message.content_parts.clone().unwrap_or_default(),
239 },
240 metadata: message
241 .metadata
242 .as_ref()
243 .and_then(serde_json::Value::as_object)
244 .cloned()
245 .unwrap_or_default(),
246 never_compress: message.never_compress,
247 }
248}
249
250async fn derive_completed_child_ids(
255 storage: &Arc<dyn Storage>,
256 parent_session_id: &str,
257 just_completed_child_id: &str,
258) -> Vec<String> {
259 let mut completed: Vec<String> = storage
260 .list_child_run_statuses(parent_session_id)
261 .await
262 .unwrap_or_default()
263 .into_iter()
264 .filter(|(_, status)| status.as_deref().is_some_and(is_terminal_child_status))
265 .map(|(id, _)| id)
266 .collect();
267 if !completed.iter().any(|id| id == just_completed_child_id) {
268 completed.push(just_completed_child_id.to_string());
269 }
270 completed.sort();
271 completed.dedup();
272 completed
273}
274
275fn read_config_snapshot(config: &Arc<RwLock<Config>>, cached_config: &StdRwLock<Config>) -> Config {
276 if let Ok(config_guard) = config.try_read() {
277 let snapshot = config_guard.clone();
278
279 if let Ok(mut cached_guard) = cached_config.try_write() {
280 *cached_guard = snapshot.clone();
281 }
282
283 snapshot
284 } else {
285 cached_config
286 .try_read()
287 .map(|guard| guard.clone())
288 .unwrap_or_default()
289 }
290}
291
292fn parent_locks() -> &'static std::sync::Mutex<HashMap<String, Arc<tokio::sync::Mutex<()>>>> {
309 static LOCKS: OnceLock<std::sync::Mutex<HashMap<String, Arc<tokio::sync::Mutex<()>>>>> =
310 OnceLock::new();
311 LOCKS.get_or_init(|| std::sync::Mutex::new(HashMap::new()))
312}
313
314fn session_resume_lock(session_id: &str) -> Arc<tokio::sync::Mutex<()>> {
321 let mut map = parent_locks().lock().recover_poison();
322 map.entry(session_id.to_string())
323 .or_insert_with(|| Arc::new(tokio::sync::Mutex::new(())))
324 .clone()
325}
326
327fn wait_policy_satisfied(
328 policy: ChildWaitPolicy,
329 wait_child_ids: &[String],
330 completed_child_ids: &[String],
331 latest_child_id: &str,
332 latest_status: &str,
333) -> bool {
334 if wait_child_ids.is_empty() {
335 return false;
336 }
337
338 match policy {
339 ChildWaitPolicy::All => wait_child_ids
340 .iter()
341 .all(|id| completed_child_ids.iter().any(|completed| completed == id)),
342 ChildWaitPolicy::Any => completed_child_ids
343 .iter()
344 .any(|id| wait_child_ids.iter().any(|wait_id| wait_id == id)),
345 ChildWaitPolicy::FirstError => {
346 (is_error_like(latest_status) && wait_child_ids.iter().any(|id| id == latest_child_id))
352 || wait_child_ids
353 .iter()
354 .all(|id| completed_child_ids.iter().any(|completed| completed == id))
355 }
356 }
357}
358
359fn child_final_assistant_text(child: &Session) -> Option<String> {
363 child
364 .messages
365 .iter()
366 .rev()
367 .find(|message| matches!(message.role, Role::Assistant))
368 .map(|message| message.content.clone())
369 .filter(|content| !content.trim().is_empty())
370}
371
372fn runtime_resume_message(
373 completion: &ChildCompletion,
374 remaining_children: usize,
375 child_final_response: Option<&str>,
376) -> Message {
377 let mut body = format!(
378 "Runtime notification: child session `{}` finished with status `{}`. Remaining child sessions: {}.",
379 completion.child_session_id, completion.status, remaining_children
380 );
381
382 let final_response = child_final_response.map(str::to_string);
388 if let Some(response) = final_response.as_deref() {
389 body.push_str("\n\nChild final response:\n");
390 body.push_str(response);
391 } else if let Some(error) = completion.error.as_deref() {
392 if !error.is_empty() {
393 body.push_str("\n\nChild error:\n");
394 body.push_str(error);
395 }
396 }
397
398 body.push_str(
399 "\n\nResume the parent task using this child result and continue from the previous plan. \
400 If you need the full child transcript, call SubAgent.get(child_session_id).",
401 );
402
403 let mut message = Message::user(body);
404 message.metadata = Some(serde_json::json!({
405 RUNTIME_RESUME_MESSAGE_HIDDEN_KEY: true,
406 RUNTIME_RESUME_MESSAGE_KIND_KEY: "child_completion_resume",
407 "child_session_id": completion.child_session_id,
408 "child_status": completion.status,
409 "child_error": completion.error,
410 "child_final_response_included": final_response.is_some(),
411 }));
412 message.never_compress = false;
416 message
417}
418
419fn guardian_resume_message(completion: &ChildCompletion, verdict: &GuardianVerdict) -> Message {
424 let mut body = if verdict.approve {
425 String::from(
426 "Guardian review APPROVED: an independent reviewer verified the work and found no blocking issues. You may finalize the task.",
427 )
428 } else {
429 String::from(
430 "Guardian review REJECTED: an independent reviewer found issues. Address every finding below before completing — do NOT declare the task complete until they are resolved.",
431 )
432 };
433 if let Some(summary) = verdict.summary.as_deref().filter(|s| !s.trim().is_empty()) {
434 body.push_str("\n\nReviewer summary: ");
435 body.push_str(summary);
436 }
437 if !verdict.findings.is_empty() {
438 body.push_str("\n\nFindings:");
439 for (idx, finding) in verdict.findings.iter().enumerate() {
440 body.push_str(&format!("\n{}. {}", idx + 1, finding));
441 }
442 }
443 body.push_str(
444 "\n\nIf you need the full guardian transcript, call SubAgent.get(child_session_id).",
445 );
446
447 let mut message = Message::user(body);
448 message.metadata = Some(serde_json::json!({
449 RUNTIME_RESUME_MESSAGE_HIDDEN_KEY: true,
450 RUNTIME_RESUME_MESSAGE_KIND_KEY: "guardian_review_resume",
451 "child_session_id": completion.child_session_id,
452 "child_status": completion.status,
453 "guardian_approved": verdict.approve,
454 }));
455 message.never_compress = false;
456 message
457}
458
459#[derive(Clone)]
460pub struct ChildCompletionCoordinator {
461 storage: Arc<dyn Storage>,
462 persistence: Arc<bamboo_storage::LockedSessionStore>,
463 sessions: crate::SessionCache,
464 agent_runners: Arc<RwLock<HashMap<String, AgentRunner>>>,
465 session_event_senders: Arc<RwLock<HashMap<String, broadcast::Sender<AgentEvent>>>>,
466 agent: Arc<Agent>,
467 config: Arc<RwLock<Config>>,
468 provider_registry: Arc<ProviderRegistry>,
469 provider_router: Arc<ProviderModelRouter>,
470 app_data_dir: std::path::PathBuf,
471 account_feed_inbox: Option<crate::execution::AccountFeedInbox>,
472 root_tools: Arc<RwLock<Option<Arc<dyn ToolExecutor>>>>,
473 guardian_spawner: Arc<RwLock<Option<Arc<dyn GuardianSpawner>>>>,
477 spawn_scheduler: Arc<RwLock<Weak<SpawnScheduler>>>,
481}
482
483impl ChildCompletionCoordinator {
484 #[allow(clippy::too_many_arguments)]
485 pub fn new(
486 storage: Arc<dyn Storage>,
487 persistence: Arc<LockedSessionStore>,
488 sessions: crate::SessionCache,
489 agent_runners: Arc<RwLock<HashMap<String, AgentRunner>>>,
490 session_event_senders: Arc<RwLock<HashMap<String, broadcast::Sender<AgentEvent>>>>,
491 agent: Arc<Agent>,
492 config: Arc<RwLock<Config>>,
493 provider_registry: Arc<ProviderRegistry>,
494 provider_router: Arc<ProviderModelRouter>,
495 app_data_dir: std::path::PathBuf,
496 account_feed_inbox: Option<crate::execution::AccountFeedInbox>,
497 ) -> Self {
498 Self {
499 storage,
500 persistence,
501 sessions,
502 agent_runners,
503 session_event_senders,
504 agent,
505 config,
506 provider_registry,
507 provider_router,
508 app_data_dir,
509 account_feed_inbox,
510 root_tools: Arc::new(RwLock::new(None)),
511 guardian_spawner: Arc::new(RwLock::new(None)),
512 spawn_scheduler: Arc::new(RwLock::new(Weak::new())),
513 }
514 }
515
516 pub async fn set_root_tools(&self, tools: Arc<dyn ToolExecutor>) {
517 *self.root_tools.write().await = Some(tools);
518 }
519
520 pub async fn set_spawn_scheduler(&self, scheduler: &Arc<SpawnScheduler>) {
521 *self.spawn_scheduler.write().await = Arc::downgrade(scheduler);
522 }
523
524 pub async fn set_guardian_spawner(&self, spawner: Arc<dyn GuardianSpawner>) {
527 *self.guardian_spawner.write().await = Some(spawner);
528 }
529
530 fn build_resume_config(
531 &self,
532 session: &Session,
533 config_snapshot: &Config,
534 ) -> ResumeConfigSnapshot {
535 crate::session_app::resolution::resolve_resume_config_snapshot(
536 config_snapshot,
537 &self.provider_registry,
538 session,
539 None,
540 )
541 }
542
543 async fn resume_parent(&self, parent_session_id: String) -> ResumeOutcome {
551 for attempt in 0..=5u8 {
552 if attempt > 0 {
553 tokio::time::sleep(Duration::from_millis(250 * attempt as u64)).await;
554 }
555
556 let Some(session) = self.load_session(&parent_session_id).await else {
557 tracing::warn!(%parent_session_id, "cannot resume parent after child completion: session not found");
558 return ResumeOutcome::NotFound;
559 };
560 let config_snapshot = self.config.read().await.clone();
561 let resume_config = self.build_resume_config(&session, &config_snapshot);
562 let outcome = resume_session_execution(self, &parent_session_id, resume_config).await;
563 tracing::info!(
564 %parent_session_id,
565 attempt,
566 outcome = outcome.as_str(),
567 "child completion requested parent resume"
568 );
569
570 if !matches!(outcome, ResumeOutcome::AlreadyRunning { .. }) {
571 return outcome;
572 }
573 }
574 tracing::error!(
580 %parent_session_id,
581 "parent resume gave up after AlreadyRunning retry budget; \
582 relying on the child-wait watchdog backstop"
583 );
584 ResumeOutcome::AlreadyRunning {
585 run_id: String::new(),
586 }
587 }
588
589 async fn save_and_cache(&self, session: &mut Session) {
590 if let Err(error) = self.persistence.merge_save_runtime(session).await {
591 tracing::warn!(session_id = %session.id, %error, "failed to persist session");
592 }
593 self.sessions.insert(
594 session.id.clone(),
595 Arc::new(parking_lot::RwLock::new(session.clone())),
596 );
597 }
598}
599
600#[async_trait]
601impl ChildCompletionHandler for ChildCompletionCoordinator {
602 async fn on_child_completed(&self, completion: ChildCompletion) {
603 if !is_terminal_child_status(&completion.status) {
612 tracing::info!(
613 parent_session_id = %completion.parent_session_id,
614 child_session_id = %completion.child_session_id,
615 status = %completion.status,
616 "non-terminal child status; leaving the parent wait armed"
617 );
618 return;
619 }
620
621 let per_parent = session_resume_lock(&completion.parent_session_id);
626 let _per_parent_guard = per_parent.lock().await;
627
628 let Some(mut parent) = self.load_session(&completion.parent_session_id).await else {
629 tracing::warn!(
630 parent_session_id = %completion.parent_session_id,
631 child_session_id = %completion.child_session_id,
632 "child completion received for missing parent"
633 );
634 return;
635 };
636
637 let mut runtime_state = read_runtime_state(&parent);
643
644 let completed_child_ids = derive_completed_child_ids(
647 &self.storage,
648 &completion.parent_session_id,
649 &completion.child_session_id,
650 )
651 .await;
652
653 let mut should_resume = false;
654 let mut remaining_children = 0usize;
655 let active_wait = runtime_state.waiting_for_children.clone();
656 if let Some(wait) = active_wait.as_ref() {
657 remaining_children = wait
658 .child_session_ids
659 .iter()
660 .filter(|id| !completed_child_ids.iter().any(|completed| completed == *id))
661 .count();
662 should_resume = wait_policy_satisfied(
663 wait.wait_for,
664 &wait.child_session_ids,
665 &completed_child_ids,
666 &completion.child_session_id,
667 &completion.status,
668 );
669 }
670
671 let reported_child_owned = match self
683 .storage
684 .load_runtime_control_plane(&completion.child_session_id)
685 .await
686 {
687 Ok(Some(control_plane)) => completion_child_is_owned(
688 &completion.parent_session_id,
689 control_plane.parent_session_id.as_deref(),
690 ),
691 _ => false,
692 };
693
694 let loaded_child = if reported_child_owned {
699 match self
700 .storage
701 .load_session(&completion.child_session_id)
702 .await
703 {
704 Ok(child) => child,
705 Err(error) => {
706 tracing::warn!(
707 child_session_id = %completion.child_session_id,
708 %error,
709 "failed to load child session for runtime resume message"
710 );
711 None
712 }
713 }
714 } else {
715 tracing::warn!(
716 parent_session_id = %completion.parent_session_id,
717 child_session_id = %completion.child_session_id,
718 "completion child is not a child of this parent; resuming with a neutral \
719 message and NOT folding its content"
720 );
721 None
722 };
723
724 let child_final_response = loaded_child.as_ref().and_then(child_final_assistant_text);
725 let guardian_resume = if should_resume {
730 let reviewed_round = runtime_state.round.current_round;
731 loaded_child.as_ref().and_then(|child| {
732 if child.subagent_type().as_deref() != Some("guardian") {
733 return None;
734 }
735 let mut guardian_state = read_guardian_state(&parent)?;
736 if guardian_state.guardian_child_id.as_deref()
737 != Some(completion.child_session_id.as_str())
738 {
739 tracing::warn!(
742 parent_session_id = %completion.parent_session_id,
743 child_session_id = %completion.child_session_id,
744 expected = ?guardian_state.guardian_child_id,
745 "guardian completion does not match recorded guardian_child_id; using generic resume"
746 );
747 return None;
748 }
749 let verdict = child_final_assistant_text(child)
757 .and_then(|text| match parse_guardian_verdict(&text) {
758 Ok(verdict) => Some(verdict),
759 Err(error) => {
760 tracing::warn!(
761 child_session_id = %completion.child_session_id,
762 %error,
763 "guardian verdict unparseable; recording a synthetic reject"
764 );
765 None
766 }
767 })
768 .unwrap_or_else(|| {
769 GuardianVerdict::rejected(vec![
770 "The guardian reviewer did not return a usable verdict (it errored or \
771 emitted unparseable output); the work has NOT been independently \
772 verified."
773 .to_string(),
774 ])
775 });
776 let approved = verdict.approve;
777 let message = guardian_resume_message(&completion, &verdict);
778 guardian_state.record_verdict(verdict, reviewed_round);
779 write_guardian_state(&mut parent, guardian_state);
780 tracing::info!(
781 parent_session_id = %completion.parent_session_id,
782 child_session_id = %completion.child_session_id,
783 approved,
784 "guardian verdict recorded; resuming parent"
785 );
786 Some(message)
787 })
788 } else {
789 None
790 };
791 let resume_message = guardian_resume.unwrap_or_else(|| {
792 runtime_resume_message(
793 &completion,
794 remaining_children,
795 child_final_response.as_deref(),
796 )
797 });
798
799 let messenger = self.agent.session_messenger().cloned();
804 let child_admission =
805 if let (Some(wait), Some(messenger)) = (active_wait.as_ref(), messenger.as_ref()) {
806 let envelope = child_completion_envelope(
807 &completion,
808 wait.registered_at,
809 child_final_response,
810 &resume_message,
811 );
812 match messenger.admit(envelope).await {
813 Ok(admission) => Some(admission),
814 Err(error) => {
815 tracing::warn!(
816 parent_session_id = %completion.parent_session_id,
817 child_session_id = %completion.child_session_id,
818 %error,
819 "child outcome SessionInbox admission failed; leaving parent wait armed"
820 );
821 return;
822 }
823 }
824 } else {
825 None
826 };
827
828 if should_resume {
829 if let (Some(messenger), Some(admission)) =
830 (messenger.as_ref(), child_admission.as_ref())
831 {
832 if let Err(error) = messenger.prepare_activation(admission).await {
833 tracing::warn!(
834 parent_session_id = %completion.parent_session_id,
835 child_session_id = %completion.child_session_id,
836 %error,
837 "child outcome activation watermark failed; leaving parent wait armed"
838 );
839 return;
840 }
841 }
842 }
843
844 if should_resume {
845 runtime_state.waiting_for_children = None;
846 runtime_state.status = AgentStatusState::Idle;
847 runtime_state.suspension = None;
848 parent.metadata.remove("runtime.suspend_reason");
849
850 if child_admission.is_none() {
851 parent.add_message(resume_message);
855 }
856 } else if runtime_state.waiting_for_children.is_some() {
857 runtime_state.status = AgentStatusState::Suspended;
858 runtime_state.suspension = Some(SuspensionState {
859 reason: "waiting_for_children".to_string(),
860 suspended_at: Utc::now(),
861 resumable: true,
862 hook_point: Some("ChildCompletion".to_string()),
863 });
864 }
865
866 parent.updated_at = Utc::now();
867 write_runtime_state(&mut parent, &runtime_state);
868 if let Err(error) = self
869 .persistence
870 .checkpoint_runtime_session(&mut parent)
871 .await
872 {
873 tracing::warn!(
874 parent_session_id = %completion.parent_session_id,
875 child_session_id = %completion.child_session_id,
876 %error,
877 "child outcome is durable but parent wait transition failed; leaving activation deferred"
878 );
879 return;
880 }
881 self.sessions.insert(
882 parent.id.clone(),
883 Arc::new(parking_lot::RwLock::new(parent.clone())),
884 );
885
886 let resume_parent_id = parent.id.clone();
891 drop(_per_parent_guard);
892
893 if should_resume {
894 if let (Some(messenger), Some(admission)) = (messenger, child_admission) {
895 if let Err(error) = messenger.activate_prepared(&admission).await {
896 tracing::warn!(
897 parent_session_id = %resume_parent_id,
898 %error,
899 "child outcome and wait transition are durable but activation failed"
900 );
901 }
902 } else {
903 self.resume_parent(resume_parent_id).await;
904 }
905 }
906 }
907}
908
909#[async_trait]
910impl ResumeExecutionPort for ChildCompletionCoordinator {
911 async fn load_session(&self, session_id: &str) -> Option<Session> {
912 match self.storage.load_session(session_id).await {
913 Ok(Some(session)) => Some(session),
914 Ok(None) => self
915 .sessions
916 .get(session_id)
917 .map(|e| e.value().clone())
918 .map(|arc| arc.read().clone()),
919 Err(error) => {
920 tracing::warn!(%session_id, %error, "failed to load session from storage");
921 self.sessions
922 .get(session_id)
923 .map(|e| e.value().clone())
924 .map(|arc| arc.read().clone())
925 }
926 }
927 }
928
929 async fn save_and_cache_session(&self, session: &mut Session) {
930 self.save_and_cache(session).await;
931 }
932
933 async fn reserve_session_execution(
934 &self,
935 session_id: &str,
936 event_sender: &broadcast::Sender<AgentEvent>,
937 ) -> SessionExecutionReserveOutcome {
938 reserve_session_execution(
939 &self.agent,
940 &self.agent_runners,
941 &self.session_event_senders,
942 session_id,
943 event_sender,
944 )
945 .await
946 }
947
948 async fn get_or_create_event_sender(&self, session_id: &str) -> broadcast::Sender<AgentEvent> {
949 crate::execution::session_events::get_or_create_event_sender(
950 &self.session_event_senders,
951 session_id,
952 )
953 .await
954 }
955
956 async fn spawn_resume_execution(&self, request: ResumeSpawnRequest) {
957 let ResumeSpawnRequest {
958 session_id,
959 session,
960 mut execution_reservation,
961 event_sender,
962 config,
963 } = request;
964 if let Err(error) = execution_reservation.ensure_registered().await {
965 tracing::warn!(
966 %session_id,
967 run_id = %execution_reservation.run_id(),
968 %error,
969 "cannot resume after child completion without exact router ownership"
970 );
971 return;
972 }
973
974 let Some(root_tools) = self.root_tools.read().await.clone() else {
975 tracing::error!(%session_id, "cannot resume parent after child completion: root tool surface is not initialized");
976 return;
977 };
978
979 let model = session.model.clone();
980 let resolved_provider_name = session_effective_model_ref(&session)
981 .map(|model_ref| model_ref.provider)
982 .unwrap_or(config.provider_name);
983 let provider_override = session_effective_model_ref(&session)
984 .and_then(|model_ref| match self.provider_router.route(&model_ref) {
985 Ok(provider) => Some(provider),
986 Err(error) => {
987 tracing::warn!(
988 session_id = %session_id,
989 provider = %model_ref.provider,
990 model = %model_ref.model,
991 error = %error,
992 "failed to resolve provider override for child-completion parent resume; falling back to runtime provider"
993 );
994 None
995 }
996 });
997 let config_snapshot = self.config.read().await.clone();
998 let resolved_fast_provider = resolve_fast_model(
999 &config_snapshot,
1000 &resolved_provider_name,
1001 &self.provider_registry,
1002 )
1003 .map(|model| model.provider);
1004 let reasoning_effort = session.reasoning_effort;
1005 let reasoning_effort_source = session
1006 .metadata
1007 .get("reasoning_effort_source")
1008 .cloned()
1009 .unwrap_or_default();
1010 let gold_config = resolve_gold_config(
1011 &config_snapshot,
1012 session
1013 .metadata
1014 .get(GOLD_CONFIG_METADATA_KEY)
1015 .map(String::as_str),
1016 )
1017 .or(config.gold_config.clone());
1018
1019 let (mpsc_tx, _forwarder) = create_event_forwarder(
1020 session_id.clone(),
1021 event_sender,
1022 self.agent_runners.clone(),
1023 self.account_feed_inbox.clone(),
1024 );
1025
1026 let config_handle = self.config.clone();
1027 let cached_config = Arc::new(StdRwLock::new(config_snapshot.clone()));
1028 let provider_registry = self.provider_registry.clone();
1029 let provider_name_for_aux = resolved_provider_name.clone();
1030 let auxiliary_model_resolver = std::sync::Arc::new(move || {
1031 let config_snapshot = read_config_snapshot(&config_handle, cached_config.as_ref());
1032 let areas = resolve_global_area_models(
1034 &config_snapshot,
1035 &provider_name_for_aux,
1036 &provider_registry,
1037 );
1038 crate::AuxiliaryModelConfig {
1039 fast_model_name: areas.fast.as_ref().map(|m| m.model_name.clone()),
1040 fast_model_provider: areas.fast.map(|m| m.provider),
1041 background_model_name: areas.background.as_ref().map(|m| m.model_name.clone()),
1042 planning_model_name: None,
1043 search_model_name: None,
1044 summarization_model_name: areas
1045 .summarization
1046 .as_ref()
1047 .map(|m| m.model_name.clone()),
1048 background_model_provider: areas.background.map(|m| m.provider),
1049 summarization_model_provider: areas.summarization.map(|m| m.provider),
1050 }
1051 });
1052 let model_roster = crate::ModelRoster {
1053 model: Some(model),
1054 provider_name: Some(resolved_provider_name),
1055 provider_type: config.provider_type.clone(),
1056 fast: crate::RoleModel::from_parts(config.fast_model, resolved_fast_provider),
1057 background: crate::RoleModel::from_parts(
1058 config.background_model,
1059 config.background_model_provider,
1060 ),
1061 summarization: crate::RoleModel::from_parts(
1062 config.summarization_model,
1063 config.summarization_model_provider,
1064 ),
1065 };
1066
1067 let guardian_config = read_guardian_config(&session);
1072 let guardian_spawner = self.guardian_spawner.read().await.clone();
1073
1074 spawn_session_execution(SessionExecutionArgs {
1075 agent: self.agent.clone(),
1076 session_id,
1077 session,
1078 execution_reservation,
1079 tools_override: Some(root_tools),
1080 provider_override,
1081 model_roster,
1082 reasoning_effort,
1083 reasoning_effort_source,
1084 auxiliary_model_resolver: Some(auxiliary_model_resolver),
1085 disabled_filter_resolver: None,
1088 disabled_tools: Some(config.disabled_tools),
1089 disabled_skill_ids: Some(config.disabled_skill_ids),
1090 selected_skill_ids: None,
1091 selected_skill_mode: None,
1092 mpsc_tx,
1093 image_fallback: config.image_fallback,
1094 gold_config,
1095 guardian_config,
1096 guardian_spawner,
1097 bash_resume_hook: {
1098 let hook: Arc<dyn BashResumeHook> = Arc::new(self.clone());
1099 Some(hook)
1100 },
1101 bash_completion_sink: {
1102 let sink: Arc<dyn BashCompletionSink> = Arc::new(self.clone());
1105 Some(sink)
1106 },
1107 app_data_dir: Some(self.app_data_dir.clone()),
1108 run_budget: None,
1111 runners: self.agent_runners.clone(),
1112 sessions_cache: self.sessions.clone(),
1113 on_complete: None,
1114 child_completion_handler: Some(Arc::new(self.clone())),
1118 });
1119 }
1120}
1121
1122#[async_trait]
1127impl SessionActivationSpawner for ChildCompletionCoordinator {
1128 async fn reserve_activation(
1129 &self,
1130 target_session_id: &str,
1131 _inbox_generation: u64,
1132 ) -> Result<SessionActivationReserveOutcome, bamboo_domain::SessionActivationError> {
1133 let Some(inbox) = self.agent.session_inbox() else {
1134 return Err(bamboo_domain::SessionActivationError::Internal(
1135 "agent runtime has no SessionInbox".to_string(),
1136 ));
1137 };
1138 let backlog = inbox
1139 .inspect(target_session_id)
1140 .await
1141 .map_err(|error| bamboo_domain::SessionActivationError::Internal(error.to_string()))?;
1142 if !backlog.activation_pending() {
1143 return Ok(SessionActivationReserveOutcome::NoWork);
1144 }
1145 let prepared = prepare_session_inbox_activation(
1148 &self.persistence,
1149 target_session_id,
1150 backlog.interrupt_pending(),
1151 )
1152 .await
1153 .map_err(|error| {
1154 bamboo_domain::SessionActivationError::Internal(format!(
1155 "persist resumable SessionInbox target: {error}"
1156 ))
1157 })?;
1158 let Some((session, ready)) = prepared else {
1159 return Ok(SessionActivationReserveOutcome::NotFound);
1160 };
1161 if !ready {
1162 tracing::info!(
1163 session_id = target_session_id,
1164 "SessionInbox backlog is activation-eligible but a specific durable wait remains armed"
1165 );
1166 return Ok(SessionActivationReserveOutcome::NoWork);
1167 }
1168
1169 enum LaunchPlan {
1170 Root(Box<ResumeConfigSnapshot>),
1171 Child {
1172 scheduler: Arc<SpawnScheduler>,
1173 parent_session_id: String,
1174 model: String,
1175 disabled_tools: Option<Vec<String>>,
1176 },
1177 }
1178
1179 let launch_plan = match session.kind {
1182 SessionKind::Root => {
1183 if self.root_tools.read().await.is_none() {
1184 return Err(bamboo_domain::SessionActivationError::Internal(
1185 "root tool surface is not initialized".to_string(),
1186 ));
1187 }
1188 let config_snapshot = self.config.read().await.clone();
1189 LaunchPlan::Root(Box::new(
1190 self.build_resume_config(&session, &config_snapshot),
1191 ))
1192 }
1193 SessionKind::Child => {
1194 let scheduler = self.spawn_scheduler.read().await.upgrade().ok_or_else(|| {
1195 bamboo_domain::SessionActivationError::Internal(
1196 "child spawn scheduler is not initialized".to_string(),
1197 )
1198 })?;
1199 let parent_session_id = session
1200 .parent_session_id
1201 .as_deref()
1202 .map(str::trim)
1203 .filter(|id| !id.is_empty())
1204 .map(ToOwned::to_owned)
1205 .ok_or_else(|| {
1206 bamboo_domain::SessionActivationError::Internal(format!(
1207 "child SessionInbox target {target_session_id} has no parent owner"
1208 ))
1209 })?;
1210 let parent = self
1211 .storage
1212 .load_session(&parent_session_id)
1213 .await
1214 .map_err(|error| {
1215 bamboo_domain::SessionActivationError::Internal(format!(
1216 "load parent owner {parent_session_id}: {error}"
1217 ))
1218 })?
1219 .ok_or_else(|| {
1220 bamboo_domain::SessionActivationError::Internal(format!(
1221 "child SessionInbox parent owner {parent_session_id} disappeared"
1222 ))
1223 })?;
1224 let child_root = if session.root_session_id.trim().is_empty() {
1225 parent_session_id.as_str()
1226 } else {
1227 session.root_session_id.as_str()
1228 };
1229 let parent_root = if parent.root_session_id.trim().is_empty() {
1230 parent.id.as_str()
1231 } else {
1232 parent.root_session_id.as_str()
1233 };
1234 if child_root != parent_root {
1235 return Err(bamboo_domain::SessionActivationError::Internal(format!(
1236 "child SessionInbox target {target_session_id} does not share its parent owner's root"
1237 )));
1238 }
1239 let model = if session.model.trim().is_empty() {
1240 parent.model.clone()
1241 } else {
1242 session.model.clone()
1243 };
1244 if model.trim().is_empty() {
1245 return Err(bamboo_domain::SessionActivationError::Internal(format!(
1246 "child SessionInbox target {target_session_id} has no executable model"
1247 )));
1248 }
1249 let disabled_tools = match session.metadata.get("disabled_tools") {
1250 None => None,
1251 Some(raw) => {
1252 let tools = serde_json::from_str::<std::collections::BTreeSet<String>>(raw)
1253 .map_err(|error| {
1254 bamboo_domain::SessionActivationError::Internal(format!(
1255 "child SessionInbox target {target_session_id} has malformed disabled_tools: {error}"
1256 ))
1257 })?;
1258 (!tools.is_empty()).then(|| tools.into_iter().collect())
1259 }
1260 };
1261 LaunchPlan::Child {
1262 scheduler,
1263 parent_session_id,
1264 model,
1265 disabled_tools,
1266 }
1267 }
1268 };
1269 let event_sender =
1270 ResumeExecutionPort::get_or_create_event_sender(self, target_session_id).await;
1271
1272 let reservation = match reserve_runner_core(
1273 &self.agent_runners,
1274 &self.session_event_senders,
1275 target_session_id,
1276 &event_sender,
1277 )
1278 .await
1279 {
1280 ReserveOutcome::Reserved(reservation) => reservation,
1281 ReserveOutcome::AlreadyRunning(run_id) => {
1282 return Ok(SessionActivationReserveOutcome::AlreadyRunning { run_id });
1283 }
1284 };
1285 let run_id = reservation.run_id.clone();
1286 let launch = match launch_plan {
1287 LaunchPlan::Root(config) => {
1288 let execution_reservation =
1289 SessionExecutionReservation::from_activation_placeholder(
1290 target_session_id,
1291 reservation,
1292 self.agent
1293 .activation_router()
1294 .expect("SessionInbox activation requires an activation router")
1295 .clone(),
1296 self.agent_runners.clone(),
1297 );
1298 let reservation_cell = Arc::new(StdMutex::new(Some(execution_reservation)));
1302 let launch_reservation = reservation_cell.clone();
1303 let rollback_reservation = reservation_cell;
1304 let coordinator = self.clone();
1305 let launch_sessions = self.sessions.clone();
1306 let launch_session_id = session.id.clone();
1307 let launch_session = session.clone();
1308 let request_session_id = target_session_id.to_string();
1309 SessionActivationLaunch::new_with_async_rollback(
1310 run_id,
1311 move || {
1312 let mut execution_reservation = launch_reservation
1313 .lock()
1314 .unwrap_or_else(std::sync::PoisonError::into_inner)
1315 .take()
1316 .expect("activation reservation launches or rolls back exactly once");
1317 execution_reservation.mark_activation_published();
1318 launch_sessions.insert(
1322 launch_session_id,
1323 Arc::new(parking_lot::RwLock::new(launch_session)),
1324 );
1325 let request = ResumeSpawnRequest {
1326 session_id: request_session_id,
1327 session,
1328 execution_reservation,
1329 event_sender,
1330 config: *config,
1331 };
1332 tokio::spawn(async move {
1333 ResumeExecutionPort::spawn_resume_execution(&coordinator, request)
1334 .await;
1335 });
1336 },
1337 move || async move {
1338 let reservation = rollback_reservation
1339 .lock()
1340 .unwrap_or_else(std::sync::PoisonError::into_inner)
1341 .take();
1342 if let Some(reservation) = reservation {
1343 reservation.rollback_unpublished_activation().await;
1344 }
1345 },
1346 )
1347 }
1348 LaunchPlan::Child {
1349 scheduler,
1350 parent_session_id,
1351 model,
1352 disabled_tools,
1353 } => {
1354 let execution_reservation =
1355 SessionExecutionReservation::from_activation_placeholder(
1356 target_session_id,
1357 reservation,
1358 self.agent
1359 .activation_router()
1360 .expect("SessionInbox activation requires an activation router")
1361 .clone(),
1362 self.agent_runners.clone(),
1363 );
1364 let reservation_cell = Arc::new(StdMutex::new(Some(execution_reservation)));
1365 let launch_reservation = reservation_cell.clone();
1366 let rollback_reservation = reservation_cell;
1367 let job = SpawnJob {
1368 parent_session_id,
1369 child_session_id: target_session_id.to_string(),
1370 model,
1371 disabled_tools,
1372 };
1373 let launch_sessions = self.sessions.clone();
1374 let launch_session_id = session.id.clone();
1375 let launch_session = session;
1376 SessionActivationLaunch::new_with_async_rollback(
1377 run_id,
1378 move || {
1379 let mut execution_reservation = launch_reservation
1380 .lock()
1381 .unwrap_or_else(std::sync::PoisonError::into_inner)
1382 .take()
1383 .expect("activation reservation launches or rolls back exactly once");
1384 execution_reservation.mark_activation_published();
1385 launch_sessions.insert(
1388 launch_session_id,
1389 Arc::new(parking_lot::RwLock::new(launch_session)),
1390 );
1391 drop(scheduler.launch_reserved(job, execution_reservation));
1395 },
1396 move || async move {
1397 let reservation = rollback_reservation
1398 .lock()
1399 .unwrap_or_else(std::sync::PoisonError::into_inner)
1400 .take();
1401 if let Some(reservation) = reservation {
1402 reservation.rollback_unpublished_activation().await;
1403 }
1404 },
1405 )
1406 }
1407 };
1408 Ok(SessionActivationReserveOutcome::Reserved(launch))
1409 }
1410}
1411
1412fn bash_completion_resume_message(bash_ids: &[String], timed_out: bool) -> Message {
1422 let body = if timed_out {
1423 format!(
1424 "Runtime notification: the background-Bash wait ceiling was reached while one or more \
1425 shell(s) ({}) may still be running. The session is being resumed so it is not \
1426 stranded; verify their actual status with BashOutput before assuming completion.",
1427 bash_ids.join(", ")
1428 )
1429 } else {
1430 format!(
1431 "Runtime notification: all background Bash shell(s) ({}) have completed. \
1432 Review their output with BashOutput and resume the task from where you left off.",
1433 bash_ids.join(", ")
1434 )
1435 };
1436 let mut message = Message::user(body);
1437 message.metadata = Some(serde_json::json!({
1438 RUNTIME_RESUME_MESSAGE_HIDDEN_KEY: true,
1439 RUNTIME_RESUME_MESSAGE_KIND_KEY: BASH_COMPLETION_RESUME_KIND,
1440 }));
1441 message.never_compress = false;
1442 message
1443}
1444
1445fn bash_resume_should_retry(outcome: &ResumeOutcome, persisted_waiting_for_bash: bool) -> bool {
1458 match outcome {
1459 ResumeOutcome::Started { .. } | ResumeOutcome::NotFound => false,
1460 ResumeOutcome::Completed | ResumeOutcome::AlreadyRunning { .. } => {
1461 persisted_waiting_for_bash
1462 }
1463 }
1464}
1465
1466fn bash_completion_should_resume(
1474 loop_suspended_on_bash: bool,
1475 all_waited_shells_done: bool,
1476) -> bool {
1477 loop_suspended_on_bash && all_waited_shells_done
1478}
1479
1480#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1481enum BashCompletionDeliveryPlan {
1482 DurableOnly,
1486 Activate,
1488 ClearWaitThenActivate,
1490}
1491
1492fn bash_completion_delivery_plan(
1493 loop_suspended_on_bash: bool,
1494 all_waited_shells_done: bool,
1495) -> BashCompletionDeliveryPlan {
1496 if bash_completion_should_resume(loop_suspended_on_bash, all_waited_shells_done) {
1497 BashCompletionDeliveryPlan::ClearWaitThenActivate
1498 } else if loop_suspended_on_bash {
1499 BashCompletionDeliveryPlan::DurableOnly
1500 } else {
1501 BashCompletionDeliveryPlan::Activate
1502 }
1503}
1504
1505fn background_bash_post_tool_payload(info: &BashCompletionInfo) -> HookPayload {
1506 let success = info.status == "completed" && info.exit_code == Some(0);
1507 let response = serde_json::json!({
1508 "bash_id": info.bash_id,
1509 "command": info.command,
1510 "exit_code": info.exit_code,
1511 "status": info.status,
1512 "output_tail": info.output_tail,
1513 });
1514 HookPayload::ToolResult {
1515 tool_name: "Bash".to_string(),
1516 tool_call_id: info.bash_id.clone(),
1517 outcome: HookToolOutcome {
1518 success,
1519 result: success.then(|| response.to_string()),
1520 error: (!success).then(|| response.to_string()),
1521 needs_human: false,
1522 duration_ms: 0,
1523 },
1524 }
1525}
1526
1527fn append_background_bash_hook_feedback(info: &mut BashCompletionInfo, feedback: Vec<String>) {
1528 let feedback = feedback
1529 .into_iter()
1530 .map(|text| text.trim().to_string())
1531 .filter(|text| !text.is_empty())
1532 .collect::<Vec<_>>();
1533 if feedback.is_empty() {
1534 return;
1535 }
1536 if !info.output_tail.is_empty() {
1537 info.output_tail.push_str("\n\n");
1538 }
1539 info.output_tail.push_str("<post_tool_use_feedback>\n");
1540 info.output_tail.push_str(&feedback.join("\n"));
1541 info.output_tail.push_str("\n</post_tool_use_feedback>");
1542}
1543
1544async fn run_background_bash_post_tool_hooks(
1545 config: &bamboo_config::LifecycleHooksConfig,
1546 fallback_cwd: Option<std::path::PathBuf>,
1547 session: &Session,
1548 info: &mut BashCompletionInfo,
1549) -> bool {
1550 let runner = crate::HookRunner::new().with_lifecycle_config(config, fallback_cwd);
1551 if !runner.has_hooks_for(AgentHookPoint::AfterToolExecution) {
1552 return false;
1553 }
1554
1555 let mut runtime_state = session
1556 .agent_runtime_state
1557 .clone()
1558 .unwrap_or_else(|| AgentRuntimeState::new(&session.id));
1559 let outcome = runner
1560 .run_observer_hooks(
1561 AgentHookPoint::AfterToolExecution,
1562 &background_bash_post_tool_payload(info),
1563 session,
1564 &mut runtime_state,
1565 None,
1566 )
1567 .await;
1568 append_background_bash_hook_feedback(info, outcome.injected_contexts);
1569 true
1570}
1571
1572fn apply_bash_resume_transition(session: &mut Session, resume_message: &Message) -> bool {
1580 let mut runtime_state = read_runtime_state(session);
1581 if runtime_state.waiting_for_bash.is_none() {
1582 return false;
1583 }
1584 runtime_state.waiting_for_bash = None;
1585 runtime_state.status = AgentStatusState::Idle;
1586 runtime_state.suspension = None;
1587 write_runtime_state(session, &runtime_state);
1588 session.metadata.remove("runtime.suspend_reason");
1589 session.add_message(resume_message.clone());
1590 true
1591}
1592
1593impl ChildCompletionCoordinator {
1595 async fn bash_self_resume(&self, session_id: String, bash_ids: Vec<String>) {
1611 let mut delay = Duration::from_secs(1);
1612 let max_delay = Duration::from_secs(30);
1613 let max_poll = Duration::from_secs(6 * 3600 + 600);
1617 let deadline = tokio::time::Instant::now() + max_poll;
1618
1619 loop {
1620 tokio::time::sleep(delay).await;
1621
1622 let Some(session) = self.load_session(&session_id).await else {
1623 tracing::info!(%session_id, "bash self-resume backstop: session gone; nothing to do");
1624 return;
1625 };
1626 if read_runtime_state(&session).waiting_for_bash.is_none() {
1627 return;
1630 }
1631
1632 let still_running =
1633 bamboo_tools::tools::bash_runtime::running_shells_for_session(&session_id);
1634 let timed_out = tokio::time::Instant::now() >= deadline;
1635 if still_running.is_empty() || timed_out {
1636 let guard = session_resume_lock(&session_id);
1641 let _held = guard.lock().await;
1642 tracing::warn!(
1643 %session_id,
1644 shell_count = bash_ids.len(),
1645 timed_out,
1646 "bash self-resume backstop engaged (push lost or wait ceiling reached)"
1647 );
1648 self.perform_bash_resume(
1649 &session_id,
1650 bash_completion_resume_message(&bash_ids, timed_out),
1651 )
1652 .await;
1653 return;
1654 }
1655
1656 delay = (delay * 2).min(max_delay);
1657 }
1658 }
1659
1660 async fn perform_bash_resume(&self, session_id: &str, resume_message: Message) {
1681 let retry_backoff = Duration::from_millis(200);
1682 const MAX_RESUME_ATTEMPTS: u8 = 5;
1683 for attempt in 0..MAX_RESUME_ATTEMPTS {
1684 if attempt > 0 {
1685 tokio::time::sleep(retry_backoff).await;
1686 }
1687
1688 let Some(mut session) = self.load_session(session_id).await else {
1689 tracing::warn!(%session_id, "bash resume: session not found; nothing to resume");
1690 return;
1691 };
1692
1693 if !apply_bash_resume_transition(&mut session, &resume_message) {
1694 tracing::info!(
1698 %session_id, attempt,
1699 "bash resume: persisted bash wait already cleared; nothing to resume"
1700 );
1701 return;
1702 }
1703 session.updated_at = Utc::now();
1704 self.save_and_cache(&mut session).await;
1705 tracing::info!(
1706 %session_id, attempt,
1707 "bash resume: cleared bash wait and appended resume message"
1708 );
1709
1710 let outcome = self.resume_parent(session_id.to_string()).await;
1711 match outcome {
1712 ResumeOutcome::Started { .. } => {
1713 tracing::info!(%session_id, attempt, "bash resume: resume fired");
1714 return;
1715 }
1716 ResumeOutcome::NotFound => {
1717 tracing::warn!(%session_id, "bash resume: session vanished during resume");
1718 return;
1719 }
1720 _ => {
1721 let clobbered = match self.load_session(session_id).await {
1727 Some(reloaded) => read_runtime_state(&reloaded).waiting_for_bash.is_some(),
1728 None => {
1729 tracing::warn!(%session_id, "bash resume: session vanished after resume");
1730 return;
1731 }
1732 };
1733 if bash_resume_should_retry(&outcome, clobbered) {
1734 tracing::warn!(
1735 %session_id, attempt,
1736 outcome = outcome.as_str(),
1737 "bash resume: persisted wait still set after resume (finalize-clobber); retrying"
1738 );
1739 continue;
1740 }
1741 tracing::info!(
1742 %session_id, attempt,
1743 outcome = outcome.as_str(),
1744 "bash resume: wait cleared and resume handled; stopping"
1745 );
1746 return;
1747 }
1748 }
1749 }
1750
1751 tracing::warn!(
1752 %session_id,
1753 attempts = MAX_RESUME_ATTEMPTS,
1754 "bash resume: exhausted clobber-retry budget without confirming resume; giving up"
1755 );
1756 }
1757}
1758
1759impl BashResumeHook for ChildCompletionCoordinator {
1760 fn arrange_bash_self_resume(&self, session_id: String, bash_ids: Vec<String>) {
1761 let coordinator = Arc::new(self.clone());
1762 tokio::spawn(async move {
1763 coordinator.bash_self_resume(session_id, bash_ids).await;
1764 });
1765 }
1766}
1767
1768fn bash_completion_injection_body(info: &BashCompletionInfo) -> String {
1773 let exit = match info.exit_code {
1774 Some(code) => code.to_string(),
1775 None => "none (signal/killed)".to_string(),
1776 };
1777 let mut body = format!(
1778 "Runtime notification: background shell `{}` (`{}`) finished — status {}, exit code {}.",
1779 info.bash_id, info.command, info.status, exit
1780 );
1781 if info.output_tail.trim().is_empty() {
1782 body.push_str(" It produced no captured output.");
1783 } else {
1784 body.push_str("\n\nOutput tail:\n");
1785 body.push_str(&info.output_tail);
1786 }
1787 body.push_str(&format!(
1788 "\n\nUse BashOutput with bash_id=\"{}\" for the full output, then continue the task.",
1789 info.bash_id
1790 ));
1791 body
1792}
1793
1794fn bash_completion_envelope(info: &BashCompletionInfo) -> bamboo_domain::SessionMessageEnvelope {
1795 let provider_message = bash_resume_message_from_info(info);
1796 let data = serde_json::json!({
1797 "session_id": info.session_id,
1798 "bash_id": info.bash_id,
1799 "command": info.command,
1800 "exit_code": info.exit_code,
1801 "status": info.status,
1802 "output_tail": info.output_tail,
1803 });
1804 let identity = data.clone();
1809 bamboo_domain::SessionMessageEnvelope {
1810 id: bamboo_domain::SessionMessageId::stable("background_bash_completion", &identity),
1811 source: bamboo_domain::SessionMessageSource::Runtime {
1812 subsystem: "background_bash".to_string(),
1813 },
1814 target_session_id: info.session_id.clone(),
1815 kind: bamboo_domain::SessionMessageKind::RuntimeInstruction,
1816 body: bamboo_domain::SessionMessageBody::RuntimeInstruction(
1817 bamboo_domain::SessionRuntimeInstruction {
1818 instruction: "background_bash_completed".to_string(),
1819 content: Some(bamboo_domain::SessionMessageContent::text(
1820 bash_completion_injection_body(info),
1821 )),
1822 data: Some(data),
1823 provider_message: Some(session_provider_message(&provider_message)),
1824 },
1825 ),
1826 created_at: Utc::now(),
1827 thread_id: None,
1828 in_reply_to: None,
1829 attempt: None,
1830 correlation_id: Some(info.bash_id.clone()),
1831 }
1832}
1833
1834async fn enqueue_bash_completion_injection(
1840 persistence: &LockedSessionStore,
1841 info: &BashCompletionInfo,
1842) -> std::io::Result<Option<Session>> {
1843 let body = bash_completion_injection_body(info);
1844 let queued = serde_json::json!({
1845 "content": body,
1846 "created_at": Utc::now(),
1847 });
1848 persistence
1849 .update_runtime_config(&info.session_id, move |session| {
1850 let mut pending = session.pending_injected_messages().unwrap_or_default();
1851 pending.push(queued);
1852 session.set_pending_injected_messages(pending);
1853 })
1854 .await
1855}
1856
1857fn bash_resume_message_from_info(info: &BashCompletionInfo) -> Message {
1865 let mut message = Message::user(bash_completion_injection_body(info));
1866 message.metadata = Some(serde_json::json!({
1867 RUNTIME_RESUME_MESSAGE_HIDDEN_KEY: true,
1868 RUNTIME_RESUME_MESSAGE_KIND_KEY: BASH_COMPLETION_RESUME_KIND,
1869 }));
1870 message.never_compress = false;
1871 message
1872}
1873
1874impl ChildCompletionCoordinator {
1875 async fn deliver_bash_completion(&self, mut info: BashCompletionInfo) {
1887 if let Some(hook_session) = self.load_session(&info.session_id).await {
1893 let config_snapshot = self.config.read().await.clone();
1894 let _ = run_background_bash_post_tool_hooks(
1895 &config_snapshot.lifecycle_hooks,
1896 Some(self.app_data_dir.clone()),
1897 &hook_session,
1898 &mut info,
1899 )
1900 .await;
1901 }
1902
1903 let guard = session_resume_lock(&info.session_id);
1904 let _held = guard.lock().await;
1905
1906 let Some(session) = self.load_session(&info.session_id).await else {
1907 tracing::warn!(
1908 session_id = %info.session_id,
1909 bash_id = %info.bash_id,
1910 "background bash completion: owning session not found; nothing to notify"
1911 );
1912 return;
1913 };
1914
1915 let waiting = read_runtime_state(&session).waiting_for_bash.is_some();
1916 let all_shells_done =
1922 bamboo_tools::tools::bash_runtime::running_shells_for_session(&info.session_id)
1923 .is_empty();
1924 let delivery_plan = bash_completion_delivery_plan(waiting, all_shells_done);
1925
1926 let messenger = self.agent.session_messenger().cloned();
1932 let admission = match messenger.as_ref() {
1933 Some(messenger) => match messenger.admit(bash_completion_envelope(&info)).await {
1934 Ok(admission) => Some(admission),
1935 Err(error) => {
1936 tracing::warn!(
1937 session_id = %info.session_id,
1938 bash_id = %info.bash_id,
1939 %error,
1940 "background bash completion: durable SessionInbox admission failed; leaving wait armed"
1941 );
1942 return;
1943 }
1944 },
1945 None => {
1946 tracing::warn!(
1947 session_id = %info.session_id,
1948 bash_id = %info.bash_id,
1949 "SessionMessenger unavailable; using compatibility injection before clearing wait"
1950 );
1951 match enqueue_bash_completion_injection(&self.persistence, &info).await {
1952 Ok(Some(_)) => None,
1953 Ok(None) => {
1954 tracing::warn!(
1955 session_id = %info.session_id,
1956 "background bash compatibility target disappeared"
1957 );
1958 return;
1959 }
1960 Err(error) => {
1961 tracing::warn!(
1962 session_id = %info.session_id,
1963 %error,
1964 "background bash compatibility admission failed; leaving wait armed"
1965 );
1966 return;
1967 }
1968 }
1969 }
1970 };
1971
1972 if delivery_plan == BashCompletionDeliveryPlan::DurableOnly {
1973 tracing::info!(
1974 session_id = %info.session_id,
1975 bash_id = %info.bash_id,
1976 "background bash completion is durable; sibling shells still run, so wait remains armed"
1977 );
1978 return;
1979 }
1980
1981 if let (Some(messenger), Some(admission)) = (messenger.as_ref(), admission.as_ref()) {
1982 if let Err(error) = messenger.prepare_activation(admission).await {
1983 tracing::warn!(
1984 session_id = %info.session_id,
1985 bash_id = %info.bash_id,
1986 %error,
1987 "background bash completion activation watermark failed; leaving wait armed"
1988 );
1989 return;
1990 }
1991 }
1992
1993 if delivery_plan == BashCompletionDeliveryPlan::ClearWaitThenActivate {
1994 tracing::info!(
1995 session_id = %info.session_id,
1996 bash_id = %info.bash_id,
1997 status = %info.status,
1998 "background bash completion: push-resuming suspended loop (event-driven)"
1999 );
2000 let mut resumable = session.clone();
2001 let mut runtime_state = read_runtime_state(&resumable);
2002 runtime_state.waiting_for_bash = None;
2003 runtime_state.status = AgentStatusState::Idle;
2004 runtime_state.suspension = None;
2005 write_runtime_state(&mut resumable, &runtime_state);
2006 resumable.metadata.remove("runtime.suspend_reason");
2007 resumable.updated_at = Utc::now();
2008 if let Err(error) = self.persistence.merge_save_runtime(&mut resumable).await {
2009 tracing::warn!(
2010 session_id = %info.session_id,
2011 %error,
2012 "background bash completion is durable but wait-state clear failed; leaving activation to the wait backstop"
2013 );
2014 return;
2015 }
2016 self.sessions.insert(
2017 resumable.id.clone(),
2018 Arc::new(parking_lot::RwLock::new(resumable)),
2019 );
2020 }
2021
2022 let (Some(messenger), Some(admission)) = (messenger, admission) else {
2026 if delivery_plan == BashCompletionDeliveryPlan::ClearWaitThenActivate {
2027 let _ = self.resume_parent(info.session_id.clone()).await;
2028 }
2029 return;
2030 };
2031 match messenger.activate_prepared(&admission).await {
2032 Ok(receipt) => tracing::info!(
2033 session_id = %info.session_id,
2034 bash_id = %info.bash_id,
2035 status = %info.status,
2036 waiting,
2037 generation = receipt.delivery.generation,
2038 activation = ?receipt.activation,
2039 "background bash completion delivered through SessionMessenger"
2040 ),
2041 Err(error) => tracing::warn!(
2042 session_id = %info.session_id,
2043 bash_id = %info.bash_id,
2044 %error,
2045 "background bash completion: SessionMessenger delivery failed"
2046 ),
2047 }
2048 }
2049}
2050
2051impl BashCompletionSink for ChildCompletionCoordinator {
2052 fn on_bash_completed(&self, info: BashCompletionInfo) {
2053 let coordinator = Arc::new(self.clone());
2057 tokio::spawn(async move {
2058 coordinator.deliver_bash_completion(info).await;
2059 });
2060 }
2061}
2062
2063const CHILD_WAIT_SWEEP_INTERVAL_SECS: u64 = 30;
2069const CHILD_WAIT_REGISTRATION_GRACE_SECS: i64 = 60;
2073const DEAD_CHILD_GRACE_SECS: i64 = 120;
2076const STALE_RUNNER_SLACK_SECS: i64 = 600;
2082
2083fn is_dead_child_candidate_status(status: Option<&str>) -> bool {
2088 match status {
2089 Some(status) => !is_terminal_child_status(status) && status != "suspended",
2092 None => true,
2093 }
2094}
2095
2096fn completion_child_is_owned(reported_parent: &str, child_parent_linkage: Option<&str>) -> bool {
2104 child_parent_linkage == Some(reported_parent)
2105}
2106
2107fn select_replay_child(terminal: &[(String, String)]) -> Option<&(String, String)> {
2111 terminal
2112 .iter()
2113 .find(|(_, status)| is_error_like(status))
2114 .or_else(|| terminal.last())
2115}
2116
2117fn child_wait_watchdog_resume_message(body: String) -> Message {
2118 let mut message = Message::user(body);
2119 message.metadata = Some(serde_json::json!({
2120 RUNTIME_RESUME_MESSAGE_HIDDEN_KEY: true,
2121 RUNTIME_RESUME_MESSAGE_KIND_KEY: "child_wait_watchdog_resume",
2122 }));
2123 message.never_compress = false;
2124 message
2125}
2126
2127fn empty_child_wait_message() -> Message {
2128 child_wait_watchdog_resume_message(
2129 "Runtime notification: this session was suspended waiting for child sessions, but the \
2130 wait tracked no children (internal inconsistency). The session has been resumed; use \
2131 SubAgent.list to inspect child state and continue the task."
2132 .to_string(),
2133 )
2134}
2135
2136fn child_wait_lease_expired_message(child_ids: &[String]) -> Message {
2137 child_wait_watchdog_resume_message(format!(
2138 "Runtime notification: the wait lease for child session(s) [{}] expired before they all \
2139 reported completion. They were NOT cancelled and may still be running or already \
2140 finished — verify their actual status with SubAgent.list / SubAgent.get before assuming \
2141 anything, then continue the task.",
2142 child_ids.join(", ")
2143 ))
2144}
2145
2146impl ChildCompletionCoordinator {
2162 pub fn spawn_child_wait_watchdog(self: &Arc<Self>) {
2165 let coordinator = Arc::clone(self);
2166 tokio::spawn(async move {
2167 use futures::FutureExt;
2168 if std::panic::AssertUnwindSafe(coordinator.reconcile_orphans_at_boot())
2169 .catch_unwind()
2170 .await
2171 .is_err()
2172 {
2173 tracing::error!("child-wait watchdog: boot reconciliation panicked");
2174 }
2175 let mut ticker = tokio::time::interval(std::time::Duration::from_secs(
2176 CHILD_WAIT_SWEEP_INTERVAL_SECS,
2177 ));
2178 ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
2179 ticker.tick().await;
2181 loop {
2182 ticker.tick().await;
2183 if std::panic::AssertUnwindSafe(coordinator.sweep_child_waits())
2184 .catch_unwind()
2185 .await
2186 .is_err()
2187 {
2188 tracing::error!("child-wait watchdog: sweep panicked; continuing");
2189 }
2190 }
2191 });
2192 }
2193
2194 async fn reconcile_orphans_at_boot(&self) {
2199 let cutoff = Utc::now();
2200
2201 let running = self
2207 .storage
2208 .list_sessions_by_run_status("running")
2209 .await
2210 .unwrap_or_default();
2211 for (child_id, parent_id) in running {
2212 let Some(parent_id) = parent_id else { continue };
2213 if self.runner_is_running(&child_id).await {
2214 continue;
2215 }
2216 let Some(control_plane) = self.load_control_plane(&child_id).await else {
2217 continue;
2218 };
2219 if control_plane.updated_at >= cutoff {
2222 continue;
2223 }
2224 tracing::warn!(
2225 child_session_id = %child_id,
2226 parent_session_id = %parent_id,
2227 "boot reconciliation: child was running when the process died; \
2228 marking it error and waking the parent"
2229 );
2230 self.synthesize_child_completion(
2231 &parent_id,
2232 &child_id,
2233 "error",
2234 Some(
2235 "orphaned by server restart: the process died while this child session \
2236 was running"
2237 .to_string(),
2238 ),
2239 )
2240 .await;
2241 }
2242
2243 let suspended = self
2247 .storage
2248 .list_sessions_by_run_status("suspended")
2249 .await
2250 .unwrap_or_default();
2251 for (session_id, _) in suspended {
2252 let Some(control_plane) = self.load_control_plane(&session_id).await else {
2253 continue;
2254 };
2255 if control_plane
2256 .metadata
2257 .get("runtime.suspend_reason")
2258 .map(String::as_str)
2259 != Some("waiting_for_bash")
2260 {
2261 continue;
2262 }
2263 if let Some(wait) = read_runtime_state(&control_plane).waiting_for_bash {
2264 tracing::warn!(
2265 %session_id,
2266 "boot reconciliation: re-arming bash self-resume backstop lost in restart"
2267 );
2268 let coordinator = self.clone();
2269 tokio::spawn(async move {
2270 coordinator
2271 .bash_self_resume(session_id, wait.bash_ids)
2272 .await;
2273 });
2274 }
2275 }
2276 }
2277
2278 async fn runner_is_running(&self, session_id: &str) -> bool {
2279 let runners = self.agent_runners.read().await;
2280 runners
2281 .get(session_id)
2282 .is_some_and(|runner| matches!(runner.status, AgentStatus::Running))
2283 }
2284
2285 async fn load_control_plane(&self, session_id: &str) -> Option<Session> {
2286 match self.storage.load_runtime_control_plane(session_id).await {
2287 Ok(session) => session,
2288 Err(error) => {
2289 tracing::warn!(
2290 %session_id,
2291 %error,
2292 "child-wait watchdog: failed to load session control plane"
2293 );
2294 None
2295 }
2296 }
2297 }
2298
2299 async fn sweep_child_waits(&self) {
2303 let suspended = match self.storage.list_sessions_by_run_status("suspended").await {
2304 Ok(entries) => entries,
2305 Err(error) => {
2306 tracing::warn!(%error, "child-wait watchdog: failed to list suspended sessions");
2307 return;
2308 }
2309 };
2310 for (session_id, _) in suspended {
2311 self.sweep_one_suspended_session(&session_id).await;
2312 }
2313 }
2314
2315 async fn sweep_one_suspended_session(&self, session_id: &str) {
2316 if self.runner_is_running(session_id).await {
2319 return;
2320 }
2321 let Some(session) = self.load_control_plane(session_id).await else {
2322 return;
2323 };
2324 let runtime_state = read_runtime_state(&session);
2325 let suspend_reason = session
2326 .metadata
2327 .get("runtime.suspend_reason")
2328 .map(String::as_str)
2329 .unwrap_or_default()
2330 .to_string();
2331 match (
2332 suspend_reason.as_str(),
2333 runtime_state.waiting_for_children.clone(),
2334 ) {
2335 ("waiting_for_bash", _)
2337 | ("awaiting_clarification", _)
2338 | ("awaiting_parent_approval", _) => {}
2339 (_, Some(wait)) => self.sweep_child_wait(session_id, wait).await,
2340 ("waiting_for_children", None) | ("", None) => {
2343 self.rescue_stranded_resume(session_id).await;
2344 }
2345 _ => {}
2346 }
2347 }
2348
2349 async fn sweep_child_wait(&self, parent_session_id: &str, wait: WaitingForChildrenState) {
2354 let now = Utc::now();
2355
2356 if wait.child_session_ids.is_empty() {
2357 tracing::warn!(
2358 %parent_session_id,
2359 "child-wait watchdog: wait armed over an empty child set; force-resuming"
2360 );
2361 self.force_resume_child_wait(parent_session_id, empty_child_wait_message())
2362 .await;
2363 return;
2364 }
2365
2366 if wait.timeout_at.is_some_and(|deadline| now >= deadline) {
2370 tracing::warn!(
2371 %parent_session_id,
2372 "child-wait watchdog: wait lease expired; force-resuming parent"
2373 );
2374 self.force_resume_child_wait(
2375 parent_session_id,
2376 child_wait_lease_expired_message(&wait.child_session_ids),
2377 )
2378 .await;
2379 return;
2380 }
2381
2382 if now.signed_duration_since(wait.registered_at).num_seconds()
2384 < CHILD_WAIT_REGISTRATION_GRACE_SECS
2385 {
2386 return;
2387 }
2388
2389 let statuses: HashMap<String, Option<String>> = self
2390 .storage
2391 .list_child_run_statuses(parent_session_id)
2392 .await
2393 .unwrap_or_default()
2394 .into_iter()
2395 .collect();
2396
2397 struct DeadChild {
2398 child_id: String,
2399 status: String,
2400 reason: String,
2401 owned: bool,
2405 }
2406
2407 let mut terminal: Vec<(String, String)> = Vec::new();
2408 let mut dead: Vec<DeadChild> = Vec::new();
2409 for child_id in &wait.child_session_ids {
2410 let status = statuses.get(child_id).and_then(|status| status.as_deref());
2411 if let Some(status) = status {
2412 if is_terminal_child_status(status) {
2413 terminal.push((child_id.clone(), status.to_string()));
2414 continue;
2415 }
2416 }
2417 if !is_dead_child_candidate_status(status) {
2418 continue;
2419 }
2420
2421 let control_plane = self.load_control_plane(child_id).await;
2428 let owned = control_plane
2429 .as_ref()
2430 .is_some_and(|cp| cp.parent_session_id.as_deref() == Some(parent_session_id));
2431 if !owned {
2432 dead.push(DeadChild {
2433 child_id: child_id.clone(),
2434 status: "error".to_string(),
2435 reason: if control_plane.is_some() {
2436 "waited-on session id is not a child of this session; clearing it \
2437 from the wait without touching that session"
2438 .to_string()
2439 } else {
2440 "waited-on child session does not exist".to_string()
2441 },
2442 owned: false,
2443 });
2444 continue;
2445 }
2446
2447 let runner = { self.agent_runners.read().await.get(child_id).cloned() };
2448 match runner {
2449 Some(runner) if matches!(runner.status, AgentStatus::Running) => {
2450 let last_activity = runner.last_event_at.unwrap_or(runner.started_at);
2456 let idle_secs = now.signed_duration_since(last_activity).num_seconds();
2457 let total_secs = now.signed_duration_since(runner.started_at).num_seconds();
2458 let policy = match &control_plane {
2459 Some(child) => {
2460 crate::runtime::execution::spawn::watchdog_policy_for_session(child)
2461 }
2462 None => Default::default(),
2463 };
2464 let idle_limit = policy.max_idle_secs.saturating_add(STALE_RUNNER_SLACK_SECS);
2465 let total_limit = policy
2466 .max_total_secs
2467 .saturating_add(STALE_RUNNER_SLACK_SECS);
2468 if idle_secs >= idle_limit || total_secs >= total_limit {
2469 runner.cancel_token.cancel();
2470 dead.push(DeadChild {
2471 child_id: child_id.clone(),
2472 status: "timeout".to_string(),
2473 reason: format!(
2474 "child runner stalled: no events for {idle_secs}s \
2475 (limit {idle_limit}s including watchdog slack); \
2476 force-finalized by the child-wait watchdog"
2477 ),
2478 owned: true,
2479 });
2480 }
2481 }
2482 _ => {
2483 let quiet_secs = control_plane
2487 .as_ref()
2488 .map(|child| now.signed_duration_since(child.updated_at).num_seconds())
2489 .unwrap_or(i64::MAX);
2490 if quiet_secs >= DEAD_CHILD_GRACE_SECS {
2491 dead.push(DeadChild {
2492 child_id: child_id.clone(),
2493 status: "error".to_string(),
2494 reason: format!(
2495 "child runner lost (crashed task, dropped spawn job, or \
2496 process restart): index status {status:?} with no live \
2497 runner driving it"
2498 ),
2499 owned: true,
2500 });
2501 }
2502 }
2503 }
2504 }
2505
2506 if !dead.is_empty() {
2507 for entry in dead {
2508 tracing::warn!(
2509 %parent_session_id,
2510 child_session_id = %entry.child_id,
2511 status = %entry.status,
2512 reason = %entry.reason,
2513 owned = entry.owned,
2514 "child-wait watchdog: synthesizing terminal completion for dead child"
2515 );
2516 if entry.owned {
2517 self.synthesize_child_completion(
2518 parent_session_id,
2519 &entry.child_id,
2520 &entry.status,
2521 Some(entry.reason),
2522 )
2523 .await;
2524 } else {
2525 self.publish_synthetic_completion(
2527 parent_session_id,
2528 &entry.child_id,
2529 &entry.status,
2530 Some(entry.reason),
2531 )
2532 .await;
2533 }
2534 }
2535 return;
2537 }
2538
2539 let terminal_ids: Vec<String> = terminal.iter().map(|(id, _)| id.clone()).collect();
2545 if let Some((child_id, status)) = select_replay_child(&terminal) {
2546 if wait_policy_satisfied(
2547 wait.wait_for,
2548 &wait.child_session_ids,
2549 &terminal_ids,
2550 child_id,
2551 status,
2552 ) {
2553 tracing::warn!(
2554 %parent_session_id,
2555 child_session_id = %child_id,
2556 "child-wait watchdog: wait already satisfied but parent still suspended \
2557 (lost wake); replaying the completion"
2558 );
2559 let error = self
2560 .load_control_plane(child_id)
2561 .await
2562 .and_then(|child| child.last_run_error());
2563 self.publish_synthetic_completion(parent_session_id, child_id, status, error)
2564 .await;
2565 }
2566 }
2567 }
2568
2569 async fn synthesize_child_completion(
2575 &self,
2576 parent_session_id: &str,
2577 child_session_id: &str,
2578 status: &str,
2579 error: Option<String>,
2580 ) {
2581 match self.storage.load_session(child_session_id).await {
2582 Ok(Some(mut child)) => {
2583 if child.parent_session_id.as_deref() != Some(parent_session_id) {
2588 tracing::warn!(
2589 %parent_session_id,
2590 child_session_id = %child.id,
2591 "child-wait watchdog: refusing to synthesize status onto a session \
2592 that is not a child of this parent"
2593 );
2594 self.publish_synthetic_completion(
2595 parent_session_id,
2596 child_session_id,
2597 status,
2598 error,
2599 )
2600 .await;
2601 return;
2602 }
2603 child.set_last_run_status(status);
2604 match &error {
2605 Some(message) => child.set_last_run_error(message.clone()),
2606 None => child.clear_last_run_error(),
2607 }
2608 child.updated_at = Utc::now();
2609 if let Err(save_error) = self.persistence.merge_save_runtime(&mut child).await {
2610 tracing::warn!(
2611 child_session_id = %child.id,
2612 %save_error,
2613 "child-wait watchdog: failed to persist synthesized terminal status"
2614 );
2615 }
2616 self.sessions
2617 .insert(child.id.clone(), Arc::new(parking_lot::RwLock::new(child)));
2618 }
2619 Ok(None) => {}
2620 Err(load_error) => {
2621 tracing::warn!(
2622 %child_session_id,
2623 %load_error,
2624 "child-wait watchdog: failed to load child for synthesized terminal status"
2625 );
2626 }
2627 }
2628 finalize_runner(
2629 &self.agent_runners,
2630 child_session_id,
2631 &Err(bamboo_agent_core::AgentError::LLM(
2632 error
2633 .clone()
2634 .unwrap_or_else(|| format!("synthesized {status}")),
2635 )),
2636 )
2637 .await;
2638 self.publish_synthetic_completion(parent_session_id, child_session_id, status, error)
2639 .await;
2640 }
2641
2642 async fn publish_synthetic_completion(
2643 &self,
2644 parent_session_id: &str,
2645 child_session_id: &str,
2646 status: &str,
2647 error: Option<String>,
2648 ) {
2649 let parent_tx = crate::execution::session_events::get_or_create_event_sender(
2650 &self.session_event_senders,
2651 parent_session_id,
2652 )
2653 .await;
2654 let handler: Arc<dyn ChildCompletionHandler> = Arc::new(self.clone());
2655 crate::runtime::execution::spawn::publish_child_completion_parts(
2656 &parent_tx,
2657 Some(handler),
2658 parent_session_id.to_string(),
2659 child_session_id.to_string(),
2660 status.to_string(),
2661 error,
2662 )
2663 .await;
2664 }
2665
2666 async fn rescue_stranded_resume(&self, session_id: &str) {
2672 let Some(session) = self.load_session(session_id).await else {
2673 return;
2674 };
2675 let pending_runtime_resume = session.messages.last().is_some_and(|message| {
2676 matches!(message.role, Role::User)
2677 && message
2678 .metadata
2679 .as_ref()
2680 .is_some_and(|meta| meta.get(RUNTIME_RESUME_MESSAGE_KIND_KEY).is_some())
2681 });
2682 if !pending_runtime_resume {
2683 return;
2684 }
2685 tracing::warn!(
2686 %session_id,
2687 "child-wait watchdog: stranded resume detected (wait cleared, resume never \
2688 spawned); resuming"
2689 );
2690 self.resume_parent(session_id.to_string()).await;
2691 }
2692
2693 async fn force_resume_child_wait(&self, session_id: &str, resume_message: Message) {
2699 const MAX_ATTEMPTS: u8 = 5;
2700 let lock = session_resume_lock(session_id);
2701 for attempt in 0..MAX_ATTEMPTS {
2702 if attempt > 0 {
2703 tokio::time::sleep(Duration::from_millis(200)).await;
2704 }
2705 {
2706 let _held = lock.lock().await;
2707 let Some(mut session) = self.load_session(session_id).await else {
2708 return;
2709 };
2710 let mut runtime_state = read_runtime_state(&session);
2711 if runtime_state.waiting_for_children.is_none() {
2712 return;
2714 }
2715 runtime_state.waiting_for_children = None;
2716 runtime_state.status = AgentStatusState::Idle;
2717 runtime_state.suspension = None;
2718 write_runtime_state(&mut session, &runtime_state);
2719 session.metadata.remove("runtime.suspend_reason");
2720 session.add_message(resume_message.clone());
2721 session.updated_at = Utc::now();
2722 self.save_and_cache(&mut session).await;
2723 }
2724 let outcome = self.resume_parent(session_id.to_string()).await;
2725 match outcome {
2726 ResumeOutcome::Started { .. } | ResumeOutcome::NotFound => return,
2727 ResumeOutcome::Completed | ResumeOutcome::AlreadyRunning { .. } => {
2728 let clobbered = self
2732 .load_session(session_id)
2733 .await
2734 .map(|session| read_runtime_state(&session).waiting_for_children.is_some())
2735 .unwrap_or(false);
2736 if !clobbered {
2737 return;
2738 }
2739 }
2740 }
2741 }
2742 tracing::error!(
2743 %session_id,
2744 "child-wait watchdog: force-resume exhausted its clobber-retry budget"
2745 );
2746 }
2747}
2748
2749#[cfg(test)]
2750mod tests {
2751 use super::*;
2752 use bamboo_agent_core::Message;
2753 use bamboo_domain::SessionInboxPort;
2754 use futures::stream;
2755 use std::sync::atomic::{AtomicUsize, Ordering};
2756
2757 struct EmptyTools;
2758
2759 #[async_trait]
2760 impl bamboo_agent_core::tools::ToolExecutor for EmptyTools {
2761 async fn execute(
2762 &self,
2763 _call: &bamboo_agent_core::tools::ToolCall,
2764 ) -> Result<bamboo_agent_core::tools::ToolResult, bamboo_agent_core::tools::ToolError>
2765 {
2766 Err(bamboo_agent_core::tools::ToolError::NotFound(
2767 "no tools".to_string(),
2768 ))
2769 }
2770
2771 fn list_tools(&self) -> Vec<bamboo_agent_core::tools::ToolSchema> {
2772 Vec::new()
2773 }
2774 }
2775
2776 struct CompletedTestProvider;
2777
2778 #[async_trait]
2779 impl bamboo_llm::LLMProvider for CompletedTestProvider {
2780 async fn chat_stream(
2781 &self,
2782 _messages: &[Message],
2783 _tools: &[bamboo_agent_core::tools::ToolSchema],
2784 _max_output_tokens: Option<u32>,
2785 _model: &str,
2786 ) -> Result<bamboo_llm::LLMStream, bamboo_llm::LLMError> {
2787 let chunks: Vec<bamboo_llm::provider::Result<bamboo_llm::LLMChunk>> = vec![
2788 Ok(bamboo_llm::LLMChunk::Token("done".to_string())),
2789 Ok(bamboo_llm::LLMChunk::Done),
2790 ];
2791 Ok(Box::pin(stream::iter(chunks)))
2792 }
2793 }
2794
2795 struct CountingActivationSpawner {
2796 reservations: Arc<AtomicUsize>,
2797 launches: Arc<AtomicUsize>,
2798 }
2799
2800 #[async_trait]
2801 impl SessionActivationSpawner for CountingActivationSpawner {
2802 async fn reserve_activation(
2803 &self,
2804 target_session_id: &str,
2805 inbox_generation: u64,
2806 ) -> Result<SessionActivationReserveOutcome, bamboo_domain::SessionActivationError>
2807 {
2808 self.reservations.fetch_add(1, Ordering::SeqCst);
2809 let launches = self.launches.clone();
2810 Ok(SessionActivationReserveOutcome::Reserved(
2811 SessionActivationLaunch::new(
2812 format!("{target_session_id}-{inbox_generation}"),
2813 move || {
2814 launches.fetch_add(1, Ordering::SeqCst);
2815 },
2816 ),
2817 ))
2818 }
2819 }
2820
2821 async fn completion_inbox_fixture() -> (
2822 tempfile::TempDir,
2823 Arc<bamboo_storage::SessionStoreV2>,
2824 Arc<dyn SessionInboxPort>,
2825 Arc<ChildCompletionCoordinator>,
2826 Arc<AtomicUsize>,
2827 Arc<AtomicUsize>,
2828 ) {
2829 let temp = tempfile::tempdir().unwrap();
2830 let store = Arc::new(
2831 bamboo_storage::SessionStoreV2::new(temp.path().to_path_buf())
2832 .await
2833 .unwrap(),
2834 );
2835 let storage: Arc<dyn Storage> = store.clone();
2836 let locked = Arc::new(LockedSessionStore::new(storage.clone()));
2837 let inbox: Arc<dyn SessionInboxPort> = Arc::new(bamboo_storage::FileSessionInbox::new(
2838 store.clone(),
2839 bamboo_domain::SessionInboxLimits::default(),
2840 ));
2841 let router = crate::SessionActivationRouter::new();
2842 let messenger = Arc::new(crate::SessionMessenger::new(
2843 storage.clone(),
2844 inbox.clone(),
2845 router.clone(),
2846 ));
2847 let reservations = Arc::new(AtomicUsize::new(0));
2848 let launches = Arc::new(AtomicUsize::new(0));
2849 router
2850 .set_spawner(Arc::new(CountingActivationSpawner {
2851 reservations: reservations.clone(),
2852 launches: launches.clone(),
2853 }))
2854 .await;
2855 let provider: Arc<dyn bamboo_llm::LLMProvider> = Arc::new(CompletedTestProvider);
2856 let config = Arc::new(RwLock::new(Config::default()));
2857 let metrics = bamboo_metrics::MetricsCollector::spawn(
2858 Arc::new(bamboo_metrics::SqliteMetricsStorage::new(
2859 temp.path().join("metrics.db"),
2860 )),
2861 7,
2862 );
2863 let tools: Arc<dyn ToolExecutor> = Arc::new(EmptyTools);
2864 let agent = Arc::new(
2865 Agent::builder()
2866 .storage(storage.clone())
2867 .persistence(locked.clone())
2868 .session_inbox(inbox.clone())
2869 .activation_router(router)
2870 .session_messenger(messenger)
2871 .attachment_reader(store.clone())
2872 .skill_manager(Arc::new(bamboo_skills::SkillManager::new()))
2873 .metrics_collector(metrics)
2874 .config(config.clone())
2875 .provider(provider.clone())
2876 .default_tools(tools)
2877 .build()
2878 .unwrap(),
2879 );
2880 let mut providers = HashMap::new();
2881 providers.insert("test".to_string(), provider);
2882 let registry = Arc::new(ProviderRegistry::new(providers, "test".to_string()));
2883 let provider_router = Arc::new(ProviderModelRouter::new(registry.clone()));
2884 let coordinator = Arc::new(ChildCompletionCoordinator::new(
2885 storage,
2886 locked,
2887 Arc::new(dashmap::DashMap::new()),
2888 Arc::new(RwLock::new(HashMap::new())),
2889 Arc::new(RwLock::new(HashMap::new())),
2890 agent,
2891 config,
2892 registry,
2893 provider_router,
2894 temp.path().to_path_buf(),
2895 None,
2896 ));
2897 (temp, store, inbox, coordinator, reservations, launches)
2898 }
2899
2900 #[test]
2903 fn dead_child_candidate_status_matrix() {
2904 assert!(is_dead_child_candidate_status(None));
2906 assert!(is_dead_child_candidate_status(Some("running")));
2909 assert!(is_dead_child_candidate_status(Some("pending")));
2910 assert!(!is_dead_child_candidate_status(Some("suspended")));
2912 for status in ["completed", "error", "timeout", "cancelled", "skipped"] {
2914 assert!(!is_dead_child_candidate_status(Some(status)), "{status}");
2915 }
2916 }
2917
2918 #[test]
2919 fn completion_child_ownership_gates_content_fold() {
2920 assert!(completion_child_is_owned("parent-1", Some("parent-1")));
2922 assert!(!completion_child_is_owned("parent-1", Some("parent-2")));
2925 assert!(!completion_child_is_owned("parent-1", None));
2927 }
2928
2929 #[test]
2930 fn replay_child_prefers_error_like_for_first_error_policy() {
2931 let terminal = vec![
2932 ("c-ok".to_string(), "completed".to_string()),
2933 ("c-err".to_string(), "timeout".to_string()),
2934 ("c-late".to_string(), "completed".to_string()),
2935 ];
2936 let (id, status) = select_replay_child(&terminal).expect("non-empty");
2937 assert_eq!(id, "c-err");
2938 assert_eq!(status, "timeout");
2939
2940 let all_ok = vec![
2941 ("c-1".to_string(), "completed".to_string()),
2942 ("c-2".to_string(), "completed".to_string()),
2943 ];
2944 let (id, _) = select_replay_child(&all_ok).expect("non-empty");
2945 assert_eq!(id, "c-2");
2946
2947 assert!(select_replay_child(&[]).is_none());
2948 }
2949
2950 #[test]
2951 fn watchdog_resume_messages_are_hidden_runtime_messages() {
2952 for message in [
2953 empty_child_wait_message(),
2954 child_wait_lease_expired_message(&["c-1".to_string(), "c-2".to_string()]),
2955 ] {
2956 assert!(matches!(message.role, Role::User));
2957 let meta = message.metadata.expect("hidden runtime metadata");
2958 assert_eq!(meta[RUNTIME_RESUME_MESSAGE_HIDDEN_KEY], true);
2959 assert_eq!(
2960 meta[RUNTIME_RESUME_MESSAGE_KIND_KEY],
2961 "child_wait_watchdog_resume"
2962 );
2963 }
2964 let lease = child_wait_lease_expired_message(&["c-1".to_string()]);
2965 assert!(lease.content.contains("NOT cancelled"));
2967 assert!(lease.content.contains("c-1"));
2968 }
2969
2970 #[test]
2973 fn non_terminal_statuses_never_satisfy_wait_policies() {
2974 assert!(!is_terminal_child_status("suspended"));
2977 assert!(!is_terminal_child_status("running"));
2978 assert!(!is_terminal_child_status("pending"));
2979 }
2980
2981 fn make_completion(status: &str) -> ChildCompletion {
2982 ChildCompletion {
2983 parent_session_id: "parent-1".to_string(),
2984 child_session_id: "child-1".to_string(),
2985 status: status.to_string(),
2986 error: None,
2987 completed_at: Utc::now(),
2988 }
2989 }
2990
2991 #[test]
2992 fn oversized_child_outcome_is_bounded_and_keeps_full_content_identity() {
2993 let mut completion = make_completion("completed");
2994 let wait_registered_at = Utc::now();
2995 let huge = format!("prefix-A-{}", "x".repeat(300 * 1024));
2996 let presentation = runtime_resume_message(&completion, 0, Some(&huge));
2997 let first = child_completion_envelope(
2998 &completion,
2999 wait_registered_at,
3000 Some(huge.clone()),
3001 &presentation,
3002 );
3003 assert!(
3004 serde_json::to_vec(&first).unwrap().len()
3005 < bamboo_domain::SessionInboxLimits::default().max_payload_bytes
3006 );
3007 let SessionMessageBody::ChildOutcome(outcome) = &first.body else {
3008 panic!("typed child outcome");
3009 };
3010 let stored = outcome.result.as_deref().unwrap();
3011 assert!(stored.contains("sha256="));
3012 assert!(stored.contains("SubAgent.get"));
3013 assert!(stored.len() < CHILD_COMPLETION_INLINE_FIELD_BYTES);
3014
3015 completion.completed_at += chrono::Duration::seconds(1);
3018 let exact_retry = child_completion_envelope(
3019 &completion,
3020 wait_registered_at,
3021 Some(huge.clone()),
3022 &runtime_resume_message(&completion, 9, Some(&huge)),
3023 );
3024 assert_eq!(exact_retry.id, first.id);
3025 let changed = format!("prefix-B-{}", "x".repeat(300 * 1024));
3026 let corrected = child_completion_envelope(
3027 &completion,
3028 wait_registered_at,
3029 Some(changed.clone()),
3030 &runtime_resume_message(&completion, 0, Some(&changed)),
3031 );
3032 assert_ne!(corrected.id, first.id);
3033 }
3034
3035 #[tokio::test]
3036 async fn oversized_child_completion_clears_wait_and_activates_exactly_once() {
3037 let (_temp, store, inbox, coordinator, reservations, launches) =
3038 completion_inbox_fixture().await;
3039 let parent_id = "oversized-parent";
3040 let child_id = "oversized-child";
3041 let now = Utc::now();
3042 let mut parent = Session::new(parent_id, "model");
3043 let mut parent_runtime = AgentRuntimeState::new("waiting-run");
3044 parent_runtime.status = AgentStatusState::Suspended;
3045 parent_runtime.waiting_for_children = Some(WaitingForChildrenState::for_children(
3046 vec![child_id.to_string()],
3047 ChildWaitPolicy::All,
3048 now,
3049 ));
3050 parent_runtime.suspension = Some(SuspensionState {
3051 reason: "waiting_for_children".to_string(),
3052 suspended_at: now,
3053 resumable: true,
3054 hook_point: Some("ChildCompletion".to_string()),
3055 });
3056 write_runtime_state(&mut parent, &parent_runtime);
3057 parent.metadata.insert(
3058 "runtime.suspend_reason".to_string(),
3059 "waiting_for_children".to_string(),
3060 );
3061 store.save_session(&parent).await.unwrap();
3062
3063 let mut child = Session::new_child(child_id, parent_id, "model", "Child");
3064 child.add_message(Message::assistant("z".repeat(300 * 1024), None));
3065 child.set_last_run_status("completed");
3066 store.save_session(&child).await.unwrap();
3067 let completion = ChildCompletion {
3068 parent_session_id: parent_id.to_string(),
3069 child_session_id: child_id.to_string(),
3070 status: "completed".to_string(),
3071 error: None,
3072 completed_at: Utc::now(),
3073 };
3074
3075 ChildCompletionHandler::on_child_completed(coordinator.as_ref(), completion.clone()).await;
3076 let durable_parent = store.load_session(parent_id).await.unwrap().unwrap();
3077 let durable_runtime = read_runtime_state(&durable_parent);
3078 assert!(durable_runtime.waiting_for_children.is_none());
3079 assert_eq!(durable_runtime.status, AgentStatusState::Idle);
3080 assert!(!durable_parent
3081 .metadata
3082 .contains_key("runtime.suspend_reason"));
3083 let backlog = inbox.inspect(parent_id).await.unwrap();
3084 assert_eq!(backlog.pending + backlog.claimed, 1);
3085 assert!(backlog.activation_pending());
3086 assert_eq!(reservations.load(Ordering::SeqCst), 1);
3087 assert_eq!(launches.load(Ordering::SeqCst), 1);
3088
3089 let claim = inbox.claim(parent_id, 1).await.unwrap().remove(0);
3090 assert!(
3091 serde_json::to_vec(&claim.envelope).unwrap().len()
3092 < bamboo_domain::SessionInboxLimits::default().max_payload_bytes
3093 );
3094 ChildCompletionHandler::on_child_completed(coordinator.as_ref(), completion).await;
3097 assert_eq!(reservations.load(Ordering::SeqCst), 1);
3098 assert_eq!(launches.load(Ordering::SeqCst), 1);
3099 assert_eq!(inbox.inspect(parent_id).await.unwrap().claimed, 1);
3100 }
3101
3102 #[tokio::test]
3103 async fn latest_locked_activation_mutation_preserves_wait_armed_after_stale_load() {
3104 let temp = tempfile::tempdir().unwrap();
3105 let store = Arc::new(
3106 bamboo_storage::SessionStoreV2::new(temp.path().to_path_buf())
3107 .await
3108 .unwrap(),
3109 );
3110 let storage: Arc<dyn Storage> = store.clone();
3111 let locked = LockedSessionStore::new(storage);
3112 let mut session = Session::new("activation-stale-wait", "model");
3113 session.agent_runtime_state = Some(AgentRuntimeState::new("old-run"));
3114 store.save_session(&session).await.unwrap();
3115
3116 let stale = store.load_session(&session.id).await.unwrap().unwrap();
3118 assert!(read_runtime_state(&stale).waiting_for_children.is_none());
3119
3120 locked
3121 .update_runtime_config(&session.id, |latest| {
3122 let mut state = read_runtime_state(latest);
3123 state.status = AgentStatusState::Suspended;
3124 state.waiting_for_children = Some(WaitingForChildrenState::for_children(
3125 vec!["child-new".to_string()],
3126 ChildWaitPolicy::All,
3127 Utc::now(),
3128 ));
3129 state.suspension = Some(SuspensionState {
3130 reason: "waiting_for_children".to_string(),
3131 suspended_at: Utc::now(),
3132 resumable: true,
3133 hook_point: None,
3134 });
3135 write_runtime_state(latest, &state);
3136 latest.metadata.insert(
3137 "runtime.suspend_reason".to_string(),
3138 "waiting_for_children".to_string(),
3139 );
3140 })
3141 .await
3142 .unwrap();
3143
3144 let (prepared, ready) = prepare_session_inbox_activation(&locked, &session.id, false)
3145 .await
3146 .unwrap()
3147 .unwrap();
3148 assert!(!ready, "latest durable specific wait must block activation");
3149 let state = read_runtime_state(&prepared);
3150 assert!(state.waiting_for_children.is_some());
3151 assert_eq!(state.status, AgentStatusState::Suspended);
3152 assert_eq!(
3153 prepared
3154 .metadata
3155 .get("runtime.suspend_reason")
3156 .map(String::as_str),
3157 Some("waiting_for_children")
3158 );
3159 }
3160
3161 #[tokio::test]
3162 async fn partial_child_and_bash_backlogs_remain_inert_after_inbox_reopen() {
3163 let temp = tempfile::tempdir().unwrap();
3164 let store = Arc::new(
3165 bamboo_storage::SessionStoreV2::new(temp.path().to_path_buf())
3166 .await
3167 .unwrap(),
3168 );
3169 store
3170 .save_session(&Session::new("restart-child-parent", "model"))
3171 .await
3172 .unwrap();
3173 store
3174 .save_session(&Session::new("restart-bash-parent", "model"))
3175 .await
3176 .unwrap();
3177 let inbox = bamboo_storage::FileSessionInbox::new(
3178 store.clone(),
3179 bamboo_domain::SessionInboxLimits::default(),
3180 );
3181
3182 let completion = ChildCompletion {
3183 parent_session_id: "restart-child-parent".to_string(),
3184 child_session_id: "child-a".to_string(),
3185 status: "completed".to_string(),
3186 error: None,
3187 completed_at: Utc::now(),
3188 };
3189 let child_resume = runtime_resume_message(&completion, 1, Some("first child"));
3190 inbox
3191 .deliver(&child_completion_envelope(
3192 &completion,
3193 Utc::now(),
3194 Some("first child".to_string()),
3195 &child_resume,
3196 ))
3197 .await
3198 .unwrap();
3199 let bash = BashCompletionInfo {
3200 session_id: "restart-bash-parent".to_string(),
3201 bash_id: "bash-a".to_string(),
3202 command: "true".to_string(),
3203 exit_code: Some(0),
3204 status: "completed".to_string(),
3205 output_tail: String::new(),
3206 };
3207 inbox
3208 .deliver(&bash_completion_envelope(&bash))
3209 .await
3210 .unwrap();
3211
3212 let reopened = bamboo_storage::FileSessionInbox::new(
3213 store,
3214 bamboo_domain::SessionInboxLimits::default(),
3215 );
3216 for session_id in ["restart-child-parent", "restart-bash-parent"] {
3217 let backlog = reopened.inspect(session_id).await.unwrap();
3218 assert_eq!(backlog.pending, 1);
3219 assert_eq!(backlog.activation_generation, 0);
3220 assert!(
3221 !backlog.activation_pending(),
3222 "startup must not run a partial wait backlog for {session_id}"
3223 );
3224 }
3225 }
3226
3227 struct StubChildIndex {
3230 children: Vec<(String, Option<String>)>,
3231 }
3232
3233 #[async_trait]
3234 impl Storage for StubChildIndex {
3235 async fn save_session(&self, _session: &Session) -> std::io::Result<()> {
3236 Ok(())
3237 }
3238 async fn load_session(&self, _id: &str) -> std::io::Result<Option<Session>> {
3239 Ok(None)
3240 }
3241 async fn delete_session(&self, _id: &str) -> std::io::Result<bool> {
3242 Ok(false)
3243 }
3244 async fn list_child_run_statuses(
3245 &self,
3246 _parent_session_id: &str,
3247 ) -> std::io::Result<Vec<(String, Option<String>)>> {
3248 Ok(self.children.clone())
3249 }
3250 }
3251
3252 #[tokio::test]
3253 async fn derive_completed_only_includes_terminal_children() {
3254 let storage: Arc<dyn Storage> = Arc::new(StubChildIndex {
3255 children: vec![
3256 ("a".into(), Some("completed".into())),
3257 ("b".into(), Some("running".into())),
3258 ("c".into(), Some("error".into())),
3259 ("d".into(), None),
3260 ],
3261 });
3262 let completed = derive_completed_child_ids(&storage, "parent-1", "b").await;
3263 assert_eq!(
3265 completed,
3266 vec!["a".to_string(), "b".to_string(), "c".to_string()]
3267 );
3268 }
3269
3270 #[tokio::test]
3271 async fn derive_completed_folds_in_just_completed_when_index_lags() {
3272 let storage: Arc<dyn Storage> = Arc::new(StubChildIndex {
3274 children: vec![("only".into(), Some("running".into()))],
3275 });
3276 let completed = derive_completed_child_ids(&storage, "parent-1", "only").await;
3277 assert_eq!(completed, vec!["only".to_string()]);
3278 }
3279
3280 #[test]
3281 fn wait_policy_all_uses_derived_completed_set() {
3282 let waited = vec!["a".to_string(), "b".to_string()];
3283 assert!(!wait_policy_satisfied(
3284 ChildWaitPolicy::All,
3285 &waited,
3286 &["a".to_string()],
3287 "a",
3288 "completed"
3289 ));
3290 assert!(wait_policy_satisfied(
3291 ChildWaitPolicy::All,
3292 &waited,
3293 &["a".to_string(), "b".to_string()],
3294 "b",
3295 "completed"
3296 ));
3297 }
3298
3299 #[test]
3300 fn wait_policy_first_error_requires_tracked_membership() {
3301 let waited = vec!["a".to_string(), "b".to_string()];
3302 assert!(wait_policy_satisfied(
3304 ChildWaitPolicy::FirstError,
3305 &waited,
3306 &["a".to_string()],
3307 "a",
3308 "error"
3309 ));
3310 assert!(!wait_policy_satisfied(
3313 ChildWaitPolicy::FirstError,
3314 &waited,
3315 &["a".to_string()],
3316 "stray-child",
3317 "timeout"
3318 ));
3319 assert!(wait_policy_satisfied(
3321 ChildWaitPolicy::FirstError,
3322 &waited,
3323 &["a".to_string(), "b".to_string()],
3324 "stray-child",
3325 "completed"
3326 ));
3327 }
3328
3329 #[test]
3330 fn child_final_assistant_text_returns_last_assistant() {
3331 let mut session = Session::new("child-1", "gpt-4");
3332 session.messages.push(Message::user("hi"));
3333 session
3334 .messages
3335 .push(Message::assistant("first answer", None));
3336 session.messages.push(Message::user("again"));
3337 session
3338 .messages
3339 .push(Message::assistant("final answer", None));
3340
3341 assert_eq!(
3342 child_final_assistant_text(&session).as_deref(),
3343 Some("final answer")
3344 );
3345 }
3346
3347 #[test]
3348 fn child_final_assistant_text_returns_none_when_blank() {
3349 let mut session = Session::new("child-1", "gpt-4");
3350 session.messages.push(Message::assistant(" ", None));
3351 assert!(child_final_assistant_text(&session).is_none());
3352 }
3353
3354 #[test]
3355 fn child_final_assistant_text_returns_none_when_no_assistant() {
3356 let mut session = Session::new("child-1", "gpt-4");
3357 session.messages.push(Message::user("hi"));
3358 assert!(child_final_assistant_text(&session).is_none());
3359 }
3360
3361 #[test]
3362 fn runtime_resume_message_folds_full_response_without_truncation() {
3363 let completion = make_completion("completed");
3366 let long: String = "a".repeat(10_000);
3367 let message = runtime_resume_message(&completion, 0, Some(&long));
3368 assert!(message.content.contains(&long));
3369 assert!(!message.content.contains("truncated"));
3370 }
3371
3372 #[test]
3373 fn runtime_resume_message_includes_child_response_when_provided() {
3374 let completion = make_completion("completed");
3375 let message = runtime_resume_message(&completion, 0, Some("the answer is 42"));
3376
3377 assert!(matches!(message.role, Role::User));
3378 assert!(!message.never_compress);
3381 assert!(message.content.contains("Child final response:"));
3382 assert!(message.content.contains("the answer is 42"));
3383
3384 let metadata = message.metadata.expect("metadata present");
3385 assert_eq!(
3386 metadata.get("hidden_from_ui").and_then(|v| v.as_bool()),
3387 Some(true)
3388 );
3389 assert_eq!(
3390 metadata.get("runtime_kind").and_then(|v| v.as_str()),
3391 Some("child_completion_resume")
3392 );
3393 assert_eq!(
3394 metadata
3395 .get("child_final_response_included")
3396 .and_then(|v| v.as_bool()),
3397 Some(true)
3398 );
3399 }
3400
3401 #[test]
3402 fn runtime_resume_message_falls_back_to_error_when_no_response() {
3403 let mut completion = make_completion("error");
3404 completion.error = Some("boom".to_string());
3405
3406 let message = runtime_resume_message(&completion, 1, None);
3407 assert!(message.content.contains("Child error:"));
3408 assert!(message.content.contains("boom"));
3409 let metadata = message.metadata.expect("metadata present");
3410 assert_eq!(
3411 metadata
3412 .get("child_final_response_included")
3413 .and_then(|v| v.as_bool()),
3414 Some(false)
3415 );
3416 }
3417
3418 #[test]
3419 fn runtime_resume_message_minimal_when_no_response_and_no_error() {
3420 let completion = make_completion("completed");
3421 let message = runtime_resume_message(&completion, 2, None);
3422 assert!(!message.content.contains("Child final response:"));
3423 assert!(!message.content.contains("Child error:"));
3424 assert!(message.content.contains("Resume the parent task"));
3425 }
3426
3427 #[test]
3428 fn read_config_snapshot_refreshes_cached_snapshot_from_live_config() {
3429 let runtime = tokio::runtime::Runtime::new().expect("runtime");
3430
3431 runtime.block_on(async {
3432 let config = Arc::new(RwLock::new(Config::default()));
3433 config.write().await.provider = "copilot".to_string();
3434 let cached_config = StdRwLock::new(Config::default());
3435
3436 let snapshot = read_config_snapshot(&config, &cached_config);
3437
3438 assert_eq!(snapshot.provider, "copilot");
3439 assert_eq!(
3440 cached_config.read().expect("cached snapshot lock").provider,
3441 "copilot"
3442 );
3443 });
3444 }
3445
3446 #[test]
3447 fn read_config_snapshot_uses_cached_snapshot_when_live_lock_is_busy() {
3448 let runtime = tokio::runtime::Runtime::new().expect("runtime");
3449
3450 runtime.block_on(async {
3451 let mut cached_snapshot = Config::default();
3452 cached_snapshot.provider = "cached-provider".to_string();
3453
3454 let config = Arc::new(RwLock::new(Config::default()));
3455 let cached_config = StdRwLock::new(cached_snapshot);
3456 let _write_guard = config.write().await;
3457
3458 let snapshot = read_config_snapshot(&config, &cached_config);
3459
3460 assert_eq!(snapshot.provider, "cached-provider");
3461 });
3462 }
3463
3464 #[test]
3467 fn bash_completion_resume_message_normal_announces_completion() {
3468 let ids = vec!["bg-1".to_string(), "bg-2".to_string()];
3469 let message = bash_completion_resume_message(&ids, false);
3470 assert!(
3472 message.content.contains("have completed"),
3473 "normal resume message must announce completion: {}",
3474 message.content
3475 );
3476 let metadata = message.metadata.expect("metadata present");
3478 assert_eq!(
3479 metadata
3480 .get(RUNTIME_RESUME_MESSAGE_HIDDEN_KEY)
3481 .and_then(|v| v.as_bool()),
3482 Some(true),
3483 "resume message must be hidden from the UI"
3484 );
3485 assert_eq!(
3486 metadata
3487 .get(RUNTIME_RESUME_MESSAGE_KIND_KEY)
3488 .and_then(|v| v.as_str()),
3489 Some(BASH_COMPLETION_RESUME_KIND),
3490 "resume message must carry the bash-completion kind discriminant"
3491 );
3492 }
3493
3494 #[test]
3495 fn bash_completion_resume_message_deadline_does_not_claim_completion() {
3496 let ids = vec!["bg-long".to_string()];
3500 let message = bash_completion_resume_message(&ids, true);
3501 assert!(
3502 !message.content.contains("have completed"),
3503 "deadline resume message must NOT claim the shells completed: {}",
3504 message.content
3505 );
3506 assert!(
3507 message.content.contains("may still be running"),
3508 "deadline resume message must warn shells may still be running: {}",
3509 message.content
3510 );
3511 assert!(
3512 message.content.contains("BashOutput"),
3513 "deadline resume message must direct verification via BashOutput: {}",
3514 message.content
3515 );
3516 let metadata = message.metadata.expect("metadata present");
3518 assert_eq!(
3519 metadata
3520 .get(RUNTIME_RESUME_MESSAGE_KIND_KEY)
3521 .and_then(|v| v.as_str()),
3522 Some(BASH_COMPLETION_RESUME_KIND)
3523 );
3524 }
3525
3526 #[test]
3527 fn bash_resume_should_retry_matrix() {
3528 assert!(!bash_resume_should_retry(
3534 &ResumeOutcome::Started { run_id: "r".into() },
3535 true
3536 ));
3537 assert!(!bash_resume_should_retry(
3538 &ResumeOutcome::Started { run_id: "r".into() },
3539 false
3540 ));
3541
3542 assert!(!bash_resume_should_retry(&ResumeOutcome::NotFound, true));
3544 assert!(!bash_resume_should_retry(&ResumeOutcome::NotFound, false));
3545
3546 assert!(bash_resume_should_retry(&ResumeOutcome::Completed, true));
3548 assert!(!bash_resume_should_retry(&ResumeOutcome::Completed, false));
3551
3552 assert!(bash_resume_should_retry(
3555 &ResumeOutcome::AlreadyRunning { run_id: "r".into() },
3556 true
3557 ));
3558 assert!(!bash_resume_should_retry(
3560 &ResumeOutcome::AlreadyRunning { run_id: "r".into() },
3561 false
3562 ));
3563 }
3564
3565 #[test]
3568 fn injection_body_includes_status_exit_command_and_tail() {
3569 let info = BashCompletionInfo {
3570 session_id: "s".into(),
3571 bash_id: "abc123".into(),
3572 command: "make build".into(),
3573 exit_code: Some(0),
3574 status: "completed".into(),
3575 output_tail: "BUILD OK".into(),
3576 };
3577 let body = bash_completion_injection_body(&info);
3578 assert!(body.contains("abc123"), "body: {body}");
3579 assert!(body.contains("make build"), "body: {body}");
3580 assert!(body.contains("completed"), "body: {body}");
3581 assert!(body.contains("exit code 0"), "body: {body}");
3582 assert!(body.contains("BUILD OK"), "body: {body}");
3583 assert!(body.contains("BashOutput"), "body: {body}");
3585 assert!(body.contains("bash_id=\"abc123\""), "body: {body}");
3586 }
3587
3588 #[test]
3589 fn injection_body_handles_no_output_and_signal_kill() {
3590 let info = BashCompletionInfo {
3591 session_id: "s".into(),
3592 bash_id: "xyz".into(),
3593 command: "sleep 99".into(),
3594 exit_code: None,
3595 status: "killed".into(),
3596 output_tail: String::new(),
3597 };
3598 let body = bash_completion_injection_body(&info);
3599 assert!(body.contains("killed"), "body: {body}");
3600 assert!(body.contains("none (signal/killed)"), "body: {body}");
3601 assert!(body.contains("no captured output"), "body: {body}");
3602 assert!(!body.contains("Output tail:"), "body: {body}");
3604 }
3605
3606 #[test]
3607 fn background_completion_builds_post_tool_use_payload_and_feedback() {
3608 let mut info = BashCompletionInfo {
3609 session_id: "s".into(),
3610 bash_id: "bg-7".into(),
3611 command: "cargo test".into(),
3612 exit_code: Some(0),
3613 status: "completed".into(),
3614 output_tail: "test result: ok".into(),
3615 };
3616
3617 let payload = background_bash_post_tool_payload(&info);
3618 match payload {
3619 HookPayload::ToolResult {
3620 tool_name,
3621 tool_call_id,
3622 outcome,
3623 } => {
3624 assert_eq!(tool_name, "Bash");
3625 assert_eq!(tool_call_id, "bg-7");
3626 assert!(outcome.success);
3627 let response: serde_json::Value =
3628 serde_json::from_str(outcome.result.as_deref().unwrap()).unwrap();
3629 assert_eq!(response["command"], "cargo test");
3630 assert_eq!(response["exit_code"], 0);
3631 assert_eq!(response["status"], "completed");
3632 assert_eq!(response["output_tail"], "test result: ok");
3633 }
3634 other => panic!("expected PostToolUse payload, got {other:?}"),
3635 }
3636
3637 append_background_bash_hook_feedback(
3638 &mut info,
3639 vec!["Run the formatter before continuing".to_string()],
3640 );
3641 assert!(info.output_tail.contains("<post_tool_use_feedback>"));
3642 assert!(info
3643 .output_tail
3644 .contains("Run the formatter before continuing"));
3645 }
3646
3647 #[tokio::test]
3648 async fn bash_completion_payload_identity_matches_file_inbox_idempotency() {
3649 let baseline = BashCompletionInfo {
3650 session_id: "session".into(),
3651 bash_id: "bg-7".into(),
3652 command: "cargo test".into(),
3653 exit_code: Some(0),
3654 status: "completed".into(),
3655 output_tail: "first tail".into(),
3656 };
3657 let mut retried = baseline.clone();
3658 retried.output_tail = "first tail\nlater bytes\n<hook feedback>".into();
3659 retried.status = "completed-after-hook".into();
3660 let baseline_envelope = bash_completion_envelope(&baseline);
3661 let exact_retry = bash_completion_envelope(&baseline);
3662 let corrected_envelope = bash_completion_envelope(&retried);
3663 assert_eq!(baseline_envelope.id, exact_retry.id);
3664 assert_ne!(
3665 baseline_envelope.id, corrected_envelope.id,
3666 "changed payload semantics must receive a distinct id"
3667 );
3668
3669 let mut other_shell = baseline.clone();
3670 other_shell.bash_id = "bg-8".into();
3671 assert_ne!(
3672 bash_completion_envelope(&baseline).id,
3673 bash_completion_envelope(&other_shell).id
3674 );
3675
3676 let temp = tempfile::tempdir().unwrap();
3677 let store = Arc::new(
3678 bamboo_storage::SessionStoreV2::new(temp.path().to_path_buf())
3679 .await
3680 .unwrap(),
3681 );
3682 store
3683 .save_session(&Session::new("session", "model"))
3684 .await
3685 .unwrap();
3686 let inbox = bamboo_storage::FileSessionInbox::new(
3687 store,
3688 bamboo_domain::SessionInboxLimits::default(),
3689 );
3690 let first = inbox.deliver(&baseline_envelope).await.unwrap();
3691 let duplicate = inbox.deliver(&exact_retry).await.unwrap();
3692 let corrected = inbox.deliver(&corrected_envelope).await.unwrap();
3693 assert_eq!(duplicate, first, "exact payload retry is idempotent");
3694 assert_ne!(corrected.id, first.id);
3695 assert_eq!(corrected.generation, first.generation + 1);
3696 assert_eq!(inbox.inspect("session").await.unwrap().pending, 2);
3697 }
3698
3699 #[cfg(unix)]
3700 #[tokio::test]
3701 async fn background_completion_fires_configured_post_tool_use_command() {
3702 use bamboo_config::{
3703 LifecycleHookCommand, LifecycleHookGroup, LifecycleHookType, LifecycleHooksConfig,
3704 DEFAULT_LIFECYCLE_HOOK_TIMEOUT_MS,
3705 };
3706
3707 let dir = tempfile::tempdir().unwrap();
3708 let output = dir.path().join("background-post-tool.json");
3709 let command = format!(
3710 "cat > '{}'; printf '%s' '{{\"additional_context\":\"inspect the completed build log\"}}'",
3711 output.display()
3712 );
3713 let config = LifecycleHooksConfig {
3714 enabled: true,
3715 post_tool_use: vec![LifecycleHookGroup {
3716 enabled: true,
3717 matcher: Some("^Bash$".to_string()),
3718 hooks: vec![LifecycleHookCommand {
3719 hook_type: LifecycleHookType::Command,
3720 command,
3721 timeout_ms: DEFAULT_LIFECYCLE_HOOK_TIMEOUT_MS,
3722 }],
3723 }],
3724 ..Default::default()
3725 };
3726 let mut session = Session::new("session-bg-hook", "test-model");
3727 session.workspace = Some(dir.path().to_string_lossy().into_owned());
3728 let mut info = BashCompletionInfo {
3729 session_id: session.id.clone(),
3730 bash_id: "bg-9".into(),
3731 command: "cargo test".into(),
3732 exit_code: Some(0),
3733 status: "completed".into(),
3734 output_tail: "test result: ok".into(),
3735 };
3736
3737 assert!(run_background_bash_post_tool_hooks(&config, None, &session, &mut info).await);
3738 let envelope: serde_json::Value =
3739 serde_json::from_str(&std::fs::read_to_string(output).unwrap()).unwrap();
3740 assert_eq!(envelope["hook_event_name"], "PostToolUse");
3741 assert_eq!(envelope["tool_name"], "Bash");
3742 assert_eq!(envelope["payload"]["tool_call_id"], "bg-9");
3743 let response = envelope["tool_response"]["result"]
3744 .as_str()
3745 .map(serde_json::from_str::<serde_json::Value>)
3746 .transpose()
3747 .unwrap()
3748 .unwrap();
3749 assert_eq!(response["command"], "cargo test");
3750 assert_eq!(response["status"], "completed");
3751 assert!(info.output_tail.contains("inspect the completed build log"));
3752 }
3753
3754 async fn temp_store() -> (tempfile::TempDir, Arc<dyn Storage>, LockedSessionStore) {
3755 let temp = tempfile::tempdir().unwrap();
3756 let storage: Arc<dyn Storage> = Arc::new(
3757 bamboo_storage::v2::SessionStoreV2::new(temp.path().to_path_buf())
3758 .await
3759 .expect("storage init"),
3760 );
3761 let persistence = LockedSessionStore::new(storage.clone());
3762 (temp, storage, persistence)
3763 }
3764
3765 #[tokio::test]
3766 async fn enqueue_writes_pending_injection_and_preserves_messages() {
3767 let (_temp, storage, persistence) = temp_store().await;
3768
3769 let mut session = Session::new("sess-enq", "test-model");
3770 session.add_message(Message::user("do the build"));
3771 storage.save_session(&session).await.unwrap();
3772
3773 let info = BashCompletionInfo {
3774 session_id: "sess-enq".into(),
3775 bash_id: "sh-1".into(),
3776 command: "make".into(),
3777 exit_code: Some(0),
3778 status: "completed".into(),
3779 output_tail: "done".into(),
3780 };
3781 let saved = enqueue_bash_completion_injection(&persistence, &info)
3782 .await
3783 .expect("enqueue io ok")
3784 .expect("session exists");
3785
3786 let pending = saved
3787 .pending_injected_messages()
3788 .expect("pending injection present");
3789 assert_eq!(pending.len(), 1);
3790 let content = pending[0].get("content").and_then(|v| v.as_str()).unwrap();
3791 assert!(content.contains("sh-1"), "content: {content}");
3792 assert!(content.contains("make"), "content: {content}");
3793 assert!(content.contains("done"), "content: {content}");
3794 assert_eq!(saved.messages.len(), 1);
3796 }
3797
3798 #[tokio::test]
3799 async fn enqueue_returns_none_for_missing_session() {
3800 let (_temp, _storage, persistence) = temp_store().await;
3801 let info = BashCompletionInfo {
3802 session_id: "does-not-exist".into(),
3803 bash_id: "x".into(),
3804 command: "true".into(),
3805 exit_code: Some(0),
3806 status: "completed".into(),
3807 output_tail: String::new(),
3808 };
3809 let result = enqueue_bash_completion_injection(&persistence, &info)
3810 .await
3811 .expect("io ok");
3812 assert!(result.is_none(), "no session → nothing enqueued");
3813 }
3814
3815 #[test]
3823 fn apply_bash_resume_transition_clears_wait_and_appends_message() {
3824 use bamboo_domain::session::runtime_state::WaitingForBashState;
3825
3826 let mut session = Session::new("sess-resume", "test-model");
3827 session.add_message(Message::user("kick off the build"));
3828 let mut rt = read_runtime_state(&session);
3829 rt.status = AgentStatusState::Running;
3830 rt.waiting_for_bash = Some(WaitingForBashState::for_bash(
3831 vec!["sh-1".into()],
3832 Utc::now(),
3833 ));
3834 write_runtime_state(&mut session, &rt);
3835 session.metadata.insert(
3836 "runtime.suspend_reason".to_string(),
3837 "waiting_for_bash".to_string(),
3838 );
3839
3840 let resume = bash_completion_resume_message(&["sh-1".to_string()], false);
3841 let did = apply_bash_resume_transition(&mut session, &resume);
3842
3843 assert!(did, "a suspended session must transition");
3844 let after = read_runtime_state(&session);
3845 assert!(
3846 after.waiting_for_bash.is_none(),
3847 "bash wait must be cleared"
3848 );
3849 assert_eq!(after.status, AgentStatusState::Idle, "runtime must be Idle");
3850 assert!(
3851 !session.metadata.contains_key("runtime.suspend_reason"),
3852 "suspend-reason marker must be removed"
3853 );
3854 assert_eq!(session.messages.len(), 2, "resume message must be appended");
3855 assert!(matches!(
3856 session.messages.last().map(|m| &m.role),
3857 Some(Role::User)
3858 ));
3859 }
3860
3861 #[test]
3865 fn apply_bash_resume_transition_noops_when_not_waiting() {
3866 let mut session = Session::new("sess-live", "test-model");
3867 session.add_message(Message::user("hi"));
3868
3869 let resume = bash_completion_resume_message(&["sh-1".to_string()], false);
3870 let did = apply_bash_resume_transition(&mut session, &resume);
3871
3872 assert!(!did, "a non-waiting session must not transition");
3873 assert_eq!(session.messages.len(), 1, "no resume message appended");
3874 }
3875
3876 #[test]
3880 fn bash_completion_should_resume_only_when_suspended_and_all_done() {
3881 assert!(bash_completion_should_resume(true, true));
3882 assert!(!bash_completion_should_resume(true, false)); assert!(!bash_completion_should_resume(false, true)); assert!(!bash_completion_should_resume(false, false));
3885 }
3886
3887 #[test]
3888 fn two_shell_delivery_stages_backlog_before_one_final_activation() {
3889 let mut waiting = true;
3890 let mut durable_backlog = 0;
3891 let mut reservations = 0;
3892
3893 durable_backlog += 1;
3895 let first = bash_completion_delivery_plan(waiting, false);
3896 assert_eq!(first, BashCompletionDeliveryPlan::DurableOnly);
3897 assert!(waiting);
3898 assert_eq!(durable_backlog, 1);
3899 assert_eq!(reservations, 0);
3900
3901 durable_backlog += 1;
3904 let last = bash_completion_delivery_plan(waiting, true);
3905 assert_eq!(last, BashCompletionDeliveryPlan::ClearWaitThenActivate);
3906 waiting = false;
3907 reservations += 1;
3908 assert!(!waiting);
3909 assert_eq!(durable_backlog, 2);
3910 assert_eq!(reservations, 1);
3911 }
3912
3913 #[test]
3917 fn bash_resume_message_from_info_carries_bashid_tail_and_kind() {
3918 let info = BashCompletionInfo {
3919 session_id: "s".into(),
3920 bash_id: "sh-42".into(),
3921 command: "cargo test".into(),
3922 exit_code: Some(0),
3923 status: "completed".into(),
3924 output_tail: "test result: ok".into(),
3925 };
3926 let msg = bash_resume_message_from_info(&info);
3927
3928 assert!(matches!(msg.role, Role::User));
3929 assert!(msg.content.contains("sh-42"), "content: {}", msg.content);
3930 assert!(
3931 msg.content.contains("cargo test"),
3932 "content: {}",
3933 msg.content
3934 );
3935 assert!(
3936 msg.content.contains("test result: ok"),
3937 "content: {}",
3938 msg.content
3939 );
3940 assert!(
3941 msg.content.contains("BashOutput"),
3942 "content: {}",
3943 msg.content
3944 );
3945 let meta = serde_json::to_string(&msg.metadata).unwrap();
3946 assert!(
3947 meta.contains(BASH_COMPLETION_RESUME_KIND),
3948 "resume message must be tagged as a bash-completion resume: {meta}"
3949 );
3950 }
3951}