1use std::collections::HashMap;
8use std::sync::{Arc, Mutex as StdMutex, OnceLock, RwLock as StdRwLock, Weak};
9use std::time::Duration;
10
11use bamboo_domain::poison::PoisonRecover;
12use bamboo_domain::{
13 AgentHookPoint, HookPayload, HookToolOutcome, SessionChildOutcome, SessionMessageBody,
14 SessionMessageContent, SessionMessageEnvelope, SessionMessageId, SessionMessageKind,
15 SessionMessageSource, SessionProviderMessage,
16};
17
18use crate::execution::{
19 create_event_forwarder, finalize_runner, reserve_runner_core, reserve_session_execution,
20 spawn_session_execution, AgentRunner, AgentStatus, ChildCompletion, ChildCompletionHandler,
21 ReserveOutcome, SessionExecutionArgs, SessionExecutionReservation,
22 SessionExecutionReserveOutcome, SpawnJob, SpawnScheduler,
23};
24use crate::runtime::config::{BashResumeHook, GuardianSpawner, BASH_COMPLETION_RESUME_KIND};
25use crate::runtime::guardian_state::{
26 parse_guardian_verdict, read_guardian_config, read_guardian_state, write_guardian_state,
27 GuardianVerdict,
28};
29use crate::Agent;
30use async_trait::async_trait;
31use bamboo_agent_core::storage::Storage;
32use bamboo_agent_core::tools::ToolExecutor;
33use bamboo_agent_core::{
34 AgentEvent, BashCompletionInfo, BashCompletionSink, Message, Role, Session, SessionKind,
35};
36use bamboo_domain::session::runtime_state::{
37 AgentRuntimeState, AgentStatusState, ChildWaitPolicy, SuspensionState, WaitingForChildrenState,
38};
39use bamboo_llm::{Config, ProviderModelRouter, ProviderRegistry};
40use bamboo_storage::LockedSessionStore;
41use chrono::Utc;
42use sha2::{Digest, Sha256};
43use tokio::sync::{broadcast, RwLock};
44
45use crate::model_areas::resolve_global_area_models;
46use crate::model_config_helper::{
47 resolve_fast_model, resolve_gold_config, resolve_provider_routing_key, GOLD_CONFIG_METADATA_KEY,
48};
49use crate::session_activation::{
50 SessionActivationLaunch, SessionActivationReserveOutcome, SessionActivationSpawner,
51};
52use crate::session_app::execute::consume_pending_clarification_resume;
53use crate::session_app::provider_model::{persist_model_ref, session_effective_model_ref};
54use crate::session_app::resume::{
55 resume_session_execution, ResumeExecutionPort, ResumeSpawnRequest,
56};
57use crate::session_app::types::{ResumeConfigSnapshot, ResumeOutcome};
58
59const AGENT_RUNTIME_STATE_METADATA_KEY: &str = "agent.runtime.state";
60const RUNTIME_RESUME_MESSAGE_HIDDEN_KEY: &str = "hidden_from_ui";
61const RUNTIME_RESUME_MESSAGE_KIND_KEY: &str = "runtime_kind";
62const CHILD_COMPLETION_INLINE_FIELD_BYTES: usize = 48 * 1024;
63const CHILD_COMPLETION_OVERSIZE_TAIL_BYTES: usize = 8 * 1024;
64
65fn read_runtime_state(session: &Session) -> AgentRuntimeState {
66 session
67 .agent_runtime_state
68 .clone()
69 .or_else(|| {
70 session
71 .metadata
72 .get(AGENT_RUNTIME_STATE_METADATA_KEY)
73 .and_then(|raw| serde_json::from_str::<AgentRuntimeState>(raw).ok())
74 })
75 .unwrap_or_else(|| AgentRuntimeState::new(format!("{}-child-wait", session.id)))
76}
77
78fn write_runtime_state(session: &mut Session, runtime_state: &AgentRuntimeState) {
79 session.agent_runtime_state = Some(runtime_state.clone());
80 if let Ok(serialized) = serde_json::to_string(runtime_state) {
81 session
82 .metadata
83 .insert(AGENT_RUNTIME_STATE_METADATA_KEY.to_string(), serialized);
84 }
85}
86
87async fn prepare_session_inbox_activation(
93 persistence: &LockedSessionStore,
94 session_id: &str,
95 interrupt_specific_wait: bool,
96) -> std::io::Result<Option<(Session, bool)>> {
97 let ready = Arc::new(std::sync::atomic::AtomicBool::new(false));
98 let ready_for_mutation = ready.clone();
99 let saved = persistence
100 .update_runtime_config(session_id, move |latest| {
101 if latest.has_pending_question() {
105 return;
106 }
107 let mut runtime_state = read_runtime_state(latest);
108 let specifically_waiting = runtime_state.waiting_for_children.is_some()
109 || runtime_state.waiting_for_bash.is_some();
110 if specifically_waiting && !interrupt_specific_wait {
111 return;
112 }
113 runtime_state.status = AgentStatusState::Idle;
119 runtime_state.suspension = None;
120 write_runtime_state(latest, &runtime_state);
121 latest.metadata.remove("runtime.suspend_reason");
122 latest.updated_at = Utc::now();
123 ready_for_mutation.store(true, std::sync::atomic::Ordering::Release);
124 })
125 .await?;
126 Ok(saved.map(|session| (session, ready.load(std::sync::atomic::Ordering::Acquire))))
127}
128
129fn is_error_like(status: &str) -> bool {
130 matches!(status, "error" | "timeout" | "cancelled")
131}
132
133fn is_terminal_child_status(status: &str) -> bool {
135 matches!(
136 status,
137 "completed" | "error" | "timeout" | "cancelled" | "skipped"
138 )
139}
140
141fn child_completion_envelope(
142 completion: &ChildCompletion,
143 wait_registered_at: chrono::DateTime<Utc>,
144 result: Option<String>,
145 provider_message: &Message,
146) -> SessionMessageEnvelope {
147 fn bounded_terminal_field(
148 label: &str,
149 child_session_id: &str,
150 value: Option<String>,
151 ) -> (Option<String>, serde_json::Value, bool) {
152 let Some(value) = value else {
153 return (None, serde_json::Value::Null, false);
154 };
155 if value.len() <= CHILD_COMPLETION_INLINE_FIELD_BYTES {
156 return (Some(value.clone()), serde_json::Value::String(value), false);
157 }
158 let digest = hex::encode(Sha256::digest(value.as_bytes()));
159 let mut tail_start = value
160 .len()
161 .saturating_sub(CHILD_COMPLETION_OVERSIZE_TAIL_BYTES);
162 while tail_start < value.len() && !value.is_char_boundary(tail_start) {
163 tail_start += 1;
164 }
165 let tail = &value[tail_start..];
166 let summary = format!(
167 "Child {label} exceeded the durable inline limit ({} UTF-8 bytes, sha256={digest}). \
168 Retrieve the full child transcript with SubAgent.get(child_session_id=\"{child_session_id}\").\
169 \n\nBounded tail:\n{tail}",
170 value.len()
171 );
172 (
173 Some(summary),
174 serde_json::json!({
175 "oversized": true,
176 "utf8_bytes": value.len(),
177 "sha256": digest,
178 }),
179 true,
180 )
181 }
182
183 let (stored_result, result_identity, _result_oversized) =
184 bounded_terminal_field("result", &completion.child_session_id, result);
185 let (stored_error, error_identity, _error_oversized) = bounded_terminal_field(
186 "error",
187 &completion.child_session_id,
188 completion.error.clone(),
189 );
190 let mut bounded_provider_message = provider_message.clone();
191 let provider_oversized = serde_json::to_vec(provider_message)
192 .map(|bytes| bytes.len() > CHILD_COMPLETION_INLINE_FIELD_BYTES)
193 .unwrap_or(true);
194 if provider_oversized {
195 let mut content = format!(
196 "Runtime notification: child session `{}` finished with status `{}`.",
197 completion.child_session_id, completion.status
198 );
199 if let Some(result) = stored_result.as_deref() {
200 content.push_str("\n\n");
201 content.push_str(result);
202 }
203 if let Some(error) = stored_error.as_deref() {
204 content.push_str("\n\n");
205 content.push_str(error);
206 }
207 bounded_provider_message.content = content;
208 bounded_provider_message.content_parts = None;
209 }
210 let body = SessionMessageBody::ChildOutcome(SessionChildOutcome {
211 child_session_id: completion.child_session_id.clone(),
212 status: completion.status.clone(),
213 result: stored_result,
214 error: stored_error,
215 provider_message: Some(session_provider_message(&bounded_provider_message)),
216 });
217 let semantic = serde_json::json!({
218 "parent_session_id": completion.parent_session_id,
219 "child_session_id": completion.child_session_id,
220 "status": completion.status,
221 "error": error_identity,
222 "result": result_identity,
223 "wait_registered_at": wait_registered_at,
224 });
225 SessionMessageEnvelope {
226 id: SessionMessageId::stable("session_child_completion", &semantic),
227 source: SessionMessageSource::Runtime {
228 subsystem: "child_completion_coordinator".to_string(),
229 },
230 target_session_id: completion.parent_session_id.clone(),
231 kind: SessionMessageKind::ChildOutcome,
232 body,
233 created_at: completion.completed_at,
234 thread_id: None,
235 in_reply_to: None,
236 attempt: None,
237 correlation_id: Some(format!("child_completion:{}", completion.child_session_id)),
238 }
239}
240
241fn session_provider_message(message: &Message) -> SessionProviderMessage {
242 SessionProviderMessage {
243 content: SessionMessageContent {
244 text: message.content.clone(),
245 parts: message.content_parts.clone().unwrap_or_default(),
246 },
247 metadata: message
248 .metadata
249 .as_ref()
250 .and_then(serde_json::Value::as_object)
251 .cloned()
252 .unwrap_or_default(),
253 never_compress: message.never_compress,
254 }
255}
256
257async fn derive_completed_child_ids(
262 storage: &Arc<dyn Storage>,
263 parent_session_id: &str,
264 just_completed_child_id: &str,
265) -> Vec<String> {
266 let mut completed: Vec<String> = storage
267 .list_child_run_statuses(parent_session_id)
268 .await
269 .unwrap_or_default()
270 .into_iter()
271 .filter(|(_, status)| status.as_deref().is_some_and(is_terminal_child_status))
272 .map(|(id, _)| id)
273 .collect();
274 if !completed.iter().any(|id| id == just_completed_child_id) {
275 completed.push(just_completed_child_id.to_string());
276 }
277 completed.sort();
278 completed.dedup();
279 completed
280}
281
282fn read_config_snapshot(config: &Arc<RwLock<Config>>, cached_config: &StdRwLock<Config>) -> Config {
283 if let Ok(config_guard) = config.try_read() {
284 let snapshot = config_guard.clone();
285
286 if let Ok(mut cached_guard) = cached_config.try_write() {
287 *cached_guard = snapshot.clone();
288 }
289
290 snapshot
291 } else {
292 cached_config
293 .try_read()
294 .map(|guard| guard.clone())
295 .unwrap_or_default()
296 }
297}
298
299fn parent_locks() -> &'static std::sync::Mutex<HashMap<String, Arc<tokio::sync::Mutex<()>>>> {
315 static LOCKS: OnceLock<std::sync::Mutex<HashMap<String, Arc<tokio::sync::Mutex<()>>>>> =
316 OnceLock::new();
317 LOCKS.get_or_init(|| std::sync::Mutex::new(HashMap::new()))
318}
319
320fn session_resume_lock(session_id: &str) -> SessionResumeLock {
327 let mut map = parent_locks().lock().recover_poison();
328 let lock = map
329 .entry(session_id.to_string())
330 .or_insert_with(|| Arc::new(tokio::sync::Mutex::new(())))
331 .clone();
332 SessionResumeLock {
333 session_id: session_id.to_string(),
334 lock: Some(lock),
335 }
336}
337
338struct SessionResumeLock {
342 session_id: String,
343 lock: Option<Arc<tokio::sync::Mutex<()>>>,
344}
345
346impl std::ops::Deref for SessionResumeLock {
347 type Target = tokio::sync::Mutex<()>;
348 fn deref(&self) -> &Self::Target {
349 self.lock.as_ref().expect("live resume-lock lease")
350 }
351}
352
353impl Drop for SessionResumeLock {
354 fn drop(&mut self) {
355 self.lock.take();
356 let mut map = parent_locks().lock().recover_poison();
357 if map
358 .get(&self.session_id)
359 .is_some_and(|lock| Arc::strong_count(lock) == 1)
360 {
361 map.remove(&self.session_id);
362 }
363 }
364}
365
366fn wait_policy_satisfied(
367 policy: ChildWaitPolicy,
368 wait_child_ids: &[String],
369 completed_child_ids: &[String],
370 latest_child_id: &str,
371 latest_status: &str,
372) -> bool {
373 if wait_child_ids.is_empty() {
374 return false;
375 }
376
377 match policy {
378 ChildWaitPolicy::All => wait_child_ids
379 .iter()
380 .all(|id| completed_child_ids.iter().any(|completed| completed == id)),
381 ChildWaitPolicy::Any => completed_child_ids
382 .iter()
383 .any(|id| wait_child_ids.iter().any(|wait_id| wait_id == id)),
384 ChildWaitPolicy::FirstError => {
385 (is_error_like(latest_status) && wait_child_ids.iter().any(|id| id == latest_child_id))
391 || wait_child_ids
392 .iter()
393 .all(|id| completed_child_ids.iter().any(|completed| completed == id))
394 }
395 }
396}
397
398fn child_final_assistant_text(child: &Session) -> Option<String> {
402 child
403 .messages
404 .iter()
405 .rev()
406 .find(|message| matches!(message.role, Role::Assistant))
407 .map(|message| message.content.clone())
408 .filter(|content| !content.trim().is_empty())
409}
410
411fn runtime_resume_message(
412 completion: &ChildCompletion,
413 remaining_children: usize,
414 child_final_response: Option<&str>,
415) -> Message {
416 let mut body = format!(
417 "Runtime notification: child session `{}` finished with status `{}`. Remaining child sessions: {}.",
418 completion.child_session_id, completion.status, remaining_children
419 );
420
421 let final_response = child_final_response.map(str::to_string);
427 if let Some(response) = final_response.as_deref() {
428 body.push_str("\n\nChild final response:\n");
429 body.push_str(response);
430 } else if let Some(error) = completion.error.as_deref() {
431 if !error.is_empty() {
432 body.push_str("\n\nChild error:\n");
433 body.push_str(error);
434 }
435 }
436
437 body.push_str(
438 "\n\nResume the parent task using this child result and continue from the previous plan. \
439 If you need the full child transcript, call SubAgent.get(child_session_id).",
440 );
441
442 let mut message = Message::user(body);
443 message.metadata = Some(serde_json::json!({
444 RUNTIME_RESUME_MESSAGE_HIDDEN_KEY: true,
445 RUNTIME_RESUME_MESSAGE_KIND_KEY: "child_completion_resume",
446 "child_session_id": completion.child_session_id,
447 "child_status": completion.status,
448 "child_error": completion.error,
449 "child_final_response_included": final_response.is_some(),
450 }));
451 message.never_compress = false;
455 message
456}
457
458fn guardian_resume_message(completion: &ChildCompletion, verdict: &GuardianVerdict) -> Message {
463 let mut body = if verdict.approve {
464 String::from(
465 "Guardian review APPROVED: an independent reviewer verified the work and found no blocking issues. You may finalize the task.",
466 )
467 } else {
468 String::from(
469 "Guardian review REJECTED: an independent reviewer found issues. Address every finding below before completing — do NOT declare the task complete until they are resolved.",
470 )
471 };
472 if let Some(summary) = verdict.summary.as_deref().filter(|s| !s.trim().is_empty()) {
473 body.push_str("\n\nReviewer summary: ");
474 body.push_str(summary);
475 }
476 if !verdict.findings.is_empty() {
477 body.push_str("\n\nFindings:");
478 for (idx, finding) in verdict.findings.iter().enumerate() {
479 body.push_str(&format!("\n{}. {}", idx + 1, finding));
480 }
481 }
482 body.push_str(
483 "\n\nIf you need the full guardian transcript, call SubAgent.get(child_session_id).",
484 );
485
486 let mut message = Message::user(body);
487 message.metadata = Some(serde_json::json!({
488 RUNTIME_RESUME_MESSAGE_HIDDEN_KEY: true,
489 RUNTIME_RESUME_MESSAGE_KIND_KEY: "guardian_review_resume",
490 "child_session_id": completion.child_session_id,
491 "child_status": completion.status,
492 "guardian_approved": verdict.approve,
493 }));
494 message.never_compress = false;
495 message
496}
497
498#[derive(Clone)]
499pub struct ChildCompletionCoordinator {
500 storage: Arc<dyn Storage>,
501 persistence: Arc<bamboo_storage::LockedSessionStore>,
502 sessions: crate::SessionCache,
503 agent_runners: Arc<RwLock<HashMap<String, AgentRunner>>>,
504 session_event_senders: Arc<RwLock<HashMap<String, broadcast::Sender<AgentEvent>>>>,
505 agent: Arc<Agent>,
506 config: Arc<RwLock<Config>>,
507 provider_registry: Arc<ProviderRegistry>,
508 provider_router: Arc<ProviderModelRouter>,
509 app_data_dir: std::path::PathBuf,
510 account_feed_inbox: Option<crate::execution::AccountFeedInbox>,
511 root_tools: Arc<RwLock<Option<Arc<dyn ToolExecutor>>>>,
512 guardian_spawner: Arc<RwLock<Option<Arc<dyn GuardianSpawner>>>>,
516 spawn_scheduler: Arc<RwLock<Weak<SpawnScheduler>>>,
520}
521
522impl ChildCompletionCoordinator {
523 #[allow(clippy::too_many_arguments)]
524 pub fn new(
525 storage: Arc<dyn Storage>,
526 persistence: Arc<LockedSessionStore>,
527 sessions: crate::SessionCache,
528 agent_runners: Arc<RwLock<HashMap<String, AgentRunner>>>,
529 session_event_senders: Arc<RwLock<HashMap<String, broadcast::Sender<AgentEvent>>>>,
530 agent: Arc<Agent>,
531 config: Arc<RwLock<Config>>,
532 provider_registry: Arc<ProviderRegistry>,
533 provider_router: Arc<ProviderModelRouter>,
534 app_data_dir: std::path::PathBuf,
535 account_feed_inbox: Option<crate::execution::AccountFeedInbox>,
536 ) -> Self {
537 Self {
538 storage,
539 persistence,
540 sessions,
541 agent_runners,
542 session_event_senders,
543 agent,
544 config,
545 provider_registry,
546 provider_router,
547 app_data_dir,
548 account_feed_inbox,
549 root_tools: Arc::new(RwLock::new(None)),
550 guardian_spawner: Arc::new(RwLock::new(None)),
551 spawn_scheduler: Arc::new(RwLock::new(Weak::new())),
552 }
553 }
554
555 pub async fn set_root_tools(&self, tools: Arc<dyn ToolExecutor>) {
556 *self.root_tools.write().await = Some(tools);
557 }
558
559 pub async fn set_spawn_scheduler(&self, scheduler: &Arc<SpawnScheduler>) {
560 *self.spawn_scheduler.write().await = Arc::downgrade(scheduler);
561 }
562
563 pub async fn set_guardian_spawner(&self, spawner: Arc<dyn GuardianSpawner>) {
566 *self.guardian_spawner.write().await = Some(spawner);
567 }
568
569 fn build_resume_config(
570 &self,
571 session: &Session,
572 config_snapshot: &Config,
573 ) -> ResumeConfigSnapshot {
574 crate::session_app::resolution::resolve_resume_config_snapshot(
575 config_snapshot,
576 &self.provider_registry,
577 session,
578 None,
579 )
580 }
581
582 async fn resume_parent(&self, parent_session_id: String) -> ResumeOutcome {
590 for attempt in 0..=5u8 {
591 if attempt > 0 {
592 tokio::time::sleep(Duration::from_millis(250 * attempt as u64)).await;
593 }
594
595 let Some(session) = self.load_session(&parent_session_id).await else {
596 tracing::warn!(%parent_session_id, "cannot resume parent after child completion: session not found");
597 return ResumeOutcome::NotFound;
598 };
599 let config_snapshot = self.config.read().await.clone();
600 let resume_config = self.build_resume_config(&session, &config_snapshot);
601 let outcome = resume_session_execution(self, &parent_session_id, resume_config).await;
602 tracing::info!(
603 %parent_session_id,
604 attempt,
605 outcome = outcome.as_str(),
606 "child completion requested parent resume"
607 );
608
609 if !matches!(outcome, ResumeOutcome::AlreadyRunning { .. }) {
610 return outcome;
611 }
612 }
613 tracing::error!(
619 %parent_session_id,
620 "parent resume gave up after AlreadyRunning retry budget; \
621 relying on the child-wait watchdog backstop"
622 );
623 ResumeOutcome::AlreadyRunning {
624 run_id: String::new(),
625 }
626 }
627
628 async fn save_and_cache(&self, session: &mut Session) {
629 if let Err(error) = self
630 .persistence
631 .merge_save_runtime_and_publish(session, |saved, _| {
632 self.sessions.insert(
633 saved.id.clone(),
634 Arc::new(crate::SessionSnapshot::new(saved.clone())),
635 );
636 })
637 .await
638 {
639 tracing::warn!(session_id = %session.id, %error, "failed to persist session");
640 }
641 }
642}
643
644#[async_trait]
645impl ChildCompletionHandler for ChildCompletionCoordinator {
646 async fn on_child_completed(&self, completion: ChildCompletion) {
647 if !is_terminal_child_status(&completion.status) {
656 tracing::info!(
657 parent_session_id = %completion.parent_session_id,
658 child_session_id = %completion.child_session_id,
659 status = %completion.status,
660 "non-terminal child status; leaving the parent wait armed"
661 );
662 return;
663 }
664
665 let per_parent = session_resume_lock(&completion.parent_session_id);
670 let _per_parent_guard = per_parent.lock().await;
671
672 let Some(mut parent) = self.load_session(&completion.parent_session_id).await else {
673 tracing::warn!(
674 parent_session_id = %completion.parent_session_id,
675 child_session_id = %completion.child_session_id,
676 "child completion received for missing parent"
677 );
678 return;
679 };
680
681 let mut runtime_state = read_runtime_state(&parent);
687
688 let completed_child_ids = derive_completed_child_ids(
691 &self.storage,
692 &completion.parent_session_id,
693 &completion.child_session_id,
694 )
695 .await;
696
697 let mut should_resume = false;
698 let mut remaining_children = 0usize;
699 let active_wait = runtime_state.waiting_for_children.clone();
700 if let Some(wait) = active_wait.as_ref() {
701 remaining_children = wait
702 .child_session_ids
703 .iter()
704 .filter(|id| !completed_child_ids.iter().any(|completed| completed == *id))
705 .count();
706 should_resume = wait_policy_satisfied(
707 wait.wait_for,
708 &wait.child_session_ids,
709 &completed_child_ids,
710 &completion.child_session_id,
711 &completion.status,
712 );
713 }
714
715 let reported_child_owned = match self
727 .storage
728 .load_runtime_control_plane(&completion.child_session_id)
729 .await
730 {
731 Ok(Some(control_plane)) => completion_child_is_owned(
732 &completion.parent_session_id,
733 control_plane.parent_session_id.as_deref(),
734 ),
735 _ => false,
736 };
737
738 let loaded_child = if reported_child_owned {
743 match self
744 .storage
745 .load_session(&completion.child_session_id)
746 .await
747 {
748 Ok(child) => child,
749 Err(error) => {
750 tracing::warn!(
751 child_session_id = %completion.child_session_id,
752 %error,
753 "failed to load child session for runtime resume message"
754 );
755 None
756 }
757 }
758 } else {
759 tracing::warn!(
760 parent_session_id = %completion.parent_session_id,
761 child_session_id = %completion.child_session_id,
762 "completion child is not a child of this parent; resuming with a neutral \
763 message and NOT folding its content"
764 );
765 None
766 };
767
768 let child_final_response = loaded_child.as_ref().and_then(child_final_assistant_text);
769 let guardian_resume = if should_resume {
774 let reviewed_round = runtime_state.round.current_round;
775 loaded_child.as_ref().and_then(|child| {
776 if child.subagent_type().as_deref() != Some("guardian") {
777 return None;
778 }
779 let mut guardian_state = read_guardian_state(&parent)?;
780 if guardian_state.guardian_child_id.as_deref()
781 != Some(completion.child_session_id.as_str())
782 {
783 tracing::warn!(
786 parent_session_id = %completion.parent_session_id,
787 child_session_id = %completion.child_session_id,
788 expected = ?guardian_state.guardian_child_id,
789 "guardian completion does not match recorded guardian_child_id; using generic resume"
790 );
791 return None;
792 }
793 let verdict = child_final_assistant_text(child)
801 .and_then(|text| match parse_guardian_verdict(&text) {
802 Ok(verdict) => Some(verdict),
803 Err(error) => {
804 tracing::warn!(
805 child_session_id = %completion.child_session_id,
806 %error,
807 "guardian verdict unparseable; recording a synthetic reject"
808 );
809 None
810 }
811 })
812 .unwrap_or_else(|| {
813 GuardianVerdict::rejected(vec![
814 "The guardian reviewer did not return a usable verdict (it errored or \
815 emitted unparseable output); the work has NOT been independently \
816 verified."
817 .to_string(),
818 ])
819 });
820 let approved = verdict.approve;
821 let message = guardian_resume_message(&completion, &verdict);
822 guardian_state.record_verdict(verdict, reviewed_round);
823 write_guardian_state(&mut parent, guardian_state);
824 tracing::info!(
825 parent_session_id = %completion.parent_session_id,
826 child_session_id = %completion.child_session_id,
827 approved,
828 "guardian verdict recorded; resuming parent"
829 );
830 Some(message)
831 })
832 } else {
833 None
834 };
835 let resume_message = guardian_resume.unwrap_or_else(|| {
836 runtime_resume_message(
837 &completion,
838 remaining_children,
839 child_final_response.as_deref(),
840 )
841 });
842
843 let messenger = self.agent.session_messenger().cloned();
848 let child_admission =
849 if let (Some(wait), Some(messenger)) = (active_wait.as_ref(), messenger.as_ref()) {
850 let envelope = child_completion_envelope(
851 &completion,
852 wait.registered_at,
853 child_final_response,
854 &resume_message,
855 );
856 match messenger.admit(envelope).await {
857 Ok(admission) => Some(admission),
858 Err(error) => {
859 tracing::warn!(
860 parent_session_id = %completion.parent_session_id,
861 child_session_id = %completion.child_session_id,
862 %error,
863 "child outcome SessionInbox admission failed; leaving parent wait armed"
864 );
865 return;
866 }
867 }
868 } else {
869 None
870 };
871
872 if should_resume {
873 if let (Some(messenger), Some(admission)) =
874 (messenger.as_ref(), child_admission.as_ref())
875 {
876 if let Err(error) = messenger.prepare_activation(admission).await {
877 tracing::warn!(
878 parent_session_id = %completion.parent_session_id,
879 child_session_id = %completion.child_session_id,
880 %error,
881 "child outcome activation watermark failed; leaving parent wait armed"
882 );
883 return;
884 }
885 }
886 }
887
888 if should_resume {
889 runtime_state.waiting_for_children = None;
890 runtime_state.status = AgentStatusState::Idle;
891 runtime_state.suspension = None;
892 parent.metadata.remove("runtime.suspend_reason");
893
894 if child_admission.is_none() {
895 parent.add_message(resume_message);
899 }
900 } else if runtime_state.waiting_for_children.is_some() {
901 runtime_state.status = AgentStatusState::Suspended;
902 runtime_state.suspension = Some(SuspensionState {
903 reason: "waiting_for_children".to_string(),
904 suspended_at: Utc::now(),
905 resumable: true,
906 hook_point: Some("ChildCompletion".to_string()),
907 });
908 }
909
910 parent.updated_at = Utc::now();
911 write_runtime_state(&mut parent, &runtime_state);
912 if let Err(error) = self
913 .persistence
914 .checkpoint_runtime_session(&mut parent)
915 .await
916 {
917 tracing::warn!(
918 parent_session_id = %completion.parent_session_id,
919 child_session_id = %completion.child_session_id,
920 %error,
921 "child outcome is durable but parent wait transition failed; leaving activation deferred"
922 );
923 return;
924 }
925 self.sessions.insert(
926 parent.id.clone(),
927 Arc::new(crate::SessionSnapshot::new(parent.clone())),
928 );
929
930 let resume_parent_id = parent.id.clone();
935 drop(_per_parent_guard);
936
937 if should_resume {
938 if let (Some(messenger), Some(admission)) = (messenger, child_admission) {
939 if let Err(error) = messenger.activate_prepared(&admission).await {
940 tracing::warn!(
941 parent_session_id = %resume_parent_id,
942 %error,
943 "child outcome and wait transition are durable but activation failed"
944 );
945 }
946 } else {
947 self.resume_parent(resume_parent_id).await;
948 }
949 }
950 }
951}
952
953#[async_trait]
954impl ResumeExecutionPort for ChildCompletionCoordinator {
955 async fn load_session(&self, session_id: &str) -> Option<Session> {
956 match self.storage.load_session(session_id).await {
957 Ok(Some(session)) => Some(session),
958 Ok(None) => self
959 .sessions
960 .get(session_id)
961 .map(|e| e.value().clone())
962 .map(|arc| arc.read().clone()),
963 Err(error) => {
964 tracing::warn!(%session_id, %error, "failed to load session from storage");
965 self.sessions
966 .get(session_id)
967 .map(|e| e.value().clone())
968 .map(|arc| arc.read().clone())
969 }
970 }
971 }
972
973 async fn save_and_cache_session(&self, session: &mut Session) {
974 self.save_and_cache(session).await;
975 }
976
977 async fn reserve_session_execution(
978 &self,
979 session_id: &str,
980 event_sender: &broadcast::Sender<AgentEvent>,
981 ) -> SessionExecutionReserveOutcome {
982 reserve_session_execution(
983 &self.agent,
984 &self.agent_runners,
985 &self.session_event_senders,
986 session_id,
987 event_sender,
988 )
989 .await
990 }
991
992 async fn get_or_create_event_sender(&self, session_id: &str) -> broadcast::Sender<AgentEvent> {
993 crate::execution::session_events::get_or_create_event_sender(
994 &self.session_event_senders,
995 session_id,
996 )
997 .await
998 }
999
1000 fn dispatch_resume_execution(
1001 &self,
1002 request: ResumeSpawnRequest,
1003 ) -> Result<(), ResumeSpawnRequest> {
1004 let owner = self.clone();
1005 tokio::spawn(async move {
1006 ResumeExecutionPort::spawn_resume_execution(&owner, request).await;
1007 });
1008 Ok(())
1009 }
1010
1011 async fn spawn_resume_execution(&self, request: ResumeSpawnRequest) {
1012 let ResumeSpawnRequest {
1013 session_id,
1014 mut session,
1015 mut execution_reservation,
1016 event_sender,
1017 config,
1018 } = request;
1019 if let Err(error) = execution_reservation.ensure_registered().await {
1020 tracing::warn!(
1021 %session_id,
1022 run_id = %execution_reservation.run_id(),
1023 %error,
1024 "cannot resume after child completion without exact router ownership"
1025 );
1026 return;
1027 }
1028
1029 let Some(root_tools) = self.root_tools.read().await.clone() else {
1030 tracing::error!(%session_id, "cannot resume parent after child completion: root tool surface is not initialized");
1031 return;
1032 };
1033
1034 let config_snapshot = self.config.read().await.clone();
1035 let model = session.model.clone();
1036 let session_model_ref = session_effective_model_ref(&session);
1037 let requested_provider = session_model_ref
1038 .as_ref()
1039 .map(|model_ref| model_ref.provider.as_str())
1040 .unwrap_or(config.provider_name.as_str());
1041 let resolved_provider_name = match resolve_provider_routing_key(
1042 &config_snapshot,
1043 requested_provider,
1044 &self.provider_registry,
1045 ) {
1046 Ok(provider) => provider,
1047 Err(error) => {
1048 tracing::error!(
1049 session_id = %session_id,
1050 provider = requested_provider,
1051 %error,
1052 "child-completion resume provider is unavailable; refusing to fall back"
1053 );
1054 execution_reservation.abandon().await;
1055 return;
1056 }
1057 };
1058 let provider_override = if let Some(mut model_ref) = session_model_ref {
1059 model_ref.provider = resolved_provider_name.clone();
1060 persist_model_ref(&mut session, &model_ref);
1061 match self.provider_router.route(&model_ref) {
1062 Ok(provider) => Some(provider),
1063 Err(error) => {
1064 tracing::error!(
1065 session_id = %session_id,
1066 provider = %model_ref.provider,
1067 model = %model_ref.model,
1068 %error,
1069 "child-completion resume provider routing failed closed"
1070 );
1071 execution_reservation.abandon().await;
1072 return;
1073 }
1074 }
1075 } else {
1076 match self.provider_registry.get(&resolved_provider_name) {
1077 Some(provider) => Some(provider),
1078 None => {
1079 tracing::error!(
1080 session_id = %session_id,
1081 provider = %resolved_provider_name,
1082 "child-completion resume provider disappeared after resolution; refusing to fall back"
1083 );
1084 execution_reservation.abandon().await;
1085 return;
1086 }
1087 }
1088 };
1089 let resolved_fast_provider = resolve_fast_model(
1090 &config_snapshot,
1091 &resolved_provider_name,
1092 &self.provider_registry,
1093 )
1094 .map(|model| model.provider);
1095 let reasoning_effort = session.reasoning_effort;
1096 let reasoning_effort_source = session
1097 .metadata
1098 .get("reasoning_effort_source")
1099 .cloned()
1100 .unwrap_or_default();
1101 let gold_config = resolve_gold_config(
1102 &config_snapshot,
1103 session
1104 .metadata
1105 .get(GOLD_CONFIG_METADATA_KEY)
1106 .map(String::as_str),
1107 )
1108 .or(config.gold_config.clone());
1109
1110 let (mpsc_tx, _forwarder) = create_event_forwarder(
1111 session_id.clone(),
1112 execution_reservation.run_id().to_string(),
1113 event_sender,
1114 self.agent_runners.clone(),
1115 self.account_feed_inbox.clone(),
1116 );
1117
1118 let config_handle = self.config.clone();
1119 let cached_config = Arc::new(StdRwLock::new(config_snapshot.clone()));
1120 let provider_registry = self.provider_registry.clone();
1121 let provider_name_for_aux = resolved_provider_name.clone();
1122 let auxiliary_model_resolver = std::sync::Arc::new(move || {
1123 let config_snapshot = read_config_snapshot(&config_handle, cached_config.as_ref());
1124 let areas = resolve_global_area_models(
1126 &config_snapshot,
1127 &provider_name_for_aux,
1128 &provider_registry,
1129 );
1130 crate::AuxiliaryModelConfig {
1131 fast_model_name: areas.fast.as_ref().map(|m| m.model_name.clone()),
1132 fast_model_provider: areas.fast.map(|m| m.provider),
1133 background_model_name: areas.background.as_ref().map(|m| m.model_name.clone()),
1134 planning_model_name: None,
1135 search_model_name: None,
1136 summarization_model_name: areas
1137 .summarization
1138 .as_ref()
1139 .map(|m| m.model_name.clone()),
1140 background_model_provider: areas.background.map(|m| m.provider),
1141 summarization_model_provider: areas.summarization.map(|m| m.provider),
1142 }
1143 });
1144 let model_roster = crate::ModelRoster {
1145 model: Some(model),
1146 provider_name: Some(resolved_provider_name),
1147 provider_type: config.provider_type.clone(),
1148 fast: crate::RoleModel::from_parts(config.fast_model, resolved_fast_provider),
1149 background: crate::RoleModel::from_parts(
1150 config.background_model,
1151 config.background_model_provider,
1152 ),
1153 summarization: crate::RoleModel::from_parts(
1154 config.summarization_model,
1155 config.summarization_model_provider,
1156 ),
1157 };
1158
1159 let guardian_config = read_guardian_config(&session);
1164 let guardian_spawner = self.guardian_spawner.read().await.clone();
1165
1166 consume_pending_clarification_resume(&mut session);
1167 spawn_session_execution(SessionExecutionArgs {
1168 agent: self.agent.clone(),
1169 session_id,
1170 session,
1171 execution_reservation,
1172 tools_override: Some(root_tools),
1173 provider_override,
1174 model_roster,
1175 reasoning_effort,
1176 reasoning_effort_source,
1177 auxiliary_model_resolver: Some(auxiliary_model_resolver),
1178 disabled_filter_resolver: None,
1181 disabled_tools: Some(config.disabled_tools),
1182 disabled_skill_ids: Some(config.disabled_skill_ids),
1183 selected_skill_ids: None,
1184 selected_skill_mode: None,
1185 mpsc_tx,
1186 image_fallback: config.image_fallback,
1187 gold_config,
1188 guardian_config,
1189 guardian_spawner,
1190 bash_resume_hook: {
1191 let hook: Arc<dyn BashResumeHook> = Arc::new(self.clone());
1192 Some(hook)
1193 },
1194 bash_completion_sink: {
1195 let sink: Arc<dyn BashCompletionSink> = Arc::new(self.clone());
1198 Some(sink)
1199 },
1200 app_data_dir: Some(self.app_data_dir.clone()),
1201 run_budget: None,
1204 runners: self.agent_runners.clone(),
1205 sessions_cache: self.sessions.clone(),
1206 on_complete: None,
1207 child_completion_handler: Some(Arc::new(self.clone())),
1211 });
1212 }
1213}
1214
1215#[async_trait]
1220impl SessionActivationSpawner for ChildCompletionCoordinator {
1221 async fn reserve_activation(
1222 &self,
1223 target_session_id: &str,
1224 _inbox_generation: u64,
1225 ) -> Result<SessionActivationReserveOutcome, bamboo_domain::SessionActivationError> {
1226 let Some(inbox) = self.agent.session_inbox() else {
1227 return Err(bamboo_domain::SessionActivationError::Internal(
1228 "agent runtime has no SessionInbox".to_string(),
1229 ));
1230 };
1231 let backlog = inbox
1232 .inspect(target_session_id)
1233 .await
1234 .map_err(|error| bamboo_domain::SessionActivationError::Internal(error.to_string()))?;
1235 if !backlog.activation_pending() {
1236 return Ok(SessionActivationReserveOutcome::NoWork);
1237 }
1238 let prepared = prepare_session_inbox_activation(
1241 &self.persistence,
1242 target_session_id,
1243 backlog.interrupt_pending(),
1244 )
1245 .await
1246 .map_err(|error| {
1247 bamboo_domain::SessionActivationError::Internal(format!(
1248 "persist resumable SessionInbox target: {error}"
1249 ))
1250 })?;
1251 let Some((session, ready)) = prepared else {
1252 return Ok(SessionActivationReserveOutcome::NotFound);
1253 };
1254 if !ready {
1255 tracing::info!(
1256 session_id = target_session_id,
1257 "SessionInbox backlog is activation-eligible but a specific durable wait remains armed"
1258 );
1259 return Ok(SessionActivationReserveOutcome::NoWork);
1260 }
1261
1262 enum LaunchPlan {
1263 Root(Box<ResumeConfigSnapshot>),
1264 Child {
1265 scheduler: Arc<SpawnScheduler>,
1266 parent_session_id: String,
1267 model: String,
1268 disabled_tools: Option<Vec<String>>,
1269 },
1270 }
1271
1272 let launch_plan = match session.kind {
1275 SessionKind::Root => {
1276 if self.root_tools.read().await.is_none() {
1277 return Err(bamboo_domain::SessionActivationError::Internal(
1278 "root tool surface is not initialized".to_string(),
1279 ));
1280 }
1281 let config_snapshot = self.config.read().await.clone();
1282 LaunchPlan::Root(Box::new(
1283 self.build_resume_config(&session, &config_snapshot),
1284 ))
1285 }
1286 SessionKind::Child => {
1287 let scheduler = self.spawn_scheduler.read().await.upgrade().ok_or_else(|| {
1288 bamboo_domain::SessionActivationError::Internal(
1289 "child spawn scheduler is not initialized".to_string(),
1290 )
1291 })?;
1292 let parent_session_id = session
1293 .parent_session_id
1294 .as_deref()
1295 .map(str::trim)
1296 .filter(|id| !id.is_empty())
1297 .map(ToOwned::to_owned)
1298 .ok_or_else(|| {
1299 bamboo_domain::SessionActivationError::Internal(format!(
1300 "child SessionInbox target {target_session_id} has no parent owner"
1301 ))
1302 })?;
1303 let parent = self
1304 .storage
1305 .load_session(&parent_session_id)
1306 .await
1307 .map_err(|error| {
1308 bamboo_domain::SessionActivationError::Internal(format!(
1309 "load parent owner {parent_session_id}: {error}"
1310 ))
1311 })?
1312 .ok_or_else(|| {
1313 bamboo_domain::SessionActivationError::Internal(format!(
1314 "child SessionInbox parent owner {parent_session_id} disappeared"
1315 ))
1316 })?;
1317 let child_root = if session.root_session_id.trim().is_empty() {
1318 parent_session_id.as_str()
1319 } else {
1320 session.root_session_id.as_str()
1321 };
1322 let parent_root = if parent.root_session_id.trim().is_empty() {
1323 parent.id.as_str()
1324 } else {
1325 parent.root_session_id.as_str()
1326 };
1327 if child_root != parent_root {
1328 return Err(bamboo_domain::SessionActivationError::Internal(format!(
1329 "child SessionInbox target {target_session_id} does not share its parent owner's root"
1330 )));
1331 }
1332 let model = if session.model.trim().is_empty() {
1333 parent.model.clone()
1334 } else {
1335 session.model.clone()
1336 };
1337 if model.trim().is_empty() {
1338 return Err(bamboo_domain::SessionActivationError::Internal(format!(
1339 "child SessionInbox target {target_session_id} has no executable model"
1340 )));
1341 }
1342 let disabled_tools = match session.metadata.get("disabled_tools") {
1343 None => None,
1344 Some(raw) => {
1345 let tools = serde_json::from_str::<std::collections::BTreeSet<String>>(raw)
1346 .map_err(|error| {
1347 bamboo_domain::SessionActivationError::Internal(format!(
1348 "child SessionInbox target {target_session_id} has malformed disabled_tools: {error}"
1349 ))
1350 })?;
1351 (!tools.is_empty()).then(|| tools.into_iter().collect())
1352 }
1353 };
1354 LaunchPlan::Child {
1355 scheduler,
1356 parent_session_id,
1357 model,
1358 disabled_tools,
1359 }
1360 }
1361 };
1362 let event_sender =
1363 ResumeExecutionPort::get_or_create_event_sender(self, target_session_id).await;
1364
1365 let reservation = match reserve_runner_core(
1366 &self.agent_runners,
1367 &self.session_event_senders,
1368 target_session_id,
1369 &event_sender,
1370 )
1371 .await
1372 {
1373 ReserveOutcome::Reserved(reservation) => reservation,
1374 ReserveOutcome::AlreadyRunning(run_id) => {
1375 return Ok(SessionActivationReserveOutcome::AlreadyRunning { run_id });
1376 }
1377 };
1378 let run_id = reservation.run_id.clone();
1379 let launch = match launch_plan {
1380 LaunchPlan::Root(config) => {
1381 let execution_reservation =
1382 SessionExecutionReservation::from_activation_placeholder(
1383 target_session_id,
1384 reservation,
1385 self.agent
1386 .activation_router()
1387 .expect("SessionInbox activation requires an activation router")
1388 .clone(),
1389 self.agent_runners.clone(),
1390 );
1391 let reservation_cell = Arc::new(StdMutex::new(Some(execution_reservation)));
1395 let launch_reservation = reservation_cell.clone();
1396 let rollback_reservation = reservation_cell;
1397 let coordinator = self.clone();
1398 let launch_sessions = self.sessions.clone();
1399 let launch_session_id = session.id.clone();
1400 let launch_session = session.clone();
1401 let request_session_id = target_session_id.to_string();
1402 SessionActivationLaunch::new_with_async_rollback(
1403 run_id,
1404 move || {
1405 let mut execution_reservation = launch_reservation
1406 .lock()
1407 .unwrap_or_else(std::sync::PoisonError::into_inner)
1408 .take()
1409 .expect("activation reservation launches or rolls back exactly once");
1410 execution_reservation.mark_activation_published();
1411 launch_sessions.insert(
1415 launch_session_id,
1416 Arc::new(crate::SessionSnapshot::new(launch_session)),
1417 );
1418 let request = ResumeSpawnRequest {
1419 session_id: request_session_id,
1420 session,
1421 execution_reservation,
1422 event_sender,
1423 config: *config,
1424 };
1425 tokio::spawn(async move {
1426 ResumeExecutionPort::spawn_resume_execution(&coordinator, request)
1427 .await;
1428 });
1429 },
1430 move || async move {
1431 let reservation = rollback_reservation
1432 .lock()
1433 .unwrap_or_else(std::sync::PoisonError::into_inner)
1434 .take();
1435 if let Some(reservation) = reservation {
1436 reservation.rollback_unpublished_activation().await;
1437 }
1438 },
1439 )
1440 }
1441 LaunchPlan::Child {
1442 scheduler,
1443 parent_session_id,
1444 model,
1445 disabled_tools,
1446 } => {
1447 let execution_reservation =
1448 SessionExecutionReservation::from_activation_placeholder(
1449 target_session_id,
1450 reservation,
1451 self.agent
1452 .activation_router()
1453 .expect("SessionInbox activation requires an activation router")
1454 .clone(),
1455 self.agent_runners.clone(),
1456 );
1457 let reservation_cell = Arc::new(StdMutex::new(Some(execution_reservation)));
1458 let launch_reservation = reservation_cell.clone();
1459 let rollback_reservation = reservation_cell;
1460 let job = SpawnJob {
1461 parent_session_id,
1462 child_session_id: target_session_id.to_string(),
1463 model,
1464 disabled_tools,
1465 };
1466 let launch_sessions = self.sessions.clone();
1467 let launch_session_id = session.id.clone();
1468 let launch_session = session;
1469 SessionActivationLaunch::new_with_async_rollback(
1470 run_id,
1471 move || {
1472 let mut execution_reservation = launch_reservation
1473 .lock()
1474 .unwrap_or_else(std::sync::PoisonError::into_inner)
1475 .take()
1476 .expect("activation reservation launches or rolls back exactly once");
1477 execution_reservation.mark_activation_published();
1478 launch_sessions.insert(
1481 launch_session_id,
1482 Arc::new(crate::SessionSnapshot::new(launch_session)),
1483 );
1484 drop(scheduler.launch_reserved(job, execution_reservation));
1488 },
1489 move || async move {
1490 let reservation = rollback_reservation
1491 .lock()
1492 .unwrap_or_else(std::sync::PoisonError::into_inner)
1493 .take();
1494 if let Some(reservation) = reservation {
1495 reservation.rollback_unpublished_activation().await;
1496 }
1497 },
1498 )
1499 }
1500 };
1501 Ok(SessionActivationReserveOutcome::Reserved(launch))
1502 }
1503}
1504
1505fn bash_completion_resume_message(bash_ids: &[String], timed_out: bool) -> Message {
1515 let body = if timed_out {
1516 format!(
1517 "Runtime notification: the background-Bash wait ceiling was reached while one or more \
1518 shell(s) ({}) may still be running. The session is being resumed so it is not \
1519 stranded; verify their actual status with BashOutput before assuming completion.",
1520 bash_ids.join(", ")
1521 )
1522 } else {
1523 format!(
1524 "Runtime notification: all background Bash shell(s) ({}) have completed. \
1525 Review their output with BashOutput and resume the task from where you left off.",
1526 bash_ids.join(", ")
1527 )
1528 };
1529 let mut message = Message::user(body);
1530 message.metadata = Some(serde_json::json!({
1531 RUNTIME_RESUME_MESSAGE_HIDDEN_KEY: true,
1532 RUNTIME_RESUME_MESSAGE_KIND_KEY: BASH_COMPLETION_RESUME_KIND,
1533 }));
1534 message.never_compress = false;
1535 message
1536}
1537
1538fn bash_resume_should_retry(outcome: &ResumeOutcome, persisted_waiting_for_bash: bool) -> bool {
1551 match outcome {
1552 ResumeOutcome::Started { .. } | ResumeOutcome::NotFound => false,
1553 ResumeOutcome::Completed | ResumeOutcome::AlreadyRunning { .. } => {
1554 persisted_waiting_for_bash
1555 }
1556 }
1557}
1558
1559fn bash_completion_should_resume(
1567 loop_suspended_on_bash: bool,
1568 all_waited_shells_done: bool,
1569) -> bool {
1570 loop_suspended_on_bash && all_waited_shells_done
1571}
1572
1573#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1574enum BashCompletionDeliveryPlan {
1575 DurableOnly,
1579 Activate,
1581 ClearWaitThenActivate,
1583}
1584
1585fn bash_completion_delivery_plan(
1586 loop_suspended_on_bash: bool,
1587 all_waited_shells_done: bool,
1588) -> BashCompletionDeliveryPlan {
1589 if bash_completion_should_resume(loop_suspended_on_bash, all_waited_shells_done) {
1590 BashCompletionDeliveryPlan::ClearWaitThenActivate
1591 } else if loop_suspended_on_bash {
1592 BashCompletionDeliveryPlan::DurableOnly
1593 } else {
1594 BashCompletionDeliveryPlan::Activate
1595 }
1596}
1597
1598fn background_bash_post_tool_payload(info: &BashCompletionInfo) -> HookPayload {
1599 let success = info.status == "completed" && info.exit_code == Some(0);
1600 let response = serde_json::json!({
1601 "bash_id": info.bash_id,
1602 "command": info.command,
1603 "exit_code": info.exit_code,
1604 "status": info.status,
1605 "output_tail": info.output_tail,
1606 });
1607 HookPayload::ToolResult {
1608 tool_name: "Bash".to_string(),
1609 tool_call_id: info.bash_id.clone(),
1610 outcome: HookToolOutcome {
1611 success,
1612 result: success.then(|| response.to_string()),
1613 error: (!success).then(|| response.to_string()),
1614 needs_human: false,
1615 duration_ms: 0,
1616 },
1617 }
1618}
1619
1620fn append_background_bash_hook_feedback(info: &mut BashCompletionInfo, feedback: Vec<String>) {
1621 let feedback = feedback
1622 .into_iter()
1623 .map(|text| text.trim().to_string())
1624 .filter(|text| !text.is_empty())
1625 .collect::<Vec<_>>();
1626 if feedback.is_empty() {
1627 return;
1628 }
1629 if !info.output_tail.is_empty() {
1630 info.output_tail.push_str("\n\n");
1631 }
1632 info.output_tail.push_str("<post_tool_use_feedback>\n");
1633 info.output_tail.push_str(&feedback.join("\n"));
1634 info.output_tail.push_str("\n</post_tool_use_feedback>");
1635}
1636
1637async fn run_background_bash_post_tool_hooks(
1638 config: &bamboo_config::LifecycleHooksConfig,
1639 fallback_cwd: Option<std::path::PathBuf>,
1640 session: &Session,
1641 info: &mut BashCompletionInfo,
1642) -> bool {
1643 let runner = crate::HookRunner::new().with_lifecycle_config(config, fallback_cwd);
1644 if !runner.has_hooks_for(AgentHookPoint::AfterToolExecution) {
1645 return false;
1646 }
1647
1648 let mut runtime_state = session
1649 .agent_runtime_state
1650 .clone()
1651 .unwrap_or_else(|| AgentRuntimeState::new(&session.id));
1652 let outcome = runner
1653 .run_observer_hooks(
1654 AgentHookPoint::AfterToolExecution,
1655 &background_bash_post_tool_payload(info),
1656 session,
1657 &mut runtime_state,
1658 None,
1659 )
1660 .await;
1661 append_background_bash_hook_feedback(info, outcome.injected_contexts);
1662 true
1663}
1664
1665fn apply_bash_resume_transition(session: &mut Session, resume_message: &Message) -> bool {
1673 let mut runtime_state = read_runtime_state(session);
1674 if runtime_state.waiting_for_bash.is_none() {
1675 return false;
1676 }
1677 runtime_state.waiting_for_bash = None;
1678 runtime_state.status = AgentStatusState::Idle;
1679 runtime_state.suspension = None;
1680 write_runtime_state(session, &runtime_state);
1681 session.metadata.remove("runtime.suspend_reason");
1682 session.add_message(resume_message.clone());
1683 true
1684}
1685
1686impl ChildCompletionCoordinator {
1688 async fn bash_self_resume(&self, session_id: String, bash_ids: Vec<String>) {
1704 let mut delay = Duration::from_secs(1);
1705 let max_delay = Duration::from_secs(30);
1706 let max_poll = Duration::from_secs(6 * 3600 + 600);
1710 let deadline = tokio::time::Instant::now() + max_poll;
1711
1712 loop {
1713 tokio::time::sleep(delay).await;
1714
1715 let Some(session) = self.load_session(&session_id).await else {
1716 tracing::info!(%session_id, "bash self-resume backstop: session gone; nothing to do");
1717 return;
1718 };
1719 if read_runtime_state(&session).waiting_for_bash.is_none() {
1720 return;
1723 }
1724
1725 let still_running =
1726 bamboo_tools::tools::bash_runtime::running_shells_for_session(&session_id);
1727 let timed_out = tokio::time::Instant::now() >= deadline;
1728 if still_running.is_empty() || timed_out {
1729 let guard = session_resume_lock(&session_id);
1734 let _held = guard.lock().await;
1735 tracing::warn!(
1736 %session_id,
1737 shell_count = bash_ids.len(),
1738 timed_out,
1739 "bash self-resume backstop engaged (push lost or wait ceiling reached)"
1740 );
1741 self.perform_bash_resume(
1742 &session_id,
1743 bash_completion_resume_message(&bash_ids, timed_out),
1744 )
1745 .await;
1746 return;
1747 }
1748
1749 delay = (delay * 2).min(max_delay);
1750 }
1751 }
1752
1753 async fn perform_bash_resume(&self, session_id: &str, resume_message: Message) {
1774 let retry_backoff = Duration::from_millis(200);
1775 const MAX_RESUME_ATTEMPTS: u8 = 5;
1776 for attempt in 0..MAX_RESUME_ATTEMPTS {
1777 if attempt > 0 {
1778 tokio::time::sleep(retry_backoff).await;
1779 }
1780
1781 let Some(mut session) = self.load_session(session_id).await else {
1782 tracing::warn!(%session_id, "bash resume: session not found; nothing to resume");
1783 return;
1784 };
1785
1786 if !apply_bash_resume_transition(&mut session, &resume_message) {
1787 tracing::info!(
1791 %session_id, attempt,
1792 "bash resume: persisted bash wait already cleared; nothing to resume"
1793 );
1794 return;
1795 }
1796 session.updated_at = Utc::now();
1797 self.save_and_cache(&mut session).await;
1798 tracing::info!(
1799 %session_id, attempt,
1800 "bash resume: cleared bash wait and appended resume message"
1801 );
1802
1803 let outcome = self.resume_parent(session_id.to_string()).await;
1804 match outcome {
1805 ResumeOutcome::Started { .. } => {
1806 tracing::info!(%session_id, attempt, "bash resume: resume fired");
1807 return;
1808 }
1809 ResumeOutcome::NotFound => {
1810 tracing::warn!(%session_id, "bash resume: session vanished during resume");
1811 return;
1812 }
1813 _ => {
1814 let clobbered = match self.load_session(session_id).await {
1820 Some(reloaded) => read_runtime_state(&reloaded).waiting_for_bash.is_some(),
1821 None => {
1822 tracing::warn!(%session_id, "bash resume: session vanished after resume");
1823 return;
1824 }
1825 };
1826 if bash_resume_should_retry(&outcome, clobbered) {
1827 tracing::warn!(
1828 %session_id, attempt,
1829 outcome = outcome.as_str(),
1830 "bash resume: persisted wait still set after resume (finalize-clobber); retrying"
1831 );
1832 continue;
1833 }
1834 tracing::info!(
1835 %session_id, attempt,
1836 outcome = outcome.as_str(),
1837 "bash resume: wait cleared and resume handled; stopping"
1838 );
1839 return;
1840 }
1841 }
1842 }
1843
1844 tracing::warn!(
1845 %session_id,
1846 attempts = MAX_RESUME_ATTEMPTS,
1847 "bash resume: exhausted clobber-retry budget without confirming resume; giving up"
1848 );
1849 }
1850}
1851
1852impl BashResumeHook for ChildCompletionCoordinator {
1853 fn arrange_bash_self_resume(&self, session_id: String, bash_ids: Vec<String>) {
1854 let coordinator = Arc::new(self.clone());
1855 tokio::spawn(async move {
1856 coordinator.bash_self_resume(session_id, bash_ids).await;
1857 });
1858 }
1859}
1860
1861fn bash_completion_injection_body(info: &BashCompletionInfo) -> String {
1866 let exit = match info.exit_code {
1867 Some(code) => code.to_string(),
1868 None => "none (signal/killed)".to_string(),
1869 };
1870 let mut body = format!(
1871 "Runtime notification: background shell `{}` (`{}`) finished — status {}, exit code {}.",
1872 info.bash_id, info.command, info.status, exit
1873 );
1874 if info.output_tail.trim().is_empty() {
1875 body.push_str(" It produced no captured output.");
1876 } else {
1877 body.push_str("\n\nOutput tail:\n");
1878 body.push_str(&info.output_tail);
1879 }
1880 body.push_str(&format!(
1881 "\n\nUse BashOutput with bash_id=\"{}\" for the full output, then continue the task.",
1882 info.bash_id
1883 ));
1884 body
1885}
1886
1887fn bash_completion_envelope(info: &BashCompletionInfo) -> bamboo_domain::SessionMessageEnvelope {
1888 let provider_message = bash_resume_message_from_info(info);
1889 let data = serde_json::json!({
1890 "session_id": info.session_id,
1891 "bash_id": info.bash_id,
1892 "command": info.command,
1893 "exit_code": info.exit_code,
1894 "status": info.status,
1895 "output_tail": info.output_tail,
1896 });
1897 let identity = data.clone();
1902 bamboo_domain::SessionMessageEnvelope {
1903 id: bamboo_domain::SessionMessageId::stable("background_bash_completion", &identity),
1904 source: bamboo_domain::SessionMessageSource::Runtime {
1905 subsystem: "background_bash".to_string(),
1906 },
1907 target_session_id: info.session_id.clone(),
1908 kind: bamboo_domain::SessionMessageKind::RuntimeInstruction,
1909 body: bamboo_domain::SessionMessageBody::RuntimeInstruction(
1910 bamboo_domain::SessionRuntimeInstruction {
1911 instruction: "background_bash_completed".to_string(),
1912 content: Some(bamboo_domain::SessionMessageContent::text(
1913 bash_completion_injection_body(info),
1914 )),
1915 data: Some(data),
1916 provider_message: Some(session_provider_message(&provider_message)),
1917 },
1918 ),
1919 created_at: Utc::now(),
1920 thread_id: None,
1921 in_reply_to: None,
1922 attempt: None,
1923 correlation_id: Some(info.bash_id.clone()),
1924 }
1925}
1926
1927async fn enqueue_bash_completion_injection(
1933 persistence: &LockedSessionStore,
1934 info: &BashCompletionInfo,
1935) -> std::io::Result<Option<Session>> {
1936 let body = bash_completion_injection_body(info);
1937 let queued = serde_json::json!({
1938 "content": body,
1939 "created_at": Utc::now(),
1940 });
1941 persistence
1942 .update_runtime_config(&info.session_id, move |session| {
1943 let mut pending = session.pending_injected_messages().unwrap_or_default();
1944 pending.push(queued);
1945 session.set_pending_injected_messages(pending);
1946 })
1947 .await
1948}
1949
1950fn bash_resume_message_from_info(info: &BashCompletionInfo) -> Message {
1958 let mut message = Message::user(bash_completion_injection_body(info));
1959 message.metadata = Some(serde_json::json!({
1960 RUNTIME_RESUME_MESSAGE_HIDDEN_KEY: true,
1961 RUNTIME_RESUME_MESSAGE_KIND_KEY: BASH_COMPLETION_RESUME_KIND,
1962 }));
1963 message.never_compress = false;
1964 message
1965}
1966
1967impl ChildCompletionCoordinator {
1968 async fn deliver_bash_completion(&self, mut info: BashCompletionInfo) {
1980 if let Some(hook_session) = self.load_session(&info.session_id).await {
1986 let config_snapshot = self.config.read().await.clone();
1987 let _ = run_background_bash_post_tool_hooks(
1988 &config_snapshot.lifecycle_hooks,
1989 Some(self.app_data_dir.clone()),
1990 &hook_session,
1991 &mut info,
1992 )
1993 .await;
1994 }
1995
1996 let guard = session_resume_lock(&info.session_id);
1997 let _held = guard.lock().await;
1998
1999 let Some(session) = self.load_session(&info.session_id).await else {
2000 tracing::warn!(
2001 session_id = %info.session_id,
2002 bash_id = %info.bash_id,
2003 "background bash completion: owning session not found; nothing to notify"
2004 );
2005 return;
2006 };
2007
2008 let waiting = read_runtime_state(&session).waiting_for_bash.is_some();
2009 let all_shells_done =
2015 bamboo_tools::tools::bash_runtime::running_shells_for_session(&info.session_id)
2016 .is_empty();
2017 let delivery_plan = bash_completion_delivery_plan(waiting, all_shells_done);
2018
2019 let messenger = self.agent.session_messenger().cloned();
2025 let admission = match messenger.as_ref() {
2026 Some(messenger) => match messenger.admit(bash_completion_envelope(&info)).await {
2027 Ok(admission) => Some(admission),
2028 Err(error) => {
2029 tracing::warn!(
2030 session_id = %info.session_id,
2031 bash_id = %info.bash_id,
2032 %error,
2033 "background bash completion: durable SessionInbox admission failed; leaving wait armed"
2034 );
2035 return;
2036 }
2037 },
2038 None => {
2039 tracing::warn!(
2040 session_id = %info.session_id,
2041 bash_id = %info.bash_id,
2042 "SessionMessenger unavailable; using compatibility injection before clearing wait"
2043 );
2044 match enqueue_bash_completion_injection(&self.persistence, &info).await {
2045 Ok(Some(_)) => None,
2046 Ok(None) => {
2047 tracing::warn!(
2048 session_id = %info.session_id,
2049 "background bash compatibility target disappeared"
2050 );
2051 return;
2052 }
2053 Err(error) => {
2054 tracing::warn!(
2055 session_id = %info.session_id,
2056 %error,
2057 "background bash compatibility admission failed; leaving wait armed"
2058 );
2059 return;
2060 }
2061 }
2062 }
2063 };
2064
2065 if delivery_plan == BashCompletionDeliveryPlan::DurableOnly {
2066 tracing::info!(
2067 session_id = %info.session_id,
2068 bash_id = %info.bash_id,
2069 "background bash completion is durable; sibling shells still run, so wait remains armed"
2070 );
2071 return;
2072 }
2073
2074 if let (Some(messenger), Some(admission)) = (messenger.as_ref(), admission.as_ref()) {
2075 if let Err(error) = messenger.prepare_activation(admission).await {
2076 tracing::warn!(
2077 session_id = %info.session_id,
2078 bash_id = %info.bash_id,
2079 %error,
2080 "background bash completion activation watermark failed; leaving wait armed"
2081 );
2082 return;
2083 }
2084 }
2085
2086 if delivery_plan == BashCompletionDeliveryPlan::ClearWaitThenActivate {
2087 tracing::info!(
2088 session_id = %info.session_id,
2089 bash_id = %info.bash_id,
2090 status = %info.status,
2091 "background bash completion: push-resuming suspended loop (event-driven)"
2092 );
2093 let mut resumable = session.clone();
2094 let mut runtime_state = read_runtime_state(&resumable);
2095 runtime_state.waiting_for_bash = None;
2096 runtime_state.status = AgentStatusState::Idle;
2097 runtime_state.suspension = None;
2098 write_runtime_state(&mut resumable, &runtime_state);
2099 resumable.metadata.remove("runtime.suspend_reason");
2100 resumable.updated_at = Utc::now();
2101 if let Err(error) = self.persistence.merge_save_runtime(&mut resumable).await {
2102 tracing::warn!(
2103 session_id = %info.session_id,
2104 %error,
2105 "background bash completion is durable but wait-state clear failed; leaving activation to the wait backstop"
2106 );
2107 return;
2108 }
2109 self.sessions.insert(
2110 resumable.id.clone(),
2111 Arc::new(crate::SessionSnapshot::new(resumable)),
2112 );
2113 }
2114
2115 let (Some(messenger), Some(admission)) = (messenger, admission) else {
2119 if delivery_plan == BashCompletionDeliveryPlan::ClearWaitThenActivate {
2120 let _ = self.resume_parent(info.session_id.clone()).await;
2121 }
2122 return;
2123 };
2124 match messenger.activate_prepared(&admission).await {
2125 Ok(receipt) => tracing::info!(
2126 session_id = %info.session_id,
2127 bash_id = %info.bash_id,
2128 status = %info.status,
2129 waiting,
2130 generation = receipt.delivery.generation,
2131 activation = ?receipt.activation,
2132 "background bash completion delivered through SessionMessenger"
2133 ),
2134 Err(error) => tracing::warn!(
2135 session_id = %info.session_id,
2136 bash_id = %info.bash_id,
2137 %error,
2138 "background bash completion: SessionMessenger delivery failed"
2139 ),
2140 }
2141 }
2142}
2143
2144impl BashCompletionSink for ChildCompletionCoordinator {
2145 fn on_bash_completed(&self, info: BashCompletionInfo) {
2146 let coordinator = Arc::new(self.clone());
2150 tokio::spawn(async move {
2151 coordinator.deliver_bash_completion(info).await;
2152 });
2153 }
2154}
2155
2156const CHILD_WAIT_SWEEP_INTERVAL_SECS: u64 = 30;
2162const CHILD_WAIT_REGISTRATION_GRACE_SECS: i64 = 60;
2166const DEAD_CHILD_GRACE_SECS: i64 = 120;
2169const STALE_RUNNER_SLACK_SECS: i64 = 600;
2175
2176fn is_dead_child_candidate_status(status: Option<&str>) -> bool {
2181 match status {
2182 Some(status) => !is_terminal_child_status(status) && status != "suspended",
2185 None => true,
2186 }
2187}
2188
2189fn completion_child_is_owned(reported_parent: &str, child_parent_linkage: Option<&str>) -> bool {
2197 child_parent_linkage == Some(reported_parent)
2198}
2199
2200fn select_replay_child(terminal: &[(String, String)]) -> Option<&(String, String)> {
2204 terminal
2205 .iter()
2206 .find(|(_, status)| is_error_like(status))
2207 .or_else(|| terminal.last())
2208}
2209
2210fn child_wait_watchdog_resume_message(body: String) -> Message {
2211 let mut message = Message::user(body);
2212 message.metadata = Some(serde_json::json!({
2213 RUNTIME_RESUME_MESSAGE_HIDDEN_KEY: true,
2214 RUNTIME_RESUME_MESSAGE_KIND_KEY: "child_wait_watchdog_resume",
2215 }));
2216 message.never_compress = false;
2217 message
2218}
2219
2220fn empty_child_wait_message() -> Message {
2221 child_wait_watchdog_resume_message(
2222 "Runtime notification: this session was suspended waiting for child sessions, but the \
2223 wait tracked no children (internal inconsistency). The session has been resumed; use \
2224 SubAgent.list to inspect child state and continue the task."
2225 .to_string(),
2226 )
2227}
2228
2229fn child_wait_lease_expired_message(child_ids: &[String]) -> Message {
2230 child_wait_watchdog_resume_message(format!(
2231 "Runtime notification: the wait lease for child session(s) [{}] expired before they all \
2232 reported completion. They were NOT cancelled and may still be running or already \
2233 finished — verify their actual status with SubAgent.list / SubAgent.get before assuming \
2234 anything, then continue the task.",
2235 child_ids.join(", ")
2236 ))
2237}
2238
2239impl ChildCompletionCoordinator {
2255 pub fn spawn_child_wait_watchdog(self: &Arc<Self>) {
2258 let coordinator = Arc::clone(self);
2259 tokio::spawn(async move {
2260 use futures::FutureExt;
2261 if std::panic::AssertUnwindSafe(coordinator.reconcile_orphans_at_boot())
2262 .catch_unwind()
2263 .await
2264 .is_err()
2265 {
2266 tracing::error!("child-wait watchdog: boot reconciliation panicked");
2267 }
2268 let mut ticker = tokio::time::interval(std::time::Duration::from_secs(
2269 CHILD_WAIT_SWEEP_INTERVAL_SECS,
2270 ));
2271 ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
2272 ticker.tick().await;
2274 loop {
2275 ticker.tick().await;
2276 if std::panic::AssertUnwindSafe(coordinator.sweep_child_waits())
2277 .catch_unwind()
2278 .await
2279 .is_err()
2280 {
2281 tracing::error!("child-wait watchdog: sweep panicked; continuing");
2282 }
2283 }
2284 });
2285 }
2286
2287 async fn reconcile_orphans_at_boot(&self) {
2292 let cutoff = Utc::now();
2293
2294 let running = self
2300 .storage
2301 .list_sessions_by_run_status("running")
2302 .await
2303 .unwrap_or_default();
2304 for (child_id, parent_id) in running {
2305 let Some(parent_id) = parent_id else { continue };
2306 if self.runner_is_running(&child_id).await {
2307 continue;
2308 }
2309 let Some(control_plane) = self.load_control_plane(&child_id).await else {
2310 continue;
2311 };
2312 if control_plane.updated_at >= cutoff {
2315 continue;
2316 }
2317 tracing::warn!(
2318 child_session_id = %child_id,
2319 parent_session_id = %parent_id,
2320 "boot reconciliation: child was running when the process died; \
2321 marking it error and waking the parent"
2322 );
2323 self.synthesize_child_completion(
2324 &parent_id,
2325 &child_id,
2326 "error",
2327 Some(
2328 "orphaned by server restart: the process died while this child session \
2329 was running"
2330 .to_string(),
2331 ),
2332 )
2333 .await;
2334 }
2335
2336 let suspended = self
2340 .storage
2341 .list_sessions_by_run_status("suspended")
2342 .await
2343 .unwrap_or_default();
2344 for (session_id, _) in suspended {
2345 let Some(control_plane) = self.load_control_plane(&session_id).await else {
2346 continue;
2347 };
2348 if control_plane
2349 .metadata
2350 .get("runtime.suspend_reason")
2351 .map(String::as_str)
2352 != Some("waiting_for_bash")
2353 {
2354 continue;
2355 }
2356 if let Some(wait) = read_runtime_state(&control_plane).waiting_for_bash {
2357 tracing::warn!(
2358 %session_id,
2359 "boot reconciliation: re-arming bash self-resume backstop lost in restart"
2360 );
2361 let coordinator = self.clone();
2362 tokio::spawn(async move {
2363 coordinator
2364 .bash_self_resume(session_id, wait.bash_ids)
2365 .await;
2366 });
2367 }
2368 }
2369 }
2370
2371 async fn runner_is_running(&self, session_id: &str) -> bool {
2372 let runners = self.agent_runners.read().await;
2373 runners
2374 .get(session_id)
2375 .is_some_and(|runner| matches!(runner.status, AgentStatus::Running))
2376 }
2377
2378 async fn load_control_plane(&self, session_id: &str) -> Option<Session> {
2379 match self.storage.load_runtime_control_plane(session_id).await {
2380 Ok(session) => session,
2381 Err(error) => {
2382 tracing::warn!(
2383 %session_id,
2384 %error,
2385 "child-wait watchdog: failed to load session control plane"
2386 );
2387 None
2388 }
2389 }
2390 }
2391
2392 async fn sweep_child_waits(&self) {
2396 let suspended = match self.storage.list_sessions_by_run_status("suspended").await {
2397 Ok(entries) => entries,
2398 Err(error) => {
2399 tracing::warn!(%error, "child-wait watchdog: failed to list suspended sessions");
2400 return;
2401 }
2402 };
2403 for (session_id, _) in suspended {
2404 self.sweep_one_suspended_session(&session_id).await;
2405 }
2406 }
2407
2408 async fn sweep_one_suspended_session(&self, session_id: &str) {
2409 if self.runner_is_running(session_id).await {
2412 return;
2413 }
2414 let Some(session) = self.load_control_plane(session_id).await else {
2415 return;
2416 };
2417 let runtime_state = read_runtime_state(&session);
2418 let suspend_reason = session
2419 .metadata
2420 .get("runtime.suspend_reason")
2421 .map(String::as_str)
2422 .unwrap_or_default()
2423 .to_string();
2424 match (
2425 suspend_reason.as_str(),
2426 runtime_state.waiting_for_children.clone(),
2427 ) {
2428 ("waiting_for_bash", _)
2430 | ("awaiting_clarification", _)
2431 | ("awaiting_parent_approval", _) => {}
2432 (_, Some(wait)) => self.sweep_child_wait(session_id, wait).await,
2433 ("waiting_for_children", None) | ("", None) => {
2436 self.rescue_stranded_resume(session_id).await;
2437 }
2438 _ => {}
2439 }
2440 }
2441
2442 async fn sweep_child_wait(&self, parent_session_id: &str, wait: WaitingForChildrenState) {
2447 let now = Utc::now();
2448
2449 if wait.child_session_ids.is_empty() {
2450 tracing::warn!(
2451 %parent_session_id,
2452 "child-wait watchdog: wait armed over an empty child set; force-resuming"
2453 );
2454 self.force_resume_child_wait(parent_session_id, empty_child_wait_message())
2455 .await;
2456 return;
2457 }
2458
2459 if wait.timeout_at.is_some_and(|deadline| now >= deadline) {
2463 tracing::warn!(
2464 %parent_session_id,
2465 "child-wait watchdog: wait lease expired; force-resuming parent"
2466 );
2467 self.force_resume_child_wait(
2468 parent_session_id,
2469 child_wait_lease_expired_message(&wait.child_session_ids),
2470 )
2471 .await;
2472 return;
2473 }
2474
2475 if now.signed_duration_since(wait.registered_at).num_seconds()
2477 < CHILD_WAIT_REGISTRATION_GRACE_SECS
2478 {
2479 return;
2480 }
2481
2482 let statuses: HashMap<String, Option<String>> = self
2483 .storage
2484 .list_child_run_statuses(parent_session_id)
2485 .await
2486 .unwrap_or_default()
2487 .into_iter()
2488 .collect();
2489
2490 struct DeadChild {
2491 child_id: String,
2492 status: String,
2493 reason: String,
2494 owned: bool,
2498 }
2499
2500 let mut terminal: Vec<(String, String)> = Vec::new();
2501 let mut dead: Vec<DeadChild> = Vec::new();
2502 for child_id in &wait.child_session_ids {
2503 let status = statuses.get(child_id).and_then(|status| status.as_deref());
2504 if let Some(status) = status {
2505 if is_terminal_child_status(status) {
2506 terminal.push((child_id.clone(), status.to_string()));
2507 continue;
2508 }
2509 }
2510 if !is_dead_child_candidate_status(status) {
2511 continue;
2512 }
2513
2514 let control_plane = self.load_control_plane(child_id).await;
2521 let owned = control_plane
2522 .as_ref()
2523 .is_some_and(|cp| cp.parent_session_id.as_deref() == Some(parent_session_id));
2524 if !owned {
2525 dead.push(DeadChild {
2526 child_id: child_id.clone(),
2527 status: "error".to_string(),
2528 reason: if control_plane.is_some() {
2529 "waited-on session id is not a child of this session; clearing it \
2530 from the wait without touching that session"
2531 .to_string()
2532 } else {
2533 "waited-on child session does not exist".to_string()
2534 },
2535 owned: false,
2536 });
2537 continue;
2538 }
2539
2540 let runner = { self.agent_runners.read().await.get(child_id).cloned() };
2541 match runner {
2542 Some(runner) if matches!(runner.status, AgentStatus::Running) => {
2543 let last_activity = runner.last_activity_at().unwrap_or(runner.started_at);
2549 let idle_secs = now.signed_duration_since(last_activity).num_seconds();
2550 let total_secs = now.signed_duration_since(runner.started_at).num_seconds();
2551 let policy = match &control_plane {
2552 Some(child) => {
2553 crate::runtime::execution::spawn::watchdog_policy_for_session(child)
2554 }
2555 None => Default::default(),
2556 };
2557 let idle_limit = policy.max_idle_secs.saturating_add(STALE_RUNNER_SLACK_SECS);
2558 let total_limit = policy
2559 .max_total_secs
2560 .saturating_add(STALE_RUNNER_SLACK_SECS);
2561 if idle_secs >= idle_limit || total_secs >= total_limit {
2562 runner.cancel_token.cancel();
2563 dead.push(DeadChild {
2564 child_id: child_id.clone(),
2565 status: "timeout".to_string(),
2566 reason: format!(
2567 "child runner stalled: no events for {idle_secs}s \
2568 (limit {idle_limit}s including watchdog slack); \
2569 force-finalized by the child-wait watchdog"
2570 ),
2571 owned: true,
2572 });
2573 }
2574 }
2575 _ => {
2576 let quiet_secs = control_plane
2580 .as_ref()
2581 .map(|child| now.signed_duration_since(child.updated_at).num_seconds())
2582 .unwrap_or(i64::MAX);
2583 if quiet_secs >= DEAD_CHILD_GRACE_SECS {
2584 dead.push(DeadChild {
2585 child_id: child_id.clone(),
2586 status: "error".to_string(),
2587 reason: format!(
2588 "child runner lost (crashed task, dropped spawn job, or \
2589 process restart): index status {status:?} with no live \
2590 runner driving it"
2591 ),
2592 owned: true,
2593 });
2594 }
2595 }
2596 }
2597 }
2598
2599 if !dead.is_empty() {
2600 for entry in dead {
2601 tracing::warn!(
2602 %parent_session_id,
2603 child_session_id = %entry.child_id,
2604 status = %entry.status,
2605 reason = %entry.reason,
2606 owned = entry.owned,
2607 "child-wait watchdog: synthesizing terminal completion for dead child"
2608 );
2609 if entry.owned {
2610 self.synthesize_child_completion(
2611 parent_session_id,
2612 &entry.child_id,
2613 &entry.status,
2614 Some(entry.reason),
2615 )
2616 .await;
2617 } else {
2618 self.publish_synthetic_completion(
2620 parent_session_id,
2621 &entry.child_id,
2622 &entry.status,
2623 Some(entry.reason),
2624 )
2625 .await;
2626 }
2627 }
2628 return;
2630 }
2631
2632 let terminal_ids: Vec<String> = terminal.iter().map(|(id, _)| id.clone()).collect();
2638 if let Some((child_id, status)) = select_replay_child(&terminal) {
2639 if wait_policy_satisfied(
2640 wait.wait_for,
2641 &wait.child_session_ids,
2642 &terminal_ids,
2643 child_id,
2644 status,
2645 ) {
2646 tracing::warn!(
2647 %parent_session_id,
2648 child_session_id = %child_id,
2649 "child-wait watchdog: wait already satisfied but parent still suspended \
2650 (lost wake); replaying the completion"
2651 );
2652 let error = self
2653 .load_control_plane(child_id)
2654 .await
2655 .and_then(|child| child.last_run_error());
2656 self.publish_synthetic_completion(parent_session_id, child_id, status, error)
2657 .await;
2658 }
2659 }
2660 }
2661
2662 async fn synthesize_child_completion(
2668 &self,
2669 parent_session_id: &str,
2670 child_session_id: &str,
2671 status: &str,
2672 error: Option<String>,
2673 ) {
2674 match self.storage.load_session(child_session_id).await {
2675 Ok(Some(mut child)) => {
2676 if child.parent_session_id.as_deref() != Some(parent_session_id) {
2681 tracing::warn!(
2682 %parent_session_id,
2683 child_session_id = %child.id,
2684 "child-wait watchdog: refusing to synthesize status onto a session \
2685 that is not a child of this parent"
2686 );
2687 self.publish_synthetic_completion(
2688 parent_session_id,
2689 child_session_id,
2690 status,
2691 error,
2692 )
2693 .await;
2694 return;
2695 }
2696 child.set_last_run_status(status);
2697 match &error {
2698 Some(message) => child.set_last_run_error(message.clone()),
2699 None => child.clear_last_run_error(),
2700 }
2701 child.updated_at = Utc::now();
2702 if let Err(save_error) = self.persistence.merge_save_runtime(&mut child).await {
2703 tracing::warn!(
2704 child_session_id = %child.id,
2705 %save_error,
2706 "child-wait watchdog: failed to persist synthesized terminal status"
2707 );
2708 }
2709 self.sessions.insert(
2710 child.id.clone(),
2711 Arc::new(crate::SessionSnapshot::new(child)),
2712 );
2713 }
2714 Ok(None) => {}
2715 Err(load_error) => {
2716 tracing::warn!(
2717 %child_session_id,
2718 %load_error,
2719 "child-wait watchdog: failed to load child for synthesized terminal status"
2720 );
2721 }
2722 }
2723 finalize_runner(
2724 &self.agent_runners,
2725 child_session_id,
2726 &Err(bamboo_agent_core::AgentError::LLM(
2727 error
2728 .clone()
2729 .unwrap_or_else(|| format!("synthesized {status}")),
2730 )),
2731 )
2732 .await;
2733 self.publish_synthetic_completion(parent_session_id, child_session_id, status, error)
2734 .await;
2735 }
2736
2737 async fn publish_synthetic_completion(
2738 &self,
2739 parent_session_id: &str,
2740 child_session_id: &str,
2741 status: &str,
2742 error: Option<String>,
2743 ) {
2744 let publisher =
2745 crate::runtime::execution::session_events::ReplayableSessionEventPublisher::new(
2746 self.agent_runners.clone(),
2747 self.session_event_senders.clone(),
2748 self.account_feed_inbox.clone(),
2749 );
2750 let handler: Arc<dyn ChildCompletionHandler> = Arc::new(self.clone());
2751 crate::runtime::execution::spawn::publish_child_completion_parts(
2752 &publisher,
2753 Some(handler),
2754 parent_session_id.to_string(),
2755 child_session_id.to_string(),
2756 status.to_string(),
2757 error,
2758 )
2759 .await;
2760 }
2761
2762 async fn rescue_stranded_resume(&self, session_id: &str) {
2768 let Some(session) = self.load_session(session_id).await else {
2769 return;
2770 };
2771 let pending_runtime_resume = session.messages.last().is_some_and(|message| {
2772 matches!(message.role, Role::User)
2773 && message
2774 .metadata
2775 .as_ref()
2776 .is_some_and(|meta| meta.get(RUNTIME_RESUME_MESSAGE_KIND_KEY).is_some())
2777 });
2778 if !pending_runtime_resume {
2779 return;
2780 }
2781 tracing::warn!(
2782 %session_id,
2783 "child-wait watchdog: stranded resume detected (wait cleared, resume never \
2784 spawned); resuming"
2785 );
2786 self.resume_parent(session_id.to_string()).await;
2787 }
2788
2789 async fn force_resume_child_wait(&self, session_id: &str, resume_message: Message) {
2795 const MAX_ATTEMPTS: u8 = 5;
2796 let lock = session_resume_lock(session_id);
2797 for attempt in 0..MAX_ATTEMPTS {
2798 if attempt > 0 {
2799 tokio::time::sleep(Duration::from_millis(200)).await;
2800 }
2801 {
2802 let _held = lock.lock().await;
2803 let Some(mut session) = self.load_session(session_id).await else {
2804 return;
2805 };
2806 let mut runtime_state = read_runtime_state(&session);
2807 if runtime_state.waiting_for_children.is_none() {
2808 return;
2810 }
2811 runtime_state.waiting_for_children = None;
2812 runtime_state.status = AgentStatusState::Idle;
2813 runtime_state.suspension = None;
2814 write_runtime_state(&mut session, &runtime_state);
2815 session.metadata.remove("runtime.suspend_reason");
2816 session.add_message(resume_message.clone());
2817 session.updated_at = Utc::now();
2818 self.save_and_cache(&mut session).await;
2819 }
2820 let outcome = self.resume_parent(session_id.to_string()).await;
2821 match outcome {
2822 ResumeOutcome::Started { .. } | ResumeOutcome::NotFound => return,
2823 ResumeOutcome::Completed | ResumeOutcome::AlreadyRunning { .. } => {
2824 let clobbered = self
2828 .load_session(session_id)
2829 .await
2830 .map(|session| read_runtime_state(&session).waiting_for_children.is_some())
2831 .unwrap_or(false);
2832 if !clobbered {
2833 return;
2834 }
2835 }
2836 }
2837 }
2838 tracing::error!(
2839 %session_id,
2840 "child-wait watchdog: force-resume exhausted its clobber-retry budget"
2841 );
2842 }
2843}
2844
2845#[cfg(test)]
2846mod tests {
2847 use super::*;
2848
2849 #[tokio::test]
2850 async fn cancelled_resume_waiters_do_not_retain_historical_parent_ids() {
2851 for index in 0..512 {
2852 let id = format!("resume-lock-reclaim-{index}");
2853 let owner = session_resume_lock(&id);
2854 let held = owner.lock().await;
2855 let waiter = session_resume_lock(&id);
2856 let mut waiting = Box::pin(waiter.lock());
2857 assert!(futures::poll!(waiting.as_mut()).is_pending());
2858 drop(held);
2859 drop(owner);
2860 drop(waiting);
2861 drop(waiter);
2862 assert!(!parent_locks().lock().recover_poison().contains_key(&id));
2863 }
2864 }
2865
2866 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
2867 async fn resume_lock_reclamation_preserves_exclusive_parent_wake_ownership() {
2868 use std::sync::atomic::{AtomicUsize, Ordering};
2869 let active = Arc::new(AtomicUsize::new(0));
2870 let mut tasks = Vec::new();
2871 for _ in 0..32 {
2872 let active = active.clone();
2873 tasks.push(tokio::spawn(async move {
2874 for _ in 0..16 {
2875 let lease = session_resume_lock("resume-lock-exclusive");
2876 let _guard = lease.lock().await;
2877 assert_eq!(active.fetch_add(1, Ordering::SeqCst), 0);
2878 tokio::task::yield_now().await;
2879 assert_eq!(active.fetch_sub(1, Ordering::SeqCst), 1);
2880 }
2881 }));
2882 }
2883 for task in tasks {
2884 task.await.unwrap();
2885 }
2886 assert!(!parent_locks()
2887 .lock()
2888 .recover_poison()
2889 .contains_key("resume-lock-exclusive"));
2890 }
2891 use bamboo_agent_core::Message;
2892 use bamboo_domain::SessionInboxPort;
2893 use futures::stream;
2894 use std::sync::atomic::{AtomicUsize, Ordering};
2895
2896 struct EmptyTools;
2897
2898 #[async_trait]
2899 impl bamboo_agent_core::tools::ToolExecutor for EmptyTools {
2900 async fn execute(
2901 &self,
2902 _call: &bamboo_agent_core::tools::ToolCall,
2903 ) -> Result<bamboo_agent_core::tools::ToolResult, bamboo_agent_core::tools::ToolError>
2904 {
2905 Err(bamboo_agent_core::tools::ToolError::NotFound(
2906 "no tools".to_string(),
2907 ))
2908 }
2909
2910 fn list_tools(&self) -> Vec<bamboo_agent_core::tools::ToolSchema> {
2911 Vec::new()
2912 }
2913 }
2914
2915 struct CompletedTestProvider;
2916
2917 #[async_trait]
2918 impl bamboo_llm::LLMProvider for CompletedTestProvider {
2919 async fn chat_stream(
2920 &self,
2921 _messages: &[Message],
2922 _tools: &[bamboo_agent_core::tools::ToolSchema],
2923 _max_output_tokens: Option<u32>,
2924 _model: &str,
2925 ) -> Result<bamboo_llm::LLMStream, bamboo_llm::LLMError> {
2926 let chunks: Vec<bamboo_llm::provider::Result<bamboo_llm::LLMChunk>> = vec![
2927 Ok(bamboo_llm::LLMChunk::Token("done".to_string())),
2928 Ok(bamboo_llm::LLMChunk::Done),
2929 ];
2930 Ok(Box::pin(stream::iter(chunks)))
2931 }
2932 }
2933
2934 struct CountingActivationSpawner {
2935 reservations: Arc<AtomicUsize>,
2936 launches: Arc<AtomicUsize>,
2937 }
2938
2939 #[async_trait]
2940 impl SessionActivationSpawner for CountingActivationSpawner {
2941 async fn reserve_activation(
2942 &self,
2943 target_session_id: &str,
2944 inbox_generation: u64,
2945 ) -> Result<SessionActivationReserveOutcome, bamboo_domain::SessionActivationError>
2946 {
2947 self.reservations.fetch_add(1, Ordering::SeqCst);
2948 let launches = self.launches.clone();
2949 Ok(SessionActivationReserveOutcome::Reserved(
2950 SessionActivationLaunch::new(
2951 format!("{target_session_id}-{inbox_generation}"),
2952 move || {
2953 launches.fetch_add(1, Ordering::SeqCst);
2954 },
2955 ),
2956 ))
2957 }
2958 }
2959
2960 async fn completion_inbox_fixture() -> (
2961 tempfile::TempDir,
2962 Arc<bamboo_storage::SessionStoreV2>,
2963 Arc<dyn SessionInboxPort>,
2964 Arc<ChildCompletionCoordinator>,
2965 Arc<AtomicUsize>,
2966 Arc<AtomicUsize>,
2967 ) {
2968 let temp = tempfile::tempdir().unwrap();
2969 let store = Arc::new(
2970 bamboo_storage::SessionStoreV2::new(temp.path().to_path_buf())
2971 .await
2972 .unwrap(),
2973 );
2974 let storage: Arc<dyn Storage> = store.clone();
2975 let locked = Arc::new(LockedSessionStore::new(storage.clone()));
2976 let inbox: Arc<dyn SessionInboxPort> = Arc::new(bamboo_storage::FileSessionInbox::new(
2977 store.clone(),
2978 bamboo_domain::SessionInboxLimits::default(),
2979 ));
2980 let router = crate::SessionActivationRouter::new();
2981 let messenger = Arc::new(crate::SessionMessenger::new(
2982 storage.clone(),
2983 inbox.clone(),
2984 router.clone(),
2985 ));
2986 let reservations = Arc::new(AtomicUsize::new(0));
2987 let launches = Arc::new(AtomicUsize::new(0));
2988 router
2989 .set_spawner(Arc::new(CountingActivationSpawner {
2990 reservations: reservations.clone(),
2991 launches: launches.clone(),
2992 }))
2993 .await;
2994 let provider: Arc<dyn bamboo_llm::LLMProvider> = Arc::new(CompletedTestProvider);
2995 let config = Arc::new(RwLock::new(Config::default()));
2996 let metrics = bamboo_metrics::MetricsCollector::spawn(
2997 Arc::new(bamboo_metrics::SqliteMetricsStorage::new(
2998 temp.path().join("metrics.db"),
2999 )),
3000 7,
3001 );
3002 let tools: Arc<dyn ToolExecutor> = Arc::new(EmptyTools);
3003 let agent = Arc::new(
3004 Agent::builder()
3005 .storage(storage.clone())
3006 .persistence(locked.clone())
3007 .session_inbox(inbox.clone())
3008 .activation_router(router)
3009 .session_messenger(messenger)
3010 .attachment_reader(store.clone())
3011 .skill_manager(Arc::new(bamboo_skills::SkillManager::new()))
3012 .metrics_collector(metrics)
3013 .config(config.clone())
3014 .provider(provider.clone())
3015 .default_tools(tools)
3016 .build()
3017 .unwrap(),
3018 );
3019 let mut providers = HashMap::new();
3020 providers.insert("test".to_string(), provider);
3021 let registry = Arc::new(ProviderRegistry::new(providers, "test".to_string()));
3022 let provider_router = Arc::new(ProviderModelRouter::new(registry.clone()));
3023 let coordinator = Arc::new(ChildCompletionCoordinator::new(
3024 storage,
3025 locked,
3026 Arc::default(),
3027 Arc::new(RwLock::new(HashMap::new())),
3028 Arc::new(RwLock::new(HashMap::new())),
3029 agent,
3030 config,
3031 registry,
3032 provider_router,
3033 temp.path().to_path_buf(),
3034 None,
3035 ));
3036 (temp, store, inbox, coordinator, reservations, launches)
3037 }
3038
3039 #[tokio::test]
3040 async fn supervisor_resume_rejects_old_incarnation_without_replacing_cache() {
3041 let (_temp, store, _inbox, coordinator, _reservations, _launches) =
3042 completion_inbox_fixture().await;
3043 let original = store
3044 .get_or_create_default_supervisor("model")
3045 .await
3046 .unwrap();
3047 let mut stale = store
3048 .load_session(&original.session_id)
3049 .await
3050 .unwrap()
3051 .unwrap();
3052 store.delete_session(&original.session_id).await.unwrap();
3053 let recreated = store
3054 .get_or_create_default_supervisor("replacement")
3055 .await
3056 .unwrap();
3057 assert_ne!(original.incarnation_id, recreated.incarnation_id);
3058 let mut current = store
3059 .load_session(&recreated.session_id)
3060 .await
3061 .unwrap()
3062 .unwrap();
3063 coordinator.save_and_cache(&mut current).await;
3064 let cached = coordinator
3065 .sessions
3066 .get(¤t.id)
3067 .unwrap()
3068 .value()
3069 .clone();
3070 coordinator.save_and_cache(&mut stale).await;
3071 assert!(Arc::ptr_eq(
3072 &cached,
3073 coordinator.sessions.get(¤t.id).unwrap().value()
3074 ));
3075 assert_eq!(
3076 store
3077 .load_session(¤t.id)
3078 .await
3079 .unwrap()
3080 .unwrap()
3081 .authority_identity,
3082 current.authority_identity
3083 );
3084 }
3085
3086 #[test]
3089 fn dead_child_candidate_status_matrix() {
3090 assert!(is_dead_child_candidate_status(None));
3092 assert!(is_dead_child_candidate_status(Some("running")));
3095 assert!(is_dead_child_candidate_status(Some("pending")));
3096 assert!(!is_dead_child_candidate_status(Some("suspended")));
3098 for status in ["completed", "error", "timeout", "cancelled", "skipped"] {
3100 assert!(!is_dead_child_candidate_status(Some(status)), "{status}");
3101 }
3102 }
3103
3104 #[test]
3105 fn completion_child_ownership_gates_content_fold() {
3106 assert!(completion_child_is_owned("parent-1", Some("parent-1")));
3108 assert!(!completion_child_is_owned("parent-1", Some("parent-2")));
3111 assert!(!completion_child_is_owned("parent-1", None));
3113 }
3114
3115 #[test]
3116 fn replay_child_prefers_error_like_for_first_error_policy() {
3117 let terminal = vec![
3118 ("c-ok".to_string(), "completed".to_string()),
3119 ("c-err".to_string(), "timeout".to_string()),
3120 ("c-late".to_string(), "completed".to_string()),
3121 ];
3122 let (id, status) = select_replay_child(&terminal).expect("non-empty");
3123 assert_eq!(id, "c-err");
3124 assert_eq!(status, "timeout");
3125
3126 let all_ok = vec![
3127 ("c-1".to_string(), "completed".to_string()),
3128 ("c-2".to_string(), "completed".to_string()),
3129 ];
3130 let (id, _) = select_replay_child(&all_ok).expect("non-empty");
3131 assert_eq!(id, "c-2");
3132
3133 assert!(select_replay_child(&[]).is_none());
3134 }
3135
3136 #[test]
3137 fn watchdog_resume_messages_are_hidden_runtime_messages() {
3138 for message in [
3139 empty_child_wait_message(),
3140 child_wait_lease_expired_message(&["c-1".to_string(), "c-2".to_string()]),
3141 ] {
3142 assert!(matches!(message.role, Role::User));
3143 let meta = message.metadata.expect("hidden runtime metadata");
3144 assert_eq!(meta[RUNTIME_RESUME_MESSAGE_HIDDEN_KEY], true);
3145 assert_eq!(
3146 meta[RUNTIME_RESUME_MESSAGE_KIND_KEY],
3147 "child_wait_watchdog_resume"
3148 );
3149 }
3150 let lease = child_wait_lease_expired_message(&["c-1".to_string()]);
3151 assert!(lease.content.contains("NOT cancelled"));
3153 assert!(lease.content.contains("c-1"));
3154 }
3155
3156 #[test]
3159 fn non_terminal_statuses_never_satisfy_wait_policies() {
3160 assert!(!is_terminal_child_status("suspended"));
3163 assert!(!is_terminal_child_status("running"));
3164 assert!(!is_terminal_child_status("pending"));
3165 }
3166
3167 fn make_completion(status: &str) -> ChildCompletion {
3168 ChildCompletion {
3169 parent_session_id: "parent-1".to_string(),
3170 child_session_id: "child-1".to_string(),
3171 status: status.to_string(),
3172 error: None,
3173 completed_at: Utc::now(),
3174 }
3175 }
3176
3177 #[test]
3178 fn oversized_child_outcome_is_bounded_and_keeps_full_content_identity() {
3179 let mut completion = make_completion("completed");
3180 let wait_registered_at = Utc::now();
3181 let huge = format!("prefix-A-{}", "x".repeat(300 * 1024));
3182 let presentation = runtime_resume_message(&completion, 0, Some(&huge));
3183 let first = child_completion_envelope(
3184 &completion,
3185 wait_registered_at,
3186 Some(huge.clone()),
3187 &presentation,
3188 );
3189 assert!(
3190 serde_json::to_vec(&first).unwrap().len()
3191 < bamboo_domain::SessionInboxLimits::default().max_payload_bytes
3192 );
3193 let SessionMessageBody::ChildOutcome(outcome) = &first.body else {
3194 panic!("typed child outcome");
3195 };
3196 let stored = outcome.result.as_deref().unwrap();
3197 assert!(stored.contains("sha256="));
3198 assert!(stored.contains("SubAgent.get"));
3199 assert!(stored.len() < CHILD_COMPLETION_INLINE_FIELD_BYTES);
3200
3201 completion.completed_at += chrono::Duration::seconds(1);
3204 let exact_retry = child_completion_envelope(
3205 &completion,
3206 wait_registered_at,
3207 Some(huge.clone()),
3208 &runtime_resume_message(&completion, 9, Some(&huge)),
3209 );
3210 assert_eq!(exact_retry.id, first.id);
3211 let changed = format!("prefix-B-{}", "x".repeat(300 * 1024));
3212 let corrected = child_completion_envelope(
3213 &completion,
3214 wait_registered_at,
3215 Some(changed.clone()),
3216 &runtime_resume_message(&completion, 0, Some(&changed)),
3217 );
3218 assert_ne!(corrected.id, first.id);
3219 }
3220
3221 #[tokio::test]
3222 async fn oversized_child_completion_clears_wait_and_activates_exactly_once() {
3223 let (_temp, store, inbox, coordinator, reservations, launches) =
3224 completion_inbox_fixture().await;
3225 let parent_id = "oversized-parent";
3226 let child_id = "oversized-child";
3227 let now = Utc::now();
3228 let mut parent = Session::new(parent_id, "model");
3229 let mut parent_runtime = AgentRuntimeState::new("waiting-run");
3230 parent_runtime.status = AgentStatusState::Suspended;
3231 parent_runtime.waiting_for_children = Some(WaitingForChildrenState::for_children(
3232 vec![child_id.to_string()],
3233 ChildWaitPolicy::All,
3234 now,
3235 ));
3236 parent_runtime.suspension = Some(SuspensionState {
3237 reason: "waiting_for_children".to_string(),
3238 suspended_at: now,
3239 resumable: true,
3240 hook_point: Some("ChildCompletion".to_string()),
3241 });
3242 write_runtime_state(&mut parent, &parent_runtime);
3243 parent.metadata.insert(
3244 "runtime.suspend_reason".to_string(),
3245 "waiting_for_children".to_string(),
3246 );
3247 store.save_session(&parent).await.unwrap();
3248
3249 let mut child = Session::new_child(child_id, parent_id, "model", "Child");
3250 child.add_message(Message::assistant("z".repeat(300 * 1024), None));
3251 child.set_last_run_status("completed");
3252 store.save_session(&child).await.unwrap();
3253 let completion = ChildCompletion {
3254 parent_session_id: parent_id.to_string(),
3255 child_session_id: child_id.to_string(),
3256 status: "completed".to_string(),
3257 error: None,
3258 completed_at: Utc::now(),
3259 };
3260
3261 ChildCompletionHandler::on_child_completed(coordinator.as_ref(), completion.clone()).await;
3262 let durable_parent = store.load_session(parent_id).await.unwrap().unwrap();
3263 let durable_runtime = read_runtime_state(&durable_parent);
3264 assert!(durable_runtime.waiting_for_children.is_none());
3265 assert_eq!(durable_runtime.status, AgentStatusState::Idle);
3266 assert!(!durable_parent
3267 .metadata
3268 .contains_key("runtime.suspend_reason"));
3269 let backlog = inbox.inspect(parent_id).await.unwrap();
3270 assert_eq!(backlog.pending + backlog.claimed, 1);
3271 assert!(backlog.activation_pending());
3272 assert_eq!(reservations.load(Ordering::SeqCst), 1);
3273 assert_eq!(launches.load(Ordering::SeqCst), 1);
3274
3275 let claim = inbox.claim(parent_id, 1).await.unwrap().remove(0);
3276 assert!(
3277 serde_json::to_vec(&claim.envelope).unwrap().len()
3278 < bamboo_domain::SessionInboxLimits::default().max_payload_bytes
3279 );
3280 ChildCompletionHandler::on_child_completed(coordinator.as_ref(), completion).await;
3283 assert_eq!(reservations.load(Ordering::SeqCst), 1);
3284 assert_eq!(launches.load(Ordering::SeqCst), 1);
3285 assert_eq!(inbox.inspect(parent_id).await.unwrap().claimed, 1);
3286 }
3287
3288 #[tokio::test]
3289 async fn latest_locked_activation_mutation_preserves_wait_armed_after_stale_load() {
3290 let temp = tempfile::tempdir().unwrap();
3291 let store = Arc::new(
3292 bamboo_storage::SessionStoreV2::new(temp.path().to_path_buf())
3293 .await
3294 .unwrap(),
3295 );
3296 let storage: Arc<dyn Storage> = store.clone();
3297 let locked = LockedSessionStore::new(storage);
3298 let mut session = Session::new("activation-stale-wait", "model");
3299 session.agent_runtime_state = Some(AgentRuntimeState::new("old-run"));
3300 store.save_session(&session).await.unwrap();
3301
3302 let stale = store.load_session(&session.id).await.unwrap().unwrap();
3304 assert!(read_runtime_state(&stale).waiting_for_children.is_none());
3305
3306 locked
3307 .update_runtime_config(&session.id, |latest| {
3308 let mut state = read_runtime_state(latest);
3309 state.status = AgentStatusState::Suspended;
3310 state.waiting_for_children = Some(WaitingForChildrenState::for_children(
3311 vec!["child-new".to_string()],
3312 ChildWaitPolicy::All,
3313 Utc::now(),
3314 ));
3315 state.suspension = Some(SuspensionState {
3316 reason: "waiting_for_children".to_string(),
3317 suspended_at: Utc::now(),
3318 resumable: true,
3319 hook_point: None,
3320 });
3321 write_runtime_state(latest, &state);
3322 latest.metadata.insert(
3323 "runtime.suspend_reason".to_string(),
3324 "waiting_for_children".to_string(),
3325 );
3326 })
3327 .await
3328 .unwrap();
3329
3330 let (prepared, ready) = prepare_session_inbox_activation(&locked, &session.id, false)
3331 .await
3332 .unwrap()
3333 .unwrap();
3334 assert!(!ready, "latest durable specific wait must block activation");
3335 let state = read_runtime_state(&prepared);
3336 assert!(state.waiting_for_children.is_some());
3337 assert_eq!(state.status, AgentStatusState::Suspended);
3338 assert_eq!(
3339 prepared
3340 .metadata
3341 .get("runtime.suspend_reason")
3342 .map(String::as_str),
3343 Some("waiting_for_children")
3344 );
3345 }
3346
3347 #[tokio::test]
3348 async fn partial_child_and_bash_backlogs_remain_inert_after_inbox_reopen() {
3349 let temp = tempfile::tempdir().unwrap();
3350 let store = Arc::new(
3351 bamboo_storage::SessionStoreV2::new(temp.path().to_path_buf())
3352 .await
3353 .unwrap(),
3354 );
3355 store
3356 .save_session(&Session::new("restart-child-parent", "model"))
3357 .await
3358 .unwrap();
3359 store
3360 .save_session(&Session::new("restart-bash-parent", "model"))
3361 .await
3362 .unwrap();
3363 let inbox = bamboo_storage::FileSessionInbox::new(
3364 store.clone(),
3365 bamboo_domain::SessionInboxLimits::default(),
3366 );
3367
3368 let completion = ChildCompletion {
3369 parent_session_id: "restart-child-parent".to_string(),
3370 child_session_id: "child-a".to_string(),
3371 status: "completed".to_string(),
3372 error: None,
3373 completed_at: Utc::now(),
3374 };
3375 let child_resume = runtime_resume_message(&completion, 1, Some("first child"));
3376 inbox
3377 .deliver(&child_completion_envelope(
3378 &completion,
3379 Utc::now(),
3380 Some("first child".to_string()),
3381 &child_resume,
3382 ))
3383 .await
3384 .unwrap();
3385 let bash = BashCompletionInfo {
3386 session_id: "restart-bash-parent".to_string(),
3387 bash_id: "bash-a".to_string(),
3388 command: "true".to_string(),
3389 exit_code: Some(0),
3390 status: "completed".to_string(),
3391 output_tail: String::new(),
3392 };
3393 inbox
3394 .deliver(&bash_completion_envelope(&bash))
3395 .await
3396 .unwrap();
3397
3398 let reopened = bamboo_storage::FileSessionInbox::new(
3399 store,
3400 bamboo_domain::SessionInboxLimits::default(),
3401 );
3402 for session_id in ["restart-child-parent", "restart-bash-parent"] {
3403 let backlog = reopened.inspect(session_id).await.unwrap();
3404 assert_eq!(backlog.pending, 1);
3405 assert_eq!(backlog.activation_generation, 0);
3406 assert!(
3407 !backlog.activation_pending(),
3408 "startup must not run a partial wait backlog for {session_id}"
3409 );
3410 }
3411 }
3412
3413 struct StubChildIndex {
3416 children: Vec<(String, Option<String>)>,
3417 }
3418
3419 #[async_trait]
3420 impl Storage for StubChildIndex {
3421 async fn save_session(&self, _session: &Session) -> std::io::Result<()> {
3422 Ok(())
3423 }
3424 async fn load_session(&self, _id: &str) -> std::io::Result<Option<Session>> {
3425 Ok(None)
3426 }
3427 async fn delete_session(&self, _id: &str) -> std::io::Result<bool> {
3428 Ok(false)
3429 }
3430 async fn list_child_run_statuses(
3431 &self,
3432 _parent_session_id: &str,
3433 ) -> std::io::Result<Vec<(String, Option<String>)>> {
3434 Ok(self.children.clone())
3435 }
3436 }
3437
3438 #[tokio::test]
3439 async fn derive_completed_only_includes_terminal_children() {
3440 let storage: Arc<dyn Storage> = Arc::new(StubChildIndex {
3441 children: vec![
3442 ("a".into(), Some("completed".into())),
3443 ("b".into(), Some("running".into())),
3444 ("c".into(), Some("error".into())),
3445 ("d".into(), None),
3446 ],
3447 });
3448 let completed = derive_completed_child_ids(&storage, "parent-1", "b").await;
3449 assert_eq!(
3451 completed,
3452 vec!["a".to_string(), "b".to_string(), "c".to_string()]
3453 );
3454 }
3455
3456 #[tokio::test]
3457 async fn derive_completed_folds_in_just_completed_when_index_lags() {
3458 let storage: Arc<dyn Storage> = Arc::new(StubChildIndex {
3460 children: vec![("only".into(), Some("running".into()))],
3461 });
3462 let completed = derive_completed_child_ids(&storage, "parent-1", "only").await;
3463 assert_eq!(completed, vec!["only".to_string()]);
3464 }
3465
3466 #[test]
3467 fn wait_policy_all_uses_derived_completed_set() {
3468 let waited = vec!["a".to_string(), "b".to_string()];
3469 assert!(!wait_policy_satisfied(
3470 ChildWaitPolicy::All,
3471 &waited,
3472 &["a".to_string()],
3473 "a",
3474 "completed"
3475 ));
3476 assert!(wait_policy_satisfied(
3477 ChildWaitPolicy::All,
3478 &waited,
3479 &["a".to_string(), "b".to_string()],
3480 "b",
3481 "completed"
3482 ));
3483 }
3484
3485 #[test]
3486 fn wait_policy_first_error_requires_tracked_membership() {
3487 let waited = vec!["a".to_string(), "b".to_string()];
3488 assert!(wait_policy_satisfied(
3490 ChildWaitPolicy::FirstError,
3491 &waited,
3492 &["a".to_string()],
3493 "a",
3494 "error"
3495 ));
3496 assert!(!wait_policy_satisfied(
3499 ChildWaitPolicy::FirstError,
3500 &waited,
3501 &["a".to_string()],
3502 "stray-child",
3503 "timeout"
3504 ));
3505 assert!(wait_policy_satisfied(
3507 ChildWaitPolicy::FirstError,
3508 &waited,
3509 &["a".to_string(), "b".to_string()],
3510 "stray-child",
3511 "completed"
3512 ));
3513 }
3514
3515 #[test]
3516 fn child_final_assistant_text_returns_last_assistant() {
3517 let mut session = Session::new("child-1", "gpt-4");
3518 session.messages.push(Message::user("hi"));
3519 session
3520 .messages
3521 .push(Message::assistant("first answer", None));
3522 session.messages.push(Message::user("again"));
3523 session
3524 .messages
3525 .push(Message::assistant("final answer", None));
3526
3527 assert_eq!(
3528 child_final_assistant_text(&session).as_deref(),
3529 Some("final answer")
3530 );
3531 }
3532
3533 #[test]
3534 fn child_final_assistant_text_returns_none_when_blank() {
3535 let mut session = Session::new("child-1", "gpt-4");
3536 session.messages.push(Message::assistant(" ", None));
3537 assert!(child_final_assistant_text(&session).is_none());
3538 }
3539
3540 #[test]
3541 fn child_final_assistant_text_returns_none_when_no_assistant() {
3542 let mut session = Session::new("child-1", "gpt-4");
3543 session.messages.push(Message::user("hi"));
3544 assert!(child_final_assistant_text(&session).is_none());
3545 }
3546
3547 #[test]
3548 fn runtime_resume_message_folds_full_response_without_truncation() {
3549 let completion = make_completion("completed");
3552 let long: String = "a".repeat(10_000);
3553 let message = runtime_resume_message(&completion, 0, Some(&long));
3554 assert!(message.content.contains(&long));
3555 assert!(!message.content.contains("truncated"));
3556 }
3557
3558 #[test]
3559 fn runtime_resume_message_includes_child_response_when_provided() {
3560 let completion = make_completion("completed");
3561 let message = runtime_resume_message(&completion, 0, Some("the answer is 42"));
3562
3563 assert!(matches!(message.role, Role::User));
3564 assert!(!message.never_compress);
3567 assert!(message.content.contains("Child final response:"));
3568 assert!(message.content.contains("the answer is 42"));
3569
3570 let metadata = message.metadata.expect("metadata present");
3571 assert_eq!(
3572 metadata.get("hidden_from_ui").and_then(|v| v.as_bool()),
3573 Some(true)
3574 );
3575 assert_eq!(
3576 metadata.get("runtime_kind").and_then(|v| v.as_str()),
3577 Some("child_completion_resume")
3578 );
3579 assert_eq!(
3580 metadata
3581 .get("child_final_response_included")
3582 .and_then(|v| v.as_bool()),
3583 Some(true)
3584 );
3585 }
3586
3587 #[test]
3588 fn runtime_resume_message_falls_back_to_error_when_no_response() {
3589 let mut completion = make_completion("error");
3590 completion.error = Some("boom".to_string());
3591
3592 let message = runtime_resume_message(&completion, 1, None);
3593 assert!(message.content.contains("Child error:"));
3594 assert!(message.content.contains("boom"));
3595 let metadata = message.metadata.expect("metadata present");
3596 assert_eq!(
3597 metadata
3598 .get("child_final_response_included")
3599 .and_then(|v| v.as_bool()),
3600 Some(false)
3601 );
3602 }
3603
3604 #[test]
3605 fn runtime_resume_message_minimal_when_no_response_and_no_error() {
3606 let completion = make_completion("completed");
3607 let message = runtime_resume_message(&completion, 2, None);
3608 assert!(!message.content.contains("Child final response:"));
3609 assert!(!message.content.contains("Child error:"));
3610 assert!(message.content.contains("Resume the parent task"));
3611 }
3612
3613 #[test]
3614 fn read_config_snapshot_refreshes_cached_snapshot_from_live_config() {
3615 let runtime = tokio::runtime::Runtime::new().expect("runtime");
3616
3617 runtime.block_on(async {
3618 let config = Arc::new(RwLock::new(Config::default()));
3619 config.write().await.provider = "copilot".to_string();
3620 let cached_config = StdRwLock::new(Config::default());
3621
3622 let snapshot = read_config_snapshot(&config, &cached_config);
3623
3624 assert_eq!(snapshot.provider, "copilot");
3625 assert_eq!(
3626 cached_config.read().expect("cached snapshot lock").provider,
3627 "copilot"
3628 );
3629 });
3630 }
3631
3632 #[test]
3633 fn read_config_snapshot_uses_cached_snapshot_when_live_lock_is_busy() {
3634 let runtime = tokio::runtime::Runtime::new().expect("runtime");
3635
3636 runtime.block_on(async {
3637 let mut cached_snapshot = Config::default();
3638 cached_snapshot.provider = "cached-provider".to_string();
3639
3640 let config = Arc::new(RwLock::new(Config::default()));
3641 let cached_config = StdRwLock::new(cached_snapshot);
3642 let _write_guard = config.write().await;
3643
3644 let snapshot = read_config_snapshot(&config, &cached_config);
3645
3646 assert_eq!(snapshot.provider, "cached-provider");
3647 });
3648 }
3649
3650 #[test]
3653 fn bash_completion_resume_message_normal_announces_completion() {
3654 let ids = vec!["bg-1".to_string(), "bg-2".to_string()];
3655 let message = bash_completion_resume_message(&ids, false);
3656 assert!(
3658 message.content.contains("have completed"),
3659 "normal resume message must announce completion: {}",
3660 message.content
3661 );
3662 let metadata = message.metadata.expect("metadata present");
3664 assert_eq!(
3665 metadata
3666 .get(RUNTIME_RESUME_MESSAGE_HIDDEN_KEY)
3667 .and_then(|v| v.as_bool()),
3668 Some(true),
3669 "resume message must be hidden from the UI"
3670 );
3671 assert_eq!(
3672 metadata
3673 .get(RUNTIME_RESUME_MESSAGE_KIND_KEY)
3674 .and_then(|v| v.as_str()),
3675 Some(BASH_COMPLETION_RESUME_KIND),
3676 "resume message must carry the bash-completion kind discriminant"
3677 );
3678 }
3679
3680 #[test]
3681 fn bash_completion_resume_message_deadline_does_not_claim_completion() {
3682 let ids = vec!["bg-long".to_string()];
3686 let message = bash_completion_resume_message(&ids, true);
3687 assert!(
3688 !message.content.contains("have completed"),
3689 "deadline resume message must NOT claim the shells completed: {}",
3690 message.content
3691 );
3692 assert!(
3693 message.content.contains("may still be running"),
3694 "deadline resume message must warn shells may still be running: {}",
3695 message.content
3696 );
3697 assert!(
3698 message.content.contains("BashOutput"),
3699 "deadline resume message must direct verification via BashOutput: {}",
3700 message.content
3701 );
3702 let metadata = message.metadata.expect("metadata present");
3704 assert_eq!(
3705 metadata
3706 .get(RUNTIME_RESUME_MESSAGE_KIND_KEY)
3707 .and_then(|v| v.as_str()),
3708 Some(BASH_COMPLETION_RESUME_KIND)
3709 );
3710 }
3711
3712 #[test]
3713 fn bash_resume_should_retry_matrix() {
3714 assert!(!bash_resume_should_retry(
3720 &ResumeOutcome::Started { run_id: "r".into() },
3721 true
3722 ));
3723 assert!(!bash_resume_should_retry(
3724 &ResumeOutcome::Started { run_id: "r".into() },
3725 false
3726 ));
3727
3728 assert!(!bash_resume_should_retry(&ResumeOutcome::NotFound, true));
3730 assert!(!bash_resume_should_retry(&ResumeOutcome::NotFound, false));
3731
3732 assert!(bash_resume_should_retry(&ResumeOutcome::Completed, true));
3734 assert!(!bash_resume_should_retry(&ResumeOutcome::Completed, false));
3737
3738 assert!(bash_resume_should_retry(
3741 &ResumeOutcome::AlreadyRunning { run_id: "r".into() },
3742 true
3743 ));
3744 assert!(!bash_resume_should_retry(
3746 &ResumeOutcome::AlreadyRunning { run_id: "r".into() },
3747 false
3748 ));
3749 }
3750
3751 #[test]
3754 fn injection_body_includes_status_exit_command_and_tail() {
3755 let info = BashCompletionInfo {
3756 session_id: "s".into(),
3757 bash_id: "abc123".into(),
3758 command: "make build".into(),
3759 exit_code: Some(0),
3760 status: "completed".into(),
3761 output_tail: "BUILD OK".into(),
3762 };
3763 let body = bash_completion_injection_body(&info);
3764 assert!(body.contains("abc123"), "body: {body}");
3765 assert!(body.contains("make build"), "body: {body}");
3766 assert!(body.contains("completed"), "body: {body}");
3767 assert!(body.contains("exit code 0"), "body: {body}");
3768 assert!(body.contains("BUILD OK"), "body: {body}");
3769 assert!(body.contains("BashOutput"), "body: {body}");
3771 assert!(body.contains("bash_id=\"abc123\""), "body: {body}");
3772 }
3773
3774 #[test]
3775 fn injection_body_handles_no_output_and_signal_kill() {
3776 let info = BashCompletionInfo {
3777 session_id: "s".into(),
3778 bash_id: "xyz".into(),
3779 command: "sleep 99".into(),
3780 exit_code: None,
3781 status: "killed".into(),
3782 output_tail: String::new(),
3783 };
3784 let body = bash_completion_injection_body(&info);
3785 assert!(body.contains("killed"), "body: {body}");
3786 assert!(body.contains("none (signal/killed)"), "body: {body}");
3787 assert!(body.contains("no captured output"), "body: {body}");
3788 assert!(!body.contains("Output tail:"), "body: {body}");
3790 }
3791
3792 #[test]
3793 fn background_completion_builds_post_tool_use_payload_and_feedback() {
3794 let mut info = BashCompletionInfo {
3795 session_id: "s".into(),
3796 bash_id: "bg-7".into(),
3797 command: "cargo test".into(),
3798 exit_code: Some(0),
3799 status: "completed".into(),
3800 output_tail: "test result: ok".into(),
3801 };
3802
3803 let payload = background_bash_post_tool_payload(&info);
3804 match payload {
3805 HookPayload::ToolResult {
3806 tool_name,
3807 tool_call_id,
3808 outcome,
3809 } => {
3810 assert_eq!(tool_name, "Bash");
3811 assert_eq!(tool_call_id, "bg-7");
3812 assert!(outcome.success);
3813 let response: serde_json::Value =
3814 serde_json::from_str(outcome.result.as_deref().unwrap()).unwrap();
3815 assert_eq!(response["command"], "cargo test");
3816 assert_eq!(response["exit_code"], 0);
3817 assert_eq!(response["status"], "completed");
3818 assert_eq!(response["output_tail"], "test result: ok");
3819 }
3820 other => panic!("expected PostToolUse payload, got {other:?}"),
3821 }
3822
3823 append_background_bash_hook_feedback(
3824 &mut info,
3825 vec!["Run the formatter before continuing".to_string()],
3826 );
3827 assert!(info.output_tail.contains("<post_tool_use_feedback>"));
3828 assert!(info
3829 .output_tail
3830 .contains("Run the formatter before continuing"));
3831 }
3832
3833 #[tokio::test]
3834 async fn bash_completion_payload_identity_matches_file_inbox_idempotency() {
3835 let baseline = BashCompletionInfo {
3836 session_id: "session".into(),
3837 bash_id: "bg-7".into(),
3838 command: "cargo test".into(),
3839 exit_code: Some(0),
3840 status: "completed".into(),
3841 output_tail: "first tail".into(),
3842 };
3843 let mut retried = baseline.clone();
3844 retried.output_tail = "first tail\nlater bytes\n<hook feedback>".into();
3845 retried.status = "completed-after-hook".into();
3846 let baseline_envelope = bash_completion_envelope(&baseline);
3847 let exact_retry = bash_completion_envelope(&baseline);
3848 let corrected_envelope = bash_completion_envelope(&retried);
3849 assert_eq!(baseline_envelope.id, exact_retry.id);
3850 assert_ne!(
3851 baseline_envelope.id, corrected_envelope.id,
3852 "changed payload semantics must receive a distinct id"
3853 );
3854
3855 let mut other_shell = baseline.clone();
3856 other_shell.bash_id = "bg-8".into();
3857 assert_ne!(
3858 bash_completion_envelope(&baseline).id,
3859 bash_completion_envelope(&other_shell).id
3860 );
3861
3862 let temp = tempfile::tempdir().unwrap();
3863 let store = Arc::new(
3864 bamboo_storage::SessionStoreV2::new(temp.path().to_path_buf())
3865 .await
3866 .unwrap(),
3867 );
3868 store
3869 .save_session(&Session::new("session", "model"))
3870 .await
3871 .unwrap();
3872 let inbox = bamboo_storage::FileSessionInbox::new(
3873 store,
3874 bamboo_domain::SessionInboxLimits::default(),
3875 );
3876 let first = inbox.deliver(&baseline_envelope).await.unwrap();
3877 let duplicate = inbox.deliver(&exact_retry).await.unwrap();
3878 let corrected = inbox.deliver(&corrected_envelope).await.unwrap();
3879 assert_eq!(duplicate, first, "exact payload retry is idempotent");
3880 assert_ne!(corrected.id, first.id);
3881 assert_eq!(corrected.generation, first.generation + 1);
3882 assert_eq!(inbox.inspect("session").await.unwrap().pending, 2);
3883 }
3884
3885 #[cfg(unix)]
3886 #[tokio::test]
3887 async fn background_completion_fires_configured_post_tool_use_command() {
3888 use bamboo_config::{
3889 LifecycleHookGroup, LifecycleHookHandler, LifecycleHooksConfig,
3890 DEFAULT_LIFECYCLE_HOOK_TIMEOUT_MS,
3891 };
3892
3893 let dir = tempfile::tempdir().unwrap();
3894 let output = dir.path().join("background-post-tool.json");
3895 let command = format!(
3896 "cat > '{}'; printf '%s' '{{\"additional_context\":\"inspect the completed build log\"}}'",
3897 output.display()
3898 );
3899 let config = LifecycleHooksConfig {
3900 enabled: true,
3901 post_tool_use: vec![LifecycleHookGroup {
3902 enabled: true,
3903 matcher: Some("^Bash$".to_string()),
3904 hooks: vec![LifecycleHookHandler::command(
3905 command,
3906 DEFAULT_LIFECYCLE_HOOK_TIMEOUT_MS,
3907 )],
3908 }],
3909 ..Default::default()
3910 };
3911 let mut session = Session::new("session-bg-hook", "test-model");
3912 session.workspace = Some(dir.path().to_string_lossy().into_owned());
3913 let mut info = BashCompletionInfo {
3914 session_id: session.id.clone(),
3915 bash_id: "bg-9".into(),
3916 command: "cargo test".into(),
3917 exit_code: Some(0),
3918 status: "completed".into(),
3919 output_tail: "test result: ok".into(),
3920 };
3921
3922 assert!(run_background_bash_post_tool_hooks(&config, None, &session, &mut info).await);
3923 let envelope: serde_json::Value =
3924 serde_json::from_str(&std::fs::read_to_string(output).unwrap()).unwrap();
3925 assert_eq!(envelope["hook_event_name"], "PostToolUse");
3926 assert_eq!(envelope["tool_name"], "Bash");
3927 assert_eq!(envelope["payload"]["tool_call_id"], "bg-9");
3928 let response = envelope["tool_response"]["result"]
3929 .as_str()
3930 .map(serde_json::from_str::<serde_json::Value>)
3931 .transpose()
3932 .unwrap()
3933 .unwrap();
3934 assert_eq!(response["command"], "cargo test");
3935 assert_eq!(response["status"], "completed");
3936 assert!(info.output_tail.contains("inspect the completed build log"));
3937 }
3938
3939 async fn temp_store() -> (tempfile::TempDir, Arc<dyn Storage>, LockedSessionStore) {
3940 let temp = tempfile::tempdir().unwrap();
3941 let storage: Arc<dyn Storage> = Arc::new(
3942 bamboo_storage::v2::SessionStoreV2::new(temp.path().to_path_buf())
3943 .await
3944 .expect("storage init"),
3945 );
3946 let persistence = LockedSessionStore::new(storage.clone());
3947 (temp, storage, persistence)
3948 }
3949
3950 #[tokio::test]
3951 async fn enqueue_writes_pending_injection_and_preserves_messages() {
3952 let (_temp, storage, persistence) = temp_store().await;
3953
3954 let mut session = Session::new("sess-enq", "test-model");
3955 session.add_message(Message::user("do the build"));
3956 storage.save_session(&session).await.unwrap();
3957
3958 let info = BashCompletionInfo {
3959 session_id: "sess-enq".into(),
3960 bash_id: "sh-1".into(),
3961 command: "make".into(),
3962 exit_code: Some(0),
3963 status: "completed".into(),
3964 output_tail: "done".into(),
3965 };
3966 let saved = enqueue_bash_completion_injection(&persistence, &info)
3967 .await
3968 .expect("enqueue io ok")
3969 .expect("session exists");
3970
3971 let pending = saved
3972 .pending_injected_messages()
3973 .expect("pending injection present");
3974 assert_eq!(pending.len(), 1);
3975 let content = pending[0].get("content").and_then(|v| v.as_str()).unwrap();
3976 assert!(content.contains("sh-1"), "content: {content}");
3977 assert!(content.contains("make"), "content: {content}");
3978 assert!(content.contains("done"), "content: {content}");
3979 assert_eq!(saved.messages.len(), 1);
3981 }
3982
3983 #[tokio::test]
3984 async fn enqueue_returns_none_for_missing_session() {
3985 let (_temp, _storage, persistence) = temp_store().await;
3986 let info = BashCompletionInfo {
3987 session_id: "does-not-exist".into(),
3988 bash_id: "x".into(),
3989 command: "true".into(),
3990 exit_code: Some(0),
3991 status: "completed".into(),
3992 output_tail: String::new(),
3993 };
3994 let result = enqueue_bash_completion_injection(&persistence, &info)
3995 .await
3996 .expect("io ok");
3997 assert!(result.is_none(), "no session → nothing enqueued");
3998 }
3999
4000 #[test]
4008 fn apply_bash_resume_transition_clears_wait_and_appends_message() {
4009 use bamboo_domain::session::runtime_state::WaitingForBashState;
4010
4011 let mut session = Session::new("sess-resume", "test-model");
4012 session.add_message(Message::user("kick off the build"));
4013 let mut rt = read_runtime_state(&session);
4014 rt.status = AgentStatusState::Running;
4015 rt.waiting_for_bash = Some(WaitingForBashState::for_bash(
4016 vec!["sh-1".into()],
4017 Utc::now(),
4018 ));
4019 write_runtime_state(&mut session, &rt);
4020 session.metadata.insert(
4021 "runtime.suspend_reason".to_string(),
4022 "waiting_for_bash".to_string(),
4023 );
4024
4025 let resume = bash_completion_resume_message(&["sh-1".to_string()], false);
4026 let did = apply_bash_resume_transition(&mut session, &resume);
4027
4028 assert!(did, "a suspended session must transition");
4029 let after = read_runtime_state(&session);
4030 assert!(
4031 after.waiting_for_bash.is_none(),
4032 "bash wait must be cleared"
4033 );
4034 assert_eq!(after.status, AgentStatusState::Idle, "runtime must be Idle");
4035 assert!(
4036 !session.metadata.contains_key("runtime.suspend_reason"),
4037 "suspend-reason marker must be removed"
4038 );
4039 assert_eq!(session.messages.len(), 2, "resume message must be appended");
4040 assert!(matches!(
4041 session.messages.last().map(|m| &m.role),
4042 Some(Role::User)
4043 ));
4044 }
4045
4046 #[test]
4050 fn apply_bash_resume_transition_noops_when_not_waiting() {
4051 let mut session = Session::new("sess-live", "test-model");
4052 session.add_message(Message::user("hi"));
4053
4054 let resume = bash_completion_resume_message(&["sh-1".to_string()], false);
4055 let did = apply_bash_resume_transition(&mut session, &resume);
4056
4057 assert!(!did, "a non-waiting session must not transition");
4058 assert_eq!(session.messages.len(), 1, "no resume message appended");
4059 }
4060
4061 #[test]
4065 fn bash_completion_should_resume_only_when_suspended_and_all_done() {
4066 assert!(bash_completion_should_resume(true, true));
4067 assert!(!bash_completion_should_resume(true, false)); assert!(!bash_completion_should_resume(false, true)); assert!(!bash_completion_should_resume(false, false));
4070 }
4071
4072 #[test]
4073 fn two_shell_delivery_stages_backlog_before_one_final_activation() {
4074 let mut waiting = true;
4075 let mut durable_backlog = 0;
4076 let mut reservations = 0;
4077
4078 durable_backlog += 1;
4080 let first = bash_completion_delivery_plan(waiting, false);
4081 assert_eq!(first, BashCompletionDeliveryPlan::DurableOnly);
4082 assert!(waiting);
4083 assert_eq!(durable_backlog, 1);
4084 assert_eq!(reservations, 0);
4085
4086 durable_backlog += 1;
4089 let last = bash_completion_delivery_plan(waiting, true);
4090 assert_eq!(last, BashCompletionDeliveryPlan::ClearWaitThenActivate);
4091 waiting = false;
4092 reservations += 1;
4093 assert!(!waiting);
4094 assert_eq!(durable_backlog, 2);
4095 assert_eq!(reservations, 1);
4096 }
4097
4098 #[test]
4102 fn bash_resume_message_from_info_carries_bashid_tail_and_kind() {
4103 let info = BashCompletionInfo {
4104 session_id: "s".into(),
4105 bash_id: "sh-42".into(),
4106 command: "cargo test".into(),
4107 exit_code: Some(0),
4108 status: "completed".into(),
4109 output_tail: "test result: ok".into(),
4110 };
4111 let msg = bash_resume_message_from_info(&info);
4112
4113 assert!(matches!(msg.role, Role::User));
4114 assert!(msg.content.contains("sh-42"), "content: {}", msg.content);
4115 assert!(
4116 msg.content.contains("cargo test"),
4117 "content: {}",
4118 msg.content
4119 );
4120 assert!(
4121 msg.content.contains("test result: ok"),
4122 "content: {}",
4123 msg.content
4124 );
4125 assert!(
4126 msg.content.contains("BashOutput"),
4127 "content: {}",
4128 msg.content
4129 );
4130 let meta = serde_json::to_string(&msg.metadata).unwrap();
4131 assert!(
4132 meta.contains(BASH_COMPLETION_RESUME_KIND),
4133 "resume message must be tagged as a bash-completion resume: {meta}"
4134 );
4135 }
4136}