1use std::collections::HashMap;
8use std::sync::{Arc, Mutex as StdMutex, OnceLock, RwLock as StdRwLock, Weak};
9use std::time::Duration;
10
11use bamboo_domain::poison::PoisonRecover;
12use bamboo_domain::{
13 AgentHookPoint, HookPayload, HookToolOutcome, SessionChildOutcome, SessionMessageBody,
14 SessionMessageContent, SessionMessageEnvelope, SessionMessageId, SessionMessageKind,
15 SessionMessageSource, SessionProviderMessage,
16};
17
18use crate::execution::{
19 finalize_runner, reserve_runner_core, reserve_session_execution, spawn_session_execution,
20 AgentRunner, AgentStatus, ChildCompletion, ChildCompletionHandler, ChildCompletionSource,
21 ReserveOutcome, SessionExecutionArgs, SessionExecutionReservation,
22 SessionExecutionReserveOutcome, SpawnJob, SpawnScheduler,
23};
24use crate::runtime::config::{BashResumeHook, GuardianSpawner, BASH_COMPLETION_RESUME_KIND};
25use crate::runtime::guardian_state::{
26 parse_guardian_verdict, read_guardian_config, read_guardian_state, write_guardian_state,
27 GuardianVerdict,
28};
29use crate::Agent;
30use async_trait::async_trait;
31use bamboo_agent_core::storage::Storage;
32use bamboo_agent_core::tools::ToolExecutor;
33use bamboo_agent_core::{
34 AgentEvent, BashCompletionInfo, BashCompletionSink, Message, Role, Session, SessionKind,
35};
36use bamboo_domain::session::runtime_state::{
37 AgentRuntimeState, AgentStatusState, ChildWaitPolicy, SuspensionState, WaitingForChildrenState,
38};
39use bamboo_llm::{Config, ProviderModelRouter, ProviderRegistry};
40use bamboo_storage::LockedSessionStore;
41use chrono::Utc;
42use sha2::{Digest, Sha256};
43use tokio::sync::{broadcast, RwLock};
44
45use crate::model_areas::resolve_global_area_models;
46use crate::model_config_helper::{
47 resolve_fast_model, resolve_gold_config, resolve_provider_routing_key, GOLD_CONFIG_METADATA_KEY,
48};
49use crate::session_activation::{
50 SessionActivationLaunch, SessionActivationReserveOutcome, SessionActivationSpawner,
51};
52use crate::session_app::child_session::result_projection::{
53 child_result_identity, decode_report, fits_child_result_budget, plain_child_report_message,
54 project_child_report, unavailable_child_result,
55};
56use crate::session_app::execute::consume_pending_clarification_resume;
57use crate::session_app::provider_model::{persist_model_ref, session_effective_model_ref};
58use crate::session_app::resume::{
59 resume_session_execution, ResumeExecutionPort, ResumeSpawnRequest,
60};
61use crate::session_app::types::{ResumeConfigSnapshot, ResumeOutcome};
62
63const AGENT_RUNTIME_STATE_METADATA_KEY: &str = "agent.runtime.state";
64const RUNTIME_RESUME_MESSAGE_HIDDEN_KEY: &str = "hidden_from_ui";
65const RUNTIME_RESUME_MESSAGE_KIND_KEY: &str = "runtime_kind";
66const CHILD_COMPLETION_INLINE_FIELD_BYTES: usize = 48 * 1024;
67const CHILD_COMPLETION_OVERSIZE_TAIL_BYTES: usize = 8 * 1024;
68
69fn read_runtime_state(session: &Session) -> AgentRuntimeState {
70 session
71 .agent_runtime_state
72 .clone()
73 .or_else(|| {
74 session
75 .metadata
76 .get(AGENT_RUNTIME_STATE_METADATA_KEY)
77 .and_then(|raw| serde_json::from_str::<AgentRuntimeState>(raw).ok())
78 })
79 .unwrap_or_else(|| AgentRuntimeState::new(format!("{}-child-wait", session.id)))
80}
81
82fn write_runtime_state(session: &mut Session, runtime_state: &AgentRuntimeState) {
83 session.agent_runtime_state = Some(runtime_state.clone());
84 if let Ok(serialized) = serde_json::to_string(runtime_state) {
85 session
86 .metadata
87 .insert(AGENT_RUNTIME_STATE_METADATA_KEY.to_string(), serialized);
88 }
89}
90
91async fn prepare_session_inbox_activation(
97 persistence: &LockedSessionStore,
98 session_id: &str,
99 interrupt_specific_wait: bool,
100) -> std::io::Result<Option<(Session, bool)>> {
101 let ready = Arc::new(std::sync::atomic::AtomicBool::new(false));
102 let ready_for_mutation = ready.clone();
103 let saved = persistence
104 .update_runtime_config(session_id, move |latest| {
105 if latest.has_pending_question() {
109 return;
110 }
111 let mut runtime_state = read_runtime_state(latest);
112 let specifically_waiting = runtime_state.waiting_for_children.is_some()
113 || runtime_state.waiting_for_bash.is_some();
114 if specifically_waiting && !interrupt_specific_wait {
115 return;
116 }
117 runtime_state.status = AgentStatusState::Idle;
123 runtime_state.suspension = None;
124 write_runtime_state(latest, &runtime_state);
125 latest.metadata.remove("runtime.suspend_reason");
126 latest.updated_at = Utc::now();
127 ready_for_mutation.store(true, std::sync::atomic::Ordering::Release);
128 })
129 .await?;
130 Ok(saved.map(|session| (session, ready.load(std::sync::atomic::Ordering::Acquire))))
131}
132
133fn is_error_like(status: &str) -> bool {
134 matches!(status, "error" | "timeout" | "cancelled")
135}
136
137fn is_terminal_child_status(status: &str) -> bool {
139 matches!(
140 status,
141 "completed" | "error" | "timeout" | "cancelled" | "skipped"
142 )
143}
144
145fn child_completion_envelope(
146 completion: &ChildCompletion,
147 wait_registered_at: chrono::DateTime<Utc>,
148 result: Option<String>,
149 provider_message: &Message,
150) -> SessionMessageEnvelope {
151 let envelope = build_child_completion_envelope(
152 completion,
153 wait_registered_at,
154 result.as_deref(),
155 provider_message,
156 false,
157 );
158 if serde_json::to_vec(&envelope).map_or(true, |bytes| {
159 bytes.len() > bamboo_domain::SessionInboxLimits::default().max_payload_bytes
160 }) {
161 return build_child_completion_envelope(
166 completion,
167 wait_registered_at,
168 result.as_deref(),
169 provider_message,
170 true,
171 );
172 }
173 envelope
174}
175
176fn build_child_completion_envelope(
177 completion: &ChildCompletion,
178 wait_registered_at: chrono::DateTime<Utc>,
179 result: Option<&str>,
180 provider_message: &Message,
181 compact: bool,
182) -> SessionMessageEnvelope {
183 fn bounded_terminal_field(
184 label: &str,
185 child_session_id: &str,
186 value: Option<&str>,
187 compact: bool,
188 ) -> (Option<String>, serde_json::Value) {
189 let Some(value) = value else {
190 return (None, serde_json::Value::Null);
191 };
192 let oversized = value.len() > CHILD_COMPLETION_INLINE_FIELD_BYTES;
193 let identity = if oversized {
194 serde_json::json!({
195 "oversized": true,
196 "utf8_bytes": value.len(),
197 "sha256": hex::encode(Sha256::digest(value.as_bytes())),
198 })
199 } else {
200 serde_json::Value::String(value.to_string())
201 };
202 if !oversized && !compact {
203 return (Some(value.to_string()), identity);
204 }
205 let digest = hex::encode(Sha256::digest(value.as_bytes()));
206 let retrieval_view = if label == "error" {
207 "diagnostics"
208 } else {
209 "result"
210 };
211 let inspection = serde_json::json!({
212 "intent": "inspect", "target": child_session_id, "message": retrieval_view,
213 });
214 let recovery = if label == "error" {
215 format!(
216 "Check observed status and recovery diagnostics by calling SubAgent with {inspection}. \
217 Detailed execution errors remain available to the authenticated inspector."
218 )
219 } else {
220 format!(
221 "Read it in bounded slices by calling SubAgent with {inspection}; use message=\"messages\" for transcript previews. \
222 Follow returned cursors with a JSON-encoded view/cursor query in message, omitting unused query fields."
223 )
224 };
225 let mut summary = format!(
226 "Child {label} could not fit in the durable inline envelope ({} UTF-8 bytes, sha256={digest}). {recovery}",
227 value.len()
228 );
229 if !compact {
230 let mut tail_start = value
231 .len()
232 .saturating_sub(CHILD_COMPLETION_OVERSIZE_TAIL_BYTES);
233 while tail_start < value.len() && !value.is_char_boundary(tail_start) {
234 tail_start += 1;
235 }
236 summary.push_str("\n\nBounded tail:\n");
237 summary.push_str(&value[tail_start..]);
238 }
239 (Some(summary), identity)
240 }
241
242 let (stored_result, result_identity) =
243 bounded_terminal_field("result", &completion.child_session_id, result, compact);
244 let (stored_error, error_identity) = bounded_terminal_field(
245 "error",
246 &completion.child_session_id,
247 completion.error.as_deref(),
248 compact,
249 );
250 let mut bounded_provider_message = provider_message.clone();
251 let provider_oversized = serde_json::to_vec(provider_message)
252 .map(|bytes| bytes.len() > CHILD_COMPLETION_INLINE_FIELD_BYTES)
253 .unwrap_or(true);
254 if provider_oversized || compact {
255 let mut content = if provider_oversized {
259 format!(
260 "Runtime notification: child session `{}` finished with status `{}`.",
261 completion.child_session_id, completion.status
262 )
263 } else {
264 bounded_provider_message.content.clone()
265 };
266 if let Some(result) = stored_result.as_deref() {
267 content.push_str("\n\n");
268 content.push_str(result);
269 }
270 if let Some(error) = stored_error.as_deref() {
271 content.push_str("\n\n");
272 content.push_str(error);
273 }
274 bounded_provider_message.content = content;
275 if provider_oversized {
276 bounded_provider_message.content_parts = None;
277 }
278 if compact {
282 let original = bounded_provider_message
283 .metadata
284 .as_ref()
285 .and_then(serde_json::Value::as_object);
286 let mut metadata = serde_json::Map::new();
287 for key in [
288 RUNTIME_RESUME_MESSAGE_HIDDEN_KEY,
289 RUNTIME_RESUME_MESSAGE_KIND_KEY,
290 "child_session_id",
291 "child_status",
292 "child_final_response_included",
293 "child_completion_source",
294 "guardian_approved",
295 ] {
296 if let Some(value) = original.and_then(|metadata| metadata.get(key)) {
297 metadata.insert(key.to_string(), value.clone());
298 }
299 }
300 if original.is_some_and(|metadata| metadata.contains_key("child_error")) {
301 metadata.insert(
302 "child_error".to_string(),
303 stored_error
304 .as_ref()
305 .map_or(serde_json::Value::Null, |error| {
306 serde_json::Value::String(error.clone())
307 }),
308 );
309 }
310 bounded_provider_message.metadata = Some(serde_json::Value::Object(metadata));
311 } else if let Some(child_error) = bounded_provider_message
312 .metadata
313 .as_mut()
314 .and_then(serde_json::Value::as_object_mut)
315 .and_then(|metadata| metadata.get_mut("child_error"))
316 {
317 *child_error = stored_error
318 .as_ref()
319 .map_or(serde_json::Value::Null, |error| {
320 serde_json::Value::String(error.clone())
321 });
322 }
323 }
324 let body = SessionMessageBody::ChildOutcome(SessionChildOutcome {
325 child_session_id: completion.child_session_id.clone(),
326 status: completion.status.clone(),
327 result: stored_result,
328 error: stored_error,
329 provider_message: Some(session_provider_message(&bounded_provider_message)),
330 });
331 let semantic = serde_json::json!({
332 "parent_session_id": completion.parent_session_id,
333 "child_session_id": completion.child_session_id,
334 "status": completion.status,
335 "error": error_identity,
336 "result": result_identity,
337 "wait_registered_at": wait_registered_at,
338 "terminal_source": completion.source,
339 });
340 SessionMessageEnvelope {
341 id: SessionMessageId::stable("session_child_completion", &semantic),
342 source: SessionMessageSource::Runtime {
343 subsystem: "child_completion_coordinator".to_string(),
344 },
345 target_session_id: completion.parent_session_id.clone(),
346 kind: SessionMessageKind::ChildOutcome,
347 body,
348 created_at: completion.completed_at,
349 thread_id: None,
350 in_reply_to: None,
351 attempt: None,
352 correlation_id: Some(format!("child_completion:{}", completion.child_session_id)),
353 }
354}
355
356fn session_provider_message(message: &Message) -> SessionProviderMessage {
357 SessionProviderMessage {
358 content: SessionMessageContent {
359 text: message.content.clone(),
360 parts: message.content_parts.clone().unwrap_or_default(),
361 },
362 metadata: message
363 .metadata
364 .as_ref()
365 .and_then(serde_json::Value::as_object)
366 .cloned()
367 .unwrap_or_default(),
368 never_compress: message.never_compress,
369 }
370}
371
372async fn derive_completed_child_ids(
377 storage: &Arc<dyn Storage>,
378 parent_session_id: &str,
379 just_completed_child_id: &str,
380) -> Vec<String> {
381 let mut completed: Vec<String> = storage
382 .list_child_run_statuses(parent_session_id)
383 .await
384 .unwrap_or_default()
385 .into_iter()
386 .filter(|(_, status)| status.as_deref().is_some_and(is_terminal_child_status))
387 .map(|(id, _)| id)
388 .collect();
389 if !completed.iter().any(|id| id == just_completed_child_id) {
390 completed.push(just_completed_child_id.to_string());
391 }
392 completed.sort();
393 completed.dedup();
394 completed
395}
396
397fn read_config_snapshot(config: &Arc<RwLock<Config>>, cached_config: &StdRwLock<Config>) -> Config {
398 if let Ok(config_guard) = config.try_read() {
399 let snapshot = config_guard.clone();
400
401 if let Ok(mut cached_guard) = cached_config.try_write() {
402 *cached_guard = snapshot.clone();
403 }
404
405 snapshot
406 } else {
407 cached_config
408 .try_read()
409 .map(|guard| guard.clone())
410 .unwrap_or_default()
411 }
412}
413
414fn parent_locks() -> &'static std::sync::Mutex<HashMap<String, Arc<tokio::sync::Mutex<()>>>> {
430 static LOCKS: OnceLock<std::sync::Mutex<HashMap<String, Arc<tokio::sync::Mutex<()>>>>> =
431 OnceLock::new();
432 LOCKS.get_or_init(|| std::sync::Mutex::new(HashMap::new()))
433}
434
435pub(crate) fn session_resume_lock(session_id: &str) -> SessionResumeLock {
442 let mut map = parent_locks().lock().recover_poison();
443 let lock = map
444 .entry(session_id.to_string())
445 .or_insert_with(|| Arc::new(tokio::sync::Mutex::new(())))
446 .clone();
447 SessionResumeLock {
448 session_id: session_id.to_string(),
449 lock: Some(lock),
450 }
451}
452
453pub(crate) struct SessionResumeLock {
457 session_id: String,
458 lock: Option<Arc<tokio::sync::Mutex<()>>>,
459}
460
461impl std::ops::Deref for SessionResumeLock {
462 type Target = tokio::sync::Mutex<()>;
463 fn deref(&self) -> &Self::Target {
464 self.lock.as_ref().expect("live resume-lock lease")
465 }
466}
467
468impl Drop for SessionResumeLock {
469 fn drop(&mut self) {
470 self.lock.take();
471 let mut map = parent_locks().lock().recover_poison();
472 if map
473 .get(&self.session_id)
474 .is_some_and(|lock| Arc::strong_count(lock) == 1)
475 {
476 map.remove(&self.session_id);
477 }
478 }
479}
480
481fn wait_policy_satisfied(
482 policy: ChildWaitPolicy,
483 wait_child_ids: &[String],
484 completed_child_ids: &[String],
485 latest_child_id: &str,
486 latest_status: &str,
487) -> bool {
488 if wait_child_ids.is_empty() {
489 return false;
490 }
491
492 match policy {
493 ChildWaitPolicy::All => wait_child_ids
494 .iter()
495 .all(|id| completed_child_ids.iter().any(|completed| completed == id)),
496 ChildWaitPolicy::Any => completed_child_ids
497 .iter()
498 .any(|id| wait_child_ids.iter().any(|wait_id| wait_id == id)),
499 ChildWaitPolicy::FirstError => {
500 (is_error_like(latest_status) && wait_child_ids.iter().any(|id| id == latest_child_id))
506 || wait_child_ids
507 .iter()
508 .all(|id| completed_child_ids.iter().any(|completed| completed == id))
509 }
510 }
511}
512
513#[cfg(test)]
517fn child_final_assistant_text(child: &Session) -> Option<String> {
518 child
519 .messages
520 .iter()
521 .rev()
522 .find(|message| matches!(message.role, Role::Assistant))
523 .map(|message| message.content.clone())
524 .filter(|content| !content.trim().is_empty())
525}
526
527fn runtime_resume_message(
528 completion: &ChildCompletion,
529 remaining_children: usize,
530 child_final_response: Option<&str>,
531) -> Message {
532 let mut body = format!(
533 "Runtime notification: child session `{}` finished with status `{}`. Remaining child sessions: {}.",
534 completion.child_session_id, completion.status, remaining_children
535 );
536
537 let final_response = child_final_response.map(str::to_string);
543 if let Some(response) = final_response.as_deref() {
544 body.push_str("\n\nChild final response:\n");
545 body.push_str(response);
546 } else if let Some(error) = completion.error.as_deref() {
547 if !error.is_empty() {
548 body.push_str("\n\nChild error:\n");
549 body.push_str(error);
550 }
551 }
552
553 let inspection = serde_json::json!({
554 "intent": "inspect", "target": completion.child_session_id, "message": "messages",
555 });
556 body.push_str(&format!(
557 "\n\nResume the parent task using this child result and continue from the previous plan. \
558 If you need transcript evidence, call SubAgent with {inspection}. \
559 Follow returned cursors with a JSON-encoded view/cursor query in message; use view=\"message\" and message_id inside that query for a selected full message. Omit unused query fields.",
560 ));
561
562 let mut message = Message::user(body);
563 message.metadata = Some(serde_json::json!({
564 RUNTIME_RESUME_MESSAGE_HIDDEN_KEY: true,
565 RUNTIME_RESUME_MESSAGE_KIND_KEY: "child_completion_resume",
566 "child_session_id": completion.child_session_id,
567 "child_status": completion.status,
568 "child_error": completion.error,
569 "child_final_response_included": final_response.is_some(),
570 "child_completion_source": completion.source,
571 }));
572 message.never_compress = false;
576 message
577}
578
579fn sealed_child_report(
583 completion: &ChildCompletion,
584 child: &Session,
585 parent: &Session,
586) -> Option<serde_json::Value> {
587 if child.subagent_type().as_deref() == Some("guardian")
588 || child
589 .metadata
590 .get("lifecycle")
591 .is_some_and(|v| v == "resident")
592 {
593 return None;
594 }
595 let source = completion.source.as_ref()?;
596 if !source.matches_completion(completion)
597 || ChildCompletionSource::from_committed_session(child).as_ref() != Some(source)
598 || !source.matches_parent(child, parent)
599 {
600 return None;
601 }
602 let binding = bamboo_domain::ChildContextBinding::from_session(child).ok()??;
603 let Some(message) = child
604 .messages
605 .iter()
606 .find(|m| Some(&m.id) == source.result_message_id.as_ref())
607 else {
608 let reason = if source.last_assistant.is_some() {
609 "report_not_current_final"
610 } else {
611 "report_absent"
612 };
613 return Some(unavailable_child_result("typed_result", reason));
614 };
615 if !plain_child_report_message(message) {
616 return Some(unavailable_child_result(
617 "typed_result",
618 "report_not_current_final",
619 ));
620 }
621 let report = match decode_report(&message.content) {
622 Ok(report) => report,
623 Err(reason) => return Some(unavailable_child_result("typed_result", reason)),
624 };
625 let mut observation =
626 child_result_identity(parent, child, &binding, source.project_id.as_deref());
627 observation.as_object_mut()?.extend(
628 serde_json::json!({"kind":"committed_terminal_source","terminal_source":source,
629 "last_run_status":source.status})
630 .as_object()?
631 .clone(),
632 );
633 Some(project_child_report(report, observation))
634}
635
636fn typed_child_resume_message(
637 completion: &ChildCompletion,
638 remaining_children: usize,
639 projection: &serde_json::Value,
640) -> Message {
641 let inspection = serde_json::json!({
642 "intent":"inspect", "target":completion.child_session_id, "message":"result",
643 });
644 let mut message = Message::user(format!(
645 "Runtime notification: child session `{}` finished with status `{}`. Remaining child sessions: {}.\n\n\
646 Child typed result:\n{}\n\n\
647 Resume the parent task. Child-reported evidence and verification are claims, not host-certified facts. \
648 For unavailable results or transcript evidence, call SubAgent with {inspection} and follow returned cursors.",
649 completion.child_session_id, completion.status, remaining_children, projection,
650 ));
651 message.metadata = Some(serde_json::json!({
652 RUNTIME_RESUME_MESSAGE_HIDDEN_KEY:true,
653 RUNTIME_RESUME_MESSAGE_KIND_KEY:"child_completion_resume",
654 "child_session_id":completion.child_session_id,"child_status":completion.status,
655 "child_final_response_included":projection["available"] == true,
656 "child_typed_result_included":true,
657 }));
658 message.never_compress = false;
659 message
660}
661
662fn bound_typed_child_delivery(
665 envelope: &mut SessionMessageEnvelope,
666 message: &mut Message,
667 projection: &mut serde_json::Value,
668 completion: &ChildCompletion,
669 remaining_children: usize,
670) -> bool {
671 fn install(envelope: &mut SessionMessageEnvelope, message: &Message) {
672 if let SessionMessageBody::ChildOutcome(outcome) = &mut envelope.body {
673 outcome.result = None;
678 outcome.error = None;
681 outcome.provider_message = Some(session_provider_message(message));
682 }
683 }
684 fn fits(envelope: &SessionMessageEnvelope, message: &Message) -> bool {
685 fits_child_result_budget(envelope)
686 && fits_child_result_budget(message)
687 && envelope
688 .to_provider_message()
689 .is_ok_and(|actual| fits_child_result_budget(&actual))
690 }
691 install(envelope, message);
692 if fits(envelope, message) {
693 return true;
694 }
695 *projection = unavailable_child_result("typed_result", "result_budget_exceeded");
696 *message = typed_child_resume_message(completion, remaining_children, projection);
697 install(envelope, message);
698 fits(envelope, message)
699}
700
701fn guardian_resume_message(completion: &ChildCompletion, verdict: &GuardianVerdict) -> Message {
706 let mut body = if verdict.approve {
707 String::from(
708 "Guardian review APPROVED: an independent reviewer verified the work and found no blocking issues. You may finalize the task.",
709 )
710 } else {
711 String::from(
712 "Guardian review REJECTED: an independent reviewer found issues. Address every finding below before completing — do NOT declare the task complete until they are resolved.",
713 )
714 };
715 if let Some(summary) = verdict.summary.as_deref().filter(|s| !s.trim().is_empty()) {
716 body.push_str("\n\nReviewer summary: ");
717 body.push_str(summary);
718 }
719 if !verdict.findings.is_empty() {
720 body.push_str("\n\nFindings:");
721 for (idx, finding) in verdict.findings.iter().enumerate() {
722 body.push_str(&format!("\n{}. {}", idx + 1, finding));
723 }
724 }
725 let inspection = serde_json::json!({
726 "intent": "inspect", "target": completion.child_session_id, "message": "messages",
727 });
728 body.push_str(&format!(
729 "\n\nIf you need guardian transcript evidence, call SubAgent with {inspection}. \
730 Follow returned cursors with a JSON-encoded view/cursor query in message, omitting unused query fields.",
731 ));
732
733 let mut message = Message::user(body);
734 message.metadata = Some(serde_json::json!({
735 RUNTIME_RESUME_MESSAGE_HIDDEN_KEY: true,
736 RUNTIME_RESUME_MESSAGE_KIND_KEY: "guardian_review_resume",
737 "child_session_id": completion.child_session_id,
738 "child_status": completion.status,
739 "guardian_approved": verdict.approve,
740 "child_completion_source": completion.source,
741 }));
742 message.never_compress = false;
743 message
744}
745
746#[derive(Clone)]
747pub struct ChildCompletionCoordinator {
748 storage: Arc<dyn Storage>,
749 persistence: Arc<bamboo_storage::LockedSessionStore>,
750 sessions: crate::SessionCache,
751 agent_runners: Arc<RwLock<HashMap<String, AgentRunner>>>,
752 session_event_senders: Arc<RwLock<HashMap<String, broadcast::Sender<AgentEvent>>>>,
753 agent: Arc<Agent>,
754 config: Arc<RwLock<Config>>,
755 provider_registry: Arc<ProviderRegistry>,
756 provider_router: Arc<ProviderModelRouter>,
757 app_data_dir: std::path::PathBuf,
758 account_feed_inbox: Option<crate::execution::AccountFeedInbox>,
759 root_account_sink: Option<Arc<crate::events::AccountEventSink>>,
760 root_tools: Arc<RwLock<Option<Arc<dyn ToolExecutor>>>>,
761 guardian_spawner: Arc<RwLock<Option<Arc<dyn GuardianSpawner>>>>,
765 spawn_scheduler: Arc<RwLock<Weak<SpawnScheduler>>>,
769}
770
771impl ChildCompletionCoordinator {
772 pub fn with_root_account_sink(mut self, sink: Arc<crate::events::AccountEventSink>) -> Self {
773 self.root_account_sink = Some(sink);
774 self
775 }
776
777 #[allow(clippy::too_many_arguments)]
778 pub fn new(
779 storage: Arc<dyn Storage>,
780 persistence: Arc<LockedSessionStore>,
781 sessions: crate::SessionCache,
782 agent_runners: Arc<RwLock<HashMap<String, AgentRunner>>>,
783 session_event_senders: Arc<RwLock<HashMap<String, broadcast::Sender<AgentEvent>>>>,
784 agent: Arc<Agent>,
785 config: Arc<RwLock<Config>>,
786 provider_registry: Arc<ProviderRegistry>,
787 provider_router: Arc<ProviderModelRouter>,
788 app_data_dir: std::path::PathBuf,
789 account_feed_inbox: Option<crate::execution::AccountFeedInbox>,
790 ) -> Self {
791 Self {
792 storage,
793 persistence,
794 sessions,
795 agent_runners,
796 session_event_senders,
797 agent,
798 config,
799 provider_registry,
800 provider_router,
801 app_data_dir,
802 account_feed_inbox,
803 root_account_sink: None,
804 root_tools: Arc::new(RwLock::new(None)),
805 guardian_spawner: Arc::new(RwLock::new(None)),
806 spawn_scheduler: Arc::new(RwLock::new(Weak::new())),
807 }
808 }
809
810 pub async fn set_root_tools(&self, tools: Arc<dyn ToolExecutor>) {
811 *self.root_tools.write().await = Some(tools);
812 }
813
814 pub async fn set_spawn_scheduler(&self, scheduler: &Arc<SpawnScheduler>) {
815 *self.spawn_scheduler.write().await = Arc::downgrade(scheduler);
816 }
817
818 pub async fn set_guardian_spawner(&self, spawner: Arc<dyn GuardianSpawner>) {
821 *self.guardian_spawner.write().await = Some(spawner);
822 }
823
824 fn build_resume_config(
825 &self,
826 session: &Session,
827 config_snapshot: &Config,
828 ) -> ResumeConfigSnapshot {
829 crate::session_app::resolution::resolve_resume_config_snapshot(
830 config_snapshot,
831 &self.provider_registry,
832 session,
833 None,
834 )
835 }
836
837 async fn resume_parent(&self, parent_session_id: String) -> ResumeOutcome {
845 for attempt in 0..=5u8 {
846 if attempt > 0 {
847 tokio::time::sleep(Duration::from_millis(250 * attempt as u64)).await;
848 }
849
850 let Some(session) = self.load_session(&parent_session_id).await else {
851 tracing::warn!(%parent_session_id, "cannot resume parent after child completion: session not found");
852 return ResumeOutcome::NotFound;
853 };
854 let config_snapshot = self.config.read().await.clone();
855 let resume_config = self.build_resume_config(&session, &config_snapshot);
856 let outcome = resume_session_execution(self, &parent_session_id, resume_config).await;
857 tracing::info!(
858 %parent_session_id,
859 attempt,
860 outcome = outcome.as_str(),
861 "child completion requested parent resume"
862 );
863
864 if !matches!(outcome, ResumeOutcome::AlreadyRunning { .. }) {
865 return outcome;
866 }
867 }
868 tracing::error!(
874 %parent_session_id,
875 "parent resume gave up after AlreadyRunning retry budget; \
876 relying on the child-wait watchdog backstop"
877 );
878 ResumeOutcome::AlreadyRunning {
879 run_id: String::new(),
880 }
881 }
882
883 async fn save_and_cache(&self, session: &mut Session) {
884 if let Err(error) = self
885 .persistence
886 .merge_save_runtime_and_publish(session, |saved, _| {
887 self.sessions.insert(
888 saved.id.clone(),
889 Arc::new(crate::SessionSnapshot::new(saved.clone())),
890 );
891 })
892 .await
893 {
894 tracing::warn!(session_id = %session.id, %error, "failed to persist session");
895 }
896 }
897}
898
899#[async_trait]
900impl ChildCompletionHandler for ChildCompletionCoordinator {
901 async fn on_child_completed(&self, completion: ChildCompletion) {
902 if !is_terminal_child_status(&completion.status) {
911 tracing::info!(
912 parent_session_id = %completion.parent_session_id,
913 child_session_id = %completion.child_session_id,
914 status = %completion.status,
915 "non-terminal child status; leaving the parent wait armed"
916 );
917 return;
918 }
919
920 let per_parent = session_resume_lock(&completion.parent_session_id);
925 let _per_parent_guard = per_parent.lock().await;
926
927 let (parent, should_resume, messenger, child_admission) = loop {
928 let Some(mut parent) = self.load_session(&completion.parent_session_id).await else {
929 tracing::warn!(
930 parent_session_id = %completion.parent_session_id,
931 child_session_id = %completion.child_session_id,
932 "child completion received for missing parent"
933 );
934 return;
935 };
936
937 let expected_parent = parent.clone();
938
939 let mut runtime_state = read_runtime_state(&parent);
945
946 let mut completed_child_ids = derive_completed_child_ids(
949 &self.storage,
950 &completion.parent_session_id,
951 &completion.child_session_id,
952 )
953 .await;
954
955 let mut should_resume = false;
956 let mut remaining_children = 0usize;
957 let active_wait = runtime_state.waiting_for_children.clone();
958 if active_wait.as_ref().is_none_or(|wait| {
959 !wait
960 .child_session_ids
961 .contains(&completion.child_session_id)
962 }) {
963 let mut unchanged = parent.clone();
967 match self
968 .persistence
969 .compare_exchange_child_wait_and_publish(
970 &parent,
971 &mut unchanged,
972 true,
973 Arc::new(|_| {}),
974 )
975 .await
976 {
977 Ok(false) => continue,
978 Ok(true) => return,
979 Err(error) => {
980 tracing::warn!(parent_session_id = %parent.id, %error,
981 "child completion could not settle no-wait observation");
982 return;
983 }
984 }
985 }
986 if let Some(wait) = active_wait.as_ref() {
989 for id in &wait.child_session_ids {
990 if id == &completion.child_session_id {
991 continue;
992 }
993 if let Ok(Some(child)) = self
994 .storage
995 .load_child_wait_session(&parent, id, false)
996 .await
997 {
998 completed_child_ids.retain(|completed| completed != id);
999 if completion_child_is_owned(&parent.id, child.parent_session_id.as_deref())
1000 && child
1001 .last_run_status()
1002 .as_deref()
1003 .is_some_and(is_terminal_child_status)
1004 {
1005 completed_child_ids.push(id.clone());
1006 }
1007 }
1008 }
1009 }
1010 if let Some(wait) = active_wait.as_ref() {
1011 remaining_children = wait
1012 .child_session_ids
1013 .iter()
1014 .filter(|id| !completed_child_ids.iter().any(|completed| completed == *id))
1015 .count();
1016 should_resume = wait_policy_satisfied(
1017 wait.wait_for,
1018 &wait.child_session_ids,
1019 &completed_child_ids,
1020 &completion.child_session_id,
1021 &completion.status,
1022 );
1023 }
1024
1025 let reported_child_owned = match self
1037 .storage
1038 .load_child_wait_session(&parent, &completion.child_session_id, false)
1039 .await
1040 {
1041 Ok(Some(control_plane)) => completion_child_is_owned(
1042 &completion.parent_session_id,
1043 control_plane.parent_session_id.as_deref(),
1044 ),
1045 _ => false,
1046 };
1047
1048 let loaded_child = if reported_child_owned {
1053 match self
1054 .storage
1055 .load_child_wait_session(&parent, &completion.child_session_id, true)
1056 .await
1057 {
1058 Ok(child) => child,
1059 Err(error) => {
1060 tracing::warn!(
1061 child_session_id = %completion.child_session_id,
1062 %error,
1063 "failed to load child session for runtime resume message"
1064 );
1065 None
1066 }
1067 }
1068 } else {
1069 tracing::warn!(
1070 parent_session_id = %completion.parent_session_id,
1071 child_session_id = %completion.child_session_id,
1072 "completion child is not a child of this parent; resuming with a neutral \
1073 message and NOT folding its content"
1074 );
1075 None
1076 };
1077
1078 if let Some(source) = completion.source.as_ref() {
1082 if loaded_child
1083 .as_ref()
1084 .and_then(ChildCompletionSource::from_committed_session)
1085 .as_ref()
1086 != Some(source)
1087 || !source.matches_completion(&completion)
1088 || loaded_child
1089 .as_ref()
1090 .is_none_or(|child| !source.matches_parent(child, &parent))
1091 {
1092 tracing::warn!(child_session_id = %completion.child_session_id,
1093 "stale/unavailable committed child completion source; leaving wait armed");
1094 return;
1095 }
1096 }
1097 let child_final_response = loaded_child
1098 .as_ref()
1099 .and_then(|child| completion.source.as_ref()?.result(child, &parent));
1100 let guardian_resume = if should_resume {
1105 let reviewed_round = runtime_state.round.current_round;
1106 loaded_child.as_ref().and_then(|child| {
1107 if child.subagent_type().as_deref() != Some("guardian") {
1108 return None;
1109 }
1110 let mut guardian_state = read_guardian_state(&parent)?;
1111 if guardian_state.guardian_child_id.as_deref()
1112 != Some(completion.child_session_id.as_str())
1113 {
1114 tracing::warn!(
1117 parent_session_id = %completion.parent_session_id,
1118 child_session_id = %completion.child_session_id,
1119 expected = ?guardian_state.guardian_child_id,
1120 "guardian completion does not match recorded guardian_child_id; using generic resume"
1121 );
1122 return None;
1123 }
1124 let verdict = child_final_response.clone()
1132 .and_then(|text| match parse_guardian_verdict(&text) {
1133 Ok(verdict) => Some(verdict),
1134 Err(error) => {
1135 tracing::warn!(
1136 child_session_id = %completion.child_session_id,
1137 %error,
1138 "guardian verdict unparseable; recording a synthetic reject"
1139 );
1140 None
1141 }
1142 })
1143 .unwrap_or_else(|| {
1144 GuardianVerdict::rejected(vec![
1145 "The guardian reviewer did not return a usable verdict (it errored or \
1146 emitted unparseable output); the work has NOT been independently \
1147 verified."
1148 .to_string(),
1149 ])
1150 });
1151 let approved = verdict.approve;
1152 let message = guardian_resume_message(&completion, &verdict);
1153 guardian_state.record_verdict(verdict, reviewed_round);
1154 write_guardian_state(&mut parent, guardian_state);
1155 tracing::info!(
1156 parent_session_id = %completion.parent_session_id,
1157 child_session_id = %completion.child_session_id,
1158 approved,
1159 "guardian verdict recorded; resuming parent"
1160 );
1161 Some(message)
1162 })
1163 } else {
1164 None
1165 };
1166 let mut typed_result = loaded_child
1167 .as_ref()
1168 .and_then(|child| sealed_child_report(&completion, child, &parent));
1169 let mut resume_message = if let Some(projection) = typed_result.as_mut() {
1170 let message =
1171 typed_child_resume_message(&completion, remaining_children, projection);
1172 if fits_child_result_budget(&message) {
1173 message
1174 } else {
1175 *projection =
1176 unavailable_child_result("typed_result", "result_budget_exceeded");
1177 typed_child_resume_message(&completion, remaining_children, projection)
1178 }
1179 } else {
1180 guardian_resume.unwrap_or_else(|| {
1181 runtime_resume_message(
1182 &completion,
1183 remaining_children,
1184 child_final_response.as_deref(),
1185 )
1186 })
1187 };
1188 if typed_result.is_some() && !fits_child_result_budget(&resume_message) {
1189 tracing::warn!(child_session_id = %completion.child_session_id,
1190 "typed child resume identity exceeds budget; leaving wait armed");
1191 return;
1192 }
1193
1194 let messenger = self.agent.session_messenger().cloned();
1199 let mut child_admission = if let (Some(wait), Some(messenger)) =
1200 (active_wait.as_ref(), messenger.as_ref())
1201 {
1202 let mut envelope = child_completion_envelope(
1203 &completion,
1204 wait.registered_at,
1205 child_final_response,
1206 &resume_message,
1207 );
1208 if let Some(run_id) = self
1209 .agent
1210 .activation_router()
1211 .map(|router| router.current_run_id(&completion.parent_session_id))
1212 {
1213 if let Some(run_id) = run_id.await {
1214 envelope.correlation_id =
1215 Some(format!("child_completion_after_run:{run_id}"));
1216 }
1217 }
1218 if let Some(projection) = typed_result.as_mut() {
1219 if !bound_typed_child_delivery(
1220 &mut envelope,
1221 &mut resume_message,
1222 projection,
1223 &completion,
1224 remaining_children,
1225 ) {
1226 tracing::warn!(child_session_id = %completion.child_session_id,
1227 "typed child outcome wrapper exceeds budget; leaving wait armed");
1228 return;
1229 }
1230 }
1231 match messenger.admit(envelope).await {
1232 Ok(admission) => Some(admission),
1233 Err(error) => {
1234 tracing::warn!(
1235 parent_session_id = %completion.parent_session_id,
1236 child_session_id = %completion.child_session_id,
1237 %error,
1238 "child outcome SessionInbox admission failed; leaving parent wait armed"
1239 );
1240 return;
1241 }
1242 }
1243 } else {
1244 None
1245 };
1246
1247 if should_resume {
1254 if let (Some(wait), Some(messenger)) = (active_wait.as_ref(), messenger.as_ref()) {
1255 let parent_run_id = if let Some(router) = self.agent.activation_router() {
1256 router.current_run_id(&completion.parent_session_id).await
1257 } else {
1258 None
1259 };
1260 for child_id in &wait.child_session_ids {
1261 if child_id == &completion.child_session_id
1262 || !completed_child_ids.contains(child_id)
1263 {
1264 continue;
1265 }
1266 let owned = match self
1270 .storage
1271 .load_child_wait_session(&parent, child_id, false)
1272 .await
1273 {
1274 Ok(Some(child)) => completion_child_is_owned(
1275 &completion.parent_session_id,
1276 child.parent_session_id.as_deref(),
1277 ),
1278 _ => false,
1279 };
1280 let sibling = if owned {
1281 self.storage
1282 .load_child_wait_session(&parent, child_id, true)
1283 .await
1284 } else {
1285 tracing::warn!(%child_id, "terminal sibling ownership unavailable; leaving parent wait armed");
1286 return;
1287 };
1288 let sibling = match sibling {
1289 Ok(Some(child))
1290 if completion_child_is_owned(
1291 &completion.parent_session_id,
1292 child.parent_session_id.as_deref(),
1293 ) && child
1294 .last_run_status()
1295 .as_deref()
1296 .is_some_and(is_terminal_child_status) =>
1297 {
1298 child
1299 }
1300 _ => {
1301 tracing::warn!(%child_id, "terminal sibling snapshot unavailable; leaving parent wait armed");
1302 return;
1303 }
1304 };
1305 if ChildCompletionSource::has_source_record(&sibling)
1306 && ChildCompletionSource::from_committed_session(&sibling).is_none()
1307 {
1308 tracing::warn!(%child_id, "terminal sibling source is invalid; leaving wait armed");
1309 return;
1310 }
1311 let sibling_completion = ChildCompletion {
1312 parent_session_id: completion.parent_session_id.clone(),
1313 child_session_id: child_id.clone(),
1314 status: sibling.last_run_status().expect("terminal status checked"),
1315 error: sibling.last_run_error(),
1316 completed_at: sibling.updated_at,
1317 source: ChildCompletionSource::from_committed_session(&sibling),
1318 };
1319 if sibling_completion
1320 .source
1321 .as_ref()
1322 .is_some_and(|source| !source.matches_parent(&sibling, &parent))
1323 {
1324 tracing::warn!(%child_id, "terminal sibling parent context is stale; leaving wait armed");
1325 return;
1326 }
1327 let result = sibling_completion
1328 .source
1329 .as_ref()
1330 .and_then(|source| source.result(&sibling, &parent));
1331 let mut typed_result =
1332 sealed_child_report(&sibling_completion, &sibling, &parent);
1333 let mut presentation = if let Some(projection) = typed_result.as_ref() {
1334 typed_child_resume_message(
1335 &sibling_completion,
1336 remaining_children,
1337 projection,
1338 )
1339 } else {
1340 runtime_resume_message(
1341 &sibling_completion,
1342 remaining_children,
1343 result.as_deref(),
1344 )
1345 };
1346 let mut envelope = child_completion_envelope(
1347 &sibling_completion,
1348 wait.registered_at,
1349 result,
1350 &presentation,
1351 );
1352 if let Some(run_id) = parent_run_id.as_ref() {
1353 envelope.correlation_id =
1354 Some(format!("child_completion_after_run:{run_id}"));
1355 }
1356 if let Some(projection) = typed_result.as_mut() {
1357 if !bound_typed_child_delivery(
1358 &mut envelope,
1359 &mut presentation,
1360 projection,
1361 &sibling_completion,
1362 remaining_children,
1363 ) {
1364 tracing::warn!(%child_id, "typed terminal sibling wrapper exceeds budget; leaving wait armed");
1365 return;
1366 }
1367 }
1368 match messenger.admit(envelope).await {
1369 Ok(admission) => {
1370 if child_admission.as_ref().is_none_or(|current| {
1374 admission.delivery.generation > current.delivery.generation
1375 }) {
1376 child_admission = Some(admission);
1377 }
1378 }
1379 Err(error) => {
1380 tracing::warn!(%child_id, %error, "terminal sibling admission failed; leaving parent wait armed");
1381 return;
1382 }
1383 }
1384 }
1385 }
1386 }
1387
1388 if should_resume {
1389 if let (Some(messenger), Some(admission)) =
1390 (messenger.as_ref(), child_admission.as_ref())
1391 {
1392 if let Err(error) = messenger.prepare_activation(admission).await {
1393 tracing::warn!(
1394 parent_session_id = %completion.parent_session_id,
1395 child_session_id = %completion.child_session_id,
1396 %error,
1397 "child outcome activation watermark failed; leaving parent wait armed"
1398 );
1399 return;
1400 }
1401 }
1402 }
1403
1404 if should_resume {
1405 runtime_state.waiting_for_children = None;
1406 runtime_state.status = AgentStatusState::Idle;
1407 runtime_state.suspension = None;
1408 parent.metadata.remove("runtime.suspend_reason");
1409
1410 if child_admission.is_none() {
1411 parent.add_message(resume_message);
1415 }
1416 } else if runtime_state.waiting_for_children.is_some() {
1417 runtime_state.status = AgentStatusState::Suspended;
1418 runtime_state.suspension = Some(SuspensionState {
1419 reason: "waiting_for_children".to_string(),
1420 suspended_at: Utc::now(),
1421 resumable: true,
1422 hook_point: Some("ChildCompletion".to_string()),
1423 });
1424 }
1425
1426 parent.updated_at = Utc::now();
1427 write_runtime_state(&mut parent, &runtime_state);
1428 let runtime_only = child_admission.is_some();
1435 let sessions = self.sessions.clone();
1436 let history = expected_parent.clone();
1437 let save = self
1438 .persistence
1439 .compare_exchange_child_wait_and_publish(
1440 &expected_parent,
1441 &mut parent,
1442 runtime_only,
1443 Arc::new(move |saved| {
1444 if runtime_only {
1445 if let Some(cached) = sessions.get(&saved.id) {
1446 cached.update(|current| {
1447 if current.created_at != saved.created_at {
1448 *current = saved.clone();
1449 current.messages = history.messages.clone();
1450 current.provider_transcript =
1451 history.provider_transcript.clone();
1452 current
1453 .runtime_metadata
1454 .get_or_insert_with(Default::default)
1455 .session_inbox_admission =
1456 history.session_inbox_admission().cloned();
1457 return;
1458 }
1459
1460 let messages = current.messages.clone();
1461 let provider_transcript = current.provider_transcript.clone();
1462 let admission =
1463 current.runtime_metadata.as_ref().and_then(|metadata| {
1464 metadata.session_inbox_admission.clone()
1465 });
1466 *current = saved.clone();
1467 current.messages = messages;
1468 current.provider_transcript = provider_transcript;
1469 current
1470 .runtime_metadata
1471 .get_or_insert_with(Default::default)
1472 .session_inbox_admission = admission;
1473 });
1474 } else {
1475 let mut snapshot = saved.clone();
1476 snapshot.messages = history.messages.clone();
1477 snapshot.provider_transcript = history.provider_transcript.clone();
1478 snapshot
1479 .runtime_metadata
1480 .get_or_insert_with(Default::default)
1481 .session_inbox_admission = history
1482 .runtime_metadata
1483 .as_ref()
1484 .and_then(|metadata| metadata.session_inbox_admission.clone());
1485 sessions.insert(
1486 snapshot.id.clone(),
1487 Arc::new(crate::SessionSnapshot::new(snapshot)),
1488 );
1489 }
1490 } else {
1491 sessions.insert(
1492 saved.id.clone(),
1493 Arc::new(crate::SessionSnapshot::new(saved.clone())),
1494 );
1495 }
1496 }),
1497 )
1498 .await;
1499 match save {
1500 Ok(false) => {
1501 continue;
1504 }
1505 Ok(true) => break (parent, should_resume, messenger, child_admission),
1506 Err(error) => {
1507 tracing::warn!(parent_session_id = %completion.parent_session_id,
1508 child_session_id = %completion.child_session_id, %error,
1509 "child outcome is durable but parent wait transition failed; leaving activation deferred");
1510 return;
1511 }
1512 }
1513 };
1514
1515 let resume_parent_id = parent.id.clone();
1520 drop(_per_parent_guard);
1521
1522 if should_resume {
1523 if let (Some(messenger), Some(admission)) = (messenger, child_admission) {
1524 if let Err(error) = messenger.activate_prepared(&admission).await {
1525 tracing::warn!(
1526 parent_session_id = %resume_parent_id,
1527 %error,
1528 "child outcome and wait transition are durable but activation failed"
1529 );
1530 }
1531 } else {
1532 self.resume_parent(resume_parent_id).await;
1533 }
1534 }
1535 }
1536}
1537
1538#[async_trait]
1539impl ResumeExecutionPort for ChildCompletionCoordinator {
1540 async fn load_session(&self, session_id: &str) -> Option<Session> {
1541 match self.storage.load_session(session_id).await {
1542 Ok(Some(session)) => Some(session),
1543 Ok(None) => self
1544 .sessions
1545 .get(session_id)
1546 .map(|e| e.value().clone())
1547 .map(|arc| arc.read().clone()),
1548 Err(error) => {
1549 tracing::warn!(%session_id, %error, "failed to load session from storage");
1550 self.sessions
1551 .get(session_id)
1552 .map(|e| e.value().clone())
1553 .map(|arc| arc.read().clone())
1554 }
1555 }
1556 }
1557
1558 async fn save_and_cache_session(&self, session: &mut Session) {
1559 self.save_and_cache(session).await;
1560 }
1561
1562 async fn reserve_session_execution(
1563 &self,
1564 session_id: &str,
1565 event_sender: &broadcast::Sender<AgentEvent>,
1566 ) -> SessionExecutionReserveOutcome {
1567 reserve_session_execution(
1568 &self.agent,
1569 &self.agent_runners,
1570 &self.session_event_senders,
1571 session_id,
1572 event_sender,
1573 )
1574 .await
1575 }
1576
1577 async fn get_or_create_event_sender(&self, session_id: &str) -> broadcast::Sender<AgentEvent> {
1578 crate::execution::session_events::get_or_create_event_sender(
1579 &self.session_event_senders,
1580 session_id,
1581 )
1582 .await
1583 }
1584
1585 fn dispatch_resume_execution(
1586 &self,
1587 request: ResumeSpawnRequest,
1588 ) -> Result<(), ResumeSpawnRequest> {
1589 let owner = self.clone();
1590 tokio::spawn(async move {
1591 ResumeExecutionPort::spawn_resume_execution(&owner, request).await;
1592 });
1593 Ok(())
1594 }
1595
1596 async fn spawn_resume_execution(&self, request: ResumeSpawnRequest) {
1597 let ResumeSpawnRequest {
1598 session_id,
1599 mut session,
1600 mut execution_reservation,
1601 event_sender,
1602 config,
1603 } = request;
1604 if let Err(error) = execution_reservation.ensure_registered().await {
1605 tracing::warn!(
1606 %session_id,
1607 run_id = %execution_reservation.run_id(),
1608 %error,
1609 "cannot resume after child completion without exact router ownership"
1610 );
1611 return;
1612 }
1613
1614 if self
1615 .agent
1616 .persistence()
1617 .root_actor_execution_required(&session)
1618 && self.root_account_sink.is_none()
1619 {
1620 tracing::warn!(%session_id, "Root resume has no actual account sink; rejecting before Actor claim");
1621 execution_reservation.abandon().await;
1622 return;
1623 }
1624 if let Err(error) = execution_reservation
1625 .bind_root_actor(&self.agent, &session)
1626 .await
1627 {
1628 tracing::warn!(%session_id, %error, "Root coordinator resume Actor binding rejected");
1629 execution_reservation.abandon().await;
1630 return;
1631 }
1632 let root_publication = execution_reservation.root_actor_writer().map(|owner| {
1633 crate::events::RootActorEventPublication::new(
1634 self.storage.clone(),
1635 self.root_account_sink
1636 .clone()
1637 .expect("required Root actual account sink checked before claim"),
1638 owner,
1639 )
1640 });
1641
1642 let Some(root_tools) = self.root_tools.read().await.clone() else {
1643 tracing::error!(%session_id, "cannot resume parent after child completion: root tool surface is not initialized");
1644 return;
1645 };
1646
1647 let config_snapshot = self.config.read().await.clone();
1648 let model = session.model.clone();
1649 let session_model_ref = session_effective_model_ref(&session);
1650 let requested_provider = session_model_ref
1651 .as_ref()
1652 .map(|model_ref| model_ref.provider.as_str())
1653 .unwrap_or(config.provider_name.as_str());
1654 let resolved_provider_name = match resolve_provider_routing_key(
1655 &config_snapshot,
1656 requested_provider,
1657 &self.provider_registry,
1658 ) {
1659 Ok(provider) => provider,
1660 Err(error) => {
1661 tracing::error!(
1662 session_id = %session_id,
1663 provider = requested_provider,
1664 %error,
1665 "child-completion resume provider is unavailable; refusing to fall back"
1666 );
1667 execution_reservation.abandon().await;
1668 return;
1669 }
1670 };
1671 let provider_override = if let Some(mut model_ref) = session_model_ref {
1672 model_ref.provider = resolved_provider_name.clone();
1673 persist_model_ref(&mut session, &model_ref);
1674 match self.provider_router.route(&model_ref) {
1675 Ok(provider) => Some(provider),
1676 Err(error) => {
1677 tracing::error!(
1678 session_id = %session_id,
1679 provider = %model_ref.provider,
1680 model = %model_ref.model,
1681 %error,
1682 "child-completion resume provider routing failed closed"
1683 );
1684 execution_reservation.abandon().await;
1685 return;
1686 }
1687 }
1688 } else {
1689 match self.provider_registry.get(&resolved_provider_name) {
1690 Some(provider) => Some(provider),
1691 None => {
1692 tracing::error!(
1693 session_id = %session_id,
1694 provider = %resolved_provider_name,
1695 "child-completion resume provider disappeared after resolution; refusing to fall back"
1696 );
1697 execution_reservation.abandon().await;
1698 return;
1699 }
1700 }
1701 };
1702 let resolved_fast_provider = resolve_fast_model(
1703 &config_snapshot,
1704 &resolved_provider_name,
1705 &self.provider_registry,
1706 )
1707 .map(|model| model.provider);
1708 let reasoning_effort = session.reasoning_effort;
1709 let reasoning_effort_source = session
1710 .metadata
1711 .get("reasoning_effort_source")
1712 .cloned()
1713 .unwrap_or_default();
1714 let gold_config = resolve_gold_config(
1715 &config_snapshot,
1716 session
1717 .metadata
1718 .get(GOLD_CONFIG_METADATA_KEY)
1719 .map(String::as_str),
1720 )
1721 .or(config.gold_config.clone());
1722
1723 let (mpsc_tx, _forwarder, history_commit_barrier) =
1724 crate::execution::create_event_forwarder_with_root_actor(
1725 session_id.clone(),
1726 execution_reservation.run_id().to_string(),
1727 event_sender,
1728 self.agent_runners.clone(),
1729 self.account_feed_inbox.clone(),
1730 root_publication,
1731 );
1732
1733 let config_handle = self.config.clone();
1734 let cached_config = Arc::new(StdRwLock::new(config_snapshot.clone()));
1735 let provider_registry = self.provider_registry.clone();
1736 let provider_name_for_aux = resolved_provider_name.clone();
1737 let auxiliary_model_resolver = std::sync::Arc::new(move || {
1738 let config_snapshot = read_config_snapshot(&config_handle, cached_config.as_ref());
1739 let areas = resolve_global_area_models(
1741 &config_snapshot,
1742 &provider_name_for_aux,
1743 &provider_registry,
1744 );
1745 crate::AuxiliaryModelConfig {
1746 fast_model_name: areas.fast.as_ref().map(|m| m.model_name.clone()),
1747 fast_model_provider: areas.fast.map(|m| m.provider),
1748 background_model_name: areas.background.as_ref().map(|m| m.model_name.clone()),
1749 planning_model_name: None,
1750 search_model_name: None,
1751 summarization_model_name: areas
1752 .summarization
1753 .as_ref()
1754 .map(|m| m.model_name.clone()),
1755 background_model_provider: areas.background.map(|m| m.provider),
1756 summarization_model_provider: areas.summarization.map(|m| m.provider),
1757 }
1758 });
1759 let model_roster = crate::ModelRoster {
1760 model: Some(model),
1761 provider_name: Some(resolved_provider_name),
1762 provider_type: config.provider_type.clone(),
1763 fast: crate::RoleModel::from_parts(config.fast_model, resolved_fast_provider),
1764 background: crate::RoleModel::from_parts(
1765 config.background_model,
1766 config.background_model_provider,
1767 ),
1768 summarization: crate::RoleModel::from_parts(
1769 config.summarization_model,
1770 config.summarization_model_provider,
1771 ),
1772 };
1773
1774 let guardian_config = read_guardian_config(&session);
1779 let guardian_spawner = self.guardian_spawner.read().await.clone();
1780
1781 consume_pending_clarification_resume(&mut session);
1782 spawn_session_execution(SessionExecutionArgs {
1783 agent: self.agent.clone(),
1784 session_id,
1785 session,
1786 execution_reservation,
1787 tools_override: Some(root_tools),
1788 provider_override,
1789 model_roster,
1790 reasoning_effort,
1791 reasoning_effort_source,
1792 auxiliary_model_resolver: Some(auxiliary_model_resolver),
1793 disabled_filter_resolver: None,
1796 disabled_tools: Some(config.disabled_tools),
1797 disabled_skill_ids: Some(config.disabled_skill_ids),
1798 selected_skill_ids: None,
1799 selected_skill_mode: None,
1800 mpsc_tx,
1801 history_commit_barrier,
1802 image_fallback: config.image_fallback,
1803 gold_config,
1804 guardian_config,
1805 guardian_spawner,
1806 bash_resume_hook: {
1807 let hook: Arc<dyn BashResumeHook> = Arc::new(self.clone());
1808 Some(hook)
1809 },
1810 bash_completion_sink: {
1811 let sink: Arc<dyn BashCompletionSink> = Arc::new(self.clone());
1814 Some(sink)
1815 },
1816 app_data_dir: Some(self.app_data_dir.clone()),
1817 run_budget: None,
1820 runners: self.agent_runners.clone(),
1821 sessions_cache: self.sessions.clone(),
1822 on_complete: None,
1823 child_completion_handler: Some(Arc::new(self.clone())),
1827 });
1828 }
1829}
1830
1831#[async_trait]
1836impl SessionActivationSpawner for ChildCompletionCoordinator {
1837 async fn reserve_activation(
1838 &self,
1839 target_session_id: &str,
1840 _inbox_generation: u64,
1841 ) -> Result<SessionActivationReserveOutcome, bamboo_domain::SessionActivationError> {
1842 let Some(inbox) = self.agent.session_inbox() else {
1843 return Err(bamboo_domain::SessionActivationError::Internal(
1844 "agent runtime has no SessionInbox".to_string(),
1845 ));
1846 };
1847 let backlog = inbox
1848 .inspect(target_session_id)
1849 .await
1850 .map_err(|error| bamboo_domain::SessionActivationError::Internal(error.to_string()))?;
1851 if !backlog.activation_pending() {
1852 return Ok(SessionActivationReserveOutcome::NoWork);
1853 }
1854 let prepared = prepare_session_inbox_activation(
1857 &self.persistence,
1858 target_session_id,
1859 backlog.interrupt_pending(),
1860 )
1861 .await
1862 .map_err(|error| {
1863 bamboo_domain::SessionActivationError::Internal(format!(
1864 "persist resumable SessionInbox target: {error}"
1865 ))
1866 })?;
1867 let Some((session, ready)) = prepared else {
1868 return Ok(SessionActivationReserveOutcome::NotFound);
1869 };
1870 if !ready {
1871 tracing::info!(
1872 session_id = target_session_id,
1873 "SessionInbox backlog is activation-eligible but a specific durable wait remains armed"
1874 );
1875 return Ok(SessionActivationReserveOutcome::NoWork);
1876 }
1877
1878 enum LaunchPlan {
1879 Root(Box<ResumeConfigSnapshot>),
1880 Child {
1881 scheduler: Arc<SpawnScheduler>,
1882 parent_session_id: String,
1883 model: String,
1884 disabled_tools: Option<Vec<String>>,
1885 },
1886 }
1887
1888 let launch_plan = match session.kind {
1891 SessionKind::Root => {
1892 if self.root_tools.read().await.is_none() {
1893 return Err(bamboo_domain::SessionActivationError::Internal(
1894 "root tool surface is not initialized".to_string(),
1895 ));
1896 }
1897 let config_snapshot = self.config.read().await.clone();
1898 LaunchPlan::Root(Box::new(
1899 self.build_resume_config(&session, &config_snapshot),
1900 ))
1901 }
1902 SessionKind::Child => {
1903 let scheduler = self.spawn_scheduler.read().await.upgrade().ok_or_else(|| {
1904 bamboo_domain::SessionActivationError::Internal(
1905 "child spawn scheduler is not initialized".to_string(),
1906 )
1907 })?;
1908 let parent_session_id = session
1909 .parent_session_id
1910 .as_deref()
1911 .map(str::trim)
1912 .filter(|id| !id.is_empty())
1913 .map(ToOwned::to_owned)
1914 .ok_or_else(|| {
1915 bamboo_domain::SessionActivationError::Internal(format!(
1916 "child SessionInbox target {target_session_id} has no parent owner"
1917 ))
1918 })?;
1919 let parent = self
1920 .storage
1921 .load_session(&parent_session_id)
1922 .await
1923 .map_err(|error| {
1924 bamboo_domain::SessionActivationError::Internal(format!(
1925 "load parent owner {parent_session_id}: {error}"
1926 ))
1927 })?
1928 .ok_or_else(|| {
1929 bamboo_domain::SessionActivationError::Internal(format!(
1930 "child SessionInbox parent owner {parent_session_id} disappeared"
1931 ))
1932 })?;
1933 let child_root = if session.root_session_id.trim().is_empty() {
1934 parent_session_id.as_str()
1935 } else {
1936 session.root_session_id.as_str()
1937 };
1938 let parent_root = if parent.root_session_id.trim().is_empty() {
1939 parent.id.as_str()
1940 } else {
1941 parent.root_session_id.as_str()
1942 };
1943 if child_root != parent_root {
1944 return Err(bamboo_domain::SessionActivationError::Internal(format!(
1945 "child SessionInbox target {target_session_id} does not share its parent owner's root"
1946 )));
1947 }
1948 let model = if session.model.trim().is_empty() {
1949 parent.model.clone()
1950 } else {
1951 session.model.clone()
1952 };
1953 if model.trim().is_empty() {
1954 return Err(bamboo_domain::SessionActivationError::Internal(format!(
1955 "child SessionInbox target {target_session_id} has no executable model"
1956 )));
1957 }
1958 let disabled_tools = match session.metadata.get("disabled_tools") {
1959 None => None,
1960 Some(raw) => {
1961 let tools = serde_json::from_str::<std::collections::BTreeSet<String>>(raw)
1962 .map_err(|error| {
1963 bamboo_domain::SessionActivationError::Internal(format!(
1964 "child SessionInbox target {target_session_id} has malformed disabled_tools: {error}"
1965 ))
1966 })?;
1967 (!tools.is_empty()).then(|| tools.into_iter().collect())
1968 }
1969 };
1970 LaunchPlan::Child {
1971 scheduler,
1972 parent_session_id,
1973 model,
1974 disabled_tools,
1975 }
1976 }
1977 };
1978 let event_sender =
1979 ResumeExecutionPort::get_or_create_event_sender(self, target_session_id).await;
1980
1981 let reservation = match reserve_runner_core(
1982 &self.agent_runners,
1983 &self.session_event_senders,
1984 target_session_id,
1985 &event_sender,
1986 )
1987 .await
1988 {
1989 ReserveOutcome::Reserved(reservation) => reservation,
1990 ReserveOutcome::AlreadyRunning(run_id) => {
1991 return Ok(SessionActivationReserveOutcome::AlreadyRunning { run_id });
1992 }
1993 };
1994 let run_id = reservation.run_id.clone();
1995 let launch = match launch_plan {
1996 LaunchPlan::Root(config) => {
1997 let mut execution_reservation =
1998 SessionExecutionReservation::from_activation_placeholder(
1999 target_session_id,
2000 reservation,
2001 self.agent
2002 .activation_router()
2003 .expect("SessionInbox activation requires an activation router")
2004 .clone(),
2005 self.agent_runners.clone(),
2006 );
2007 if self
2011 .agent
2012 .persistence()
2013 .root_actor_execution_required(&session)
2014 && self.root_account_sink.is_none()
2015 {
2016 execution_reservation
2017 .rollback_unpublished_activation()
2018 .await;
2019 return Err(bamboo_domain::SessionActivationError::Internal(
2020 "Root activation has no actual account sink".into(),
2021 ));
2022 }
2023 if let Err(error) = execution_reservation
2024 .bind_root_actor(&self.agent, &session)
2025 .await
2026 {
2027 execution_reservation
2028 .rollback_unpublished_activation()
2029 .await;
2030 return Err(bamboo_domain::SessionActivationError::Internal(format!(
2031 "Root Actor reservation rejected: {error}"
2032 )));
2033 }
2034 let reservation_cell = Arc::new(StdMutex::new(Some(execution_reservation)));
2038 let launch_reservation = reservation_cell.clone();
2039 let rollback_reservation = reservation_cell;
2040 let coordinator = self.clone();
2041 let launch_sessions = self.sessions.clone();
2042 let launch_session_id = session.id.clone();
2043 let launch_session = session.clone();
2044 let request_session_id = target_session_id.to_string();
2045 SessionActivationLaunch::new_with_async_rollback(
2046 run_id,
2047 move || {
2048 let mut execution_reservation = launch_reservation
2049 .lock()
2050 .unwrap_or_else(std::sync::PoisonError::into_inner)
2051 .take()
2052 .expect("activation reservation launches or rolls back exactly once");
2053 execution_reservation.mark_activation_published();
2054 if execution_reservation.root_actor_writer().is_none() {
2058 launch_sessions.insert(
2059 launch_session_id,
2060 Arc::new(crate::SessionSnapshot::new(launch_session)),
2061 );
2062 }
2063 let request = ResumeSpawnRequest {
2064 session_id: request_session_id,
2065 session,
2066 execution_reservation,
2067 event_sender,
2068 config: *config,
2069 };
2070 tokio::spawn(async move {
2071 ResumeExecutionPort::spawn_resume_execution(&coordinator, request)
2072 .await;
2073 });
2074 },
2075 move || async move {
2076 let reservation = rollback_reservation
2077 .lock()
2078 .unwrap_or_else(std::sync::PoisonError::into_inner)
2079 .take();
2080 if let Some(reservation) = reservation {
2081 reservation.rollback_unpublished_activation().await;
2082 }
2083 },
2084 )
2085 }
2086 LaunchPlan::Child {
2087 scheduler,
2088 parent_session_id,
2089 model,
2090 disabled_tools,
2091 } => {
2092 let execution_reservation =
2093 SessionExecutionReservation::from_activation_placeholder(
2094 target_session_id,
2095 reservation,
2096 self.agent
2097 .activation_router()
2098 .expect("SessionInbox activation requires an activation router")
2099 .clone(),
2100 self.agent_runners.clone(),
2101 );
2102 let reservation_cell = Arc::new(StdMutex::new(Some(execution_reservation)));
2103 let launch_reservation = reservation_cell.clone();
2104 let rollback_reservation = reservation_cell;
2105 let job = SpawnJob {
2106 parent_session_id,
2107 child_session_id: target_session_id.to_string(),
2108 model,
2109 disabled_tools,
2110 };
2111 let launch_sessions = self.sessions.clone();
2112 let launch_session_id = session.id.clone();
2113 let launch_session = session;
2114 SessionActivationLaunch::new_with_async_rollback(
2115 run_id,
2116 move || {
2117 let mut execution_reservation = launch_reservation
2118 .lock()
2119 .unwrap_or_else(std::sync::PoisonError::into_inner)
2120 .take()
2121 .expect("activation reservation launches or rolls back exactly once");
2122 execution_reservation.mark_activation_published();
2123 launch_sessions.insert(
2126 launch_session_id,
2127 Arc::new(crate::SessionSnapshot::new(launch_session)),
2128 );
2129 drop(scheduler.launch_reserved(job, execution_reservation));
2133 },
2134 move || async move {
2135 let reservation = rollback_reservation
2136 .lock()
2137 .unwrap_or_else(std::sync::PoisonError::into_inner)
2138 .take();
2139 if let Some(reservation) = reservation {
2140 reservation.rollback_unpublished_activation().await;
2141 }
2142 },
2143 )
2144 }
2145 };
2146 Ok(SessionActivationReserveOutcome::Reserved(launch))
2147 }
2148}
2149
2150fn bash_completion_resume_message(bash_ids: &[String], timed_out: bool) -> Message {
2160 let body = if timed_out {
2161 format!(
2162 "Runtime notification: the background-Bash wait ceiling was reached while one or more \
2163 shell(s) ({}) may still be running. The session is being resumed so it is not \
2164 stranded; verify their actual status with BashOutput before assuming completion.",
2165 bash_ids.join(", ")
2166 )
2167 } else {
2168 format!(
2169 "Runtime notification: all background Bash shell(s) ({}) have completed. \
2170 Review their output with BashOutput and resume the task from where you left off.",
2171 bash_ids.join(", ")
2172 )
2173 };
2174 let mut message = Message::user(body);
2175 message.metadata = Some(serde_json::json!({
2176 RUNTIME_RESUME_MESSAGE_HIDDEN_KEY: true,
2177 RUNTIME_RESUME_MESSAGE_KIND_KEY: BASH_COMPLETION_RESUME_KIND,
2178 }));
2179 message.never_compress = false;
2180 message
2181}
2182
2183fn bash_resume_should_retry(outcome: &ResumeOutcome, persisted_waiting_for_bash: bool) -> bool {
2196 match outcome {
2197 ResumeOutcome::Started { .. } | ResumeOutcome::NotFound => false,
2198 ResumeOutcome::Completed | ResumeOutcome::AlreadyRunning { .. } => {
2199 persisted_waiting_for_bash
2200 }
2201 }
2202}
2203
2204fn bash_completion_should_resume(
2212 loop_suspended_on_bash: bool,
2213 all_waited_shells_done: bool,
2214) -> bool {
2215 loop_suspended_on_bash && all_waited_shells_done
2216}
2217
2218#[derive(Debug, Clone, Copy, PartialEq, Eq)]
2219enum BashCompletionDeliveryPlan {
2220 DurableOnly,
2224 Activate,
2226 ClearWaitThenActivate,
2228}
2229
2230fn bash_completion_delivery_plan(
2231 loop_suspended_on_bash: bool,
2232 all_waited_shells_done: bool,
2233) -> BashCompletionDeliveryPlan {
2234 if bash_completion_should_resume(loop_suspended_on_bash, all_waited_shells_done) {
2235 BashCompletionDeliveryPlan::ClearWaitThenActivate
2236 } else if loop_suspended_on_bash {
2237 BashCompletionDeliveryPlan::DurableOnly
2238 } else {
2239 BashCompletionDeliveryPlan::Activate
2240 }
2241}
2242
2243fn background_bash_post_tool_payload(info: &BashCompletionInfo) -> HookPayload {
2244 let success = info.status == "completed" && info.exit_code == Some(0);
2245 let response = serde_json::json!({
2246 "bash_id": info.bash_id,
2247 "command": info.command,
2248 "exit_code": info.exit_code,
2249 "status": info.status,
2250 "output_tail": info.output_tail,
2251 });
2252 HookPayload::ToolResult {
2253 tool_name: "Bash".to_string(),
2254 tool_call_id: info.bash_id.clone(),
2255 outcome: HookToolOutcome {
2256 success,
2257 result: success.then(|| response.to_string()),
2258 error: (!success).then(|| response.to_string()),
2259 needs_human: false,
2260 duration_ms: 0,
2261 },
2262 }
2263}
2264
2265fn append_background_bash_hook_feedback(info: &mut BashCompletionInfo, feedback: Vec<String>) {
2266 let feedback = feedback
2267 .into_iter()
2268 .map(|text| text.trim().to_string())
2269 .filter(|text| !text.is_empty())
2270 .collect::<Vec<_>>();
2271 if feedback.is_empty() {
2272 return;
2273 }
2274 if !info.output_tail.is_empty() {
2275 info.output_tail.push_str("\n\n");
2276 }
2277 info.output_tail.push_str("<post_tool_use_feedback>\n");
2278 info.output_tail.push_str(&feedback.join("\n"));
2279 info.output_tail.push_str("\n</post_tool_use_feedback>");
2280}
2281
2282async fn run_background_bash_post_tool_hooks(
2283 config: &bamboo_config::LifecycleHooksConfig,
2284 fallback_cwd: Option<std::path::PathBuf>,
2285 session: &Session,
2286 info: &mut BashCompletionInfo,
2287) -> bool {
2288 let runner = crate::HookRunner::new().with_lifecycle_config(config, fallback_cwd);
2289 if !runner.has_hooks_for(AgentHookPoint::AfterToolExecution) {
2290 return false;
2291 }
2292
2293 let mut runtime_state = session
2294 .agent_runtime_state
2295 .clone()
2296 .unwrap_or_else(|| AgentRuntimeState::new(&session.id));
2297 let outcome = runner
2298 .run_observer_hooks(
2299 AgentHookPoint::AfterToolExecution,
2300 &background_bash_post_tool_payload(info),
2301 session,
2302 &mut runtime_state,
2303 None,
2304 )
2305 .await;
2306 append_background_bash_hook_feedback(info, outcome.injected_contexts);
2307 true
2308}
2309
2310fn apply_bash_resume_transition(session: &mut Session, resume_message: &Message) -> bool {
2318 let mut runtime_state = read_runtime_state(session);
2319 if runtime_state.waiting_for_bash.is_none() {
2320 return false;
2321 }
2322 runtime_state.waiting_for_bash = None;
2323 runtime_state.status = AgentStatusState::Idle;
2324 runtime_state.suspension = None;
2325 write_runtime_state(session, &runtime_state);
2326 session.metadata.remove("runtime.suspend_reason");
2327 session.add_message(resume_message.clone());
2328 true
2329}
2330
2331impl ChildCompletionCoordinator {
2333 async fn bash_self_resume(&self, session_id: String, bash_ids: Vec<String>) {
2349 let mut delay = Duration::from_secs(1);
2350 let max_delay = Duration::from_secs(30);
2351 let max_poll = Duration::from_secs(6 * 3600 + 600);
2355 let deadline = tokio::time::Instant::now() + max_poll;
2356
2357 loop {
2358 tokio::time::sleep(delay).await;
2359
2360 let Some(session) = self.load_session(&session_id).await else {
2361 tracing::info!(%session_id, "bash self-resume backstop: session gone; nothing to do");
2362 return;
2363 };
2364 if read_runtime_state(&session).waiting_for_bash.is_none() {
2365 return;
2368 }
2369
2370 let still_running =
2371 bamboo_tools::tools::bash_runtime::running_shells_for_session(&session_id);
2372 let timed_out = tokio::time::Instant::now() >= deadline;
2373 if still_running.is_empty() || timed_out {
2374 let guard = session_resume_lock(&session_id);
2379 let _held = guard.lock().await;
2380 tracing::warn!(
2381 %session_id,
2382 shell_count = bash_ids.len(),
2383 timed_out,
2384 "bash self-resume backstop engaged (push lost or wait ceiling reached)"
2385 );
2386 self.perform_bash_resume(
2387 &session_id,
2388 bash_completion_resume_message(&bash_ids, timed_out),
2389 )
2390 .await;
2391 return;
2392 }
2393
2394 delay = (delay * 2).min(max_delay);
2395 }
2396 }
2397
2398 async fn perform_bash_resume(&self, session_id: &str, resume_message: Message) {
2419 let retry_backoff = Duration::from_millis(200);
2420 const MAX_RESUME_ATTEMPTS: u8 = 5;
2421 for attempt in 0..MAX_RESUME_ATTEMPTS {
2422 if attempt > 0 {
2423 tokio::time::sleep(retry_backoff).await;
2424 }
2425
2426 let Some(mut session) = self.load_session(session_id).await else {
2427 tracing::warn!(%session_id, "bash resume: session not found; nothing to resume");
2428 return;
2429 };
2430
2431 if !apply_bash_resume_transition(&mut session, &resume_message) {
2432 tracing::info!(
2436 %session_id, attempt,
2437 "bash resume: persisted bash wait already cleared; nothing to resume"
2438 );
2439 return;
2440 }
2441 session.updated_at = Utc::now();
2442 self.save_and_cache(&mut session).await;
2443 tracing::info!(
2444 %session_id, attempt,
2445 "bash resume: cleared bash wait and appended resume message"
2446 );
2447
2448 let outcome = self.resume_parent(session_id.to_string()).await;
2449 match outcome {
2450 ResumeOutcome::Started { .. } => {
2451 tracing::info!(%session_id, attempt, "bash resume: resume fired");
2452 return;
2453 }
2454 ResumeOutcome::NotFound => {
2455 tracing::warn!(%session_id, "bash resume: session vanished during resume");
2456 return;
2457 }
2458 _ => {
2459 let clobbered = match self.load_session(session_id).await {
2465 Some(reloaded) => read_runtime_state(&reloaded).waiting_for_bash.is_some(),
2466 None => {
2467 tracing::warn!(%session_id, "bash resume: session vanished after resume");
2468 return;
2469 }
2470 };
2471 if bash_resume_should_retry(&outcome, clobbered) {
2472 tracing::warn!(
2473 %session_id, attempt,
2474 outcome = outcome.as_str(),
2475 "bash resume: persisted wait still set after resume (finalize-clobber); retrying"
2476 );
2477 continue;
2478 }
2479 tracing::info!(
2480 %session_id, attempt,
2481 outcome = outcome.as_str(),
2482 "bash resume: wait cleared and resume handled; stopping"
2483 );
2484 return;
2485 }
2486 }
2487 }
2488
2489 tracing::warn!(
2490 %session_id,
2491 attempts = MAX_RESUME_ATTEMPTS,
2492 "bash resume: exhausted clobber-retry budget without confirming resume; giving up"
2493 );
2494 }
2495}
2496
2497impl BashResumeHook for ChildCompletionCoordinator {
2498 fn arrange_bash_self_resume(&self, session_id: String, bash_ids: Vec<String>) {
2499 let coordinator = Arc::new(self.clone());
2500 tokio::spawn(async move {
2501 coordinator.bash_self_resume(session_id, bash_ids).await;
2502 });
2503 }
2504}
2505
2506fn bash_completion_injection_body(info: &BashCompletionInfo) -> String {
2511 let exit = match info.exit_code {
2512 Some(code) => code.to_string(),
2513 None => "none (signal/killed)".to_string(),
2514 };
2515 let mut body = format!(
2516 "Runtime notification: background shell `{}` (`{}`) finished — status {}, exit code {}.",
2517 info.bash_id, info.command, info.status, exit
2518 );
2519 if info.output_tail.trim().is_empty() {
2520 body.push_str(" It produced no captured output.");
2521 } else {
2522 body.push_str("\n\nOutput tail:\n");
2523 body.push_str(&info.output_tail);
2524 }
2525 body.push_str(&format!(
2526 "\n\nUse BashOutput with bash_id=\"{}\" for the full output, then continue the task.",
2527 info.bash_id
2528 ));
2529 body
2530}
2531
2532fn bash_completion_envelope(info: &BashCompletionInfo) -> bamboo_domain::SessionMessageEnvelope {
2533 let provider_message = bash_resume_message_from_info(info);
2534 let data = serde_json::json!({
2535 "session_id": info.session_id,
2536 "bash_id": info.bash_id,
2537 "command": info.command,
2538 "exit_code": info.exit_code,
2539 "status": info.status,
2540 "output_tail": info.output_tail,
2541 });
2542 let identity = data.clone();
2547 bamboo_domain::SessionMessageEnvelope {
2548 id: bamboo_domain::SessionMessageId::stable("background_bash_completion", &identity),
2549 source: bamboo_domain::SessionMessageSource::Runtime {
2550 subsystem: "background_bash".to_string(),
2551 },
2552 target_session_id: info.session_id.clone(),
2553 kind: bamboo_domain::SessionMessageKind::RuntimeInstruction,
2554 body: bamboo_domain::SessionMessageBody::RuntimeInstruction(
2555 bamboo_domain::SessionRuntimeInstruction {
2556 instruction: "background_bash_completed".to_string(),
2557 content: Some(bamboo_domain::SessionMessageContent::text(
2558 bash_completion_injection_body(info),
2559 )),
2560 data: Some(data),
2561 provider_message: Some(session_provider_message(&provider_message)),
2562 },
2563 ),
2564 created_at: Utc::now(),
2565 thread_id: None,
2566 in_reply_to: None,
2567 attempt: None,
2568 correlation_id: Some(info.bash_id.clone()),
2569 }
2570}
2571
2572async fn enqueue_bash_completion_injection(
2578 persistence: &LockedSessionStore,
2579 info: &BashCompletionInfo,
2580) -> std::io::Result<Option<Session>> {
2581 let body = bash_completion_injection_body(info);
2582 let queued = serde_json::json!({
2583 "content": body,
2584 "created_at": Utc::now(),
2585 });
2586 persistence
2587 .update_runtime_config(&info.session_id, move |session| {
2588 let mut pending = session.pending_injected_messages().unwrap_or_default();
2589 pending.push(queued);
2590 session.set_pending_injected_messages(pending);
2591 })
2592 .await
2593}
2594
2595fn bash_resume_message_from_info(info: &BashCompletionInfo) -> Message {
2603 let mut message = Message::user(bash_completion_injection_body(info));
2604 message.metadata = Some(serde_json::json!({
2605 RUNTIME_RESUME_MESSAGE_HIDDEN_KEY: true,
2606 RUNTIME_RESUME_MESSAGE_KIND_KEY: BASH_COMPLETION_RESUME_KIND,
2607 }));
2608 message.never_compress = false;
2609 message
2610}
2611
2612impl ChildCompletionCoordinator {
2613 async fn deliver_bash_completion(&self, mut info: BashCompletionInfo) {
2625 if let Some(hook_session) = self.load_session(&info.session_id).await {
2631 let config_snapshot = self.config.read().await.clone();
2632 let _ = run_background_bash_post_tool_hooks(
2633 &config_snapshot.lifecycle_hooks,
2634 Some(self.app_data_dir.clone()),
2635 &hook_session,
2636 &mut info,
2637 )
2638 .await;
2639 }
2640
2641 let guard = session_resume_lock(&info.session_id);
2642 let _held = guard.lock().await;
2643
2644 let Some(session) = self.load_session(&info.session_id).await else {
2645 tracing::warn!(
2646 session_id = %info.session_id,
2647 bash_id = %info.bash_id,
2648 "background bash completion: owning session not found; nothing to notify"
2649 );
2650 return;
2651 };
2652
2653 let waiting = read_runtime_state(&session).waiting_for_bash.is_some();
2654 let all_shells_done =
2660 bamboo_tools::tools::bash_runtime::running_shells_for_session(&info.session_id)
2661 .is_empty();
2662 let delivery_plan = bash_completion_delivery_plan(waiting, all_shells_done);
2663
2664 let messenger = self.agent.session_messenger().cloned();
2670 let admission = match messenger.as_ref() {
2671 Some(messenger) => match messenger.admit(bash_completion_envelope(&info)).await {
2672 Ok(admission) => Some(admission),
2673 Err(error) => {
2674 tracing::warn!(
2675 session_id = %info.session_id,
2676 bash_id = %info.bash_id,
2677 %error,
2678 "background bash completion: durable SessionInbox admission failed; leaving wait armed"
2679 );
2680 return;
2681 }
2682 },
2683 None => {
2684 tracing::warn!(
2685 session_id = %info.session_id,
2686 bash_id = %info.bash_id,
2687 "SessionMessenger unavailable; using compatibility injection before clearing wait"
2688 );
2689 match enqueue_bash_completion_injection(&self.persistence, &info).await {
2690 Ok(Some(_)) => None,
2691 Ok(None) => {
2692 tracing::warn!(
2693 session_id = %info.session_id,
2694 "background bash compatibility target disappeared"
2695 );
2696 return;
2697 }
2698 Err(error) => {
2699 tracing::warn!(
2700 session_id = %info.session_id,
2701 %error,
2702 "background bash compatibility admission failed; leaving wait armed"
2703 );
2704 return;
2705 }
2706 }
2707 }
2708 };
2709
2710 if delivery_plan == BashCompletionDeliveryPlan::DurableOnly {
2711 tracing::info!(
2712 session_id = %info.session_id,
2713 bash_id = %info.bash_id,
2714 "background bash completion is durable; sibling shells still run, so wait remains armed"
2715 );
2716 return;
2717 }
2718
2719 if let (Some(messenger), Some(admission)) = (messenger.as_ref(), admission.as_ref()) {
2720 if let Err(error) = messenger.prepare_activation(admission).await {
2721 tracing::warn!(
2722 session_id = %info.session_id,
2723 bash_id = %info.bash_id,
2724 %error,
2725 "background bash completion activation watermark failed; leaving wait armed"
2726 );
2727 return;
2728 }
2729 }
2730
2731 if delivery_plan == BashCompletionDeliveryPlan::ClearWaitThenActivate {
2732 tracing::info!(
2733 session_id = %info.session_id,
2734 bash_id = %info.bash_id,
2735 status = %info.status,
2736 "background bash completion: push-resuming suspended loop (event-driven)"
2737 );
2738 let mut resumable = session.clone();
2739 let mut runtime_state = read_runtime_state(&resumable);
2740 runtime_state.waiting_for_bash = None;
2741 runtime_state.status = AgentStatusState::Idle;
2742 runtime_state.suspension = None;
2743 write_runtime_state(&mut resumable, &runtime_state);
2744 resumable.metadata.remove("runtime.suspend_reason");
2745 resumable.updated_at = Utc::now();
2746 if let Err(error) = self.persistence.merge_save_runtime(&mut resumable).await {
2747 tracing::warn!(
2748 session_id = %info.session_id,
2749 %error,
2750 "background bash completion is durable but wait-state clear failed; leaving activation to the wait backstop"
2751 );
2752 return;
2753 }
2754 self.sessions.insert(
2755 resumable.id.clone(),
2756 Arc::new(crate::SessionSnapshot::new(resumable)),
2757 );
2758 }
2759
2760 let (Some(messenger), Some(admission)) = (messenger, admission) else {
2764 if delivery_plan == BashCompletionDeliveryPlan::ClearWaitThenActivate {
2765 let _ = self.resume_parent(info.session_id.clone()).await;
2766 }
2767 return;
2768 };
2769 match messenger.activate_prepared(&admission).await {
2770 Ok(receipt) => tracing::info!(
2771 session_id = %info.session_id,
2772 bash_id = %info.bash_id,
2773 status = %info.status,
2774 waiting,
2775 generation = receipt.delivery.generation,
2776 activation = ?receipt.activation,
2777 "background bash completion delivered through SessionMessenger"
2778 ),
2779 Err(error) => tracing::warn!(
2780 session_id = %info.session_id,
2781 bash_id = %info.bash_id,
2782 %error,
2783 "background bash completion: SessionMessenger delivery failed"
2784 ),
2785 }
2786 }
2787}
2788
2789impl BashCompletionSink for ChildCompletionCoordinator {
2790 fn on_bash_completed(&self, info: BashCompletionInfo) {
2791 let coordinator = Arc::new(self.clone());
2795 tokio::spawn(async move {
2796 coordinator.deliver_bash_completion(info).await;
2797 });
2798 }
2799}
2800
2801const CHILD_WAIT_SWEEP_INTERVAL_SECS: u64 = 30;
2807const CHILD_WAIT_REGISTRATION_GRACE_SECS: i64 = 60;
2811const DEAD_CHILD_GRACE_SECS: i64 = 120;
2814const STALE_RUNNER_SLACK_SECS: i64 = 600;
2820
2821fn is_dead_child_candidate_status(status: Option<&str>) -> bool {
2826 match status {
2827 Some(status) => !is_terminal_child_status(status) && status != "suspended",
2830 None => true,
2831 }
2832}
2833
2834fn completion_child_is_owned(reported_parent: &str, child_parent_linkage: Option<&str>) -> bool {
2842 child_parent_linkage == Some(reported_parent)
2843}
2844
2845fn select_replay_child(terminal: &[(String, String)]) -> Option<&(String, String)> {
2849 terminal
2850 .iter()
2851 .find(|(_, status)| is_error_like(status))
2852 .or_else(|| terminal.last())
2853}
2854
2855fn child_wait_watchdog_resume_message(body: String) -> Message {
2856 let mut message = Message::user(body);
2857 message.metadata = Some(serde_json::json!({
2858 RUNTIME_RESUME_MESSAGE_HIDDEN_KEY: true,
2859 RUNTIME_RESUME_MESSAGE_KIND_KEY: "child_wait_watchdog_resume",
2860 }));
2861 message.never_compress = false;
2862 message
2863}
2864
2865fn empty_child_wait_message() -> Message {
2866 let inspection = serde_json::json!({"intent": "inspect", "message": "tree"});
2867 child_wait_watchdog_resume_message(format!(
2868 "Runtime notification: this session was suspended waiting for child sessions, but the \
2869 wait tracked no children (internal inconsistency). The session has been resumed; use \
2870 SubAgent with {inspection} to inspect child state and continue the task.",
2871 ))
2872}
2873
2874fn child_wait_lease_expired_message(child_ids: &[String]) -> Message {
2875 let inspection = serde_json::json!({"intent": "inspect", "message": "tree"});
2876 child_wait_watchdog_resume_message(format!(
2877 "Runtime notification: the wait lease for child session(s) [{}] expired before they all \
2878 reported completion. They were NOT cancelled and may still be running or already \
2879 finished — verify their actual status by calling SubAgent with {inspection} before assuming \
2880 anything, then continue the task.",
2881 child_ids.join(", ")
2882 ))
2883}
2884
2885impl ChildCompletionCoordinator {
2901 pub fn spawn_child_wait_watchdog(self: &Arc<Self>) {
2904 let coordinator = Arc::clone(self);
2905 tokio::spawn(async move {
2906 use futures::FutureExt;
2907 if std::panic::AssertUnwindSafe(coordinator.reconcile_orphans_at_boot())
2908 .catch_unwind()
2909 .await
2910 .is_err()
2911 {
2912 tracing::error!("child-wait watchdog: boot reconciliation panicked");
2913 }
2914 let mut ticker = tokio::time::interval(std::time::Duration::from_secs(
2915 CHILD_WAIT_SWEEP_INTERVAL_SECS,
2916 ));
2917 ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
2918 ticker.tick().await;
2920 loop {
2921 ticker.tick().await;
2922 if std::panic::AssertUnwindSafe(coordinator.sweep_child_waits())
2923 .catch_unwind()
2924 .await
2925 .is_err()
2926 {
2927 tracing::error!("child-wait watchdog: sweep panicked; continuing");
2928 }
2929 }
2930 });
2931 }
2932
2933 async fn reconcile_orphans_at_boot(&self) {
2938 let cutoff = Utc::now();
2939
2940 let running = self
2946 .storage
2947 .list_sessions_by_run_status("running")
2948 .await
2949 .unwrap_or_default();
2950 for (child_id, parent_id) in running {
2951 let Some(parent_id) = parent_id else { continue };
2952 if self.runner_is_running(&child_id).await {
2953 continue;
2954 }
2955 let Some(control_plane) = self.load_control_plane(&child_id).await else {
2956 continue;
2957 };
2958 if control_plane.updated_at >= cutoff {
2961 continue;
2962 }
2963 tracing::warn!(
2964 child_session_id = %child_id,
2965 parent_session_id = %parent_id,
2966 "boot reconciliation: child was running when the process died; \
2967 marking it error and waking the parent"
2968 );
2969 self.synthesize_child_completion(
2970 &parent_id,
2971 &child_id,
2972 "error",
2973 Some(
2974 "orphaned by server restart: the process died while this child session \
2975 was running"
2976 .to_string(),
2977 ),
2978 )
2979 .await;
2980 }
2981
2982 let suspended = self
2986 .storage
2987 .list_sessions_by_run_status("suspended")
2988 .await
2989 .unwrap_or_default();
2990 for (session_id, _) in suspended {
2991 let Some(control_plane) = self.load_control_plane(&session_id).await else {
2992 continue;
2993 };
2994 if control_plane
2995 .metadata
2996 .get("runtime.suspend_reason")
2997 .map(String::as_str)
2998 != Some("waiting_for_bash")
2999 {
3000 continue;
3001 }
3002 if let Some(wait) = read_runtime_state(&control_plane).waiting_for_bash {
3003 tracing::warn!(
3004 %session_id,
3005 "boot reconciliation: re-arming bash self-resume backstop lost in restart"
3006 );
3007 let coordinator = self.clone();
3008 tokio::spawn(async move {
3009 coordinator
3010 .bash_self_resume(session_id, wait.bash_ids)
3011 .await;
3012 });
3013 }
3014 }
3015 }
3016
3017 async fn runner_is_running(&self, session_id: &str) -> bool {
3018 let runners = self.agent_runners.read().await;
3019 runners
3020 .get(session_id)
3021 .is_some_and(|runner| matches!(runner.status, AgentStatus::Running))
3022 }
3023
3024 async fn load_control_plane(&self, session_id: &str) -> Option<Session> {
3025 match self.storage.load_runtime_control_plane(session_id).await {
3026 Ok(session) => session,
3027 Err(error) => {
3028 tracing::warn!(
3029 %session_id,
3030 %error,
3031 "child-wait watchdog: failed to load session control plane"
3032 );
3033 None
3034 }
3035 }
3036 }
3037
3038 async fn sweep_child_waits(&self) {
3042 let suspended = match self.storage.list_sessions_by_run_status("suspended").await {
3043 Ok(entries) => entries,
3044 Err(error) => {
3045 tracing::warn!(%error, "child-wait watchdog: failed to list suspended sessions");
3046 return;
3047 }
3048 };
3049 for (session_id, _) in suspended {
3050 self.sweep_one_suspended_session(&session_id).await;
3051 }
3052 }
3053
3054 async fn sweep_one_suspended_session(&self, session_id: &str) {
3055 if self.runner_is_running(session_id).await {
3058 return;
3059 }
3060 let Some(session) = self.load_control_plane(session_id).await else {
3061 return;
3062 };
3063 let runtime_state = read_runtime_state(&session);
3064 let suspend_reason = session
3065 .metadata
3066 .get("runtime.suspend_reason")
3067 .map(String::as_str)
3068 .unwrap_or_default()
3069 .to_string();
3070 match (
3071 suspend_reason.as_str(),
3072 runtime_state.waiting_for_children.clone(),
3073 ) {
3074 ("waiting_for_bash", _)
3076 | ("awaiting_clarification", _)
3077 | ("awaiting_parent_approval", _) => {}
3078 (_, Some(wait)) => self.sweep_child_wait(session_id, wait).await,
3079 ("waiting_for_children", None) | ("", None) => {
3082 self.rescue_stranded_resume(session_id).await;
3083 }
3084 _ => {}
3085 }
3086 }
3087
3088 async fn sweep_child_wait(&self, parent_session_id: &str, wait: WaitingForChildrenState) {
3093 let now = Utc::now();
3094
3095 if wait.child_session_ids.is_empty() {
3096 tracing::warn!(
3097 %parent_session_id,
3098 "child-wait watchdog: wait armed over an empty child set; force-resuming"
3099 );
3100 self.force_resume_child_wait(parent_session_id, empty_child_wait_message())
3101 .await;
3102 return;
3103 }
3104
3105 if wait.timeout_at.is_some_and(|deadline| now >= deadline) {
3109 tracing::warn!(
3110 %parent_session_id,
3111 "child-wait watchdog: wait lease expired; force-resuming parent"
3112 );
3113 self.force_resume_child_wait(
3114 parent_session_id,
3115 child_wait_lease_expired_message(&wait.child_session_ids),
3116 )
3117 .await;
3118 return;
3119 }
3120
3121 if now.signed_duration_since(wait.registered_at).num_seconds()
3123 < CHILD_WAIT_REGISTRATION_GRACE_SECS
3124 {
3125 return;
3126 }
3127
3128 let statuses: HashMap<String, Option<String>> = self
3129 .storage
3130 .list_child_run_statuses(parent_session_id)
3131 .await
3132 .unwrap_or_default()
3133 .into_iter()
3134 .collect();
3135
3136 struct DeadChild {
3137 child_id: String,
3138 status: String,
3139 reason: String,
3140 owned: bool,
3144 }
3145
3146 let mut terminal: Vec<(String, String)> = Vec::new();
3147 let mut dead: Vec<DeadChild> = Vec::new();
3148 for child_id in &wait.child_session_ids {
3149 let status = statuses.get(child_id).and_then(|status| status.as_deref());
3150 if let Some(status) = status {
3151 if is_terminal_child_status(status) {
3152 terminal.push((child_id.clone(), status.to_string()));
3153 continue;
3154 }
3155 }
3156 if !is_dead_child_candidate_status(status) {
3157 continue;
3158 }
3159
3160 let control_plane = self.load_control_plane(child_id).await;
3167 let owned = control_plane
3168 .as_ref()
3169 .is_some_and(|cp| cp.parent_session_id.as_deref() == Some(parent_session_id));
3170 if !owned {
3171 dead.push(DeadChild {
3172 child_id: child_id.clone(),
3173 status: "error".to_string(),
3174 reason: if control_plane.is_some() {
3175 "waited-on session id is not a child of this session; clearing it \
3176 from the wait without touching that session"
3177 .to_string()
3178 } else {
3179 "waited-on child session does not exist".to_string()
3180 },
3181 owned: false,
3182 });
3183 continue;
3184 }
3185
3186 let runner = { self.agent_runners.read().await.get(child_id).cloned() };
3187 match runner {
3188 Some(runner) if matches!(runner.status, AgentStatus::Running) => {
3189 let last_activity = runner.last_activity_at().unwrap_or(runner.started_at);
3195 let idle_secs = now.signed_duration_since(last_activity).num_seconds();
3196 let total_secs = now.signed_duration_since(runner.started_at).num_seconds();
3197 let policy = match &control_plane {
3198 Some(child) => {
3199 crate::runtime::execution::spawn::watchdog_policy_for_session(child)
3200 }
3201 None => Default::default(),
3202 };
3203 let idle_limit = policy.max_idle_secs.saturating_add(STALE_RUNNER_SLACK_SECS);
3204 let total_limit = policy
3205 .max_total_secs
3206 .saturating_add(STALE_RUNNER_SLACK_SECS);
3207 if idle_secs >= idle_limit || total_secs >= total_limit {
3208 runner.cancel_token.cancel();
3209 dead.push(DeadChild {
3210 child_id: child_id.clone(),
3211 status: "timeout".to_string(),
3212 reason: format!(
3213 "child runner stalled: no events for {idle_secs}s \
3214 (limit {idle_limit}s including watchdog slack); \
3215 force-finalized by the child-wait watchdog"
3216 ),
3217 owned: true,
3218 });
3219 }
3220 }
3221 _ => {
3222 let quiet_secs = control_plane
3226 .as_ref()
3227 .map(|child| now.signed_duration_since(child.updated_at).num_seconds())
3228 .unwrap_or(i64::MAX);
3229 if quiet_secs >= DEAD_CHILD_GRACE_SECS {
3230 dead.push(DeadChild {
3231 child_id: child_id.clone(),
3232 status: "error".to_string(),
3233 reason: format!(
3234 "child runner lost (crashed task, dropped spawn job, or \
3235 process restart): index status {status:?} with no live \
3236 runner driving it"
3237 ),
3238 owned: true,
3239 });
3240 }
3241 }
3242 }
3243 }
3244
3245 if !dead.is_empty() {
3246 for entry in dead {
3247 tracing::warn!(
3248 %parent_session_id,
3249 child_session_id = %entry.child_id,
3250 status = %entry.status,
3251 reason = %entry.reason,
3252 owned = entry.owned,
3253 "child-wait watchdog: synthesizing terminal completion for dead child"
3254 );
3255 if entry.owned {
3256 self.synthesize_child_completion(
3257 parent_session_id,
3258 &entry.child_id,
3259 &entry.status,
3260 Some(entry.reason),
3261 )
3262 .await;
3263 } else {
3264 self.publish_synthetic_completion(
3266 parent_session_id,
3267 &entry.child_id,
3268 &entry.status,
3269 Some(entry.reason),
3270 )
3271 .await;
3272 }
3273 }
3274 return;
3276 }
3277
3278 let terminal_ids: Vec<String> = terminal.iter().map(|(id, _)| id.clone()).collect();
3284 if let Some((child_id, status)) = select_replay_child(&terminal) {
3285 if wait_policy_satisfied(
3286 wait.wait_for,
3287 &wait.child_session_ids,
3288 &terminal_ids,
3289 child_id,
3290 status,
3291 ) {
3292 tracing::warn!(
3293 %parent_session_id,
3294 child_session_id = %child_id,
3295 "child-wait watchdog: wait already satisfied but parent still suspended \
3296 (lost wake); replaying the completion"
3297 );
3298 let control_plane = self.load_control_plane(child_id).await;
3299 let mut completion = ChildCompletion {
3300 parent_session_id: parent_session_id.into(),
3301 child_session_id: child_id.clone(),
3302 status: status.clone(),
3303 error: control_plane.as_ref().and_then(Session::last_run_error),
3304 completed_at: Utc::now(),
3305 source: None,
3306 };
3307 if control_plane.as_ref().is_some_and(|child| {
3310 child.parent_session_id.as_deref() == Some(parent_session_id)
3311 }) {
3312 let Ok(Some(child)) = self.storage.load_session(child_id).await else {
3313 return;
3314 };
3315 completion.source = ChildCompletionSource::from_committed_session(&child)
3316 .filter(|source| source.matches_completion(&completion));
3317 if ChildCompletionSource::has_source_record(&child)
3318 && completion.source.is_none()
3319 {
3320 tracing::warn!(%child_id,
3321 "unavailable committed replay source; leaving wait armed");
3322 return;
3323 }
3324 completion.completed_at = child.updated_at;
3325 }
3326 self.publish_completion(completion).await;
3327 }
3328 }
3329 }
3330
3331 async fn synthesize_child_completion(
3337 &self,
3338 parent_session_id: &str,
3339 child_session_id: &str,
3340 status: &str,
3341 error: Option<String>,
3342 ) {
3343 match self.storage.load_session(child_session_id).await {
3344 Ok(Some(mut child)) => {
3345 if child.parent_session_id.as_deref() != Some(parent_session_id) {
3350 tracing::warn!(
3351 %parent_session_id,
3352 child_session_id = %child.id,
3353 "child-wait watchdog: refusing to synthesize status onto a session \
3354 that is not a child of this parent"
3355 );
3356 self.publish_synthetic_completion(
3357 parent_session_id,
3358 child_session_id,
3359 status,
3360 error,
3361 )
3362 .await;
3363 return;
3364 }
3365 child.set_last_run_status(status);
3366 match &error {
3367 Some(message) => child.set_last_run_error(message.clone()),
3368 None => child.clear_last_run_error(),
3369 }
3370 child.updated_at = Utc::now();
3371 if let Err(save_error) = self.persistence.merge_save_runtime(&mut child).await {
3372 tracing::warn!(
3373 child_session_id = %child.id,
3374 %save_error,
3375 "child-wait watchdog: failed to persist synthesized terminal status"
3376 );
3377 }
3378 self.sessions.insert(
3379 child.id.clone(),
3380 Arc::new(crate::SessionSnapshot::new(child)),
3381 );
3382 }
3383 Ok(None) => {}
3384 Err(load_error) => {
3385 tracing::warn!(
3386 %child_session_id,
3387 %load_error,
3388 "child-wait watchdog: failed to load child for synthesized terminal status"
3389 );
3390 }
3391 }
3392 finalize_runner(
3393 &self.agent_runners,
3394 child_session_id,
3395 &Err(bamboo_agent_core::AgentError::LLM(
3396 error
3397 .clone()
3398 .unwrap_or_else(|| format!("synthesized {status}")),
3399 )),
3400 )
3401 .await;
3402 self.publish_synthetic_completion(parent_session_id, child_session_id, status, error)
3403 .await;
3404 }
3405
3406 async fn publish_synthetic_completion(
3407 &self,
3408 parent_session_id: &str,
3409 child_session_id: &str,
3410 status: &str,
3411 error: Option<String>,
3412 ) {
3413 self.publish_completion(ChildCompletion {
3414 parent_session_id: parent_session_id.into(),
3415 child_session_id: child_session_id.into(),
3416 status: status.into(),
3417 error,
3418 completed_at: Utc::now(),
3419 source: None,
3420 })
3421 .await;
3422 }
3423
3424 async fn publish_completion(&self, completion: ChildCompletion) {
3425 let publisher =
3426 crate::runtime::execution::session_events::ReplayableSessionEventPublisher::new(
3427 self.agent_runners.clone(),
3428 self.session_event_senders.clone(),
3429 self.account_feed_inbox.clone(),
3430 );
3431 let handler: Arc<dyn ChildCompletionHandler> = Arc::new(self.clone());
3432 crate::runtime::execution::spawn::publish_child_completion(
3433 &publisher,
3434 Some(handler),
3435 completion,
3436 )
3437 .await;
3438 }
3439
3440 async fn rescue_stranded_resume(&self, session_id: &str) {
3446 let Some(session) = self.load_session(session_id).await else {
3447 return;
3448 };
3449 let pending_runtime_resume = session.messages.last().is_some_and(|message| {
3450 matches!(message.role, Role::User)
3451 && message
3452 .metadata
3453 .as_ref()
3454 .is_some_and(|meta| meta.get(RUNTIME_RESUME_MESSAGE_KIND_KEY).is_some())
3455 });
3456 if !pending_runtime_resume {
3457 return;
3458 }
3459 tracing::warn!(
3460 %session_id,
3461 "child-wait watchdog: stranded resume detected (wait cleared, resume never \
3462 spawned); resuming"
3463 );
3464 self.resume_parent(session_id.to_string()).await;
3465 }
3466
3467 async fn force_resume_child_wait(&self, session_id: &str, resume_message: Message) {
3473 const MAX_ATTEMPTS: u8 = 5;
3474 let lock = session_resume_lock(session_id);
3475 for attempt in 0..MAX_ATTEMPTS {
3476 if attempt > 0 {
3477 tokio::time::sleep(Duration::from_millis(200)).await;
3478 }
3479 {
3480 let _held = lock.lock().await;
3481 let Some(mut session) = self.load_session(session_id).await else {
3482 return;
3483 };
3484 let mut runtime_state = read_runtime_state(&session);
3485 if runtime_state.waiting_for_children.is_none() {
3486 return;
3488 }
3489 runtime_state.waiting_for_children = None;
3490 runtime_state.status = AgentStatusState::Idle;
3491 runtime_state.suspension = None;
3492 write_runtime_state(&mut session, &runtime_state);
3493 session.metadata.remove("runtime.suspend_reason");
3494 session.add_message(resume_message.clone());
3495 session.updated_at = Utc::now();
3496 self.save_and_cache(&mut session).await;
3497 }
3498 let outcome = self.resume_parent(session_id.to_string()).await;
3499 match outcome {
3500 ResumeOutcome::Started { .. } | ResumeOutcome::NotFound => return,
3501 ResumeOutcome::Completed | ResumeOutcome::AlreadyRunning { .. } => {
3502 let clobbered = self
3506 .load_session(session_id)
3507 .await
3508 .map(|session| read_runtime_state(&session).waiting_for_children.is_some())
3509 .unwrap_or(false);
3510 if !clobbered {
3511 return;
3512 }
3513 }
3514 }
3515 }
3516 tracing::error!(
3517 %session_id,
3518 "child-wait watchdog: force-resume exhausted its clobber-retry budget"
3519 );
3520 }
3521}
3522
3523#[cfg(test)]
3524mod tests {
3525 use super::*;
3526
3527 #[tokio::test]
3528 async fn cancelled_resume_waiters_do_not_retain_historical_parent_ids() {
3529 for index in 0..512 {
3530 let id = format!("resume-lock-reclaim-{index}");
3531 let owner = session_resume_lock(&id);
3532 let held = owner.lock().await;
3533 let waiter = session_resume_lock(&id);
3534 let mut waiting = Box::pin(waiter.lock());
3535 assert!(futures::poll!(waiting.as_mut()).is_pending());
3536 drop(held);
3537 drop(owner);
3538 drop(waiting);
3539 drop(waiter);
3540 assert!(!parent_locks().lock().recover_poison().contains_key(&id));
3541 }
3542 }
3543
3544 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
3545 async fn resume_lock_reclamation_preserves_exclusive_parent_wake_ownership() {
3546 use std::sync::atomic::{AtomicUsize, Ordering};
3547 let active = Arc::new(AtomicUsize::new(0));
3548 let mut tasks = Vec::new();
3549 for _ in 0..32 {
3550 let active = active.clone();
3551 tasks.push(tokio::spawn(async move {
3552 for _ in 0..16 {
3553 let lease = session_resume_lock("resume-lock-exclusive");
3554 let _guard = lease.lock().await;
3555 assert_eq!(active.fetch_add(1, Ordering::SeqCst), 0);
3556 tokio::task::yield_now().await;
3557 assert_eq!(active.fetch_sub(1, Ordering::SeqCst), 1);
3558 }
3559 }));
3560 }
3561 for task in tasks {
3562 task.await.unwrap();
3563 }
3564 assert!(!parent_locks()
3565 .lock()
3566 .recover_poison()
3567 .contains_key("resume-lock-exclusive"));
3568 }
3569 use bamboo_agent_core::Message;
3570 use bamboo_domain::SessionInboxPort;
3571 use futures::stream;
3572 use std::sync::atomic::{AtomicUsize, Ordering};
3573
3574 struct FailingRuntimeSaveStorage {
3575 inner: Arc<bamboo_storage::SessionStoreV2>,
3576 }
3577
3578 #[async_trait]
3579 impl Storage for FailingRuntimeSaveStorage {
3580 async fn save_session(&self, session: &Session) -> std::io::Result<()> {
3581 self.inner.save_session(session).await
3582 }
3583
3584 async fn load_session(&self, id: &str) -> std::io::Result<Option<Session>> {
3585 self.inner.load_session(id).await
3586 }
3587
3588 async fn delete_session(&self, id: &str) -> std::io::Result<bool> {
3589 self.inner.delete_session(id).await
3590 }
3591
3592 async fn load_runtime_control_plane(&self, id: &str) -> std::io::Result<Option<Session>> {
3593 self.inner.load_runtime_control_plane(id).await
3594 }
3595
3596 async fn save_runtime_state(&self, _session: &Session) -> std::io::Result<()> {
3597 Err(std::io::Error::other("injected sidecar save failure"))
3598 }
3599 }
3600
3601 struct EmptyTools;
3602
3603 #[async_trait]
3604 impl bamboo_agent_core::tools::ToolExecutor for EmptyTools {
3605 async fn execute(
3606 &self,
3607 _call: &bamboo_agent_core::tools::ToolCall,
3608 ) -> Result<bamboo_agent_core::tools::ToolResult, bamboo_agent_core::tools::ToolError>
3609 {
3610 Err(bamboo_agent_core::tools::ToolError::NotFound(
3611 "no tools".to_string(),
3612 ))
3613 }
3614
3615 fn list_tools(&self) -> Vec<bamboo_agent_core::tools::ToolSchema> {
3616 Vec::new()
3617 }
3618 }
3619
3620 struct CompletedTestProvider;
3621
3622 #[async_trait]
3623 impl bamboo_llm::LLMProvider for CompletedTestProvider {
3624 async fn chat_stream(
3625 &self,
3626 _messages: &[Message],
3627 _tools: &[bamboo_agent_core::tools::ToolSchema],
3628 _max_output_tokens: Option<u32>,
3629 _model: &str,
3630 ) -> Result<bamboo_llm::LLMStream, bamboo_llm::LLMError> {
3631 let chunks: Vec<bamboo_llm::provider::Result<bamboo_llm::LLMChunk>> = vec![
3632 Ok(bamboo_llm::LLMChunk::Token("done".to_string())),
3633 Ok(bamboo_llm::LLMChunk::Done),
3634 ];
3635 Ok(Box::pin(stream::iter(chunks)))
3636 }
3637 }
3638
3639 struct CountingActivationSpawner {
3640 reservations: Arc<AtomicUsize>,
3641 launches: Arc<AtomicUsize>,
3642 }
3643
3644 #[async_trait]
3645 impl SessionActivationSpawner for CountingActivationSpawner {
3646 async fn reserve_activation(
3647 &self,
3648 target_session_id: &str,
3649 inbox_generation: u64,
3650 ) -> Result<SessionActivationReserveOutcome, bamboo_domain::SessionActivationError>
3651 {
3652 self.reservations.fetch_add(1, Ordering::SeqCst);
3653 let launches = self.launches.clone();
3654 Ok(SessionActivationReserveOutcome::Reserved(
3655 SessionActivationLaunch::new(
3656 format!("{target_session_id}-{inbox_generation}"),
3657 move || {
3658 launches.fetch_add(1, Ordering::SeqCst);
3659 },
3660 ),
3661 ))
3662 }
3663 }
3664
3665 async fn save_sourced_test_child(store: &bamboo_storage::SessionStoreV2, child: &Session) {
3666 let mut saved = child.clone();
3667 ChildCompletionSource::prepare(
3668 &mut saved,
3669 &format!("test-run-{}", child.id),
3670 &Default::default(),
3671 );
3672 store.save_session(&saved).await.unwrap();
3673 }
3674
3675 async fn test_child_source(
3676 store: &bamboo_storage::SessionStoreV2,
3677 id: &str,
3678 ) -> Option<ChildCompletionSource> {
3679 ChildCompletionSource::from_committed_session(
3680 &store.load_session(id).await.unwrap().unwrap(),
3681 )
3682 }
3683
3684 fn typed_test_report() -> serde_json::Value {
3685 serde_json::json!({"version":1,"outcome":"blocked","summary":"Quoted \"claim\" \\ 用户🪷\n",
3686 "reported_evidence":[{"description":"reported only","reference":"/never/read/reported-ref","sha256":null}],
3687 "reported_verification":[{"check":"fixture","reported_status":"passed","details":"a model claim"}],
3688 "proposals":["Try the next step"],"blockers":["Need approval"],"open_decisions":["Which route?"]})
3689 }
3690
3691 fn typed_test_child(parent: &Session, id: &str, report: &str) -> Session {
3692 let mut child = Session::new_child(id, &parent.id, "model", "Child");
3693 let packet = bamboo_domain::ChildContextPacket {
3694 version: 1,
3695 objective: "bounded task".into(),
3696 constraints: vec![],
3697 acceptance: vec!["report only".into()],
3698 non_goals: vec![],
3699 necessary_user_instructions: vec![],
3700 recorded_decisions: vec![],
3701 source_user_message_ids: vec![parent.messages[0].id.clone()],
3702 background_message_ids: vec![],
3703 };
3704 let resolved = packet.resolve(parent, "bounded task").unwrap();
3705 let binding = bamboo_domain::ChildContextBinding::new(
3706 parent,
3707 id,
3708 resolved.required_brief.clone(),
3709 resolved,
3710 )
3711 .unwrap();
3712 binding.install(&mut child).unwrap();
3713 child.add_message(binding.assignment_message());
3714 child.add_message(Message::assistant(report, None));
3715 child.set_last_run_status("completed");
3716 ChildCompletionSource::prepare(&mut child, "sealed-report-run", &Default::default());
3717 child
3718 }
3719
3720 fn typed_test_completion(child: &Session) -> ChildCompletion {
3721 ChildCompletion {
3722 parent_session_id: child.parent_session_id.clone().unwrap(),
3723 child_session_id: child.id.clone(),
3724 status: child.last_run_status().unwrap(),
3725 error: child.last_run_error(),
3726 completed_at: child.updated_at,
3727 source: ChildCompletionSource::from_committed_session(child),
3728 }
3729 }
3730
3731 fn unusable_typed_test_child(
3732 parent: &Session,
3733 id: &str,
3734 case: &str,
3735 ) -> (Session, &'static str) {
3736 let mut child = typed_test_child(parent, id, &typed_test_report().to_string());
3737 let mut prior = Default::default();
3738 let reason = match case {
3739 "prose" => {
3740 child.messages.last_mut().unwrap().content = "UNUSABLE_RAW_PROSE".repeat(64);
3741 "report_malformed"
3742 }
3743 "malformed" => {
3744 child.messages.last_mut().unwrap().content =
3745 "{\"version\":1,UNUSABLE_RAW_JSON".into();
3746 "report_malformed"
3747 }
3748 "oversized" => {
3749 child
3750 .messages
3751 .last_mut()
3752 .unwrap()
3753 .content
3754 .push_str(&" ".repeat(8193));
3755 "result_budget_exceeded"
3756 }
3757 "multipart" => {
3758 child.messages.last_mut().unwrap().content_parts = Some(
3759 serde_json::from_value(
3760 serde_json::json!([{"type":"text","text":"UNUSABLE_RAW_PART"}]),
3761 )
3762 .unwrap(),
3763 );
3764 "report_not_current_final"
3765 }
3766 "commentary" => {
3767 child.messages.last_mut().unwrap().phase =
3768 Some(bamboo_domain::MessagePhase::Commentary);
3769 "report_not_current_final"
3770 }
3771 "blank" => {
3772 child.messages.last_mut().unwrap().content = " ".into();
3773 "report_not_current_final"
3774 }
3775 "old_final" => {
3776 prior = child.messages.iter().map(|m| m.id.clone()).collect();
3777 "report_not_current_final"
3778 }
3779 "error" => {
3780 prior = child.messages.iter().map(|m| m.id.clone()).collect();
3781 child.set_last_run_status("error");
3782 child.set_last_run_error("UNUSABLE_RAW_ERROR\"\\\n".repeat(4096));
3783 "report_not_current_final"
3784 }
3785 "absent" => {
3786 child.messages.pop();
3787 "report_absent"
3788 }
3789 _ => panic!("unknown unusable report case"),
3790 };
3791 ChildCompletionSource::prepare(&mut child, "unusable-report-run", &prior);
3792 (child, reason)
3793 }
3794
3795 #[test]
3796 fn sealed_unusable_reports_keep_current_source_boundary_and_escape_budget() {
3797 let mut parent = Session::new("unavailable-parent", "model");
3798 parent.add_message(Message::user("required source"));
3799 for case in [
3800 "prose",
3801 "malformed",
3802 "oversized",
3803 "multipart",
3804 "commentary",
3805 "blank",
3806 "old_final",
3807 "error",
3808 "absent",
3809 ] {
3810 let (child, reason) = unusable_typed_test_child(&parent, "unavailable-child", case);
3811 let completion = typed_test_completion(&child);
3812 assert!(completion.source.is_some(), "valid seal: {case}");
3813 let mut projection = sealed_child_report(&completion, &child, &parent).unwrap();
3814 assert_eq!(
3815 projection,
3816 unavailable_child_result("typed_result", reason),
3817 "{case}"
3818 );
3819 let mut message = typed_child_resume_message(&completion, 0, &projection);
3820 let raw = completion.source.as_ref().unwrap().result(&child, &parent);
3821 let mut envelope =
3822 child_completion_envelope(&completion, Utc::now(), raw.clone(), &message);
3823 assert!(bound_typed_child_delivery(
3824 &mut envelope,
3825 &mut message,
3826 &mut projection,
3827 &completion,
3828 0
3829 ));
3830 assert!(fits_child_result_budget(&envelope));
3831 assert!(fits_child_result_budget(&message));
3832 assert!(fits_child_result_budget(
3833 &envelope.to_provider_message().unwrap()
3834 ));
3835 assert!(message.content.contains("SubAgent"));
3836 assert_eq!(
3837 message.metadata.as_ref().unwrap()["child_final_response_included"],
3838 false
3839 );
3840 assert!(projection.get("child_report").is_none());
3841 assert!(projection.get("host_observation").is_none());
3842 let SessionMessageBody::ChildOutcome(outcome) = envelope.body else {
3843 panic!("child outcome")
3844 };
3845 assert!(outcome.result.is_none());
3846 assert!(outcome.error.is_none());
3847 if let Some(raw) = raw {
3848 assert!(!message.content.contains(&raw), "no raw report: {case}");
3849 }
3850 let mut stale = child.clone();
3853 stale
3854 .messages
3855 .last_mut()
3856 .unwrap()
3857 .content
3858 .push_str("changed");
3859 assert!(sealed_child_report(&completion, &stale, &parent).is_none());
3860 }
3861 }
3862
3863 #[test]
3864 fn unavailable_delivery_counts_escaped_wrappers_at_exact_limit() {
3865 let mut parent = Session::new("unavailable-parent", "model");
3866 parent.add_message(Message::user("required source"));
3867 let (child, _) = unusable_typed_test_child(&parent, "unavailable-child", "oversized");
3868 let completion = typed_test_completion(&child);
3869 let mut projection = sealed_child_report(&completion, &child, &parent).unwrap();
3870 let mut message = typed_child_resume_message(&completion, 0, &projection);
3871 let mut base = child_completion_envelope(&completion, Utc::now(), None, &message);
3872 assert!(bound_typed_child_delivery(
3873 &mut base,
3874 &mut message,
3875 &mut projection,
3876 &completion,
3877 0
3878 ));
3879 let bytes = serde_json::to_vec(&base).unwrap().len().max(
3880 serde_json::to_vec(&base.to_provider_message().unwrap())
3881 .unwrap()
3882 .len(),
3883 );
3884 for extra in [8192 - bytes, 8193 - bytes] {
3885 let mut envelope = base.clone();
3886 let padding = format!("{}{}", "\"".repeat(extra / 2), "x".repeat(extra % 2));
3888 envelope.correlation_id.as_mut().unwrap().push_str(&padding);
3889 let mut projection = projection.clone();
3890 let mut message = message.clone();
3891 assert_eq!(
3892 bound_typed_child_delivery(
3893 &mut envelope,
3894 &mut message,
3895 &mut projection,
3896 &completion,
3897 0
3898 ),
3899 bytes + extra == 8192
3900 );
3901 assert_eq!(
3902 serde_json::to_vec(&envelope).unwrap().len().max(
3903 serde_json::to_vec(&envelope.to_provider_message().unwrap())
3904 .unwrap()
3905 .len()
3906 ),
3907 bytes + extra
3908 );
3909 assert!(fits_child_result_budget(&message));
3910 }
3911 }
3912
3913 #[test]
3914 fn sealed_typed_projection_keeps_host_source_distinct_from_all_reported_claims() {
3915 let mut parent = Session::new("typed-parent", "model");
3916 parent.add_message(Message::user("required user constraint"));
3917 let report = typed_test_report();
3918 let child = typed_test_child(&parent, "typed-child", &report.to_string());
3919 let completion = typed_test_completion(&child);
3920 let projection = sealed_child_report(&completion, &child, &parent).unwrap();
3921 assert_eq!(projection["child_report"], report);
3922 assert_eq!(
3923 projection["host_observation"]["terminal_source"],
3924 serde_json::to_value(&completion.source).unwrap()
3925 );
3926 assert_eq!(
3927 projection["host_observation"]["kind"],
3928 "committed_terminal_source"
3929 );
3930 assert_eq!(
3931 projection["host_observation"]["last_run_status"],
3932 "completed"
3933 );
3934 assert_eq!(projection["child_report"]["outcome"], "blocked");
3935 assert_eq!(
3936 projection["child_report"]["reported_verification"][0]["reported_status"],
3937 "passed"
3938 );
3939 assert!(projection["host_observation"].get("verified").is_none());
3940 for excluded in ["guardian", "resident", "legacy", "missing_source"] {
3941 let mut excluded_child = child.clone();
3942 match excluded {
3943 "guardian" => {
3944 excluded_child
3945 .metadata
3946 .insert("subagent_type".into(), "guardian".into());
3947 }
3948 "resident" => {
3949 excluded_child
3950 .metadata
3951 .insert("lifecycle".into(), "resident".into());
3952 }
3953 "legacy" => {
3954 excluded_child =
3955 Session::new_child("typed-child", &parent.id, "model", "Child");
3956 excluded_child.add_message(Message::assistant(report.to_string(), None));
3957 excluded_child.set_last_run_status("completed");
3958 }
3959 _ => {}
3960 }
3961 ChildCompletionSource::prepare(
3962 &mut excluded_child,
3963 "excluded-run",
3964 &Default::default(),
3965 );
3966 let mut excluded_completion = typed_test_completion(&excluded_child);
3967 if excluded == "missing_source" {
3968 excluded_completion.source = None;
3969 }
3970 assert!(
3971 sealed_child_report(&excluded_completion, &excluded_child, &parent).is_none(),
3972 "{excluded}"
3973 );
3974 }
3975 }
3976
3977 #[tokio::test]
3978 async fn typed_delivery_counts_complete_escaped_envelope_and_final_correlation() {
3979 let mut parent = Session::new("typed-parent", "model");
3980 parent.add_message(Message::user("required source"));
3981 let child = typed_test_child(&parent, "typed-child", &typed_test_report().to_string());
3982 let completion = typed_test_completion(&child);
3983 let original = sealed_child_report(&completion, &child, &parent).unwrap();
3984 let presentation = typed_child_resume_message(&completion, 0, &original);
3985 let mut base = child_completion_envelope(
3986 &completion,
3987 Utc::now(),
3988 completion.source.as_ref().unwrap().result(&child, &parent),
3989 &presentation,
3990 );
3991 let mut projection = original.clone();
3992 let mut message = presentation.clone();
3993 assert!(bound_typed_child_delivery(
3994 &mut base,
3995 &mut message,
3996 &mut projection,
3997 &completion,
3998 0
3999 ));
4000 assert_eq!(projection, original);
4001 let temp = tempfile::tempdir().unwrap();
4002 let store = Arc::new(
4003 bamboo_storage::SessionStoreV2::new(temp.path().to_path_buf())
4004 .await
4005 .unwrap(),
4006 );
4007 store.save_session(&parent).await.unwrap();
4008 let inbox = bamboo_storage::FileSessionInbox::new(store, Default::default());
4009 let first = inbox.deliver(&base).await.unwrap();
4010 let stable_id = base.id.clone();
4011 let base_bytes = serde_json::to_vec(&base.to_provider_message().unwrap())
4012 .unwrap()
4013 .len();
4014 assert!(base_bytes < 8192);
4015 let correlation = base.correlation_id.as_ref().unwrap().clone();
4016 for (extra, available) in [(8192 - base_bytes, true), (8193 - base_bytes, false)] {
4017 let mut envelope = base.clone();
4018 envelope.correlation_id = Some(format!("{correlation}{}", "x".repeat(extra)));
4019 assert_eq!(
4020 serde_json::to_vec(&envelope.to_provider_message().unwrap())
4021 .unwrap()
4022 .len(),
4023 base_bytes + extra
4024 );
4025 let mut message = presentation.clone();
4026 let mut projection = original.clone();
4027 assert!(bound_typed_child_delivery(
4028 &mut envelope,
4029 &mut message,
4030 &mut projection,
4031 &completion,
4032 0
4033 ));
4034 assert_eq!(projection["available"], available);
4035 assert!(fits_child_result_budget(&envelope));
4036 assert!(fits_child_result_budget(&message));
4037 assert!(fits_child_result_budget(
4038 &envelope.to_provider_message().unwrap()
4039 ));
4040 assert_eq!(
4041 envelope.idempotency_semantics(),
4042 base.idempotency_semantics()
4043 );
4044 assert_eq!(envelope.id, stable_id);
4045 assert_eq!(
4046 inbox.deliver(&envelope).await.unwrap(),
4047 first,
4048 "presentation overflow cannot change delivery semantics"
4049 );
4050 if !available {
4051 assert_eq!(projection["reason"], "result_budget_exceeded");
4052 assert!(projection.get("child_report").is_none());
4053 assert!(message.content.contains("SubAgent"));
4054 assert!(!message.content.contains("Bounded tail"));
4055 }
4056 }
4057 let mut large = typed_test_report();
4058 large["summary"] = serde_json::json!("\"\\\n".repeat(500));
4059 large["proposals"] = serde_json::json!(["\"\\\n".repeat(250)]);
4060 let child = typed_test_child(&parent, "escaped-child", &large.to_string());
4061 assert!(decode_report(&large.to_string()).is_ok());
4062 let completion = typed_test_completion(&child);
4063 let mut projection = sealed_child_report(&completion, &child, &parent).unwrap();
4064 let mut message = typed_child_resume_message(&completion, 0, &projection);
4065 let mut envelope =
4066 child_completion_envelope(&completion, Utc::now(), Some(large.to_string()), &message);
4067 assert!(serde_json::to_vec(&envelope).unwrap().len() > 8192);
4068 assert!(bound_typed_child_delivery(
4069 &mut envelope,
4070 &mut message,
4071 &mut projection,
4072 &completion,
4073 0
4074 ));
4075 assert_eq!(projection["available"], false);
4076 assert!(fits_child_result_budget(&envelope));
4077 assert!(fits_child_result_budget(&message));
4078 assert!(fits_child_result_budget(
4079 &envelope.to_provider_message().unwrap()
4080 ));
4081 }
4082
4083 #[tokio::test]
4084 async fn typed_source_mutations_reject_before_admission_and_wait_clear() {
4085 for mutation in ["birth", "assignment", "input", "result", "parent_source"] {
4086 let (_temp, store, inbox, coordinator, _, launches) = completion_inbox_fixture().await;
4087 save_waiting_test_parent(
4088 &store,
4089 "typed-parent",
4090 vec!["typed-child".into()],
4091 ChildWaitPolicy::All,
4092 )
4093 .await;
4094 let mut parent = store.load_session("typed-parent").await.unwrap().unwrap();
4095 parent.add_message(Message::user("required source"));
4096 store.save_session(&parent).await.unwrap();
4097 let mut child =
4098 typed_test_child(&parent, "typed-child", &typed_test_report().to_string());
4099 store.save_session(&child).await.unwrap();
4100 let mut completion = typed_test_completion(&child);
4101 match mutation {
4102 "birth" => {
4103 completion.source.as_mut().unwrap().child_created_at +=
4104 chrono::Duration::seconds(1)
4105 }
4106 "assignment" => {
4107 child.messages[0].content.push_str("changed assignment");
4108 }
4109 "input" => child.add_message(Message::user("new user input")),
4110 "result" => child
4111 .messages
4112 .last_mut()
4113 .unwrap()
4114 .content
4115 .push_str("changed"),
4116 _ => {
4117 parent.messages[0].content.push_str("changed");
4118 store.save_session(&parent).await.unwrap();
4119 }
4120 }
4121 store.save_session(&child).await.unwrap();
4122 coordinator.on_child_completed(completion).await;
4123 assert_eq!(
4124 inbox.inspect(&parent.id).await.unwrap().pending,
4125 0,
4126 "{mutation}"
4127 );
4128 assert_eq!(launches.load(Ordering::SeqCst), 0, "{mutation}");
4129 assert!(
4130 read_runtime_state(&store.load_session(&parent.id).await.unwrap().unwrap())
4131 .waiting_for_children
4132 .is_some(),
4133 "{mutation}"
4134 );
4135 }
4136 }
4137
4138 #[tokio::test]
4139 async fn typed_cold_watchdog_and_terminal_siblings_admit_each_source_once() {
4140 for case in [
4141 "valid",
4142 "prose",
4143 "malformed",
4144 "oversized",
4145 "multipart",
4146 "commentary",
4147 "blank",
4148 "old_final",
4149 "error",
4150 "absent",
4151 ] {
4152 for cold_watchdog in [false, true] {
4153 let (_temp, store, inbox, coordinator, reservations, launches) =
4154 completion_inbox_fixture().await;
4155 let ids = vec!["typed-a".into(), "typed-b".into()];
4156 save_waiting_test_parent(&store, "typed-parent", ids.clone(), ChildWaitPolicy::All)
4157 .await;
4158 let mut parent = store.load_session("typed-parent").await.unwrap().unwrap();
4159 parent.add_message(Message::user("required source"));
4160 store.save_session(&parent).await.unwrap();
4161 let mut completions = Vec::new();
4162 for id in &ids {
4163 let child = if case == "valid" {
4164 typed_test_child(&parent, id, &typed_test_report().to_string())
4165 } else {
4166 unusable_typed_test_child(&parent, id, case).0
4167 };
4168 store.save_session(&child).await.unwrap();
4169 completions.push(typed_test_completion(&child));
4170 }
4171 if cold_watchdog {
4172 let mut wait = read_runtime_state(&parent).waiting_for_children.unwrap();
4174 wait.registered_at = Utc::now() - chrono::Duration::seconds(61);
4175 coordinator.sweep_child_wait(&parent.id, wait.clone()).await;
4176 coordinator.sweep_child_wait(&parent.id, wait).await;
4177 } else {
4178 coordinator.on_child_completed(completions[0].clone()).await;
4180 }
4181 for completion in completions {
4182 coordinator.on_child_completed(completion).await;
4183 }
4184 let claims = inbox.claim(&parent.id, 8).await.unwrap();
4185 assert_eq!(claims.len(), 2);
4186 assert_eq!(reservations.load(Ordering::SeqCst), 1);
4187 assert_eq!(launches.load(Ordering::SeqCst), 1);
4188 for claim in claims {
4189 assert!(fits_child_result_budget(&claim.envelope));
4190 assert!(fits_child_result_budget(
4191 &claim.envelope.to_provider_message().unwrap()
4192 ));
4193 let actual = claim.envelope.to_provider_message().unwrap();
4194 assert!(!actual.content.contains("UNUSABLE_RAW"));
4195 let SessionMessageBody::ChildOutcome(outcome) = claim.envelope.body else {
4196 panic!("child outcome");
4197 };
4198 let projection: serde_json::Value = serde_json::from_str(
4199 outcome
4200 .provider_message
4201 .as_ref()
4202 .unwrap()
4203 .content
4204 .text
4205 .split_once("Child typed result:\n")
4206 .unwrap()
4207 .1
4208 .lines()
4209 .next()
4210 .unwrap(),
4211 )
4212 .unwrap();
4213 if case != "valid" {
4214 let expected =
4215 unusable_typed_test_child(&parent, &outcome.child_session_id, case).1;
4216 assert_eq!(
4217 projection,
4218 unavailable_child_result("typed_result", expected)
4219 );
4220 assert!(outcome.result.is_none());
4221 assert!(outcome.error.is_none());
4222 assert!(fits_child_result_budget(&outcome.provider_message));
4223 continue;
4224 }
4225 assert_eq!(projection["child_report"], typed_test_report());
4226 assert_eq!(
4227 projection["host_observation"]["terminal_source"]["child_session_id"],
4228 outcome.child_session_id
4229 );
4230 assert_eq!(
4231 projection["host_observation"]["last_run_status"],
4232 "completed"
4233 );
4234 assert!(fits_child_result_budget(&outcome.provider_message));
4235 }
4236 }
4237 }
4238 }
4239
4240 async fn completion_inbox_fixture() -> (
4241 tempfile::TempDir,
4242 Arc<bamboo_storage::SessionStoreV2>,
4243 Arc<dyn SessionInboxPort>,
4244 Arc<ChildCompletionCoordinator>,
4245 Arc<AtomicUsize>,
4246 Arc<AtomicUsize>,
4247 ) {
4248 completion_inbox_fixture_with_limits(bamboo_domain::SessionInboxLimits::default()).await
4249 }
4250
4251 async fn completion_inbox_fixture_with_limits(
4252 limits: bamboo_domain::SessionInboxLimits,
4253 ) -> (
4254 tempfile::TempDir,
4255 Arc<bamboo_storage::SessionStoreV2>,
4256 Arc<dyn SessionInboxPort>,
4257 Arc<ChildCompletionCoordinator>,
4258 Arc<AtomicUsize>,
4259 Arc<AtomicUsize>,
4260 ) {
4261 let temp = tempfile::tempdir().unwrap();
4262 let store = Arc::new(
4263 bamboo_storage::SessionStoreV2::new(temp.path().to_path_buf())
4264 .await
4265 .unwrap(),
4266 );
4267 let storage: Arc<dyn Storage> = store.clone();
4268 let locked = Arc::new(LockedSessionStore::new(storage.clone()));
4269 let inbox: Arc<dyn SessionInboxPort> =
4270 Arc::new(bamboo_storage::FileSessionInbox::new(store.clone(), limits));
4271 let router = crate::SessionActivationRouter::new();
4272 let messenger = Arc::new(crate::SessionMessenger::new(
4273 storage.clone(),
4274 inbox.clone(),
4275 router.clone(),
4276 ));
4277 let reservations = Arc::new(AtomicUsize::new(0));
4278 let launches = Arc::new(AtomicUsize::new(0));
4279 router
4280 .set_spawner(Arc::new(CountingActivationSpawner {
4281 reservations: reservations.clone(),
4282 launches: launches.clone(),
4283 }))
4284 .await;
4285 let provider: Arc<dyn bamboo_llm::LLMProvider> = Arc::new(CompletedTestProvider);
4286 let config = Arc::new(RwLock::new(Config::default()));
4287 let metrics = bamboo_metrics::MetricsCollector::spawn(
4288 Arc::new(bamboo_metrics::SqliteMetricsStorage::new(
4289 temp.path().join("metrics.db"),
4290 )),
4291 7,
4292 );
4293 let tools: Arc<dyn ToolExecutor> = Arc::new(EmptyTools);
4294 let agent = Arc::new(
4295 Agent::builder()
4296 .storage(storage.clone())
4297 .persistence(locked.clone())
4298 .session_inbox(inbox.clone())
4299 .activation_router(router)
4300 .session_messenger(messenger)
4301 .attachment_reader(store.clone())
4302 .skill_manager(Arc::new(bamboo_skills::SkillManager::new()))
4303 .metrics_collector(metrics)
4304 .config(config.clone())
4305 .provider(provider.clone())
4306 .default_tools(tools)
4307 .build()
4308 .unwrap(),
4309 );
4310 let mut providers = HashMap::new();
4311 providers.insert("test".to_string(), provider);
4312 let registry = Arc::new(ProviderRegistry::new(providers, "test".to_string()));
4313 let provider_router = Arc::new(ProviderModelRouter::new(registry.clone()));
4314 let coordinator = Arc::new(ChildCompletionCoordinator::new(
4315 storage,
4316 locked,
4317 Arc::default(),
4318 Arc::new(RwLock::new(HashMap::new())),
4319 Arc::new(RwLock::new(HashMap::new())),
4320 agent,
4321 config,
4322 registry,
4323 provider_router,
4324 temp.path().to_path_buf(),
4325 None,
4326 ));
4327 (temp, store, inbox, coordinator, reservations, launches)
4328 }
4329
4330 async fn save_waiting_test_parent(
4331 store: &bamboo_storage::SessionStoreV2,
4332 parent_id: &str,
4333 ids: Vec<String>,
4334 policy: ChildWaitPolicy,
4335 ) {
4336 let mut parent = Session::new(parent_id, "model");
4337 let mut runtime = AgentRuntimeState::new("waiting-run");
4338 runtime.status = AgentStatusState::Suspended;
4339 runtime.waiting_for_children = Some(WaitingForChildrenState::for_children(
4340 ids,
4341 policy,
4342 Utc::now(),
4343 ));
4344 runtime.suspension = Some(SuspensionState {
4345 reason: "waiting_for_children".into(),
4346 suspended_at: Utc::now(),
4347 resumable: true,
4348 hook_point: Some("ChildCompletion".into()),
4349 });
4350 write_runtime_state(&mut parent, &runtime);
4351 parent.metadata.insert(
4352 "runtime.suspend_reason".into(),
4353 "waiting_for_children".into(),
4354 );
4355 store.save_session(&parent).await.unwrap();
4356 }
4357
4358 #[tokio::test]
4359 async fn terminal_index_ahead_of_callbacks_preserves_every_waited_child_outcome() {
4360 for partial_first in [false, true] {
4361 let (_temp, store, inbox, coordinator, reservations, launches) =
4362 completion_inbox_fixture().await;
4363 let parent_id = "parallel-parent";
4364 let ids: Vec<_> = (0..4)
4365 .map(|slot| format!("parallel-child-{slot}"))
4366 .collect();
4367 save_waiting_test_parent(&store, parent_id, ids.clone(), ChildWaitPolicy::All).await;
4368 for (slot, id) in ids.iter().enumerate() {
4369 let mut child = Session::new_child(id, parent_id, "model", "Child");
4370 child.add_message(Message::assistant(format!("result-{id}"), None));
4371 child.set_last_run_status(if partial_first && slot > 0 {
4372 "running"
4373 } else {
4374 "completed"
4375 });
4376 save_sourced_test_child(&store, &child).await;
4377 }
4378 let mut sources = HashMap::new();
4379 for id in &ids {
4380 sources.insert(id.clone(), test_child_source(&store, id).await);
4381 }
4382 let completion = |id: &str| ChildCompletion {
4383 parent_session_id: parent_id.into(),
4384 child_session_id: id.into(),
4385 status: "completed".into(),
4386 error: None,
4387 completed_at: Utc::now(),
4388 source: sources.get(id).cloned().flatten(),
4389 };
4390 if partial_first {
4391 ChildCompletionHandler::on_child_completed(
4392 coordinator.as_ref(),
4393 completion(&ids[0]),
4394 )
4395 .await;
4396 assert!(
4397 read_runtime_state(&store.load_session(parent_id).await.unwrap().unwrap())
4398 .waiting_for_children
4399 .is_some()
4400 );
4401 assert_eq!(inbox.inspect(parent_id).await.unwrap().pending, 1);
4402 assert_eq!(launches.load(Ordering::SeqCst), 0);
4403 for id in &ids[1..] {
4404 let mut child = store.load_session(id).await.unwrap().unwrap();
4405 child.set_last_run_status("completed");
4406 save_sourced_test_child(&store, &child).await;
4407 }
4408 }
4409 for id in &ids {
4412 sources.insert(id.clone(), test_child_source(&store, id).await);
4413 }
4414 let completion = |id: &str| ChildCompletion {
4415 parent_session_id: parent_id.into(),
4416 child_session_id: id.into(),
4417 status: "completed".into(),
4418 error: None,
4419 completed_at: Utc::now(),
4420 source: sources.get(id).cloned().flatten(),
4421 };
4422 ChildCompletionHandler::on_child_completed(coordinator.as_ref(), completion(&ids[1]))
4423 .await;
4424 assert!(
4425 read_runtime_state(&store.load_session(parent_id).await.unwrap().unwrap())
4426 .waiting_for_children
4427 .is_none()
4428 );
4429 assert_eq!(inbox.inspect(parent_id).await.unwrap().pending, 4);
4430 assert_eq!(reservations.load(Ordering::SeqCst), 1);
4431 assert_eq!(launches.load(Ordering::SeqCst), 1);
4432 for id in &ids {
4433 ChildCompletionHandler::on_child_completed(coordinator.as_ref(), completion(id))
4434 .await;
4435 }
4436 let claims = inbox.claim(parent_id, 8).await.unwrap();
4437 assert_eq!(claims.len(), 4, "late callbacks cannot duplicate outcomes");
4438 assert_eq!(reservations.load(Ordering::SeqCst), 1);
4439 for id in &ids {
4440 let matching: Vec<_> = claims
4441 .iter()
4442 .filter_map(|claim| match &claim.envelope.body {
4443 SessionMessageBody::ChildOutcome(outcome)
4444 if &outcome.child_session_id == id =>
4445 {
4446 Some(outcome)
4447 }
4448 _ => None,
4449 })
4450 .collect();
4451 assert_eq!(matching.len(), 1);
4452 assert_eq!(
4453 matching[0].result.as_deref(),
4454 Some(format!("result-{id}").as_str())
4455 );
4456 }
4457 }
4458 }
4459
4460 #[tokio::test]
4461 async fn saved_a_callback_barrier_never_attributes_completed_b_to_a() {
4462 for guardian in [false, true] {
4463 let (_temp, store, inbox, coordinator, _, launches) = completion_inbox_fixture().await;
4464 let parent_id = "source-barrier-parent";
4465 let child_id = "source-barrier-child";
4466 save_waiting_test_parent(
4467 &store,
4468 parent_id,
4469 vec![child_id.into()],
4470 ChildWaitPolicy::All,
4471 )
4472 .await;
4473 if guardian {
4474 let mut parent = store.load_session(parent_id).await.unwrap().unwrap();
4475 let mut state = crate::runtime::guardian_state::ensure_guardian_state(&parent);
4476 state.record_spawn(child_id);
4477 write_guardian_state(&mut parent, state);
4478 store.save_session(&parent).await.unwrap();
4479 }
4480 let mut child = Session::new_child(child_id, parent_id, "model", "Child");
4481 if guardian {
4482 child
4483 .metadata
4484 .insert("subagent_type".into(), "guardian".into());
4485 }
4486 child.add_message(Message::user("assignment A"));
4487 store.save_session(&child).await.unwrap();
4488 let prior = child.messages.iter().map(|m| m.id.clone()).collect();
4489 child.add_message(Message::assistant("answer A", None));
4490 child.set_last_run_status("completed");
4491 ChildCompletionSource::prepare(&mut child, "actual-run-A", &prior);
4492 coordinator
4493 .agent
4494 .persistence()
4495 .save_runtime_session(&mut child)
4496 .await
4497 .unwrap();
4498 let completion_a = ChildCompletion {
4499 parent_session_id: parent_id.into(),
4500 child_session_id: child_id.into(),
4501 status: "completed".into(),
4502 error: None,
4503 completed_at: Utc::now(),
4504 source: ChildCompletionSource::after_final_save(&child, true),
4505 };
4506 assert_eq!(
4507 completion_a.source.as_ref().unwrap().activation_run_id,
4508 "actual-run-A"
4509 );
4510 let (release, barrier) = tokio::sync::oneshot::channel();
4513 let target = coordinator.clone();
4514 let callback = tokio::spawn(async move {
4515 barrier.await.unwrap();
4516 target.on_child_completed(completion_a).await;
4517 });
4518 let prior = child.messages.iter().map(|m| m.id.clone()).collect();
4519 child.add_message(Message::user("assignment B"));
4520 let answer_b = if guardian {
4521 r#"{"approve":true,"summary":"B approved","findings":[]}"#
4522 } else {
4523 "answer B"
4524 };
4525 child.add_message(Message::assistant(answer_b, None));
4526 child.set_last_run_status("completed");
4527 ChildCompletionSource::prepare(&mut child, "actual-run-B", &prior);
4528 coordinator
4529 .agent
4530 .persistence()
4531 .save_runtime_session(&mut child)
4532 .await
4533 .unwrap();
4534 release.send(()).unwrap();
4535 callback.await.unwrap();
4536 assert_eq!(inbox.inspect(parent_id).await.unwrap().pending, 0);
4537 let parent = store.load_session(parent_id).await.unwrap().unwrap();
4538 assert!(read_runtime_state(&parent).waiting_for_children.is_some());
4539 assert_eq!(launches.load(Ordering::SeqCst), 0);
4540 if guardian {
4541 assert!(read_guardian_state(&parent).unwrap().phase.is_pending());
4542 }
4543
4544 let completion_b = ChildCompletion {
4545 parent_session_id: parent_id.into(),
4546 child_session_id: child_id.into(),
4547 status: "completed".into(),
4548 error: None,
4549 completed_at: Utc::now(),
4550 source: ChildCompletionSource::after_final_save(&child, true),
4551 };
4552 coordinator.on_child_completed(completion_b.clone()).await;
4553 coordinator.on_child_completed(completion_b).await;
4554 let claims = inbox.claim(parent_id, 8).await.unwrap();
4555 assert_eq!(
4556 claims.len(),
4557 1,
4558 "B is admitted once with B's own provenance"
4559 );
4560 let SessionMessageBody::ChildOutcome(outcome) = &claims[0].envelope.body else {
4561 panic!("expected ChildOutcome");
4562 };
4563 assert_eq!(outcome.result.as_deref(), Some(answer_b));
4564 assert_eq!(launches.load(Ordering::SeqCst), 1);
4565 }
4566 }
4567
4568 #[tokio::test]
4569 async fn missing_source_cannot_import_latest_guardian_approval() {
4570 let (_temp, store, inbox, coordinator, _, _) = completion_inbox_fixture().await;
4571 let parent_id = "unavailable-guardian-parent";
4572 let child_id = "unavailable-guardian-child";
4573 save_waiting_test_parent(
4574 &store,
4575 parent_id,
4576 vec![child_id.into()],
4577 ChildWaitPolicy::All,
4578 )
4579 .await;
4580 let mut parent = store.load_session(parent_id).await.unwrap().unwrap();
4581 let mut state = crate::runtime::guardian_state::ensure_guardian_state(&parent);
4582 state.record_spawn(child_id);
4583 write_guardian_state(&mut parent, state);
4584 store.save_session(&parent).await.unwrap();
4585 let mut child = Session::new_child(child_id, parent_id, "model", "Child");
4586 child
4587 .metadata
4588 .insert("subagent_type".into(), "guardian".into());
4589 child.add_message(Message::assistant(
4590 r#"{"approve":true,"summary":"unbound latest","findings":[]}"#,
4591 None,
4592 ));
4593 child.set_last_run_status("completed");
4594 store.save_session(&child).await.unwrap();
4595 coordinator
4596 .on_child_completed(ChildCompletion {
4597 parent_session_id: parent_id.into(),
4598 child_session_id: child_id.into(),
4599 status: "completed".into(),
4600 error: None,
4601 completed_at: Utc::now(),
4602 source: None,
4603 })
4604 .await;
4605 let parent = store.load_session(parent_id).await.unwrap().unwrap();
4606 let state = read_guardian_state(&parent).unwrap();
4607 assert!(state.phase.is_reviewed());
4608 assert!(
4609 !state.last_approved(),
4610 "source-less latest approval is never authoritative"
4611 );
4612 let claims = inbox.claim(parent_id, 8).await.unwrap();
4613 let SessionMessageBody::ChildOutcome(outcome) = &claims[0].envelope.body else {
4614 panic!("expected ChildOutcome");
4615 };
4616 assert!(outcome.result.is_none());
4617 }
4618
4619 #[tokio::test]
4620 async fn lost_callback_replay_uses_the_committed_seal() {
4621 for valid in [true, false] {
4622 let (_temp, store, inbox, coordinator, _, _) = completion_inbox_fixture().await;
4623 save_waiting_test_parent(
4624 &store,
4625 "replay-parent",
4626 vec!["replay-child".into()],
4627 ChildWaitPolicy::All,
4628 )
4629 .await;
4630 let mut child = Session::new_child("replay-child", "replay-parent", "model", "Child");
4631 child.add_message(Message::assistant("sealed replay result", None));
4632 child.set_last_run_status("completed");
4633 save_sourced_test_child(&store, &child).await;
4634 if !valid {
4635 let mut child = store.load_session("replay-child").await.unwrap().unwrap();
4636 child.messages[0].content = "unsealed successor result".into();
4637 store.save_session(&child).await.unwrap();
4638 }
4639 let parent = store.load_session("replay-parent").await.unwrap().unwrap();
4640 let mut wait = read_runtime_state(&parent).waiting_for_children.unwrap();
4641 wait.registered_at = Utc::now() - chrono::Duration::seconds(61);
4642 coordinator.sweep_child_wait("replay-parent", wait).await;
4643 let claims = inbox.claim("replay-parent", 1).await.unwrap();
4644 if !valid {
4645 assert!(claims.is_empty());
4646 let parent = store.load_session("replay-parent").await.unwrap().unwrap();
4647 assert!(read_runtime_state(&parent).waiting_for_children.is_some());
4648 continue;
4649 }
4650 let SessionMessageBody::ChildOutcome(outcome) = &claims[0].envelope.body else {
4651 panic!("expected ChildOutcome");
4652 };
4653 assert_eq!(outcome.result.as_deref(), Some("sealed replay result"));
4654 }
4655 }
4656
4657 #[tokio::test]
4658 async fn partial_admission_then_source_mutation_ignores_stale_retry() {
4659 for parent_changed in [false, true] {
4660 let (_temp, store, inbox, coordinator, _, launches) = completion_inbox_fixture().await;
4661 let parent_id = "partial-source-parent";
4662 let child_id = "partial-source-child";
4663 save_waiting_test_parent(
4664 &store,
4665 parent_id,
4666 vec![child_id.into(), "still-running-child".into()],
4667 ChildWaitPolicy::All,
4668 )
4669 .await;
4670 let mut child = Session::new_child(child_id, parent_id, "model", "Child");
4671 if parent_changed {
4672 let mut parent = store.load_session(parent_id).await.unwrap().unwrap();
4673 let mut source = Message::user("required parent constraint");
4674 source.id = "parent-source".into();
4675 parent.add_message(source);
4676 store.save_session(&parent).await.unwrap();
4677 let packet = bamboo_domain::ChildContextPacket {
4678 version: 1,
4679 objective: "task".into(),
4680 constraints: vec![],
4681 acceptance: vec!["source remains attributable".into()],
4682 non_goals: vec![],
4683 necessary_user_instructions: vec![],
4684 recorded_decisions: vec![],
4685 source_user_message_ids: vec!["parent-source".into()],
4686 background_message_ids: vec![],
4687 };
4688 let resolved = packet.resolve(&parent, "task").unwrap();
4689 let binding = bamboo_domain::ChildContextBinding::new(
4690 &parent,
4691 child_id,
4692 resolved.required_brief.clone(),
4693 resolved,
4694 )
4695 .unwrap();
4696 binding.install(&mut child).unwrap();
4697 child.add_message(binding.assignment_message());
4698 }
4699 child.add_message(Message::assistant("original source", None));
4700 child.set_last_run_status("completed");
4701 save_sourced_test_child(&store, &child).await;
4702 let completion = ChildCompletion {
4703 parent_session_id: parent_id.into(),
4704 child_session_id: child_id.into(),
4705 status: "completed".into(),
4706 error: None,
4707 completed_at: Utc::now(),
4708 source: test_child_source(&store, child_id).await,
4709 };
4710 coordinator.on_child_completed(completion.clone()).await;
4711 assert_eq!(inbox.inspect(parent_id).await.unwrap().pending, 1);
4712 if parent_changed {
4713 let mut parent = store.load_session(parent_id).await.unwrap().unwrap();
4714 parent.messages[0].content = "modified parent source".into();
4715 store.save_session(&parent).await.unwrap();
4716 } else {
4717 child = store.load_session(child_id).await.unwrap().unwrap();
4718 child.messages[0].content = "modified after admission".into();
4719 store.save_session(&child).await.unwrap();
4720 }
4721 coordinator.on_child_completed(completion).await;
4722 assert_eq!(inbox.inspect(parent_id).await.unwrap().pending, 1);
4723 assert_eq!(launches.load(Ordering::SeqCst), 0);
4724 assert!(
4725 read_runtime_state(&store.load_session(parent_id).await.unwrap().unwrap())
4726 .waiting_for_children
4727 .is_some()
4728 );
4729 let mut sibling =
4730 Session::new_child("still-running-child", parent_id, "model", "Child");
4731 sibling.add_message(Message::assistant("sibling result", None));
4732 sibling.set_last_run_status("completed");
4733 save_sourced_test_child(&store, &sibling).await;
4734 coordinator
4735 .on_child_completed(ChildCompletion {
4736 parent_session_id: parent_id.into(),
4737 child_session_id: sibling.id.clone(),
4738 status: "completed".into(),
4739 error: None,
4740 completed_at: Utc::now(),
4741 source: test_child_source(&store, &sibling.id).await,
4742 })
4743 .await;
4744 assert_eq!(
4745 inbox.inspect(parent_id).await.unwrap().pending,
4746 2,
4747 "sibling completion cannot restage mutated A as a neutral outcome"
4748 );
4749 assert_eq!(launches.load(Ordering::SeqCst), 0);
4750 assert!(
4751 read_runtime_state(&store.load_session(parent_id).await.unwrap().unwrap())
4752 .waiting_for_children
4753 .is_some()
4754 );
4755 assert_eq!(
4756 inbox.inspect(parent_id).await.unwrap().claimed,
4757 0,
4758 "partial outcomes remain inert until the wait is durably released"
4759 );
4760 }
4761 }
4762
4763 #[tokio::test]
4764 async fn short_circuit_wait_stages_terminal_siblings_without_waiting_or_foreign_content() {
4765 for policy in [ChildWaitPolicy::Any, ChildWaitPolicy::FirstError] {
4766 let (_temp, store, inbox, coordinator, _, launches) = completion_inbox_fixture().await;
4767 let parent_id = "short-circuit-parent";
4768 let ids = vec![
4769 "failed-child".into(),
4770 "done-child".into(),
4771 "running-child".into(),
4772 "foreign-child".into(),
4773 ];
4774 save_waiting_test_parent(&store, parent_id, ids, policy).await;
4775 for (id, owner, status, result) in [
4776 ("failed-child", parent_id, "error", "failed-owned-result"),
4777 ("done-child", parent_id, "completed", "done-owned-result"),
4778 ("running-child", parent_id, "running", "not-terminal"),
4779 (
4780 "foreign-child",
4781 "another-parent",
4782 "completed",
4783 "foreign-secret",
4784 ),
4785 ] {
4786 let mut child = Session::new_child(id, owner, "model", "Child");
4787 child.add_message(Message::assistant(result, None));
4788 child.set_last_run_status(status);
4789 save_sourced_test_child(&store, &child).await;
4790 }
4791 ChildCompletionHandler::on_child_completed(
4792 coordinator.as_ref(),
4793 ChildCompletion {
4794 parent_session_id: parent_id.into(),
4795 child_session_id: "failed-child".into(),
4796 status: "error".into(),
4797 error: None,
4798 completed_at: Utc::now(),
4799 source: test_child_source(&store, "failed-child").await,
4800 },
4801 )
4802 .await;
4803 assert!(
4804 read_runtime_state(&store.load_session(parent_id).await.unwrap().unwrap())
4805 .waiting_for_children
4806 .is_none()
4807 );
4808 let claims = inbox.claim(parent_id, 8).await.unwrap();
4809 assert_eq!(claims.len(), 2);
4810 let content = serde_json::to_string(
4811 &claims
4812 .iter()
4813 .map(|claim| &claim.envelope)
4814 .collect::<Vec<_>>(),
4815 )
4816 .unwrap();
4817 assert!(content.contains("done-owned-result"));
4818 assert!(!content.contains("foreign-secret"));
4819 assert!(!content.contains("not-terminal"));
4820 assert_eq!(
4821 store
4822 .load_session("running-child")
4823 .await
4824 .unwrap()
4825 .unwrap()
4826 .last_run_status()
4827 .as_deref(),
4828 Some("running")
4829 );
4830 assert_eq!(launches.load(Ordering::SeqCst), 1);
4831 }
4832 }
4833
4834 #[tokio::test]
4835 async fn sibling_admission_failure_keeps_wait_armed_without_activation() {
4836 let limits = bamboo_domain::SessionInboxLimits {
4837 max_backlog: 1,
4838 ..Default::default()
4839 };
4840 let (_temp, store, inbox, coordinator, reservations, launches) =
4841 completion_inbox_fixture_with_limits(limits).await;
4842 let parent_id = "partial-stage-parent";
4843 let ids = vec!["first-child".into(), "second-child".into()];
4844 save_waiting_test_parent(&store, parent_id, ids, ChildWaitPolicy::All).await;
4845 for id in ["first-child", "second-child"] {
4846 let mut child = Session::new_child(id, parent_id, "model", "Child");
4847 child.add_message(Message::assistant(format!("result-{id}"), None));
4848 child.set_last_run_status("completed");
4849 save_sourced_test_child(&store, &child).await;
4850 }
4851 ChildCompletionHandler::on_child_completed(
4852 coordinator.as_ref(),
4853 ChildCompletion {
4854 parent_session_id: parent_id.into(),
4855 child_session_id: "first-child".into(),
4856 status: "completed".into(),
4857 error: None,
4858 completed_at: Utc::now(),
4859 source: test_child_source(&store, "first-child").await,
4860 },
4861 )
4862 .await;
4863 let durable = store.load_session(parent_id).await.unwrap().unwrap();
4864 assert!(read_runtime_state(&durable).waiting_for_children.is_some());
4865 assert_eq!(
4866 read_runtime_state(&durable).status,
4867 AgentStatusState::Suspended
4868 );
4869 assert_eq!(inbox.inspect(parent_id).await.unwrap().pending, 1);
4870 assert!(!inbox.inspect(parent_id).await.unwrap().activation_pending());
4871 assert_eq!(reservations.load(Ordering::SeqCst), 0);
4872 assert_eq!(launches.load(Ordering::SeqCst), 0);
4873 }
4874
4875 #[tokio::test]
4876 async fn supervisor_resume_rejects_old_incarnation_without_replacing_cache() {
4877 let (_temp, store, _inbox, coordinator, _reservations, _launches) =
4878 completion_inbox_fixture().await;
4879 let original = store
4880 .get_or_create_default_supervisor("model")
4881 .await
4882 .unwrap();
4883 let mut stale = store
4884 .load_session(&original.session_id)
4885 .await
4886 .unwrap()
4887 .unwrap();
4888 store.delete_session(&original.session_id).await.unwrap();
4889 let recreated = store
4890 .get_or_create_default_supervisor("replacement")
4891 .await
4892 .unwrap();
4893 assert_ne!(original.incarnation_id, recreated.incarnation_id);
4894 let mut current = store
4895 .load_session(&recreated.session_id)
4896 .await
4897 .unwrap()
4898 .unwrap();
4899 coordinator.save_and_cache(&mut current).await;
4900 let cached = coordinator
4901 .sessions
4902 .get(¤t.id)
4903 .unwrap()
4904 .value()
4905 .clone();
4906 coordinator.save_and_cache(&mut stale).await;
4907 assert!(Arc::ptr_eq(
4908 &cached,
4909 coordinator.sessions.get(¤t.id).unwrap().value()
4910 ));
4911 assert_eq!(
4912 store
4913 .load_session(¤t.id)
4914 .await
4915 .unwrap()
4916 .unwrap()
4917 .authority_identity,
4918 current.authority_identity
4919 );
4920 }
4921
4922 #[test]
4925 fn dead_child_candidate_status_matrix() {
4926 assert!(is_dead_child_candidate_status(None));
4928 assert!(is_dead_child_candidate_status(Some("running")));
4931 assert!(is_dead_child_candidate_status(Some("pending")));
4932 assert!(!is_dead_child_candidate_status(Some("suspended")));
4934 for status in ["completed", "error", "timeout", "cancelled", "skipped"] {
4936 assert!(!is_dead_child_candidate_status(Some(status)), "{status}");
4937 }
4938 }
4939
4940 #[test]
4941 fn completion_child_ownership_gates_content_fold() {
4942 assert!(completion_child_is_owned("parent-1", Some("parent-1")));
4944 assert!(!completion_child_is_owned("parent-1", Some("parent-2")));
4947 assert!(!completion_child_is_owned("parent-1", None));
4949 }
4950
4951 #[test]
4952 fn replay_child_prefers_error_like_for_first_error_policy() {
4953 let terminal = vec![
4954 ("c-ok".to_string(), "completed".to_string()),
4955 ("c-err".to_string(), "timeout".to_string()),
4956 ("c-late".to_string(), "completed".to_string()),
4957 ];
4958 let (id, status) = select_replay_child(&terminal).expect("non-empty");
4959 assert_eq!(id, "c-err");
4960 assert_eq!(status, "timeout");
4961
4962 let all_ok = vec![
4963 ("c-1".to_string(), "completed".to_string()),
4964 ("c-2".to_string(), "completed".to_string()),
4965 ];
4966 let (id, _) = select_replay_child(&all_ok).expect("non-empty");
4967 assert_eq!(id, "c-2");
4968
4969 assert!(select_replay_child(&[]).is_none());
4970 }
4971
4972 #[test]
4973 fn watchdog_resume_messages_are_hidden_runtime_messages() {
4974 for message in [
4975 empty_child_wait_message(),
4976 child_wait_lease_expired_message(&["c-1".to_string(), "c-2".to_string()]),
4977 ] {
4978 assert!(matches!(message.role, Role::User));
4979 let meta = message.metadata.expect("hidden runtime metadata");
4980 assert_eq!(meta[RUNTIME_RESUME_MESSAGE_HIDDEN_KEY], true);
4981 assert_eq!(
4982 meta[RUNTIME_RESUME_MESSAGE_KIND_KEY],
4983 "child_wait_watchdog_resume"
4984 );
4985 }
4986 let lease = child_wait_lease_expired_message(&["c-1".to_string()]);
4987 assert!(lease.content.contains("NOT cancelled"));
4989 assert!(lease.content.contains("c-1"));
4990 }
4991
4992 #[test]
4993 fn runtime_child_guidance_uses_compact_inspection_calls() {
4994 let completion = make_completion("completed");
4995 let child_inspection = serde_json::json!({
4996 "intent": "inspect", "target": "child-1", "message": "messages",
4997 })
4998 .to_string();
4999 let tree_inspection =
5000 serde_json::json!({"intent": "inspect", "message": "tree"}).to_string();
5001 for (message, expected_call) in [
5002 (
5003 runtime_resume_message(&completion, 0, Some("evidence")),
5004 &child_inspection,
5005 ),
5006 (
5007 guardian_resume_message(&completion, &GuardianVerdict::approved()),
5008 &child_inspection,
5009 ),
5010 (empty_child_wait_message(), &tree_inspection),
5011 (
5012 child_wait_lease_expired_message(&["child-1".to_string()]),
5013 &tree_inspection,
5014 ),
5015 ] {
5016 assert!(message.content.contains(expected_call));
5017 for legacy in [
5018 "SubAgent.get",
5019 "SubAgent.list",
5020 "SubAgent.wait",
5021 "child_session_id=",
5022 ] {
5023 assert!(!message.content.contains(legacy), "stale hint: {legacy}");
5024 }
5025 }
5026 }
5027
5028 #[test]
5031 fn non_terminal_statuses_never_satisfy_wait_policies() {
5032 assert!(!is_terminal_child_status("suspended"));
5035 assert!(!is_terminal_child_status("running"));
5036 assert!(!is_terminal_child_status("pending"));
5037 }
5038
5039 fn make_completion(status: &str) -> ChildCompletion {
5040 ChildCompletion {
5041 parent_session_id: "parent-1".to_string(),
5042 child_session_id: "child-1".to_string(),
5043 status: status.to_string(),
5044 error: None,
5045 completed_at: Utc::now(),
5046 source: None,
5047 }
5048 }
5049
5050 #[test]
5051 fn oversized_child_outcome_is_bounded_and_keeps_full_content_identity() {
5052 let mut completion = make_completion("completed");
5053 let wait_registered_at = Utc::now();
5054 let huge = format!("prefix-A-{}", "x".repeat(300 * 1024));
5055 let presentation = runtime_resume_message(&completion, 0, Some(&huge));
5056 let first = child_completion_envelope(
5057 &completion,
5058 wait_registered_at,
5059 Some(huge.clone()),
5060 &presentation,
5061 );
5062 assert!(
5063 serde_json::to_vec(&first).unwrap().len()
5064 < bamboo_domain::SessionInboxLimits::default().max_payload_bytes
5065 );
5066 let SessionMessageBody::ChildOutcome(outcome) = &first.body else {
5067 panic!("typed child outcome");
5068 };
5069 let stored = outcome.result.as_deref().unwrap();
5070 assert!(stored.contains("sha256="));
5071 assert!(stored.contains("\"intent\":\"inspect\""));
5072 assert!(stored.contains("\"target\":\"child-1\""));
5073 assert!(stored.contains("\"message\":\"result\""));
5074 assert!(!stored.contains("SubAgent.get"));
5075 assert!(stored.len() < CHILD_COMPLETION_INLINE_FIELD_BYTES);
5076
5077 completion.completed_at += chrono::Duration::seconds(1);
5080 let exact_retry = child_completion_envelope(
5081 &completion,
5082 wait_registered_at,
5083 Some(huge.clone()),
5084 &runtime_resume_message(&completion, 9, Some(&huge)),
5085 );
5086 assert_eq!(exact_retry.id, first.id);
5087 let changed = format!("prefix-B-{}", "x".repeat(300 * 1024));
5088 let corrected = child_completion_envelope(
5089 &completion,
5090 wait_registered_at,
5091 Some(changed.clone()),
5092 &runtime_resume_message(&completion, 0, Some(&changed)),
5093 );
5094 assert_ne!(corrected.id, first.id);
5095 }
5096
5097 #[test]
5098 fn oversized_child_error_points_to_bounded_error_view() {
5099 let mut completion = make_completion("error");
5100 completion.error = Some("失败🙂".repeat(30_000));
5101 let presentation = runtime_resume_message(&completion, 0, None);
5102 let envelope = child_completion_envelope(&completion, Utc::now(), None, &presentation);
5103 assert!(
5104 serde_json::to_vec(&envelope).unwrap().len()
5105 < bamboo_domain::SessionInboxLimits::default().max_payload_bytes
5106 );
5107 let SessionMessageBody::ChildOutcome(outcome) = &envelope.body else {
5108 panic!("typed child outcome");
5109 };
5110 let stored = outcome.error.as_deref().expect("bounded error");
5111 assert!(stored.contains(r#""message":"diagnostics""#));
5112 assert!(stored.contains("authenticated inspector"));
5113 assert!(!stored.contains("bounded slices"));
5114 assert!(!stored.contains("cursors"));
5115 assert!(stored.len() < CHILD_COMPLETION_INLINE_FIELD_BYTES);
5116 let metadata_error = outcome
5117 .provider_message
5118 .as_ref()
5119 .and_then(|provider| provider.metadata.get("child_error"))
5120 .and_then(serde_json::Value::as_str)
5121 .expect("bounded metadata error");
5122 assert_eq!(metadata_error, stored);
5123 assert!(!metadata_error.contains(&"失败🙂".repeat(30_000)));
5124 }
5125
5126 #[test]
5127 fn escaped_and_mixed_child_fields_fit_the_serialized_inbox_budget() {
5128 let mut completion = make_completion("error");
5129 completion.error = Some("\u{0001}".repeat(CHILD_COMPLETION_INLINE_FIELD_BYTES));
5130 let result = "\n".repeat(CHILD_COMPLETION_INLINE_FIELD_BYTES);
5131 let wait_registered_at = Utc::now();
5132 let presentation = runtime_resume_message(&completion, 0, Some(&result));
5133 let envelope = child_completion_envelope(
5134 &completion,
5135 wait_registered_at,
5136 Some(result.clone()),
5137 &presentation,
5138 );
5139 assert!(
5140 serde_json::to_vec(&envelope).unwrap().len()
5141 < bamboo_domain::SessionInboxLimits::default().max_payload_bytes
5142 );
5143 let SessionMessageBody::ChildOutcome(outcome) = &envelope.body else {
5144 panic!("typed child outcome");
5145 };
5146 let error = outcome.error.as_deref().unwrap();
5147 let stored_result = outcome.result.as_deref().unwrap();
5148 assert!(error.contains(r#""message":"diagnostics""#));
5149 assert!(stored_result.contains(r#""message":"result""#));
5150 assert!(!error.contains("Bounded tail"));
5151 assert!(!stored_result.contains("Bounded tail"));
5152 assert_eq!(
5153 outcome
5154 .provider_message
5155 .as_ref()
5156 .and_then(|provider| provider.metadata.get("child_error"))
5157 .and_then(serde_json::Value::as_str),
5158 Some(error)
5159 );
5160 let retry = child_completion_envelope(
5161 &completion,
5162 wait_registered_at,
5163 Some(result),
5164 &runtime_resume_message(&completion, 8, None),
5165 );
5166 assert_eq!(envelope.id, retry.id);
5167 }
5168
5169 #[test]
5170 fn compact_guardian_outcome_keeps_rejected_verdict_and_result_retrieval() {
5171 let completion = make_completion("completed");
5172 let result = format!(
5173 "{}{{\"approve\":false,\"findings\":[\"needs fix\"]}}",
5174 "\u{0001}".repeat(CHILD_COMPLETION_INLINE_FIELD_BYTES - 80)
5175 );
5176 assert!(result.len() <= CHILD_COMPLETION_INLINE_FIELD_BYTES);
5177 let verdict = parse_guardian_verdict(&result).expect("trailing guardian verdict");
5178 let presentation = guardian_resume_message(&completion, &verdict);
5179 assert!(
5180 serde_json::to_vec(&presentation).unwrap().len() <= CHILD_COMPLETION_INLINE_FIELD_BYTES,
5181 "the original guardian presentation must fit the inline cap"
5182 );
5183 let wait_registered_at = Utc::now();
5184 let expanded = build_child_completion_envelope(
5185 &completion,
5186 wait_registered_at,
5187 Some(&result),
5188 &presentation,
5189 false,
5190 );
5191 assert!(
5192 serde_json::to_vec(&expanded).unwrap().len()
5193 > bamboo_domain::SessionInboxLimits::default().max_payload_bytes,
5194 "JSON escaping must trigger the compact fallback"
5195 );
5196
5197 let envelope =
5198 child_completion_envelope(&completion, wait_registered_at, Some(result), &presentation);
5199 assert!(
5200 serde_json::to_vec(&envelope).unwrap().len()
5201 <= bamboo_domain::SessionInboxLimits::default().max_payload_bytes
5202 );
5203 let SessionMessageBody::ChildOutcome(outcome) = &envelope.body else {
5204 panic!("typed child outcome");
5205 };
5206 let provider = outcome.provider_message.as_ref().unwrap();
5207 assert!(provider.content.text.contains("Guardian review REJECTED"));
5208 assert!(provider.content.text.contains(r#""message":"result""#));
5209 assert_eq!(
5210 provider.metadata.get("guardian_approved"),
5211 Some(&serde_json::json!(false))
5212 );
5213 assert!(envelope
5214 .to_provider_message()
5215 .unwrap()
5216 .content
5217 .contains("Guardian review REJECTED"));
5218 }
5219
5220 #[tokio::test]
5221 async fn oversized_multibyte_child_error_is_durable_and_replayable() {
5222 let home = tempfile::tempdir().unwrap();
5223 let store = Arc::new(
5224 bamboo_storage::SessionStoreV2::new(home.path().to_path_buf())
5225 .await
5226 .unwrap(),
5227 );
5228 let parent_id = "large-error-parent";
5229 store
5230 .save_session(&Session::new(parent_id, "model"))
5231 .await
5232 .unwrap();
5233 let inbox = bamboo_storage::FileSessionInbox::new(
5234 store.clone(),
5235 bamboo_domain::SessionInboxLimits::default(),
5236 );
5237 let mut completion = make_completion("error");
5238 completion.parent_session_id = parent_id.to_string();
5239 completion.error = Some("失败🙂".repeat(30_000));
5240 let presentation = runtime_resume_message(&completion, 0, None);
5241 let envelope = child_completion_envelope(&completion, Utc::now(), None, &presentation);
5242 inbox.deliver(&envelope).await.unwrap();
5243 completion.child_session_id = "escaped-error-child".to_string();
5244 completion.error = Some("\u{0001}".repeat(CHILD_COMPLETION_INLINE_FIELD_BYTES));
5245 let result = "\n".repeat(CHILD_COMPLETION_INLINE_FIELD_BYTES);
5246 let mixed = child_completion_envelope(
5247 &completion,
5248 Utc::now(),
5249 Some(result.clone()),
5250 &runtime_resume_message(&completion, 0, Some(&result)),
5251 );
5252 let receipt = inbox.deliver(&mixed).await.unwrap();
5253 inbox
5254 .mark_activation_eligible(
5255 parent_id,
5256 receipt.generation,
5257 bamboo_domain::SessionActivationPolicy::RespectSpecificWait,
5258 )
5259 .await
5260 .unwrap();
5261 drop(inbox);
5262 drop(store);
5263
5264 let reopened_store = Arc::new(
5265 bamboo_storage::SessionStoreV2::new(home.path().to_path_buf())
5266 .await
5267 .unwrap(),
5268 );
5269 let reopened = bamboo_storage::FileSessionInbox::new(
5270 reopened_store,
5271 bamboo_domain::SessionInboxLimits::default(),
5272 );
5273 let claims = reopened.claim(parent_id, 2).await.unwrap();
5274 assert_eq!(claims.len(), 2);
5275 assert_eq!(claims[0].envelope.id, envelope.id);
5276 assert_eq!(claims[1].envelope.id, mixed.id);
5277 for claim in claims {
5278 assert!(
5279 serde_json::to_vec(&claim.envelope).unwrap().len()
5280 < bamboo_domain::SessionInboxLimits::default().max_payload_bytes
5281 );
5282 }
5283 }
5284
5285 #[tokio::test]
5286 async fn failed_child_wait_sidecar_save_keeps_cache_wait_and_transcript() {
5287 let (_temp, store, inbox, mut coordinator, reservations, launches) =
5288 completion_inbox_fixture().await;
5289 let parent_id = "failed-sidecar-parent";
5290 let child_id = "failed-sidecar-child";
5291 let now = Utc::now();
5292 let mut parent = Session::new(parent_id, "model");
5293 parent.add_message(Message::user("durable root turn"));
5294 let mut runtime = AgentRuntimeState::new("waiting-run");
5295 runtime.status = AgentStatusState::Suspended;
5296 runtime.waiting_for_children = Some(WaitingForChildrenState::for_children(
5297 vec![child_id.to_string()],
5298 ChildWaitPolicy::All,
5299 now,
5300 ));
5301 write_runtime_state(&mut parent, &runtime);
5302 parent.metadata.insert(
5303 "runtime.suspend_reason".to_string(),
5304 "waiting_for_children".to_string(),
5305 );
5306 store.save_session(&parent).await.unwrap();
5307 let mut cached = parent.clone();
5308 cached.add_message(Message::assistant("newer cached root turn", None));
5309 coordinator.sessions.insert(
5310 parent_id.to_string(),
5311 Arc::new(crate::SessionSnapshot::new(cached)),
5312 );
5313
5314 let mut child = Session::new_child(child_id, parent_id, "model", "Child");
5315 child.set_last_run_status("completed");
5316 save_sourced_test_child(&store, &child).await;
5317 Arc::get_mut(&mut coordinator).unwrap().persistence = Arc::new(LockedSessionStore::new(
5318 Arc::new(FailingRuntimeSaveStorage {
5319 inner: store.clone(),
5320 }),
5321 ));
5322
5323 ChildCompletionHandler::on_child_completed(
5324 coordinator.as_ref(),
5325 ChildCompletion {
5326 parent_session_id: parent_id.to_string(),
5327 child_session_id: child_id.to_string(),
5328 status: "completed".to_string(),
5329 error: None,
5330 completed_at: Utc::now(),
5331 source: None,
5332 },
5333 )
5334 .await;
5335
5336 let durable = store.load_session(parent_id).await.unwrap().unwrap();
5337 assert!(read_runtime_state(&durable).waiting_for_children.is_some());
5338 assert_eq!(durable.messages.len(), 1);
5339 assert_eq!(durable.messages[0].content, "durable root turn");
5340 let cached = coordinator.sessions.get(parent_id).unwrap();
5341 let cached = cached.value().read();
5342 assert!(read_runtime_state(&cached).waiting_for_children.is_some());
5343 assert!(cached
5344 .messages
5345 .iter()
5346 .any(|message| message.content == "newer cached root turn"));
5347 let backlog = inbox.inspect(parent_id).await.unwrap();
5348 assert_eq!(backlog.pending + backlog.claimed, 1);
5349 assert_eq!(reservations.load(Ordering::SeqCst), 0);
5350 assert_eq!(launches.load(Ordering::SeqCst), 0);
5351 }
5352
5353 #[tokio::test]
5354 async fn oversized_child_completion_clears_wait_and_activates_exactly_once() {
5355 let (_temp, store, inbox, coordinator, reservations, launches) =
5356 completion_inbox_fixture().await;
5357 let parent_id = "oversized-parent";
5358 let child_id = "oversized-child";
5359 let now = Utc::now();
5360 let mut parent = Session::new(parent_id, "model");
5361 let mut parent_runtime = AgentRuntimeState::new("waiting-run");
5362 parent_runtime.status = AgentStatusState::Suspended;
5363 parent_runtime.waiting_for_children = Some(WaitingForChildrenState::for_children(
5364 vec![child_id.to_string()],
5365 ChildWaitPolicy::All,
5366 now,
5367 ));
5368 parent_runtime.suspension = Some(SuspensionState {
5369 reason: "waiting_for_children".to_string(),
5370 suspended_at: now,
5371 resumable: true,
5372 hook_point: Some("ChildCompletion".to_string()),
5373 });
5374 write_runtime_state(&mut parent, &parent_runtime);
5375 parent.metadata.insert(
5376 "runtime.suspend_reason".to_string(),
5377 "waiting_for_children".to_string(),
5378 );
5379 store.save_session(&parent).await.unwrap();
5380 let mut cached_parent = parent.clone();
5384 cached_parent.add_message(Message::assistant("concurrent root turn", None));
5385 coordinator.sessions.insert(
5386 parent_id.to_string(),
5387 Arc::new(crate::SessionSnapshot::new(cached_parent)),
5388 );
5389
5390 let mut child = Session::new_child(child_id, parent_id, "model", "Child");
5391 child.add_message(Message::assistant("z".repeat(300 * 1024), None));
5392 child.set_last_run_status("completed");
5393 save_sourced_test_child(&store, &child).await;
5394 let completion = ChildCompletion {
5395 parent_session_id: parent_id.to_string(),
5396 child_session_id: child_id.to_string(),
5397 status: "completed".to_string(),
5398 error: None,
5399 completed_at: Utc::now(),
5400 source: test_child_source(&store, child_id).await,
5401 };
5402
5403 ChildCompletionHandler::on_child_completed(coordinator.as_ref(), completion.clone()).await;
5404 let durable_parent = store.load_session(parent_id).await.unwrap().unwrap();
5405 let durable_runtime = read_runtime_state(&durable_parent);
5406 assert!(durable_runtime.waiting_for_children.is_none());
5407 assert_eq!(durable_runtime.status, AgentStatusState::Idle);
5408 assert!(!durable_parent
5409 .metadata
5410 .contains_key("runtime.suspend_reason"));
5411 let cached_parent = coordinator.sessions.get(parent_id).unwrap();
5412 let cached_parent = cached_parent.value().read();
5413 assert!(cached_parent
5414 .messages
5415 .iter()
5416 .any(|message| message.content == "concurrent root turn"));
5417 assert!(read_runtime_state(&cached_parent)
5418 .waiting_for_children
5419 .is_none());
5420 let backlog = inbox.inspect(parent_id).await.unwrap();
5421 assert_eq!(backlog.pending + backlog.claimed, 1);
5422 assert!(backlog.activation_pending());
5423 assert_eq!(reservations.load(Ordering::SeqCst), 1);
5424 assert_eq!(launches.load(Ordering::SeqCst), 1);
5425
5426 let claim = inbox.claim(parent_id, 1).await.unwrap().remove(0);
5427 assert!(
5428 serde_json::to_vec(&claim.envelope).unwrap().len()
5429 < bamboo_domain::SessionInboxLimits::default().max_payload_bytes
5430 );
5431 ChildCompletionHandler::on_child_completed(coordinator.as_ref(), completion).await;
5434 assert_eq!(reservations.load(Ordering::SeqCst), 1);
5435 assert_eq!(launches.load(Ordering::SeqCst), 1);
5436 assert_eq!(inbox.inspect(parent_id).await.unwrap().claimed, 1);
5437 }
5438
5439 #[tokio::test]
5440 async fn completion_before_parent_wait_does_not_checkpoint_stale_root() {
5441 let (_temp, store, inbox, coordinator, reservations, launches) =
5442 completion_inbox_fixture().await;
5443 let parent_id = "parent-without-wait";
5444 let child_id = "already-finished-child";
5445 let mut parent = Session::new(parent_id, "model");
5446 parent.add_message(Message::assistant("root is still reasoning", None));
5447 store.save_session(&parent).await.unwrap();
5448 let before = store.load_session(parent_id).await.unwrap().unwrap();
5449
5450 let mut child = Session::new_child(child_id, parent_id, "model", "Child");
5451 child.set_last_run_status("completed");
5452 save_sourced_test_child(&store, &child).await;
5453 ChildCompletionHandler::on_child_completed(
5454 coordinator.as_ref(),
5455 ChildCompletion {
5456 parent_session_id: parent_id.to_string(),
5457 child_session_id: child_id.to_string(),
5458 status: "completed".to_string(),
5459 error: None,
5460 completed_at: Utc::now(),
5461 source: None,
5462 },
5463 )
5464 .await;
5465
5466 let after = store.load_session(parent_id).await.unwrap().unwrap();
5467 assert_eq!(after.updated_at, before.updated_at);
5468 assert_eq!(
5469 serde_json::to_value(&after.messages).unwrap(),
5470 serde_json::to_value(&before.messages).unwrap()
5471 );
5472 assert!(read_runtime_state(&after).waiting_for_children.is_none());
5473 assert_eq!(inbox.inspect(parent_id).await.unwrap().pending, 0);
5474 assert_eq!(reservations.load(Ordering::SeqCst), 0);
5475 assert_eq!(launches.load(Ordering::SeqCst), 0);
5476 }
5477
5478 #[tokio::test]
5479 async fn latest_locked_activation_mutation_preserves_wait_armed_after_stale_load() {
5480 let temp = tempfile::tempdir().unwrap();
5481 let store = Arc::new(
5482 bamboo_storage::SessionStoreV2::new(temp.path().to_path_buf())
5483 .await
5484 .unwrap(),
5485 );
5486 let storage: Arc<dyn Storage> = store.clone();
5487 let locked = LockedSessionStore::new(storage);
5488 let mut session = Session::new("activation-stale-wait", "model");
5489 session.agent_runtime_state = Some(AgentRuntimeState::new("old-run"));
5490 store.save_session(&session).await.unwrap();
5491
5492 let stale = store.load_session(&session.id).await.unwrap().unwrap();
5494 assert!(read_runtime_state(&stale).waiting_for_children.is_none());
5495
5496 locked
5497 .update_runtime_config(&session.id, |latest| {
5498 let mut state = read_runtime_state(latest);
5499 state.status = AgentStatusState::Suspended;
5500 state.waiting_for_children = Some(WaitingForChildrenState::for_children(
5501 vec!["child-new".to_string()],
5502 ChildWaitPolicy::All,
5503 Utc::now(),
5504 ));
5505 state.suspension = Some(SuspensionState {
5506 reason: "waiting_for_children".to_string(),
5507 suspended_at: Utc::now(),
5508 resumable: true,
5509 hook_point: None,
5510 });
5511 write_runtime_state(latest, &state);
5512 latest.metadata.insert(
5513 "runtime.suspend_reason".to_string(),
5514 "waiting_for_children".to_string(),
5515 );
5516 })
5517 .await
5518 .unwrap();
5519
5520 let (prepared, ready) = prepare_session_inbox_activation(&locked, &session.id, false)
5521 .await
5522 .unwrap()
5523 .unwrap();
5524 assert!(!ready, "latest durable specific wait must block activation");
5525 let state = read_runtime_state(&prepared);
5526 assert!(state.waiting_for_children.is_some());
5527 assert_eq!(state.status, AgentStatusState::Suspended);
5528 assert_eq!(
5529 prepared
5530 .metadata
5531 .get("runtime.suspend_reason")
5532 .map(String::as_str),
5533 Some("waiting_for_children")
5534 );
5535 }
5536
5537 #[tokio::test]
5538 async fn partial_child_and_bash_backlogs_remain_inert_after_inbox_reopen() {
5539 let temp = tempfile::tempdir().unwrap();
5540 let store = Arc::new(
5541 bamboo_storage::SessionStoreV2::new(temp.path().to_path_buf())
5542 .await
5543 .unwrap(),
5544 );
5545 store
5546 .save_session(&Session::new("restart-child-parent", "model"))
5547 .await
5548 .unwrap();
5549 store
5550 .save_session(&Session::new("restart-bash-parent", "model"))
5551 .await
5552 .unwrap();
5553 let inbox = bamboo_storage::FileSessionInbox::new(
5554 store.clone(),
5555 bamboo_domain::SessionInboxLimits::default(),
5556 );
5557
5558 let completion = ChildCompletion {
5559 parent_session_id: "restart-child-parent".to_string(),
5560 child_session_id: "child-a".to_string(),
5561 status: "completed".to_string(),
5562 error: None,
5563 completed_at: Utc::now(),
5564 source: None,
5565 };
5566 let child_resume = runtime_resume_message(&completion, 1, Some("first child"));
5567 inbox
5568 .deliver(&child_completion_envelope(
5569 &completion,
5570 Utc::now(),
5571 Some("first child".to_string()),
5572 &child_resume,
5573 ))
5574 .await
5575 .unwrap();
5576 let bash = BashCompletionInfo {
5577 session_id: "restart-bash-parent".to_string(),
5578 bash_id: "bash-a".to_string(),
5579 command: "true".to_string(),
5580 exit_code: Some(0),
5581 status: "completed".to_string(),
5582 output_tail: String::new(),
5583 };
5584 inbox
5585 .deliver(&bash_completion_envelope(&bash))
5586 .await
5587 .unwrap();
5588
5589 let reopened = bamboo_storage::FileSessionInbox::new(
5590 store,
5591 bamboo_domain::SessionInboxLimits::default(),
5592 );
5593 for session_id in ["restart-child-parent", "restart-bash-parent"] {
5594 let backlog = reopened.inspect(session_id).await.unwrap();
5595 assert_eq!(backlog.pending, 1);
5596 assert_eq!(backlog.activation_generation, 0);
5597 assert!(
5598 !backlog.activation_pending(),
5599 "startup must not run a partial wait backlog for {session_id}"
5600 );
5601 }
5602 }
5603
5604 struct StubChildIndex {
5607 children: Vec<(String, Option<String>)>,
5608 }
5609
5610 #[async_trait]
5611 impl Storage for StubChildIndex {
5612 async fn save_session(&self, _session: &Session) -> std::io::Result<()> {
5613 Ok(())
5614 }
5615 async fn load_session(&self, _id: &str) -> std::io::Result<Option<Session>> {
5616 Ok(None)
5617 }
5618 async fn delete_session(&self, _id: &str) -> std::io::Result<bool> {
5619 Ok(false)
5620 }
5621 async fn list_child_run_statuses(
5622 &self,
5623 _parent_session_id: &str,
5624 ) -> std::io::Result<Vec<(String, Option<String>)>> {
5625 Ok(self.children.clone())
5626 }
5627 }
5628
5629 #[tokio::test]
5630 async fn derive_completed_only_includes_terminal_children() {
5631 let storage: Arc<dyn Storage> = Arc::new(StubChildIndex {
5632 children: vec![
5633 ("a".into(), Some("completed".into())),
5634 ("b".into(), Some("running".into())),
5635 ("c".into(), Some("error".into())),
5636 ("d".into(), None),
5637 ],
5638 });
5639 let completed = derive_completed_child_ids(&storage, "parent-1", "b").await;
5640 assert_eq!(
5642 completed,
5643 vec!["a".to_string(), "b".to_string(), "c".to_string()]
5644 );
5645 }
5646
5647 #[tokio::test]
5648 async fn derive_completed_folds_in_just_completed_when_index_lags() {
5649 let storage: Arc<dyn Storage> = Arc::new(StubChildIndex {
5651 children: vec![("only".into(), Some("running".into()))],
5652 });
5653 let completed = derive_completed_child_ids(&storage, "parent-1", "only").await;
5654 assert_eq!(completed, vec!["only".to_string()]);
5655 }
5656
5657 #[test]
5658 fn wait_policy_all_uses_derived_completed_set() {
5659 let waited = vec!["a".to_string(), "b".to_string()];
5660 assert!(!wait_policy_satisfied(
5661 ChildWaitPolicy::All,
5662 &waited,
5663 &["a".to_string()],
5664 "a",
5665 "completed"
5666 ));
5667 assert!(wait_policy_satisfied(
5668 ChildWaitPolicy::All,
5669 &waited,
5670 &["a".to_string(), "b".to_string()],
5671 "b",
5672 "completed"
5673 ));
5674 }
5675
5676 #[test]
5677 fn wait_policy_first_error_requires_tracked_membership() {
5678 let waited = vec!["a".to_string(), "b".to_string()];
5679 assert!(wait_policy_satisfied(
5681 ChildWaitPolicy::FirstError,
5682 &waited,
5683 &["a".to_string()],
5684 "a",
5685 "error"
5686 ));
5687 assert!(!wait_policy_satisfied(
5690 ChildWaitPolicy::FirstError,
5691 &waited,
5692 &["a".to_string()],
5693 "stray-child",
5694 "timeout"
5695 ));
5696 assert!(wait_policy_satisfied(
5698 ChildWaitPolicy::FirstError,
5699 &waited,
5700 &["a".to_string(), "b".to_string()],
5701 "stray-child",
5702 "completed"
5703 ));
5704 }
5705
5706 #[test]
5707 fn child_final_assistant_text_returns_last_assistant() {
5708 let mut session = Session::new("child-1", "gpt-4");
5709 session.messages.push(Message::user("hi"));
5710 session
5711 .messages
5712 .push(Message::assistant("first answer", None));
5713 session.messages.push(Message::user("again"));
5714 session
5715 .messages
5716 .push(Message::assistant("final answer", None));
5717
5718 assert_eq!(
5719 child_final_assistant_text(&session).as_deref(),
5720 Some("final answer")
5721 );
5722 }
5723
5724 #[test]
5725 fn child_final_assistant_text_returns_none_when_blank() {
5726 let mut session = Session::new("child-1", "gpt-4");
5727 session.messages.push(Message::assistant(" ", None));
5728 assert!(child_final_assistant_text(&session).is_none());
5729 }
5730
5731 #[test]
5732 fn child_final_assistant_text_returns_none_when_no_assistant() {
5733 let mut session = Session::new("child-1", "gpt-4");
5734 session.messages.push(Message::user("hi"));
5735 assert!(child_final_assistant_text(&session).is_none());
5736 }
5737
5738 #[test]
5739 fn runtime_resume_message_folds_full_response_without_truncation() {
5740 let completion = make_completion("completed");
5743 let long: String = "a".repeat(10_000);
5744 let message = runtime_resume_message(&completion, 0, Some(&long));
5745 assert!(message.content.contains(&long));
5746 assert!(!message.content.contains("truncated"));
5747 }
5748
5749 #[test]
5750 fn runtime_resume_message_includes_child_response_when_provided() {
5751 let completion = make_completion("completed");
5752 let message = runtime_resume_message(&completion, 0, Some("the answer is 42"));
5753
5754 assert!(matches!(message.role, Role::User));
5755 assert!(!message.never_compress);
5758 assert!(message.content.contains("Child final response:"));
5759 assert!(message.content.contains("the answer is 42"));
5760
5761 let metadata = message.metadata.expect("metadata present");
5762 assert_eq!(
5763 metadata.get("hidden_from_ui").and_then(|v| v.as_bool()),
5764 Some(true)
5765 );
5766 assert_eq!(
5767 metadata.get("runtime_kind").and_then(|v| v.as_str()),
5768 Some("child_completion_resume")
5769 );
5770 assert_eq!(
5771 metadata
5772 .get("child_final_response_included")
5773 .and_then(|v| v.as_bool()),
5774 Some(true)
5775 );
5776 }
5777
5778 #[test]
5779 fn runtime_resume_message_falls_back_to_error_when_no_response() {
5780 let mut completion = make_completion("error");
5781 completion.error = Some("boom".to_string());
5782
5783 let message = runtime_resume_message(&completion, 1, None);
5784 assert!(message.content.contains("Child error:"));
5785 assert!(message.content.contains("boom"));
5786 let metadata = message.metadata.expect("metadata present");
5787 assert_eq!(
5788 metadata
5789 .get("child_final_response_included")
5790 .and_then(|v| v.as_bool()),
5791 Some(false)
5792 );
5793 }
5794
5795 #[test]
5796 fn runtime_resume_message_minimal_when_no_response_and_no_error() {
5797 let completion = make_completion("completed");
5798 let message = runtime_resume_message(&completion, 2, None);
5799 assert!(!message.content.contains("Child final response:"));
5800 assert!(!message.content.contains("Child error:"));
5801 assert!(message.content.contains("Resume the parent task"));
5802 }
5803
5804 #[test]
5805 fn read_config_snapshot_refreshes_cached_snapshot_from_live_config() {
5806 let runtime = tokio::runtime::Runtime::new().expect("runtime");
5807
5808 runtime.block_on(async {
5809 let config = Arc::new(RwLock::new(Config::default()));
5810 config.write().await.provider = "copilot".to_string();
5811 let cached_config = StdRwLock::new(Config::default());
5812
5813 let snapshot = read_config_snapshot(&config, &cached_config);
5814
5815 assert_eq!(snapshot.provider, "copilot");
5816 assert_eq!(
5817 cached_config.read().expect("cached snapshot lock").provider,
5818 "copilot"
5819 );
5820 });
5821 }
5822
5823 #[test]
5824 fn read_config_snapshot_uses_cached_snapshot_when_live_lock_is_busy() {
5825 let runtime = tokio::runtime::Runtime::new().expect("runtime");
5826
5827 runtime.block_on(async {
5828 let mut cached_snapshot = Config::default();
5829 cached_snapshot.provider = "cached-provider".to_string();
5830
5831 let config = Arc::new(RwLock::new(Config::default()));
5832 let cached_config = StdRwLock::new(cached_snapshot);
5833 let _write_guard = config.write().await;
5834
5835 let snapshot = read_config_snapshot(&config, &cached_config);
5836
5837 assert_eq!(snapshot.provider, "cached-provider");
5838 });
5839 }
5840
5841 #[test]
5844 fn bash_completion_resume_message_normal_announces_completion() {
5845 let ids = vec!["bg-1".to_string(), "bg-2".to_string()];
5846 let message = bash_completion_resume_message(&ids, false);
5847 assert!(
5849 message.content.contains("have completed"),
5850 "normal resume message must announce completion: {}",
5851 message.content
5852 );
5853 let metadata = message.metadata.expect("metadata present");
5855 assert_eq!(
5856 metadata
5857 .get(RUNTIME_RESUME_MESSAGE_HIDDEN_KEY)
5858 .and_then(|v| v.as_bool()),
5859 Some(true),
5860 "resume message must be hidden from the UI"
5861 );
5862 assert_eq!(
5863 metadata
5864 .get(RUNTIME_RESUME_MESSAGE_KIND_KEY)
5865 .and_then(|v| v.as_str()),
5866 Some(BASH_COMPLETION_RESUME_KIND),
5867 "resume message must carry the bash-completion kind discriminant"
5868 );
5869 }
5870
5871 #[test]
5872 fn bash_completion_resume_message_deadline_does_not_claim_completion() {
5873 let ids = vec!["bg-long".to_string()];
5877 let message = bash_completion_resume_message(&ids, true);
5878 assert!(
5879 !message.content.contains("have completed"),
5880 "deadline resume message must NOT claim the shells completed: {}",
5881 message.content
5882 );
5883 assert!(
5884 message.content.contains("may still be running"),
5885 "deadline resume message must warn shells may still be running: {}",
5886 message.content
5887 );
5888 assert!(
5889 message.content.contains("BashOutput"),
5890 "deadline resume message must direct verification via BashOutput: {}",
5891 message.content
5892 );
5893 let metadata = message.metadata.expect("metadata present");
5895 assert_eq!(
5896 metadata
5897 .get(RUNTIME_RESUME_MESSAGE_KIND_KEY)
5898 .and_then(|v| v.as_str()),
5899 Some(BASH_COMPLETION_RESUME_KIND)
5900 );
5901 }
5902
5903 #[test]
5904 fn bash_resume_should_retry_matrix() {
5905 assert!(!bash_resume_should_retry(
5911 &ResumeOutcome::Started { run_id: "r".into() },
5912 true
5913 ));
5914 assert!(!bash_resume_should_retry(
5915 &ResumeOutcome::Started { run_id: "r".into() },
5916 false
5917 ));
5918
5919 assert!(!bash_resume_should_retry(&ResumeOutcome::NotFound, true));
5921 assert!(!bash_resume_should_retry(&ResumeOutcome::NotFound, false));
5922
5923 assert!(bash_resume_should_retry(&ResumeOutcome::Completed, true));
5925 assert!(!bash_resume_should_retry(&ResumeOutcome::Completed, false));
5928
5929 assert!(bash_resume_should_retry(
5932 &ResumeOutcome::AlreadyRunning { run_id: "r".into() },
5933 true
5934 ));
5935 assert!(!bash_resume_should_retry(
5937 &ResumeOutcome::AlreadyRunning { run_id: "r".into() },
5938 false
5939 ));
5940 }
5941
5942 #[test]
5945 fn injection_body_includes_status_exit_command_and_tail() {
5946 let info = BashCompletionInfo {
5947 session_id: "s".into(),
5948 bash_id: "abc123".into(),
5949 command: "make build".into(),
5950 exit_code: Some(0),
5951 status: "completed".into(),
5952 output_tail: "BUILD OK".into(),
5953 };
5954 let body = bash_completion_injection_body(&info);
5955 assert!(body.contains("abc123"), "body: {body}");
5956 assert!(body.contains("make build"), "body: {body}");
5957 assert!(body.contains("completed"), "body: {body}");
5958 assert!(body.contains("exit code 0"), "body: {body}");
5959 assert!(body.contains("BUILD OK"), "body: {body}");
5960 assert!(body.contains("BashOutput"), "body: {body}");
5962 assert!(body.contains("bash_id=\"abc123\""), "body: {body}");
5963 }
5964
5965 #[test]
5966 fn injection_body_handles_no_output_and_signal_kill() {
5967 let info = BashCompletionInfo {
5968 session_id: "s".into(),
5969 bash_id: "xyz".into(),
5970 command: "sleep 99".into(),
5971 exit_code: None,
5972 status: "killed".into(),
5973 output_tail: String::new(),
5974 };
5975 let body = bash_completion_injection_body(&info);
5976 assert!(body.contains("killed"), "body: {body}");
5977 assert!(body.contains("none (signal/killed)"), "body: {body}");
5978 assert!(body.contains("no captured output"), "body: {body}");
5979 assert!(!body.contains("Output tail:"), "body: {body}");
5981 }
5982
5983 #[test]
5984 fn background_completion_builds_post_tool_use_payload_and_feedback() {
5985 let mut info = BashCompletionInfo {
5986 session_id: "s".into(),
5987 bash_id: "bg-7".into(),
5988 command: "cargo test".into(),
5989 exit_code: Some(0),
5990 status: "completed".into(),
5991 output_tail: "test result: ok".into(),
5992 };
5993
5994 let payload = background_bash_post_tool_payload(&info);
5995 match payload {
5996 HookPayload::ToolResult {
5997 tool_name,
5998 tool_call_id,
5999 outcome,
6000 } => {
6001 assert_eq!(tool_name, "Bash");
6002 assert_eq!(tool_call_id, "bg-7");
6003 assert!(outcome.success);
6004 let response: serde_json::Value =
6005 serde_json::from_str(outcome.result.as_deref().unwrap()).unwrap();
6006 assert_eq!(response["command"], "cargo test");
6007 assert_eq!(response["exit_code"], 0);
6008 assert_eq!(response["status"], "completed");
6009 assert_eq!(response["output_tail"], "test result: ok");
6010 }
6011 other => panic!("expected PostToolUse payload, got {other:?}"),
6012 }
6013
6014 append_background_bash_hook_feedback(
6015 &mut info,
6016 vec!["Run the formatter before continuing".to_string()],
6017 );
6018 assert!(info.output_tail.contains("<post_tool_use_feedback>"));
6019 assert!(info
6020 .output_tail
6021 .contains("Run the formatter before continuing"));
6022 }
6023
6024 #[tokio::test]
6025 async fn bash_completion_payload_identity_matches_file_inbox_idempotency() {
6026 let baseline = BashCompletionInfo {
6027 session_id: "session".into(),
6028 bash_id: "bg-7".into(),
6029 command: "cargo test".into(),
6030 exit_code: Some(0),
6031 status: "completed".into(),
6032 output_tail: "first tail".into(),
6033 };
6034 let mut retried = baseline.clone();
6035 retried.output_tail = "first tail\nlater bytes\n<hook feedback>".into();
6036 retried.status = "completed-after-hook".into();
6037 let baseline_envelope = bash_completion_envelope(&baseline);
6038 let exact_retry = bash_completion_envelope(&baseline);
6039 let corrected_envelope = bash_completion_envelope(&retried);
6040 assert_eq!(baseline_envelope.id, exact_retry.id);
6041 assert_ne!(
6042 baseline_envelope.id, corrected_envelope.id,
6043 "changed payload semantics must receive a distinct id"
6044 );
6045
6046 let mut other_shell = baseline.clone();
6047 other_shell.bash_id = "bg-8".into();
6048 assert_ne!(
6049 bash_completion_envelope(&baseline).id,
6050 bash_completion_envelope(&other_shell).id
6051 );
6052
6053 let temp = tempfile::tempdir().unwrap();
6054 let store = Arc::new(
6055 bamboo_storage::SessionStoreV2::new(temp.path().to_path_buf())
6056 .await
6057 .unwrap(),
6058 );
6059 store
6060 .save_session(&Session::new("session", "model"))
6061 .await
6062 .unwrap();
6063 let inbox = bamboo_storage::FileSessionInbox::new(
6064 store,
6065 bamboo_domain::SessionInboxLimits::default(),
6066 );
6067 let first = inbox.deliver(&baseline_envelope).await.unwrap();
6068 let duplicate = inbox.deliver(&exact_retry).await.unwrap();
6069 let corrected = inbox.deliver(&corrected_envelope).await.unwrap();
6070 assert_eq!(duplicate, first, "exact payload retry is idempotent");
6071 assert_ne!(corrected.id, first.id);
6072 assert_eq!(corrected.generation, first.generation + 1);
6073 assert_eq!(inbox.inspect("session").await.unwrap().pending, 2);
6074 }
6075
6076 #[cfg(unix)]
6077 #[tokio::test]
6078 async fn background_completion_fires_configured_post_tool_use_command() {
6079 use bamboo_config::{
6080 LifecycleHookGroup, LifecycleHookHandler, LifecycleHooksConfig,
6081 DEFAULT_LIFECYCLE_HOOK_TIMEOUT_MS,
6082 };
6083
6084 let dir = tempfile::tempdir().unwrap();
6085 let output = dir.path().join("background-post-tool.json");
6086 let command = format!(
6087 "cat > '{}'; printf '%s' '{{\"additional_context\":\"inspect the completed build log\"}}'",
6088 output.display()
6089 );
6090 let config = LifecycleHooksConfig {
6091 enabled: true,
6092 post_tool_use: vec![LifecycleHookGroup {
6093 enabled: true,
6094 matcher: Some("^Bash$".to_string()),
6095 hooks: vec![LifecycleHookHandler::command(
6096 command,
6097 DEFAULT_LIFECYCLE_HOOK_TIMEOUT_MS,
6098 )],
6099 }],
6100 ..Default::default()
6101 };
6102 let mut session = Session::new("session-bg-hook", "test-model");
6103 session.workspace = Some(dir.path().to_string_lossy().into_owned());
6104 let mut info = BashCompletionInfo {
6105 session_id: session.id.clone(),
6106 bash_id: "bg-9".into(),
6107 command: "cargo test".into(),
6108 exit_code: Some(0),
6109 status: "completed".into(),
6110 output_tail: "test result: ok".into(),
6111 };
6112
6113 assert!(run_background_bash_post_tool_hooks(&config, None, &session, &mut info).await);
6114 let envelope: serde_json::Value =
6115 serde_json::from_str(&std::fs::read_to_string(output).unwrap()).unwrap();
6116 assert_eq!(envelope["hook_event_name"], "PostToolUse");
6117 assert_eq!(envelope["tool_name"], "Bash");
6118 assert_eq!(envelope["payload"]["tool_call_id"], "bg-9");
6119 let response = envelope["tool_response"]["result"]
6120 .as_str()
6121 .map(serde_json::from_str::<serde_json::Value>)
6122 .transpose()
6123 .unwrap()
6124 .unwrap();
6125 assert_eq!(response["command"], "cargo test");
6126 assert_eq!(response["status"], "completed");
6127 assert!(info.output_tail.contains("inspect the completed build log"));
6128 }
6129
6130 async fn temp_store() -> (tempfile::TempDir, Arc<dyn Storage>, LockedSessionStore) {
6131 let temp = tempfile::tempdir().unwrap();
6132 let storage: Arc<dyn Storage> = Arc::new(
6133 bamboo_storage::v2::SessionStoreV2::new(temp.path().to_path_buf())
6134 .await
6135 .expect("storage init"),
6136 );
6137 let persistence = LockedSessionStore::new(storage.clone());
6138 (temp, storage, persistence)
6139 }
6140
6141 #[tokio::test]
6142 async fn enqueue_writes_pending_injection_and_preserves_messages() {
6143 let (_temp, storage, persistence) = temp_store().await;
6144
6145 let mut session = Session::new("sess-enq", "test-model");
6146 session.add_message(Message::user("do the build"));
6147 storage.save_session(&session).await.unwrap();
6148
6149 let info = BashCompletionInfo {
6150 session_id: "sess-enq".into(),
6151 bash_id: "sh-1".into(),
6152 command: "make".into(),
6153 exit_code: Some(0),
6154 status: "completed".into(),
6155 output_tail: "done".into(),
6156 };
6157 let saved = enqueue_bash_completion_injection(&persistence, &info)
6158 .await
6159 .expect("enqueue io ok")
6160 .expect("session exists");
6161
6162 let pending = saved
6163 .pending_injected_messages()
6164 .expect("pending injection present");
6165 assert_eq!(pending.len(), 1);
6166 let content = pending[0].get("content").and_then(|v| v.as_str()).unwrap();
6167 assert!(content.contains("sh-1"), "content: {content}");
6168 assert!(content.contains("make"), "content: {content}");
6169 assert!(content.contains("done"), "content: {content}");
6170 assert_eq!(saved.messages.len(), 1);
6172 }
6173
6174 #[tokio::test]
6175 async fn enqueue_returns_none_for_missing_session() {
6176 let (_temp, _storage, persistence) = temp_store().await;
6177 let info = BashCompletionInfo {
6178 session_id: "does-not-exist".into(),
6179 bash_id: "x".into(),
6180 command: "true".into(),
6181 exit_code: Some(0),
6182 status: "completed".into(),
6183 output_tail: String::new(),
6184 };
6185 let result = enqueue_bash_completion_injection(&persistence, &info)
6186 .await
6187 .expect("io ok");
6188 assert!(result.is_none(), "no session → nothing enqueued");
6189 }
6190
6191 #[test]
6199 fn apply_bash_resume_transition_clears_wait_and_appends_message() {
6200 use bamboo_domain::session::runtime_state::WaitingForBashState;
6201
6202 let mut session = Session::new("sess-resume", "test-model");
6203 session.add_message(Message::user("kick off the build"));
6204 let mut rt = read_runtime_state(&session);
6205 rt.status = AgentStatusState::Running;
6206 rt.waiting_for_bash = Some(WaitingForBashState::for_bash(
6207 vec!["sh-1".into()],
6208 Utc::now(),
6209 ));
6210 write_runtime_state(&mut session, &rt);
6211 session.metadata.insert(
6212 "runtime.suspend_reason".to_string(),
6213 "waiting_for_bash".to_string(),
6214 );
6215
6216 let resume = bash_completion_resume_message(&["sh-1".to_string()], false);
6217 let did = apply_bash_resume_transition(&mut session, &resume);
6218
6219 assert!(did, "a suspended session must transition");
6220 let after = read_runtime_state(&session);
6221 assert!(
6222 after.waiting_for_bash.is_none(),
6223 "bash wait must be cleared"
6224 );
6225 assert_eq!(after.status, AgentStatusState::Idle, "runtime must be Idle");
6226 assert!(
6227 !session.metadata.contains_key("runtime.suspend_reason"),
6228 "suspend-reason marker must be removed"
6229 );
6230 assert_eq!(session.messages.len(), 2, "resume message must be appended");
6231 assert!(matches!(
6232 session.messages.last().map(|m| &m.role),
6233 Some(Role::User)
6234 ));
6235 }
6236
6237 #[test]
6241 fn apply_bash_resume_transition_noops_when_not_waiting() {
6242 let mut session = Session::new("sess-live", "test-model");
6243 session.add_message(Message::user("hi"));
6244
6245 let resume = bash_completion_resume_message(&["sh-1".to_string()], false);
6246 let did = apply_bash_resume_transition(&mut session, &resume);
6247
6248 assert!(!did, "a non-waiting session must not transition");
6249 assert_eq!(session.messages.len(), 1, "no resume message appended");
6250 }
6251
6252 #[test]
6256 fn bash_completion_should_resume_only_when_suspended_and_all_done() {
6257 assert!(bash_completion_should_resume(true, true));
6258 assert!(!bash_completion_should_resume(true, false)); assert!(!bash_completion_should_resume(false, true)); assert!(!bash_completion_should_resume(false, false));
6261 }
6262
6263 #[test]
6264 fn two_shell_delivery_stages_backlog_before_one_final_activation() {
6265 let mut waiting = true;
6266 let mut durable_backlog = 0;
6267 let mut reservations = 0;
6268
6269 durable_backlog += 1;
6271 let first = bash_completion_delivery_plan(waiting, false);
6272 assert_eq!(first, BashCompletionDeliveryPlan::DurableOnly);
6273 assert!(waiting);
6274 assert_eq!(durable_backlog, 1);
6275 assert_eq!(reservations, 0);
6276
6277 durable_backlog += 1;
6280 let last = bash_completion_delivery_plan(waiting, true);
6281 assert_eq!(last, BashCompletionDeliveryPlan::ClearWaitThenActivate);
6282 waiting = false;
6283 reservations += 1;
6284 assert!(!waiting);
6285 assert_eq!(durable_backlog, 2);
6286 assert_eq!(reservations, 1);
6287 }
6288
6289 #[test]
6293 fn bash_resume_message_from_info_carries_bashid_tail_and_kind() {
6294 let info = BashCompletionInfo {
6295 session_id: "s".into(),
6296 bash_id: "sh-42".into(),
6297 command: "cargo test".into(),
6298 exit_code: Some(0),
6299 status: "completed".into(),
6300 output_tail: "test result: ok".into(),
6301 };
6302 let msg = bash_resume_message_from_info(&info);
6303
6304 assert!(matches!(msg.role, Role::User));
6305 assert!(msg.content.contains("sh-42"), "content: {}", msg.content);
6306 assert!(
6307 msg.content.contains("cargo test"),
6308 "content: {}",
6309 msg.content
6310 );
6311 assert!(
6312 msg.content.contains("test result: ok"),
6313 "content: {}",
6314 msg.content
6315 );
6316 assert!(
6317 msg.content.contains("BashOutput"),
6318 "content: {}",
6319 msg.content
6320 );
6321 let meta = serde_json::to_string(&msg.metadata).unwrap();
6322 assert!(
6323 meta.contains(BASH_COMPLETION_RESUME_KIND),
6324 "resume message must be tagged as a bash-completion resume: {meta}"
6325 );
6326 }
6327}