1use std::collections::HashMap;
8use std::sync::{Arc, Mutex as StdMutex, OnceLock, RwLock as StdRwLock, Weak};
9use std::time::Duration;
10
11use bamboo_domain::poison::PoisonRecover;
12use bamboo_domain::{
13 AgentHookPoint, HookPayload, HookToolOutcome, SessionActivationPolicy, SessionChildOutcome,
14 SessionMessageBody, SessionMessageContent, SessionMessageEnvelope, SessionMessageId,
15 SessionMessageKind, SessionMessageSource, SessionProviderMessage,
16};
17
18use crate::execution::{
19 finalize_runner, reserve_runner_core, reserve_session_execution, spawn_session_execution,
20 AgentRunner, AgentStatus, ChildCompletion, ChildCompletionHandler, ChildCompletionSource,
21 ReserveOutcome, SessionExecutionArgs, SessionExecutionReservation,
22 SessionExecutionReserveOutcome, SpawnJob, SpawnScheduler,
23};
24use crate::runtime::config::{BashResumeHook, GuardianSpawner, BASH_COMPLETION_RESUME_KIND};
25use crate::runtime::guardian_state::{
26 parse_guardian_verdict, read_guardian_config, read_guardian_state, write_guardian_state,
27 GuardianVerdict,
28};
29use crate::Agent;
30use async_trait::async_trait;
31use bamboo_agent_core::storage::Storage;
32use bamboo_agent_core::tools::ToolExecutor;
33use bamboo_agent_core::{
34 AgentEvent, BashCompletionInfo, BashCompletionSink, Message, Role, Session, SessionKind,
35};
36use bamboo_domain::session::runtime_state::{
37 AgentRuntimeState, AgentStatusState, ChildWaitPolicy, WaitingForChildrenState,
38};
39use bamboo_llm::{Config, ProviderModelRouter, ProviderRegistry};
40use bamboo_storage::LockedSessionStore;
41use chrono::Utc;
42use sha2::{Digest, Sha256};
43use tokio::sync::{broadcast, RwLock};
44
45use crate::model_areas::resolve_global_area_models;
46use crate::model_config_helper::{
47 resolve_fast_model, resolve_gold_config, resolve_provider_routing_key, GOLD_CONFIG_METADATA_KEY,
48};
49use crate::session_activation::{
50 SessionActivationLaunch, SessionActivationReserveOutcome, SessionActivationSpawner,
51};
52use crate::session_app::child_session::result_projection::{
53 child_result_identity, decode_report, fits_child_result_budget, plain_child_report_message,
54 project_child_report, unavailable_child_result,
55};
56use crate::session_app::execute::consume_pending_clarification_resume;
57use crate::session_app::provider_model::{persist_model_ref, session_effective_model_ref};
58use crate::session_app::resume::{
59 resume_session_execution, ResumeExecutionPort, ResumeSpawnRequest,
60};
61use crate::session_app::types::{ResumeConfigSnapshot, ResumeOutcome};
62
63const AGENT_RUNTIME_STATE_METADATA_KEY: &str = "agent.runtime.state";
64const RUNTIME_RESUME_MESSAGE_HIDDEN_KEY: &str = "hidden_from_ui";
65const RUNTIME_RESUME_MESSAGE_KIND_KEY: &str = "runtime_kind";
66const CHILD_COMPLETION_INLINE_FIELD_BYTES: usize = 48 * 1024;
67const CHILD_COMPLETION_OVERSIZE_TAIL_BYTES: usize = 8 * 1024;
68
69fn read_runtime_state(session: &Session) -> AgentRuntimeState {
70 session
71 .agent_runtime_state
72 .clone()
73 .or_else(|| {
74 session
75 .metadata
76 .get(AGENT_RUNTIME_STATE_METADATA_KEY)
77 .and_then(|raw| serde_json::from_str::<AgentRuntimeState>(raw).ok())
78 })
79 .unwrap_or_else(|| AgentRuntimeState::new(format!("{}-child-wait", session.id)))
80}
81
82fn write_runtime_state(session: &mut Session, runtime_state: &AgentRuntimeState) {
83 session.agent_runtime_state = Some(runtime_state.clone());
84 if let Ok(serialized) = serde_json::to_string(runtime_state) {
85 session
86 .metadata
87 .insert(AGENT_RUNTIME_STATE_METADATA_KEY.to_string(), serialized);
88 }
89}
90
91async fn prepare_session_inbox_activation(
97 persistence: &LockedSessionStore,
98 session_id: &str,
99 interrupt_specific_wait: bool,
100) -> std::io::Result<Option<(Session, bool)>> {
101 let ready = Arc::new(std::sync::atomic::AtomicBool::new(false));
102 let ready_for_mutation = ready.clone();
103 let saved = persistence
104 .update_runtime_config(session_id, move |latest| {
105 if latest.has_pending_question() {
109 return;
110 }
111 let mut runtime_state = read_runtime_state(latest);
112 let specifically_waiting = runtime_state.waiting_for_children.is_some()
113 || runtime_state.waiting_for_bash.is_some();
114 if specifically_waiting && !interrupt_specific_wait {
115 return;
116 }
117 runtime_state.status = AgentStatusState::Idle;
123 runtime_state.suspension = None;
124 write_runtime_state(latest, &runtime_state);
125 latest.metadata.remove("runtime.suspend_reason");
126 latest.updated_at = Utc::now();
127 ready_for_mutation.store(true, std::sync::atomic::Ordering::Release);
128 })
129 .await?;
130 Ok(saved.map(|session| (session, ready.load(std::sync::atomic::Ordering::Acquire))))
131}
132
133fn is_error_like(status: &str) -> bool {
134 matches!(status, "error" | "timeout" | "cancelled")
135}
136
137fn is_terminal_child_status(status: &str) -> bool {
139 matches!(
140 status,
141 "completed" | "error" | "timeout" | "cancelled" | "skipped"
142 )
143}
144
145fn child_completion_envelope(
146 completion: &ChildCompletion,
147 wait_registered_at: chrono::DateTime<Utc>,
148 result: Option<String>,
149 provider_message: &Message,
150) -> SessionMessageEnvelope {
151 let envelope = build_child_completion_envelope(
152 completion,
153 wait_registered_at,
154 result.as_deref(),
155 provider_message,
156 false,
157 );
158 if serde_json::to_vec(&envelope).map_or(true, |bytes| {
159 bytes.len() > bamboo_domain::SessionInboxLimits::default().max_payload_bytes
160 }) {
161 return build_child_completion_envelope(
166 completion,
167 wait_registered_at,
168 result.as_deref(),
169 provider_message,
170 true,
171 );
172 }
173 envelope
174}
175
176fn child_completion_id(
177 completion: &ChildCompletion,
178 wait_registered_at: chrono::DateTime<Utc>,
179 result: Option<&str>,
180 legacy_wait_identity: bool,
181) -> SessionMessageId {
182 fn identity(value: Option<&str>) -> serde_json::Value {
183 match value {
184 Some(value) if value.len() > CHILD_COMPLETION_INLINE_FIELD_BYTES => serde_json::json!({
185 "oversized": true, "utf8_bytes": value.len(),
186 "sha256": hex::encode(Sha256::digest(value.as_bytes())),
187 }),
188 value => serde_json::json!(value),
189 }
190 }
191 SessionMessageId::stable(
192 "session_child_completion",
193 &serde_json::json!({
194 "parent_session_id": completion.parent_session_id,
195 "child_session_id": completion.child_session_id,
196 "status": completion.status,
197 "error": identity(completion.error.as_deref()), "result": identity(result),
198 "wait_registered_at": (legacy_wait_identity || completion.source.is_none()).then_some(wait_registered_at),
199 "terminal_source": completion.source,
200 }),
201 )
202}
203
204fn build_child_completion_envelope(
205 completion: &ChildCompletion,
206 wait_registered_at: chrono::DateTime<Utc>,
207 result: Option<&str>,
208 provider_message: &Message,
209 compact: bool,
210) -> SessionMessageEnvelope {
211 fn bounded_terminal_field(
212 label: &str,
213 child_session_id: &str,
214 value: Option<&str>,
215 compact: bool,
216 ) -> (Option<String>, serde_json::Value) {
217 let Some(value) = value else {
218 return (None, serde_json::Value::Null);
219 };
220 let oversized = value.len() > CHILD_COMPLETION_INLINE_FIELD_BYTES;
221 let identity = if oversized {
222 serde_json::json!({
223 "oversized": true,
224 "utf8_bytes": value.len(),
225 "sha256": hex::encode(Sha256::digest(value.as_bytes())),
226 })
227 } else {
228 serde_json::Value::String(value.to_string())
229 };
230 if !oversized && !compact {
231 return (Some(value.to_string()), identity);
232 }
233 let digest = hex::encode(Sha256::digest(value.as_bytes()));
234 let retrieval_view = if label == "error" {
235 "diagnostics"
236 } else {
237 "result"
238 };
239 let inspection = serde_json::json!({
240 "intent": "inspect", "target": child_session_id, "message": retrieval_view,
241 });
242 let recovery = if label == "error" {
243 format!(
244 "Check observed status and recovery diagnostics by calling SubAgent with {inspection}. \
245 Detailed execution errors remain available to the authenticated inspector."
246 )
247 } else {
248 format!(
249 "Read it in bounded slices by calling SubAgent with {inspection}; use message=\"messages\" for transcript previews. \
250 Follow returned cursors with a JSON-encoded view/cursor query in message, omitting unused query fields."
251 )
252 };
253 let mut summary = format!(
254 "Child {label} could not fit in the durable inline envelope ({} UTF-8 bytes, sha256={digest}). {recovery}",
255 value.len()
256 );
257 if !compact {
258 let mut tail_start = value
259 .len()
260 .saturating_sub(CHILD_COMPLETION_OVERSIZE_TAIL_BYTES);
261 while tail_start < value.len() && !value.is_char_boundary(tail_start) {
262 tail_start += 1;
263 }
264 summary.push_str("\n\nBounded tail:\n");
265 summary.push_str(&value[tail_start..]);
266 }
267 (Some(summary), identity)
268 }
269
270 let (stored_result, _) =
271 bounded_terminal_field("result", &completion.child_session_id, result, compact);
272 let (stored_error, _) = bounded_terminal_field(
273 "error",
274 &completion.child_session_id,
275 completion.error.as_deref(),
276 compact,
277 );
278 let mut bounded_provider_message = provider_message.clone();
279 let provider_oversized = serde_json::to_vec(provider_message)
280 .map(|bytes| bytes.len() > CHILD_COMPLETION_INLINE_FIELD_BYTES)
281 .unwrap_or(true);
282 if provider_oversized || compact {
283 let mut content = if provider_oversized {
287 format!(
288 "Runtime notification: child session `{}` finished with status `{}`.",
289 completion.child_session_id, completion.status
290 )
291 } else {
292 bounded_provider_message.content.clone()
293 };
294 if let Some(result) = stored_result.as_deref() {
295 content.push_str("\n\n");
296 content.push_str(result);
297 }
298 if let Some(error) = stored_error.as_deref() {
299 content.push_str("\n\n");
300 content.push_str(error);
301 }
302 bounded_provider_message.content = content;
303 if provider_oversized {
304 bounded_provider_message.content_parts = None;
305 }
306 if compact {
310 let original = bounded_provider_message
311 .metadata
312 .as_ref()
313 .and_then(serde_json::Value::as_object);
314 let mut metadata = serde_json::Map::new();
315 for key in [
316 RUNTIME_RESUME_MESSAGE_HIDDEN_KEY,
317 RUNTIME_RESUME_MESSAGE_KIND_KEY,
318 "child_session_id",
319 "child_status",
320 "child_final_response_included",
321 "child_completion_source",
322 "child_completion_source_sha256",
323 "guardian_approved",
324 ] {
325 if let Some(value) = original.and_then(|metadata| metadata.get(key)) {
326 metadata.insert(key.to_string(), value.clone());
327 }
328 }
329 if original.is_some_and(|metadata| metadata.contains_key("child_error")) {
330 metadata.insert(
331 "child_error".to_string(),
332 stored_error
333 .as_ref()
334 .map_or(serde_json::Value::Null, |error| {
335 serde_json::Value::String(error.clone())
336 }),
337 );
338 }
339 bounded_provider_message.metadata = Some(serde_json::Value::Object(metadata));
340 } else if let Some(child_error) = bounded_provider_message
341 .metadata
342 .as_mut()
343 .and_then(serde_json::Value::as_object_mut)
344 .and_then(|metadata| metadata.get_mut("child_error"))
345 {
346 *child_error = stored_error
347 .as_ref()
348 .map_or(serde_json::Value::Null, |error| {
349 serde_json::Value::String(error.clone())
350 });
351 }
352 }
353 let body = SessionMessageBody::ChildOutcome(SessionChildOutcome {
354 child_session_id: completion.child_session_id.clone(),
355 status: completion.status.clone(),
356 result: stored_result,
357 error: stored_error,
358 provider_message: Some(session_provider_message(&bounded_provider_message)),
359 });
360 SessionMessageEnvelope {
361 id: child_completion_id(completion, wait_registered_at, result, false),
364 source: SessionMessageSource::Runtime {
365 subsystem: "child_completion_coordinator".to_string(),
366 },
367 target_session_id: completion.parent_session_id.clone(),
368 kind: SessionMessageKind::ChildOutcome,
369 body,
370 created_at: completion.completed_at,
371 thread_id: None,
372 in_reply_to: None,
373 attempt: None,
374 correlation_id: Some(format!("child_completion:{}", completion.child_session_id)),
375 }
376}
377
378fn session_provider_message(message: &Message) -> SessionProviderMessage {
379 SessionProviderMessage {
380 content: SessionMessageContent {
381 text: message.content.clone(),
382 parts: message.content_parts.clone().unwrap_or_default(),
383 skill_request: None,
384 },
385 metadata: message
386 .metadata
387 .as_ref()
388 .and_then(serde_json::Value::as_object)
389 .cloned()
390 .unwrap_or_default(),
391 never_compress: message.never_compress,
392 }
393}
394
395async fn derive_completed_child_ids(
400 storage: &Arc<dyn Storage>,
401 parent_session_id: &str,
402 just_completed_child_id: &str,
403) -> Vec<String> {
404 let mut completed: Vec<String> = storage
405 .list_child_run_statuses(parent_session_id)
406 .await
407 .unwrap_or_default()
408 .into_iter()
409 .filter(|(_, status)| status.as_deref().is_some_and(is_terminal_child_status))
410 .map(|(id, _)| id)
411 .collect();
412 if !completed.iter().any(|id| id == just_completed_child_id) {
413 completed.push(just_completed_child_id.to_string());
414 }
415 completed.sort();
416 completed.dedup();
417 completed
418}
419
420fn read_config_snapshot(config: &Arc<RwLock<Config>>, cached_config: &StdRwLock<Config>) -> Config {
421 if let Ok(config_guard) = config.try_read() {
422 let snapshot = config_guard.clone();
423
424 if let Ok(mut cached_guard) = cached_config.try_write() {
425 *cached_guard = snapshot.clone();
426 }
427
428 snapshot
429 } else {
430 cached_config
431 .try_read()
432 .map(|guard| guard.clone())
433 .unwrap_or_default()
434 }
435}
436
437fn parent_locks() -> &'static std::sync::Mutex<HashMap<String, Arc<tokio::sync::Mutex<()>>>> {
453 static LOCKS: OnceLock<std::sync::Mutex<HashMap<String, Arc<tokio::sync::Mutex<()>>>>> =
454 OnceLock::new();
455 LOCKS.get_or_init(|| std::sync::Mutex::new(HashMap::new()))
456}
457
458pub(crate) fn session_resume_lock(session_id: &str) -> SessionResumeLock {
465 let mut map = parent_locks().lock().recover_poison();
466 let lock = map
467 .entry(session_id.to_string())
468 .or_insert_with(|| Arc::new(tokio::sync::Mutex::new(())))
469 .clone();
470 SessionResumeLock {
471 session_id: session_id.to_string(),
472 lock: Some(lock),
473 }
474}
475
476pub(crate) struct SessionResumeLock {
480 session_id: String,
481 lock: Option<Arc<tokio::sync::Mutex<()>>>,
482}
483
484impl std::ops::Deref for SessionResumeLock {
485 type Target = tokio::sync::Mutex<()>;
486 fn deref(&self) -> &Self::Target {
487 self.lock.as_ref().expect("live resume-lock lease")
488 }
489}
490
491impl Drop for SessionResumeLock {
492 fn drop(&mut self) {
493 self.lock.take();
494 let mut map = parent_locks().lock().recover_poison();
495 if map
496 .get(&self.session_id)
497 .is_some_and(|lock| Arc::strong_count(lock) == 1)
498 {
499 map.remove(&self.session_id);
500 }
501 }
502}
503
504fn wait_policy_satisfied(
505 policy: ChildWaitPolicy,
506 wait_child_ids: &[String],
507 completed_child_ids: &[String],
508 latest_child_id: &str,
509 latest_status: &str,
510) -> bool {
511 if wait_child_ids.is_empty() {
512 return false;
513 }
514
515 match policy {
516 ChildWaitPolicy::All => wait_child_ids
517 .iter()
518 .all(|id| completed_child_ids.iter().any(|completed| completed == id)),
519 ChildWaitPolicy::Any => completed_child_ids
520 .iter()
521 .any(|id| wait_child_ids.iter().any(|wait_id| wait_id == id)),
522 ChildWaitPolicy::FirstError => {
523 (is_error_like(latest_status) && wait_child_ids.iter().any(|id| id == latest_child_id))
529 || wait_child_ids
530 .iter()
531 .all(|id| completed_child_ids.iter().any(|completed| completed == id))
532 }
533 }
534}
535
536#[cfg(test)]
540fn child_final_assistant_text(child: &Session) -> Option<String> {
541 child
542 .messages
543 .iter()
544 .rev()
545 .find(|message| matches!(message.role, Role::Assistant))
546 .map(|message| message.content.clone())
547 .filter(|content| !content.trim().is_empty())
548}
549
550fn runtime_resume_message(
551 completion: &ChildCompletion,
552 remaining_children: usize,
553 child_final_response: Option<&str>,
554) -> Message {
555 let mut body = format!(
556 "Runtime notification: child session `{}` finished with status `{}`. Remaining child sessions: {}.",
557 completion.child_session_id, completion.status, remaining_children
558 );
559
560 let final_response = child_final_response.map(str::to_string);
566 if let Some(response) = final_response.as_deref() {
567 body.push_str("\n\nChild final response:\n");
568 body.push_str(response);
569 } else if let Some(error) = completion.error.as_deref() {
570 if !error.is_empty() {
571 body.push_str("\n\nChild error:\n");
572 body.push_str(error);
573 }
574 }
575
576 let inspection = serde_json::json!({
577 "intent": "inspect", "target": completion.child_session_id, "message": "messages",
578 });
579 body.push_str(&format!(
580 "\n\nResume the parent task using this child result and continue from the previous plan. \
581 If you need transcript evidence, call SubAgent with {inspection}. \
582 Follow returned cursors with a JSON-encoded view/cursor query in message; use view=\"message\" and message_id inside that query for a selected full message. Omit unused query fields.",
583 ));
584
585 let mut message = Message::user(body);
586 message.metadata = Some(serde_json::json!({
587 RUNTIME_RESUME_MESSAGE_HIDDEN_KEY: true,
588 RUNTIME_RESUME_MESSAGE_KIND_KEY: "child_completion_resume",
589 "child_session_id": completion.child_session_id,
590 "child_status": completion.status,
591 "child_error": completion.error,
592 "child_final_response_included": final_response.is_some(),
593 "child_completion_source": completion.source,
594 }));
595 message.never_compress = false;
599 message
600}
601
602fn sealed_child_report(
606 completion: &ChildCompletion,
607 child: &Session,
608 parent: &Session,
609) -> Option<serde_json::Value> {
610 if child.subagent_type().as_deref() == Some("guardian")
611 || child
612 .metadata
613 .get("lifecycle")
614 .is_some_and(|v| v == "resident")
615 {
616 return None;
617 }
618 let source = completion.source.as_ref()?;
619 if !source.matches_completion(completion)
620 || ChildCompletionSource::from_committed_session(child).as_ref() != Some(source)
621 || !source.matches_parent(child, parent)
622 {
623 return None;
624 }
625 let binding = bamboo_domain::ChildContextBinding::from_session(child).ok()??;
626 let Some(message) = child
627 .messages
628 .iter()
629 .find(|m| Some(&m.id) == source.result_message_id.as_ref())
630 else {
631 let reason = if source.last_assistant.is_some() {
632 "report_not_current_final"
633 } else {
634 "report_absent"
635 };
636 return Some(unavailable_child_result("typed_result", reason));
637 };
638 if !plain_child_report_message(message) {
639 return Some(unavailable_child_result(
640 "typed_result",
641 "report_not_current_final",
642 ));
643 }
644 let report = match decode_report(&message.content) {
645 Ok(report) => report,
646 Err(reason) => return Some(unavailable_child_result("typed_result", reason)),
647 };
648 let mut observation =
649 child_result_identity(parent, child, &binding, source.project_id.as_deref());
650 observation.as_object_mut()?.extend(
651 serde_json::json!({"kind":"committed_terminal_source","terminal_source":source,
652 "last_run_status":source.status})
653 .as_object()?
654 .clone(),
655 );
656 Some(project_child_report(report, observation))
657}
658
659fn child_completion_source_sha256(completion: &ChildCompletion) -> Option<String> {
660 completion.source.as_ref().map(|source| {
661 hex::encode(Sha256::digest(
662 serde_json::to_vec(source).expect("sealed Child source serializes"),
663 ))
664 })
665}
666
667fn typed_child_resume_message(
668 completion: &ChildCompletion,
669 remaining_children: usize,
670 projection: &serde_json::Value,
671) -> Message {
672 let inspection = serde_json::json!({
673 "intent":"inspect", "target":completion.child_session_id, "message":"result",
674 });
675 let mut message = Message::user(format!(
676 "Runtime notification: child session `{}` finished with status `{}`. Remaining child sessions: {}.\n\n\
677 Child typed result:\n{}\n\n\
678 Resume the parent task. Child-reported evidence and verification are claims, not host-certified facts. \
679 For unavailable results or transcript evidence, call SubAgent with {inspection} and follow returned cursors.",
680 completion.child_session_id, completion.status, remaining_children, projection,
681 ));
682 message.metadata = Some(serde_json::json!({
683 RUNTIME_RESUME_MESSAGE_HIDDEN_KEY:true,
684 RUNTIME_RESUME_MESSAGE_KIND_KEY:"child_completion_resume",
685 "child_session_id":completion.child_session_id,"child_status":completion.status,
686 "child_final_response_included":projection["available"] == true,
687 "child_typed_result_included":true,
688 "child_completion_source_sha256":child_completion_source_sha256(completion),
689 }));
690 message.never_compress = false;
691 message
692}
693
694fn bound_typed_child_delivery(
697 envelope: &mut SessionMessageEnvelope,
698 message: &mut Message,
699 projection: &mut serde_json::Value,
700 completion: &ChildCompletion,
701 remaining_children: usize,
702) -> bool {
703 fn install(envelope: &mut SessionMessageEnvelope, message: &Message) {
704 if let SessionMessageBody::ChildOutcome(outcome) = &mut envelope.body {
705 outcome.result = None;
710 outcome.error = None;
713 outcome.provider_message = Some(session_provider_message(message));
714 }
715 }
716 fn fits(envelope: &SessionMessageEnvelope, message: &Message) -> bool {
717 fits_child_result_budget(envelope)
718 && fits_child_result_budget(message)
719 && envelope
720 .to_provider_message()
721 .is_ok_and(|actual| fits_child_result_budget(&actual))
722 }
723 install(envelope, message);
724 if fits(envelope, message) {
725 return true;
726 }
727 *projection = unavailable_child_result("typed_result", "result_budget_exceeded");
728 *message = typed_child_resume_message(completion, remaining_children, projection);
729 install(envelope, message);
730 fits(envelope, message)
731}
732
733fn guardian_resume_message(completion: &ChildCompletion, verdict: &GuardianVerdict) -> Message {
738 let mut body = if verdict.approve {
739 String::from(
740 "Guardian review APPROVED: an independent reviewer verified the work and found no blocking issues. You may finalize the task.",
741 )
742 } else {
743 String::from(
744 "Guardian review REJECTED: an independent reviewer found issues. Address every finding below before completing — do NOT declare the task complete until they are resolved.",
745 )
746 };
747 if let Some(summary) = verdict.summary.as_deref().filter(|s| !s.trim().is_empty()) {
748 body.push_str("\n\nReviewer summary: ");
749 body.push_str(summary);
750 }
751 if !verdict.findings.is_empty() {
752 body.push_str("\n\nFindings:");
753 for (idx, finding) in verdict.findings.iter().enumerate() {
754 body.push_str(&format!("\n{}. {}", idx + 1, finding));
755 }
756 }
757 let inspection = serde_json::json!({
758 "intent": "inspect", "target": completion.child_session_id, "message": "messages",
759 });
760 body.push_str(&format!(
761 "\n\nIf you need guardian transcript evidence, call SubAgent with {inspection}. \
762 Follow returned cursors with a JSON-encoded view/cursor query in message, omitting unused query fields.",
763 ));
764
765 let mut message = Message::user(body);
766 message.metadata = Some(serde_json::json!({
767 RUNTIME_RESUME_MESSAGE_HIDDEN_KEY: true,
768 RUNTIME_RESUME_MESSAGE_KIND_KEY: "guardian_review_resume",
769 "child_session_id": completion.child_session_id,
770 "child_status": completion.status,
771 "guardian_approved": verdict.approve,
772 "child_completion_source": completion.source,
773 }));
774 message.never_compress = false;
775 message
776}
777
778pub type RootToolSurfaceObserver = Arc<
780 dyn Fn() -> futures::future::BoxFuture<'static, Option<Arc<dyn ToolExecutor>>> + Send + Sync,
781>;
782
783pub type ReservedRootExecutionAdapter = Arc<
784 dyn Fn(
785 Arc<Agent>,
786 &Session,
787 &SessionExecutionReservation,
788 Arc<dyn ToolExecutor>,
789 ) -> Result<(Arc<Agent>, Arc<dyn ToolExecutor>), bamboo_agent_core::AgentError>
790 + Send
791 + Sync,
792>;
793
794#[derive(Clone)]
795pub struct ChildCompletionCoordinator {
796 storage: Arc<dyn Storage>,
797 persistence: Arc<bamboo_storage::LockedSessionStore>,
798 sessions: crate::SessionCache,
799 agent_runners: Arc<RwLock<HashMap<String, AgentRunner>>>,
800 session_event_senders: Arc<RwLock<HashMap<String, broadcast::Sender<AgentEvent>>>>,
801 agent: Arc<Agent>,
802 config: Arc<RwLock<Config>>,
803 provider_registry: Arc<ProviderRegistry>,
804 provider_router: Arc<ProviderModelRouter>,
805 app_data_dir: std::path::PathBuf,
806 account_feed_inbox: Option<crate::execution::AccountFeedInbox>,
807 root_account_sink: Option<Arc<crate::events::AccountEventSink>>,
808 root_tools: Arc<RwLock<Option<Arc<dyn ToolExecutor>>>>,
809 reserved_root_adapter: Arc<StdRwLock<Option<ReservedRootExecutionAdapter>>>,
810 guardian_spawner: Arc<RwLock<Option<Arc<dyn GuardianSpawner>>>>,
814 spawn_scheduler: Arc<RwLock<Weak<SpawnScheduler>>>,
818}
819
820impl ChildCompletionCoordinator {
821 pub fn with_root_account_sink(mut self, sink: Arc<crate::events::AccountEventSink>) -> Self {
822 self.root_account_sink = Some(sink);
823 self
824 }
825
826 #[allow(clippy::too_many_arguments)]
827 pub fn new(
828 storage: Arc<dyn Storage>,
829 persistence: Arc<LockedSessionStore>,
830 sessions: crate::SessionCache,
831 agent_runners: Arc<RwLock<HashMap<String, AgentRunner>>>,
832 session_event_senders: Arc<RwLock<HashMap<String, broadcast::Sender<AgentEvent>>>>,
833 agent: Arc<Agent>,
834 config: Arc<RwLock<Config>>,
835 provider_registry: Arc<ProviderRegistry>,
836 provider_router: Arc<ProviderModelRouter>,
837 app_data_dir: std::path::PathBuf,
838 account_feed_inbox: Option<crate::execution::AccountFeedInbox>,
839 ) -> Self {
840 Self {
841 storage,
842 persistence,
843 sessions,
844 agent_runners,
845 session_event_senders,
846 agent,
847 config,
848 provider_registry,
849 provider_router,
850 app_data_dir,
851 account_feed_inbox,
852 root_account_sink: None,
853 root_tools: Arc::new(RwLock::new(None)),
854 reserved_root_adapter: Arc::new(StdRwLock::new(None)),
855 guardian_spawner: Arc::new(RwLock::new(None)),
856 spawn_scheduler: Arc::new(RwLock::new(Weak::new())),
857 }
858 }
859
860 pub fn set_reserved_root_execution_adapter(&self, adapter: ReservedRootExecutionAdapter) {
863 *self
864 .reserved_root_adapter
865 .write()
866 .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(adapter);
867 }
868
869 pub fn root_tool_surface_observer(&self) -> RootToolSurfaceObserver {
870 let source = Arc::downgrade(&self.root_tools);
871 Arc::new(move || {
872 let source = source.clone();
873 Box::pin(async move {
874 let source = source.upgrade()?;
875 let current = source.read().await.clone();
876 current
877 })
878 })
879 }
880
881 pub async fn set_root_tools(&self, tools: Arc<dyn ToolExecutor>) {
882 *self.root_tools.write().await = Some(tools);
883 }
884
885 pub async fn set_spawn_scheduler(&self, scheduler: &Arc<SpawnScheduler>) {
886 *self.spawn_scheduler.write().await = Arc::downgrade(scheduler);
887 }
888
889 pub async fn set_guardian_spawner(&self, spawner: Arc<dyn GuardianSpawner>) {
892 *self.guardian_spawner.write().await = Some(spawner);
893 }
894
895 fn build_resume_config(
896 &self,
897 session: &Session,
898 config_snapshot: &Config,
899 ) -> ResumeConfigSnapshot {
900 crate::session_app::resolution::resolve_resume_config_snapshot(
901 config_snapshot,
902 &self.provider_registry,
903 session,
904 None,
905 )
906 }
907
908 async fn resume_parent(&self, parent_session_id: String) -> ResumeOutcome {
916 for attempt in 0..=5u8 {
917 if attempt > 0 {
918 tokio::time::sleep(Duration::from_millis(250 * attempt as u64)).await;
919 }
920
921 let Some(session) = self.load_session(&parent_session_id).await else {
922 tracing::warn!(%parent_session_id, "cannot resume parent after child completion: session not found");
923 return ResumeOutcome::NotFound;
924 };
925 let config_snapshot = self.config.read().await.clone();
926 let resume_config = self.build_resume_config(&session, &config_snapshot);
927 let outcome = resume_session_execution(self, &parent_session_id, resume_config).await;
928 tracing::info!(
929 %parent_session_id,
930 attempt,
931 outcome = outcome.as_str(),
932 "child completion requested parent resume"
933 );
934
935 if !matches!(outcome, ResumeOutcome::AlreadyRunning { .. }) {
936 return outcome;
937 }
938 }
939 tracing::error!(
945 %parent_session_id,
946 "parent resume gave up after AlreadyRunning retry budget; \
947 relying on the child-wait watchdog backstop"
948 );
949 ResumeOutcome::AlreadyRunning {
950 run_id: String::new(),
951 }
952 }
953
954 async fn save_and_cache(&self, session: &mut Session) {
955 if let Err(error) = self
956 .persistence
957 .merge_save_runtime_and_publish(session, |saved, _| {
958 self.sessions.insert(
959 saved.id.clone(),
960 Arc::new(crate::SessionSnapshot::new(saved.clone())),
961 );
962 })
963 .await
964 {
965 tracing::warn!(session_id = %session.id, %error, "failed to persist session");
966 }
967 }
968
969 async fn reuse_legacy_child_delivery(
970 &self,
971 envelope: &mut SessionMessageEnvelope,
972 completion: &ChildCompletion,
973 wait: Option<&WaitingForChildrenState>,
974 result: Option<&str>,
975 ) -> Result<(), bamboo_domain::SessionInboxError> {
976 if let (Some(wait), Some(inbox), Some(_)) =
977 (wait, self.agent.session_inbox(), completion.source.as_ref())
978 {
979 let mut legacy = envelope.clone();
980 legacy.id = child_completion_id(completion, wait.registered_at, result, true);
981 if inbox.existing_staged_delivery(&legacy).await?.is_some() {
982 *envelope = legacy;
983 }
984 }
985 Ok(())
986 }
987
988 async fn legacy_child_delivery(
989 &self,
990 parent: &Session,
991 completion: &ChildCompletion,
992 ) -> Result<Option<(SessionMessageEnvelope, bool)>, bamboo_domain::SessionInboxError> {
993 let (Some(source), Some(inbox)) = (completion.source.as_ref(), self.agent.session_inbox())
994 else {
995 return Ok(None);
996 };
997 let source_sha256 = child_completion_source_sha256(completion);
998 let source = serde_json::to_value(source).expect("sealed Child source serializes");
999 let matches_source = |message: &Message, envelope: &SessionMessageEnvelope| {
1000 if !matches!(&envelope.source, SessionMessageSource::Runtime { subsystem }
1001 if subsystem == "child_completion_coordinator")
1002 || envelope.target_session_id != parent.id
1003 || !matches!(&envelope.body, SessionMessageBody::ChildOutcome(outcome)
1004 if outcome.child_session_id == completion.child_session_id && outcome.status == completion.status)
1005 || !bamboo_domain::is_matching_session_message(message, envelope)
1006 {
1007 return false;
1008 }
1009 let Some(metadata) = message.metadata.as_ref() else {
1010 return false;
1011 };
1012 let typed_source = message
1016 .content
1017 .split_once("\n\nChild typed result:\n")
1018 .and_then(|(_, text)| text.lines().next())
1019 .filter(|line| line.len() <= 8192)
1020 .and_then(|line| serde_json::from_str::<serde_json::Value>(line).ok())
1021 .and_then(|projection| {
1022 projection
1023 .get("host_observation")?
1024 .get("terminal_source")
1025 .cloned()
1026 });
1027 metadata.get("child_completion_source") == Some(&source)
1028 || source_sha256.as_ref().is_some_and(|digest| {
1029 metadata
1030 .get("child_completion_source_sha256")
1031 .and_then(serde_json::Value::as_str)
1032 == Some(digest.as_str())
1033 })
1034 || typed_source.as_ref() == Some(&source)
1035 };
1036 for message in &parent.messages {
1037 let Some(marker) = message
1038 .metadata
1039 .as_ref()
1040 .and_then(|m| m.get("session_message"))
1041 else {
1042 continue;
1043 };
1044 let Ok(envelope) = serde_json::from_value::<SessionMessageEnvelope>(marker.clone())
1045 else {
1046 continue;
1047 };
1048 if matches_source(message, &envelope)
1049 && inbox.was_admitted(&parent.id, &envelope.id).await?
1050 {
1051 return Ok(Some((envelope, true)));
1052 }
1053 }
1054 for envelope in inbox
1058 .pending_child_outcomes(&parent.id, &completion.child_session_id)
1059 .await?
1060 {
1061 if envelope
1062 .to_provider_message()
1063 .is_ok_and(|message| matches_source(&message, &envelope))
1064 {
1065 return Ok(Some((envelope, false)));
1066 }
1067 }
1068 Ok(None)
1069 }
1070}
1071
1072#[async_trait]
1073impl ChildCompletionHandler for ChildCompletionCoordinator {
1074 async fn on_child_completed(&self, completion: ChildCompletion) {
1075 if !is_terminal_child_status(&completion.status) {
1084 tracing::info!(
1085 parent_session_id = %completion.parent_session_id,
1086 child_session_id = %completion.child_session_id,
1087 status = %completion.status,
1088 "non-terminal child status; leaving the parent wait armed"
1089 );
1090 return;
1091 }
1092
1093 let per_parent = session_resume_lock(&completion.parent_session_id);
1098 let _per_parent_guard = per_parent.lock().await;
1099
1100 let (parent, should_resume, messenger, child_admission, child_admissions) = loop {
1101 let Some(mut parent) = self.load_session(&completion.parent_session_id).await else {
1102 tracing::warn!(
1103 parent_session_id = %completion.parent_session_id,
1104 child_session_id = %completion.child_session_id,
1105 "child completion received for missing parent"
1106 );
1107 return;
1108 };
1109
1110 let expected_parent = parent.clone();
1111
1112 let mut runtime_state = read_runtime_state(&parent);
1118
1119 let mut completed_child_ids = derive_completed_child_ids(
1122 &self.storage,
1123 &completion.parent_session_id,
1124 &completion.child_session_id,
1125 )
1126 .await;
1127
1128 let mut should_resume = false;
1129 let mut remaining_children = 0usize;
1130 let active_wait = runtime_state.waiting_for_children.clone();
1131 let tracked_wait = active_wait.as_ref().is_some_and(|wait| {
1132 wait.child_session_ids
1133 .contains(&completion.child_session_id)
1134 });
1135 if !tracked_wait {
1136 let mut unchanged = parent.clone();
1140 match self
1141 .persistence
1142 .compare_exchange_child_wait_and_publish(
1143 &parent,
1144 &mut unchanged,
1145 true,
1146 Arc::new(|_| {}),
1147 )
1148 .await
1149 {
1150 Ok(false) => continue,
1151 Ok(true) if completion.source.is_none() => return,
1152 Ok(true) => {}
1153 Err(error) => {
1154 tracing::warn!(parent_session_id = %parent.id, %error,
1155 "child completion could not settle no-wait observation");
1156 return;
1157 }
1158 }
1159 }
1160 if let Some(wait) = active_wait.as_ref() {
1163 for id in &wait.child_session_ids {
1164 if id == &completion.child_session_id {
1165 continue;
1166 }
1167 if let Ok(Some(child)) = self
1168 .storage
1169 .load_child_wait_session(&parent, id, false)
1170 .await
1171 {
1172 completed_child_ids.retain(|completed| completed != id);
1173 if completion_child_is_owned(&parent.id, child.parent_session_id.as_deref())
1174 && child
1175 .last_run_status()
1176 .as_deref()
1177 .is_some_and(is_terminal_child_status)
1178 {
1179 completed_child_ids.push(id.clone());
1180 }
1181 }
1182 }
1183 }
1184 if let Some(wait) = active_wait.as_ref() {
1185 remaining_children = wait
1186 .child_session_ids
1187 .iter()
1188 .filter(|id| !completed_child_ids.iter().any(|completed| completed == *id))
1189 .count();
1190 should_resume = tracked_wait
1191 && wait_policy_satisfied(
1192 wait.wait_for,
1193 &wait.child_session_ids,
1194 &completed_child_ids,
1195 &completion.child_session_id,
1196 &completion.status,
1197 );
1198 }
1199
1200 let reported_child_owned = match self
1212 .storage
1213 .load_child_wait_session(&parent, &completion.child_session_id, false)
1214 .await
1215 {
1216 Ok(Some(control_plane)) => completion_child_is_owned(
1217 &completion.parent_session_id,
1218 control_plane.parent_session_id.as_deref(),
1219 ),
1220 _ => false,
1221 };
1222
1223 let loaded_child = if reported_child_owned {
1228 match self
1229 .storage
1230 .load_child_wait_session(&parent, &completion.child_session_id, true)
1231 .await
1232 {
1233 Ok(child) => child,
1234 Err(error) => {
1235 tracing::warn!(
1236 child_session_id = %completion.child_session_id,
1237 %error,
1238 "failed to load child session for runtime resume message"
1239 );
1240 None
1241 }
1242 }
1243 } else {
1244 tracing::warn!(
1245 parent_session_id = %completion.parent_session_id,
1246 child_session_id = %completion.child_session_id,
1247 "completion child is not a child of this parent; resuming with a neutral \
1248 message and NOT folding its content"
1249 );
1250 None
1251 };
1252
1253 if let Some(source) = completion.source.as_ref() {
1257 if loaded_child
1258 .as_ref()
1259 .and_then(ChildCompletionSource::from_committed_session)
1260 .as_ref()
1261 != Some(source)
1262 || !source.matches_completion(&completion)
1263 || loaded_child
1264 .as_ref()
1265 .is_none_or(|child| !source.matches_parent(child, &parent))
1266 {
1267 tracing::warn!(child_session_id = %completion.child_session_id,
1268 "stale/unavailable committed child completion source; leaving wait armed");
1269 return;
1270 }
1271 }
1272 if !tracked_wait
1273 && crate::ticket_runtime::is_independent_ticket_child(
1274 self.storage.as_ref(),
1275 &parent,
1276 &completion.child_session_id,
1277 )
1278 .await
1279 {
1280 return;
1281 }
1282 let previous_delivery = match self.legacy_child_delivery(&parent, &completion).await {
1283 Ok(envelope) => envelope,
1284 Err(error) => {
1285 tracing::warn!(parent_session_id = %parent.id, %error,
1286 "cannot verify historical Child delivery; leaving replay recoverable");
1287 return;
1288 }
1289 };
1290 let previously_admitted = previous_delivery
1291 .as_ref()
1292 .is_some_and(|(_, admitted)| *admitted);
1293 let child_final_response = loaded_child
1294 .as_ref()
1295 .and_then(|child| completion.source.as_ref()?.result(child, &parent));
1296 let guardian_resume = if completion.source.is_some() || should_resume {
1301 let reviewed_round = runtime_state.round.current_round;
1302 loaded_child.as_ref().and_then(|child| {
1303 if child.subagent_type().as_deref() != Some("guardian") {
1304 return None;
1305 }
1306 let mut guardian_state = read_guardian_state(&parent)?;
1307 if guardian_state.guardian_child_id.as_deref()
1308 != Some(completion.child_session_id.as_str())
1309 {
1310 tracing::warn!(
1313 parent_session_id = %completion.parent_session_id,
1314 child_session_id = %completion.child_session_id,
1315 expected = ?guardian_state.guardian_child_id,
1316 "guardian completion does not match recorded guardian_child_id; using generic resume"
1317 );
1318 return None;
1319 }
1320 if guardian_state.phase.is_reviewed() {
1321 return guardian_state.last_verdict.as_ref()
1324 .map(|verdict| guardian_resume_message(&completion, verdict));
1325 }
1326 let verdict = child_final_response.clone()
1334 .and_then(|text| match parse_guardian_verdict(&text) {
1335 Ok(verdict) => Some(verdict),
1336 Err(error) => {
1337 tracing::warn!(
1338 child_session_id = %completion.child_session_id,
1339 %error,
1340 "guardian verdict unparseable; recording a synthetic reject"
1341 );
1342 None
1343 }
1344 })
1345 .unwrap_or_else(|| {
1346 GuardianVerdict::rejected(vec![
1347 "The guardian reviewer did not return a usable verdict (it errored or \
1348 emitted unparseable output); the work has NOT been independently \
1349 verified."
1350 .to_string(),
1351 ])
1352 });
1353 let approved = verdict.approve;
1354 let message = guardian_resume_message(&completion, &verdict);
1355 guardian_state.record_verdict(verdict, reviewed_round);
1356 write_guardian_state(&mut parent, guardian_state);
1357 tracing::info!(
1358 parent_session_id = %completion.parent_session_id,
1359 child_session_id = %completion.child_session_id,
1360 approved,
1361 "guardian verdict recorded; resuming parent"
1362 );
1363 Some(message)
1364 })
1365 } else {
1366 None
1367 };
1368 let guardian_control_changed = parent.metadata.get("guardian.state")
1369 != expected_parent.metadata.get("guardian.state");
1370 if previously_admitted && !should_resume && !guardian_control_changed {
1373 return;
1374 }
1375 let activation_policy = if should_resume || guardian_control_changed {
1376 SessionActivationPolicy::RespectSpecificWait
1377 } else {
1378 SessionActivationPolicy::InterruptSpecificWait
1379 };
1380 let mut typed_result = loaded_child
1381 .as_ref()
1382 .and_then(|child| sealed_child_report(&completion, child, &parent));
1383 let mut resume_message = if let Some(projection) = typed_result.as_mut() {
1384 let message =
1385 typed_child_resume_message(&completion, remaining_children, projection);
1386 if fits_child_result_budget(&message) {
1387 message
1388 } else {
1389 *projection =
1390 unavailable_child_result("typed_result", "result_budget_exceeded");
1391 typed_child_resume_message(&completion, remaining_children, projection)
1392 }
1393 } else {
1394 guardian_resume.unwrap_or_else(|| {
1395 runtime_resume_message(
1396 &completion,
1397 remaining_children,
1398 child_final_response.as_deref(),
1399 )
1400 })
1401 };
1402 if typed_result.is_some() && !fits_child_result_budget(&resume_message) {
1403 tracing::warn!(child_session_id = %completion.child_session_id,
1404 "typed child resume identity exceeds budget; leaving wait armed");
1405 return;
1406 }
1407
1408 let messenger = self.agent.session_messenger().cloned();
1412 let mut child_admission = if let Some(messenger) = messenger.as_ref() {
1413 let mut envelope = child_completion_envelope(
1414 &completion,
1415 active_wait
1416 .as_ref()
1417 .map_or(completion.completed_at, |wait| wait.registered_at),
1418 child_final_response.clone(),
1419 &resume_message,
1420 );
1421 if let Some(run_id) = self
1422 .agent
1423 .activation_router()
1424 .map(|router| router.current_run_id(&completion.parent_session_id))
1425 {
1426 if let Some(run_id) = run_id.await {
1427 envelope.correlation_id =
1428 Some(format!("child_completion_after_run:{run_id}"));
1429 }
1430 }
1431 if let Some(projection) = typed_result.as_mut() {
1432 if !bound_typed_child_delivery(
1433 &mut envelope,
1434 &mut resume_message,
1435 projection,
1436 &completion,
1437 remaining_children,
1438 ) {
1439 tracing::warn!(child_session_id = %completion.child_session_id,
1440 "typed child outcome wrapper exceeds budget; leaving wait armed");
1441 return;
1442 }
1443 }
1444 if let Some((previous, _)) = previous_delivery.as_ref() {
1445 envelope = previous.clone();
1446 } else if let Err(error) = self
1447 .reuse_legacy_child_delivery(
1448 &mut envelope,
1449 &completion,
1450 active_wait.as_ref(),
1451 child_final_response.as_deref(),
1452 )
1453 .await
1454 {
1455 tracing::warn!(parent_session_id = %parent.id, %error,
1456 "cannot verify existing Child delivery; leaving wait armed");
1457 return;
1458 }
1459 match messenger.admit(envelope).await {
1460 Ok(admission) => {
1461 if !should_resume && !guardian_control_changed {
1462 let mut observed = expected_parent.clone();
1463 match self
1464 .persistence
1465 .compare_exchange_child_wait_and_publish(
1466 &expected_parent,
1467 &mut observed,
1468 true,
1469 Arc::new(|_| {}),
1470 )
1471 .await
1472 {
1473 Ok(false) => continue,
1474 Ok(true) => {}
1475 Err(error) => {
1476 tracing::warn!(parent_session_id = %parent.id, %error,
1477 "partial Child delivery lost its parent authority; leaving staged outcome inert");
1478 return;
1479 }
1480 }
1481 }
1482 if should_resume || !guardian_control_changed {
1486 if let Err(error) = messenger
1487 .prepare_message_activation(&admission, activation_policy)
1488 .await
1489 {
1490 tracing::warn!(parent_session_id = %completion.parent_session_id,
1491 child_session_id = %completion.child_session_id, %error,
1492 "child outcome notification permission failed; leaving wait armed");
1493 return;
1494 }
1495 }
1496 Some(admission)
1497 }
1498 Err(error) => {
1499 tracing::warn!(
1500 parent_session_id = %completion.parent_session_id,
1501 child_session_id = %completion.child_session_id,
1502 %error,
1503 "child outcome SessionInbox admission failed; leaving parent wait armed"
1504 );
1505 return;
1506 }
1507 }
1508 } else {
1509 None
1510 };
1511
1512 let mut child_admissions = child_admission.iter().cloned().collect::<Vec<_>>();
1513
1514 if should_resume {
1521 if let (Some(wait), Some(messenger)) = (active_wait.as_ref(), messenger.as_ref()) {
1522 let parent_run_id = if let Some(router) = self.agent.activation_router() {
1523 router.current_run_id(&completion.parent_session_id).await
1524 } else {
1525 None
1526 };
1527 for child_id in &wait.child_session_ids {
1528 if child_id == &completion.child_session_id
1529 || !completed_child_ids.contains(child_id)
1530 {
1531 continue;
1532 }
1533 let owned = match self
1537 .storage
1538 .load_child_wait_session(&parent, child_id, false)
1539 .await
1540 {
1541 Ok(Some(child)) => completion_child_is_owned(
1542 &completion.parent_session_id,
1543 child.parent_session_id.as_deref(),
1544 ),
1545 _ => false,
1546 };
1547 let sibling = if owned {
1548 self.storage
1549 .load_child_wait_session(&parent, child_id, true)
1550 .await
1551 } else {
1552 tracing::warn!(%child_id, "terminal sibling ownership unavailable; leaving parent wait armed");
1553 return;
1554 };
1555 let sibling = match sibling {
1556 Ok(Some(child))
1557 if completion_child_is_owned(
1558 &completion.parent_session_id,
1559 child.parent_session_id.as_deref(),
1560 ) && child
1561 .last_run_status()
1562 .as_deref()
1563 .is_some_and(is_terminal_child_status) =>
1564 {
1565 child
1566 }
1567 _ => {
1568 tracing::warn!(%child_id, "terminal sibling snapshot unavailable; leaving parent wait armed");
1569 return;
1570 }
1571 };
1572 if ChildCompletionSource::has_source_record(&sibling)
1573 && ChildCompletionSource::from_committed_session(&sibling).is_none()
1574 {
1575 tracing::warn!(%child_id, "terminal sibling source is invalid; leaving wait armed");
1576 return;
1577 }
1578 let sibling_completion = ChildCompletion {
1579 parent_session_id: completion.parent_session_id.clone(),
1580 child_session_id: child_id.clone(),
1581 status: sibling.last_run_status().expect("terminal status checked"),
1582 error: sibling.last_run_error(),
1583 completed_at: sibling.updated_at,
1584 source: ChildCompletionSource::from_committed_session(&sibling),
1585 };
1586 if sibling_completion
1587 .source
1588 .as_ref()
1589 .is_some_and(|source| !source.matches_parent(&sibling, &parent))
1590 {
1591 tracing::warn!(%child_id, "terminal sibling parent context is stale; leaving wait armed");
1592 return;
1593 }
1594 let result = sibling_completion
1595 .source
1596 .as_ref()
1597 .and_then(|source| source.result(&sibling, &parent));
1598 let mut typed_result =
1599 sealed_child_report(&sibling_completion, &sibling, &parent);
1600 let mut presentation = if let Some(projection) = typed_result.as_ref() {
1601 typed_child_resume_message(
1602 &sibling_completion,
1603 remaining_children,
1604 projection,
1605 )
1606 } else {
1607 runtime_resume_message(
1608 &sibling_completion,
1609 remaining_children,
1610 result.as_deref(),
1611 )
1612 };
1613 let mut envelope = child_completion_envelope(
1614 &sibling_completion,
1615 wait.registered_at,
1616 result.clone(),
1617 &presentation,
1618 );
1619 if let Some(run_id) = parent_run_id.as_ref() {
1620 envelope.correlation_id =
1621 Some(format!("child_completion_after_run:{run_id}"));
1622 }
1623 if let Some(projection) = typed_result.as_mut() {
1624 if !bound_typed_child_delivery(
1625 &mut envelope,
1626 &mut presentation,
1627 projection,
1628 &sibling_completion,
1629 remaining_children,
1630 ) {
1631 tracing::warn!(%child_id, "typed terminal sibling wrapper exceeds budget; leaving wait armed");
1632 return;
1633 }
1634 }
1635 let previous_delivery = match self
1636 .legacy_child_delivery(&parent, &sibling_completion)
1637 .await
1638 {
1639 Ok(delivery) => delivery,
1640 Err(error) => {
1641 tracing::warn!(%child_id, %error,
1642 "cannot verify historical sibling delivery; leaving wait armed");
1643 return;
1644 }
1645 };
1646 if let Some((previous, _)) = previous_delivery {
1647 envelope = previous;
1648 } else if let Err(error) = self
1649 .reuse_legacy_child_delivery(
1650 &mut envelope,
1651 &sibling_completion,
1652 Some(wait),
1653 result.as_deref(),
1654 )
1655 .await
1656 {
1657 tracing::warn!(%child_id, %error,
1658 "cannot verify existing sibling delivery; leaving wait armed");
1659 return;
1660 }
1661 match messenger.admit(envelope).await {
1662 Ok(admission) => {
1663 if let Err(error) = messenger
1664 .prepare_message_activation(
1665 &admission,
1666 SessionActivationPolicy::RespectSpecificWait,
1667 )
1668 .await
1669 {
1670 tracing::warn!(%child_id, %error,
1671 "terminal sibling notification permission failed; leaving wait armed");
1672 return;
1673 }
1674 child_admissions.push(admission.clone());
1675 if child_admission.as_ref().is_none_or(|current| {
1678 admission.delivery.generation > current.delivery.generation
1679 }) {
1680 child_admission = Some(admission);
1681 }
1682 }
1683 Err(error) => {
1684 tracing::warn!(%child_id, %error, "terminal sibling admission failed; leaving parent wait armed");
1685 return;
1686 }
1687 }
1688 }
1689 }
1690 }
1691
1692 if !should_resume && !guardian_control_changed && child_admission.is_some() {
1696 break (parent, true, messenger, child_admission, child_admissions);
1697 }
1698
1699 if should_resume {
1700 runtime_state.waiting_for_children = None;
1701 runtime_state.status = AgentStatusState::Idle;
1702 runtime_state.suspension = None;
1703 parent.metadata.remove("runtime.suspend_reason");
1704
1705 if child_admission.is_none() {
1706 parent.add_message(resume_message);
1710 }
1711 }
1712
1713 parent.updated_at = Utc::now();
1714 write_runtime_state(&mut parent, &runtime_state);
1715 let runtime_only = child_admission.is_some();
1722 let sessions = self.sessions.clone();
1723 let history = expected_parent.clone();
1724 let save = self
1725 .persistence
1726 .compare_exchange_child_wait_and_publish(
1727 &expected_parent,
1728 &mut parent,
1729 runtime_only,
1730 Arc::new(move |saved| {
1731 if runtime_only {
1732 if let Some(cached) = sessions.get(&saved.id) {
1733 cached.update(|current| {
1734 if current.created_at != saved.created_at {
1735 *current = saved.clone();
1736 current.messages = history.messages.clone();
1737 current.provider_transcript =
1738 history.provider_transcript.clone();
1739 current
1740 .runtime_metadata
1741 .get_or_insert_with(Default::default)
1742 .session_inbox_admission =
1743 history.session_inbox_admission().cloned();
1744 return;
1745 }
1746
1747 let messages = current.messages.clone();
1748 let provider_transcript = current.provider_transcript.clone();
1749 let admission =
1750 current.runtime_metadata.as_ref().and_then(|metadata| {
1751 metadata.session_inbox_admission.clone()
1752 });
1753 *current = saved.clone();
1754 current.messages = messages;
1755 current.provider_transcript = provider_transcript;
1756 current
1757 .runtime_metadata
1758 .get_or_insert_with(Default::default)
1759 .session_inbox_admission = admission;
1760 });
1761 } else {
1762 let mut snapshot = saved.clone();
1763 snapshot.messages = history.messages.clone();
1764 snapshot.provider_transcript = history.provider_transcript.clone();
1765 snapshot
1766 .runtime_metadata
1767 .get_or_insert_with(Default::default)
1768 .session_inbox_admission = history
1769 .runtime_metadata
1770 .as_ref()
1771 .and_then(|metadata| metadata.session_inbox_admission.clone());
1772 sessions.insert(
1773 snapshot.id.clone(),
1774 Arc::new(crate::SessionSnapshot::new(snapshot)),
1775 );
1776 }
1777 } else {
1778 sessions.insert(
1779 saved.id.clone(),
1780 Arc::new(crate::SessionSnapshot::new(saved.clone())),
1781 );
1782 }
1783 }),
1784 )
1785 .await;
1786 match save {
1787 Ok(false) => {
1788 continue;
1791 }
1792 Ok(true) => {
1793 break (
1794 parent,
1795 should_resume || (guardian_control_changed && !previously_admitted),
1796 messenger,
1797 child_admission,
1798 child_admissions,
1799 )
1800 }
1801 Err(error) => {
1802 tracing::warn!(parent_session_id = %completion.parent_session_id,
1803 child_session_id = %completion.child_session_id, %error,
1804 "child outcome is durable but parent wait transition failed; leaving activation deferred");
1805 return;
1806 }
1807 }
1808 };
1809
1810 let resume_parent_id = parent.id.clone();
1815 drop(_per_parent_guard);
1816
1817 if should_resume {
1818 if let (Some(messenger), Some(admission)) = (messenger, child_admission) {
1819 for outcome in &child_admissions {
1823 if let Err(error) = messenger
1824 .prepare_message_activation(
1825 outcome,
1826 SessionActivationPolicy::InterruptSpecificWait,
1827 )
1828 .await
1829 {
1830 tracing::warn!(parent_session_id = %resume_parent_id, %error,
1831 "child control transition committed but notification permission failed");
1832 return;
1833 }
1834 }
1835 if let Err(error) = messenger.activate_prepared(&admission).await {
1836 tracing::warn!(
1837 parent_session_id = %resume_parent_id,
1838 %error,
1839 "child outcome and wait transition are durable but activation failed"
1840 );
1841 }
1842 } else {
1843 self.resume_parent(resume_parent_id).await;
1844 }
1845 }
1846 }
1847}
1848
1849#[async_trait]
1850impl ResumeExecutionPort for ChildCompletionCoordinator {
1851 async fn load_session(&self, session_id: &str) -> Option<Session> {
1852 match self.storage.load_session(session_id).await {
1853 Ok(Some(session)) => Some(session),
1854 Ok(None) => self
1855 .sessions
1856 .get(session_id)
1857 .map(|e| e.value().clone())
1858 .map(|arc| arc.read().clone()),
1859 Err(error) => {
1860 tracing::warn!(%session_id, %error, "failed to load session from storage");
1861 self.sessions
1862 .get(session_id)
1863 .map(|e| e.value().clone())
1864 .map(|arc| arc.read().clone())
1865 }
1866 }
1867 }
1868
1869 async fn save_and_cache_session(&self, session: &mut Session) {
1870 self.save_and_cache(session).await;
1871 }
1872
1873 async fn reserve_session_execution(
1874 &self,
1875 session_id: &str,
1876 event_sender: &broadcast::Sender<AgentEvent>,
1877 ) -> SessionExecutionReserveOutcome {
1878 reserve_session_execution(
1879 &self.agent,
1880 &self.agent_runners,
1881 &self.session_event_senders,
1882 session_id,
1883 event_sender,
1884 )
1885 .await
1886 }
1887
1888 async fn get_or_create_event_sender(&self, session_id: &str) -> broadcast::Sender<AgentEvent> {
1889 crate::execution::session_events::get_or_create_event_sender(
1890 &self.session_event_senders,
1891 session_id,
1892 )
1893 .await
1894 }
1895
1896 fn dispatch_resume_execution(
1897 &self,
1898 request: ResumeSpawnRequest,
1899 ) -> Result<(), ResumeSpawnRequest> {
1900 let owner = self.clone();
1901 tokio::spawn(async move {
1902 ResumeExecutionPort::spawn_resume_execution(&owner, request).await;
1903 });
1904 Ok(())
1905 }
1906
1907 async fn spawn_resume_execution(&self, request: ResumeSpawnRequest) {
1908 let ResumeSpawnRequest {
1909 session_id,
1910 mut session,
1911 mut execution_reservation,
1912 event_sender,
1913 config,
1914 } = request;
1915 if let Err(error) = execution_reservation.ensure_registered().await {
1916 tracing::warn!(
1917 %session_id,
1918 run_id = %execution_reservation.run_id(),
1919 %error,
1920 "cannot resume after child completion without exact router ownership"
1921 );
1922 return;
1923 }
1924
1925 if self
1926 .agent
1927 .persistence()
1928 .root_actor_execution_required(&session)
1929 && self.root_account_sink.is_none()
1930 {
1931 tracing::warn!(%session_id, "Root resume has no actual account sink; rejecting before Actor claim");
1932 execution_reservation.abandon().await;
1933 return;
1934 }
1935 if let Err(error) = execution_reservation
1936 .bind_root_actor(&self.agent, &session)
1937 .await
1938 {
1939 tracing::warn!(%session_id, %error, "Root coordinator resume Actor binding rejected");
1940 execution_reservation.abandon().await;
1941 return;
1942 }
1943 let root_publication = execution_reservation.root_actor_writer().map(|owner| {
1944 crate::events::RootActorEventPublication::new(
1945 self.storage.clone(),
1946 self.root_account_sink
1947 .clone()
1948 .expect("required Root actual account sink checked before claim"),
1949 owner,
1950 )
1951 });
1952
1953 let Some(root_tools) = self.root_tools.read().await.clone() else {
1954 tracing::error!(%session_id, "cannot resume parent after child completion: root tool surface is not initialized");
1955 return;
1956 };
1957
1958 let adapter = self
1959 .reserved_root_adapter
1960 .read()
1961 .unwrap_or_else(std::sync::PoisonError::into_inner)
1962 .clone();
1963 let (agent, root_tools) = if let Some(adapter) = adapter {
1964 match adapter(
1965 self.agent.clone(),
1966 &session,
1967 &execution_reservation,
1968 root_tools,
1969 ) {
1970 Ok(bound) => bound,
1971 Err(error) => {
1972 tracing::error!(%session_id, %error, "registered Root execution producer rejected resume");
1973 execution_reservation.abandon().await;
1974 return;
1975 }
1976 }
1977 } else {
1978 (self.agent.clone(), root_tools)
1979 };
1980
1981 let config_snapshot = self.config.read().await.clone();
1982 let model = session.model.clone();
1983 let session_model_ref = session_effective_model_ref(&session);
1984 let requested_provider = session_model_ref
1985 .as_ref()
1986 .map(|model_ref| model_ref.provider.as_str())
1987 .unwrap_or(config.provider_name.as_str());
1988 let resolved_provider_name = match resolve_provider_routing_key(
1989 &config_snapshot,
1990 requested_provider,
1991 &self.provider_registry,
1992 ) {
1993 Ok(provider) => provider,
1994 Err(error) => {
1995 tracing::error!(
1996 session_id = %session_id,
1997 provider = requested_provider,
1998 %error,
1999 "child-completion resume provider is unavailable; refusing to fall back"
2000 );
2001 execution_reservation.abandon().await;
2002 return;
2003 }
2004 };
2005 let provider_override = if let Some(mut model_ref) = session_model_ref {
2006 model_ref.provider = resolved_provider_name.clone();
2007 persist_model_ref(&mut session, &model_ref);
2008 match self.provider_router.route(&model_ref) {
2009 Ok(provider) => Some(provider),
2010 Err(error) => {
2011 tracing::error!(
2012 session_id = %session_id,
2013 provider = %model_ref.provider,
2014 model = %model_ref.model,
2015 %error,
2016 "child-completion resume provider routing failed closed"
2017 );
2018 execution_reservation.abandon().await;
2019 return;
2020 }
2021 }
2022 } else {
2023 match self.provider_registry.get(&resolved_provider_name) {
2024 Some(provider) => Some(provider),
2025 None => {
2026 tracing::error!(
2027 session_id = %session_id,
2028 provider = %resolved_provider_name,
2029 "child-completion resume provider disappeared after resolution; refusing to fall back"
2030 );
2031 execution_reservation.abandon().await;
2032 return;
2033 }
2034 }
2035 };
2036 let resolved_fast_provider = resolve_fast_model(
2037 &config_snapshot,
2038 &resolved_provider_name,
2039 &self.provider_registry,
2040 )
2041 .map(|model| model.provider);
2042 let reasoning_effort = session.reasoning_effort;
2043 let reasoning_effort_source = session
2044 .metadata
2045 .get("reasoning_effort_source")
2046 .cloned()
2047 .unwrap_or_default();
2048 let gold_config = resolve_gold_config(
2049 &config_snapshot,
2050 session
2051 .metadata
2052 .get(GOLD_CONFIG_METADATA_KEY)
2053 .map(String::as_str),
2054 )
2055 .or(config.gold_config.clone());
2056
2057 let (mpsc_tx, _forwarder, history_commit_barrier) =
2058 crate::execution::create_event_forwarder_with_root_actor(
2059 session_id.clone(),
2060 execution_reservation.run_id().to_string(),
2061 event_sender,
2062 self.agent_runners.clone(),
2063 self.account_feed_inbox.clone(),
2064 root_publication,
2065 );
2066
2067 let config_handle = self.config.clone();
2068 let cached_config = Arc::new(StdRwLock::new(config_snapshot.clone()));
2069 let provider_registry = self.provider_registry.clone();
2070 let provider_name_for_aux = resolved_provider_name.clone();
2071 let auxiliary_model_resolver = std::sync::Arc::new(move || {
2072 let config_snapshot = read_config_snapshot(&config_handle, cached_config.as_ref());
2073 let areas = resolve_global_area_models(
2075 &config_snapshot,
2076 &provider_name_for_aux,
2077 &provider_registry,
2078 );
2079 crate::AuxiliaryModelConfig {
2080 fast_model_name: areas.fast.as_ref().map(|m| m.model_name.clone()),
2081 fast_model_provider: areas.fast.map(|m| m.provider),
2082 background_model_name: areas.background.as_ref().map(|m| m.model_name.clone()),
2083 planning_model_name: None,
2084 search_model_name: None,
2085 summarization_model_name: areas
2086 .summarization
2087 .as_ref()
2088 .map(|m| m.model_name.clone()),
2089 background_model_provider: areas.background.map(|m| m.provider),
2090 summarization_model_provider: areas.summarization.map(|m| m.provider),
2091 }
2092 });
2093 let model_roster = crate::ModelRoster {
2094 model: Some(model),
2095 provider_name: Some(resolved_provider_name),
2096 provider_type: config.provider_type.clone(),
2097 fast: crate::RoleModel::from_parts(config.fast_model, resolved_fast_provider),
2098 background: crate::RoleModel::from_parts(
2099 config.background_model,
2100 config.background_model_provider,
2101 ),
2102 summarization: crate::RoleModel::from_parts(
2103 config.summarization_model,
2104 config.summarization_model_provider,
2105 ),
2106 };
2107
2108 let guardian_config = read_guardian_config(&session);
2113 let guardian_spawner = self.guardian_spawner.read().await.clone();
2114
2115 consume_pending_clarification_resume(&mut session);
2116 spawn_session_execution(SessionExecutionArgs {
2117 agent,
2118 session_id,
2119 session,
2120 execution_reservation,
2121 tools_override: Some(root_tools),
2122 provider_override,
2123 model_roster,
2124 reasoning_effort,
2125 reasoning_effort_source,
2126 auxiliary_model_resolver: Some(auxiliary_model_resolver),
2127 disabled_filter_resolver: None,
2130 disabled_tools: Some(config.disabled_tools),
2131 disabled_skill_ids: Some(config.disabled_skill_ids),
2132 selected_skill_ids: None,
2133 selected_skill_mode: None,
2134 mpsc_tx,
2135 history_commit_barrier,
2136 image_fallback: config.image_fallback,
2137 gold_config,
2138 guardian_config,
2139 guardian_spawner,
2140 bash_resume_hook: {
2141 let hook: Arc<dyn BashResumeHook> = Arc::new(self.clone());
2142 Some(hook)
2143 },
2144 bash_completion_sink: {
2145 let sink: Arc<dyn BashCompletionSink> = Arc::new(self.clone());
2148 Some(sink)
2149 },
2150 app_data_dir: Some(self.app_data_dir.clone()),
2151 run_budget: None,
2154 runners: self.agent_runners.clone(),
2155 sessions_cache: self.sessions.clone(),
2156 on_complete: None,
2157 child_completion_handler: Some(Arc::new(self.clone())),
2161 });
2162 }
2163}
2164
2165#[async_trait]
2170impl SessionActivationSpawner for ChildCompletionCoordinator {
2171 async fn reserve_activation(
2172 &self,
2173 target_session_id: &str,
2174 _inbox_generation: u64,
2175 ) -> Result<SessionActivationReserveOutcome, bamboo_domain::SessionActivationError> {
2176 let Some(inbox) = self.agent.session_inbox() else {
2177 return Err(bamboo_domain::SessionActivationError::Internal(
2178 "agent runtime has no SessionInbox".to_string(),
2179 ));
2180 };
2181 let backlog = inbox
2182 .inspect(target_session_id)
2183 .await
2184 .map_err(|error| bamboo_domain::SessionActivationError::Internal(error.to_string()))?;
2185 if !backlog.activation_pending() {
2186 return Ok(SessionActivationReserveOutcome::NoWork);
2187 }
2188 let prepared = prepare_session_inbox_activation(
2191 &self.persistence,
2192 target_session_id,
2193 backlog.interrupt_pending(),
2194 )
2195 .await
2196 .map_err(|error| {
2197 bamboo_domain::SessionActivationError::Internal(format!(
2198 "persist resumable SessionInbox target: {error}"
2199 ))
2200 })?;
2201 let Some((session, ready)) = prepared else {
2202 return Ok(SessionActivationReserveOutcome::NotFound);
2203 };
2204 if !ready {
2205 tracing::info!(
2206 session_id = target_session_id,
2207 "SessionInbox backlog is activation-eligible but a specific durable wait remains armed"
2208 );
2209 return Ok(SessionActivationReserveOutcome::NoWork);
2210 }
2211
2212 enum LaunchPlan {
2213 Root(Box<ResumeConfigSnapshot>),
2214 Child {
2215 scheduler: Arc<SpawnScheduler>,
2216 parent_session_id: String,
2217 model: String,
2218 disabled_tools: Option<Vec<String>>,
2219 },
2220 }
2221
2222 let launch_plan = match session.kind {
2225 SessionKind::Root => {
2226 if self.root_tools.read().await.is_none() {
2227 return Err(bamboo_domain::SessionActivationError::Internal(
2228 "root tool surface is not initialized".to_string(),
2229 ));
2230 }
2231 let config_snapshot = self.config.read().await.clone();
2232 LaunchPlan::Root(Box::new(
2233 self.build_resume_config(&session, &config_snapshot),
2234 ))
2235 }
2236 SessionKind::Child => {
2237 let scheduler = self.spawn_scheduler.read().await.upgrade().ok_or_else(|| {
2238 bamboo_domain::SessionActivationError::Internal(
2239 "child spawn scheduler is not initialized".to_string(),
2240 )
2241 })?;
2242 let parent_session_id = session
2243 .parent_session_id
2244 .as_deref()
2245 .map(str::trim)
2246 .filter(|id| !id.is_empty())
2247 .map(ToOwned::to_owned)
2248 .ok_or_else(|| {
2249 bamboo_domain::SessionActivationError::Internal(format!(
2250 "child SessionInbox target {target_session_id} has no parent owner"
2251 ))
2252 })?;
2253 let parent = self
2254 .storage
2255 .load_session(&parent_session_id)
2256 .await
2257 .map_err(|error| {
2258 bamboo_domain::SessionActivationError::Internal(format!(
2259 "load parent owner {parent_session_id}: {error}"
2260 ))
2261 })?
2262 .ok_or_else(|| {
2263 bamboo_domain::SessionActivationError::Internal(format!(
2264 "child SessionInbox parent owner {parent_session_id} disappeared"
2265 ))
2266 })?;
2267 let child_root = if session.root_session_id.trim().is_empty() {
2268 parent_session_id.as_str()
2269 } else {
2270 session.root_session_id.as_str()
2271 };
2272 let parent_root = if parent.root_session_id.trim().is_empty() {
2273 parent.id.as_str()
2274 } else {
2275 parent.root_session_id.as_str()
2276 };
2277 if child_root != parent_root {
2278 return Err(bamboo_domain::SessionActivationError::Internal(format!(
2279 "child SessionInbox target {target_session_id} does not share its parent owner's root"
2280 )));
2281 }
2282 let model = if session.model.trim().is_empty() {
2283 parent.model.clone()
2284 } else {
2285 session.model.clone()
2286 };
2287 if model.trim().is_empty() {
2288 return Err(bamboo_domain::SessionActivationError::Internal(format!(
2289 "child SessionInbox target {target_session_id} has no executable model"
2290 )));
2291 }
2292 let disabled_tools = match session.metadata.get("disabled_tools") {
2293 None => None,
2294 Some(raw) => {
2295 let tools = serde_json::from_str::<std::collections::BTreeSet<String>>(raw)
2296 .map_err(|error| {
2297 bamboo_domain::SessionActivationError::Internal(format!(
2298 "child SessionInbox target {target_session_id} has malformed disabled_tools: {error}"
2299 ))
2300 })?;
2301 (!tools.is_empty()).then(|| tools.into_iter().collect())
2302 }
2303 };
2304 LaunchPlan::Child {
2305 scheduler,
2306 parent_session_id,
2307 model,
2308 disabled_tools,
2309 }
2310 }
2311 };
2312 let event_sender =
2313 ResumeExecutionPort::get_or_create_event_sender(self, target_session_id).await;
2314
2315 let reservation = match reserve_runner_core(
2316 &self.agent_runners,
2317 &self.session_event_senders,
2318 target_session_id,
2319 &event_sender,
2320 )
2321 .await
2322 {
2323 ReserveOutcome::Reserved(reservation) => reservation,
2324 ReserveOutcome::AlreadyRunning(run_id) => {
2325 return Ok(SessionActivationReserveOutcome::AlreadyRunning { run_id });
2326 }
2327 };
2328 let run_id = reservation.run_id.clone();
2329 let launch = match launch_plan {
2330 LaunchPlan::Root(config) => {
2331 let mut execution_reservation =
2332 SessionExecutionReservation::from_activation_placeholder(
2333 target_session_id,
2334 reservation,
2335 self.agent
2336 .activation_router()
2337 .expect("SessionInbox activation requires an activation router")
2338 .clone(),
2339 self.agent_runners.clone(),
2340 );
2341 if self
2345 .agent
2346 .persistence()
2347 .root_actor_execution_required(&session)
2348 && self.root_account_sink.is_none()
2349 {
2350 execution_reservation
2351 .rollback_unpublished_activation()
2352 .await;
2353 return Err(bamboo_domain::SessionActivationError::Internal(
2354 "Root activation has no actual account sink".into(),
2355 ));
2356 }
2357 if let Err(error) = execution_reservation
2358 .bind_root_actor(&self.agent, &session)
2359 .await
2360 {
2361 execution_reservation
2362 .rollback_unpublished_activation()
2363 .await;
2364 return Err(bamboo_domain::SessionActivationError::Internal(format!(
2365 "Root Actor reservation rejected: {error}"
2366 )));
2367 }
2368 let reservation_cell = Arc::new(StdMutex::new(Some(execution_reservation)));
2372 let launch_reservation = reservation_cell.clone();
2373 let rollback_reservation = reservation_cell;
2374 let coordinator = self.clone();
2375 let launch_sessions = self.sessions.clone();
2376 let launch_session_id = session.id.clone();
2377 let launch_session = session.clone();
2378 let request_session_id = target_session_id.to_string();
2379 SessionActivationLaunch::new_with_async_rollback(
2380 run_id,
2381 move || {
2382 let mut execution_reservation = launch_reservation
2383 .lock()
2384 .unwrap_or_else(std::sync::PoisonError::into_inner)
2385 .take()
2386 .expect("activation reservation launches or rolls back exactly once");
2387 execution_reservation.mark_activation_published();
2388 if execution_reservation.root_actor_writer().is_none() {
2392 launch_sessions.insert(
2393 launch_session_id,
2394 Arc::new(crate::SessionSnapshot::new(launch_session)),
2395 );
2396 }
2397 let request = ResumeSpawnRequest {
2398 session_id: request_session_id,
2399 session,
2400 execution_reservation,
2401 event_sender,
2402 config: *config,
2403 };
2404 tokio::spawn(async move {
2405 ResumeExecutionPort::spawn_resume_execution(&coordinator, request)
2406 .await;
2407 });
2408 },
2409 move || async move {
2410 let reservation = rollback_reservation
2411 .lock()
2412 .unwrap_or_else(std::sync::PoisonError::into_inner)
2413 .take();
2414 if let Some(reservation) = reservation {
2415 reservation.rollback_unpublished_activation().await;
2416 }
2417 },
2418 )
2419 }
2420 LaunchPlan::Child {
2421 scheduler,
2422 parent_session_id,
2423 model,
2424 disabled_tools,
2425 } => {
2426 let execution_reservation =
2427 SessionExecutionReservation::from_activation_placeholder(
2428 target_session_id,
2429 reservation,
2430 self.agent
2431 .activation_router()
2432 .expect("SessionInbox activation requires an activation router")
2433 .clone(),
2434 self.agent_runners.clone(),
2435 );
2436 let reservation_cell = Arc::new(StdMutex::new(Some(execution_reservation)));
2437 let launch_reservation = reservation_cell.clone();
2438 let rollback_reservation = reservation_cell;
2439 let job = SpawnJob {
2440 parent_session_id,
2441 child_session_id: target_session_id.to_string(),
2442 model,
2443 disabled_tools,
2444 };
2445 let launch_sessions = self.sessions.clone();
2446 let launch_session_id = session.id.clone();
2447 let launch_session = session;
2448 SessionActivationLaunch::new_with_async_rollback(
2449 run_id,
2450 move || {
2451 let mut execution_reservation = launch_reservation
2452 .lock()
2453 .unwrap_or_else(std::sync::PoisonError::into_inner)
2454 .take()
2455 .expect("activation reservation launches or rolls back exactly once");
2456 execution_reservation.mark_activation_published();
2457 launch_sessions.insert(
2460 launch_session_id,
2461 Arc::new(crate::SessionSnapshot::new(launch_session)),
2462 );
2463 drop(scheduler.launch_reserved(job, execution_reservation));
2467 },
2468 move || async move {
2469 let reservation = rollback_reservation
2470 .lock()
2471 .unwrap_or_else(std::sync::PoisonError::into_inner)
2472 .take();
2473 if let Some(reservation) = reservation {
2474 reservation.rollback_unpublished_activation().await;
2475 }
2476 },
2477 )
2478 }
2479 };
2480 Ok(SessionActivationReserveOutcome::Reserved(launch))
2481 }
2482}
2483
2484fn bash_completion_resume_message(bash_ids: &[String], timed_out: bool) -> Message {
2494 let body = if timed_out {
2495 format!(
2496 "Runtime notification: the background-Bash wait ceiling was reached while one or more \
2497 shell(s) ({}) may still be running. The session is being resumed so it is not \
2498 stranded; verify their actual status with BashOutput before assuming completion.",
2499 bash_ids.join(", ")
2500 )
2501 } else {
2502 format!(
2503 "Runtime notification: all background Bash shell(s) ({}) have completed. \
2504 Review their output with BashOutput and resume the task from where you left off.",
2505 bash_ids.join(", ")
2506 )
2507 };
2508 let mut message = Message::user(body);
2509 message.metadata = Some(serde_json::json!({
2510 RUNTIME_RESUME_MESSAGE_HIDDEN_KEY: true,
2511 RUNTIME_RESUME_MESSAGE_KIND_KEY: BASH_COMPLETION_RESUME_KIND,
2512 }));
2513 message.never_compress = false;
2514 message
2515}
2516
2517fn bash_resume_should_retry(outcome: &ResumeOutcome, persisted_waiting_for_bash: bool) -> bool {
2530 match outcome {
2531 ResumeOutcome::Started { .. } | ResumeOutcome::NotFound => false,
2532 ResumeOutcome::Completed | ResumeOutcome::AlreadyRunning { .. } => {
2533 persisted_waiting_for_bash
2534 }
2535 }
2536}
2537
2538fn bash_completion_should_resume(
2546 loop_suspended_on_bash: bool,
2547 all_waited_shells_done: bool,
2548) -> bool {
2549 loop_suspended_on_bash && all_waited_shells_done
2550}
2551
2552#[derive(Debug, Clone, Copy, PartialEq, Eq)]
2553enum BashCompletionDeliveryPlan {
2554 DurableOnly,
2558 Activate,
2560 ClearWaitThenActivate,
2562}
2563
2564fn bash_completion_delivery_plan(
2565 loop_suspended_on_bash: bool,
2566 all_waited_shells_done: bool,
2567) -> BashCompletionDeliveryPlan {
2568 if bash_completion_should_resume(loop_suspended_on_bash, all_waited_shells_done) {
2569 BashCompletionDeliveryPlan::ClearWaitThenActivate
2570 } else if loop_suspended_on_bash {
2571 BashCompletionDeliveryPlan::DurableOnly
2572 } else {
2573 BashCompletionDeliveryPlan::Activate
2574 }
2575}
2576
2577fn background_bash_post_tool_payload(info: &BashCompletionInfo) -> HookPayload {
2578 let success = info.status == "completed" && info.exit_code == Some(0);
2579 let response = serde_json::json!({
2580 "bash_id": info.bash_id,
2581 "command": info.command,
2582 "exit_code": info.exit_code,
2583 "status": info.status,
2584 "output_tail": info.output_tail,
2585 });
2586 HookPayload::ToolResult {
2587 tool_name: "Bash".to_string(),
2588 tool_call_id: info.bash_id.clone(),
2589 outcome: HookToolOutcome {
2590 success,
2591 result: success.then(|| response.to_string()),
2592 error: (!success).then(|| response.to_string()),
2593 needs_human: false,
2594 duration_ms: 0,
2595 },
2596 }
2597}
2598
2599fn append_background_bash_hook_feedback(info: &mut BashCompletionInfo, feedback: Vec<String>) {
2600 let feedback = feedback
2601 .into_iter()
2602 .map(|text| text.trim().to_string())
2603 .filter(|text| !text.is_empty())
2604 .collect::<Vec<_>>();
2605 if feedback.is_empty() {
2606 return;
2607 }
2608 if !info.output_tail.is_empty() {
2609 info.output_tail.push_str("\n\n");
2610 }
2611 info.output_tail.push_str("<post_tool_use_feedback>\n");
2612 info.output_tail.push_str(&feedback.join("\n"));
2613 info.output_tail.push_str("\n</post_tool_use_feedback>");
2614}
2615
2616async fn run_background_bash_post_tool_hooks(
2617 config: &bamboo_config::LifecycleHooksConfig,
2618 fallback_cwd: Option<std::path::PathBuf>,
2619 session: &Session,
2620 info: &mut BashCompletionInfo,
2621) -> bool {
2622 let runner = crate::HookRunner::new().with_lifecycle_config(config, fallback_cwd);
2623 if !runner.has_hooks_for(AgentHookPoint::AfterToolExecution) {
2624 return false;
2625 }
2626
2627 let mut runtime_state = session
2628 .agent_runtime_state
2629 .clone()
2630 .unwrap_or_else(|| AgentRuntimeState::new(&session.id));
2631 let outcome = runner
2632 .run_observer_hooks(
2633 AgentHookPoint::AfterToolExecution,
2634 &background_bash_post_tool_payload(info),
2635 session,
2636 &mut runtime_state,
2637 None,
2638 )
2639 .await;
2640 append_background_bash_hook_feedback(info, outcome.injected_contexts);
2641 true
2642}
2643
2644fn apply_bash_resume_transition(session: &mut Session, resume_message: &Message) -> bool {
2652 let mut runtime_state = read_runtime_state(session);
2653 if runtime_state.waiting_for_bash.is_none() {
2654 return false;
2655 }
2656 runtime_state.waiting_for_bash = None;
2657 runtime_state.status = AgentStatusState::Idle;
2658 runtime_state.suspension = None;
2659 write_runtime_state(session, &runtime_state);
2660 session.metadata.remove("runtime.suspend_reason");
2661 session.add_message(resume_message.clone());
2662 true
2663}
2664
2665impl ChildCompletionCoordinator {
2667 async fn bash_self_resume(&self, session_id: String, bash_ids: Vec<String>) {
2683 let mut delay = Duration::from_secs(1);
2684 let max_delay = Duration::from_secs(30);
2685 let max_poll = Duration::from_secs(6 * 3600 + 600);
2689 let deadline = tokio::time::Instant::now() + max_poll;
2690
2691 loop {
2692 tokio::time::sleep(delay).await;
2693
2694 let Some(session) = self.load_session(&session_id).await else {
2695 tracing::info!(%session_id, "bash self-resume backstop: session gone; nothing to do");
2696 return;
2697 };
2698 if read_runtime_state(&session).waiting_for_bash.is_none() {
2699 return;
2702 }
2703
2704 let still_running =
2705 bamboo_tools::tools::bash_runtime::running_shells_for_session(&session_id);
2706 let timed_out = tokio::time::Instant::now() >= deadline;
2707 if still_running.is_empty() || timed_out {
2708 let guard = session_resume_lock(&session_id);
2713 let _held = guard.lock().await;
2714 tracing::warn!(
2715 %session_id,
2716 shell_count = bash_ids.len(),
2717 timed_out,
2718 "bash self-resume backstop engaged (push lost or wait ceiling reached)"
2719 );
2720 self.perform_bash_resume(
2721 &session_id,
2722 bash_completion_resume_message(&bash_ids, timed_out),
2723 )
2724 .await;
2725 return;
2726 }
2727
2728 delay = (delay * 2).min(max_delay);
2729 }
2730 }
2731
2732 async fn perform_bash_resume(&self, session_id: &str, resume_message: Message) {
2753 let retry_backoff = Duration::from_millis(200);
2754 const MAX_RESUME_ATTEMPTS: u8 = 5;
2755 for attempt in 0..MAX_RESUME_ATTEMPTS {
2756 if attempt > 0 {
2757 tokio::time::sleep(retry_backoff).await;
2758 }
2759
2760 let Some(mut session) = self.load_session(session_id).await else {
2761 tracing::warn!(%session_id, "bash resume: session not found; nothing to resume");
2762 return;
2763 };
2764
2765 if !apply_bash_resume_transition(&mut session, &resume_message) {
2766 tracing::info!(
2770 %session_id, attempt,
2771 "bash resume: persisted bash wait already cleared; nothing to resume"
2772 );
2773 return;
2774 }
2775 session.updated_at = Utc::now();
2776 self.save_and_cache(&mut session).await;
2777 tracing::info!(
2778 %session_id, attempt,
2779 "bash resume: cleared bash wait and appended resume message"
2780 );
2781
2782 let outcome = self.resume_parent(session_id.to_string()).await;
2783 match outcome {
2784 ResumeOutcome::Started { .. } => {
2785 tracing::info!(%session_id, attempt, "bash resume: resume fired");
2786 return;
2787 }
2788 ResumeOutcome::NotFound => {
2789 tracing::warn!(%session_id, "bash resume: session vanished during resume");
2790 return;
2791 }
2792 _ => {
2793 let clobbered = match self.load_session(session_id).await {
2799 Some(reloaded) => read_runtime_state(&reloaded).waiting_for_bash.is_some(),
2800 None => {
2801 tracing::warn!(%session_id, "bash resume: session vanished after resume");
2802 return;
2803 }
2804 };
2805 if bash_resume_should_retry(&outcome, clobbered) {
2806 tracing::warn!(
2807 %session_id, attempt,
2808 outcome = outcome.as_str(),
2809 "bash resume: persisted wait still set after resume (finalize-clobber); retrying"
2810 );
2811 continue;
2812 }
2813 tracing::info!(
2814 %session_id, attempt,
2815 outcome = outcome.as_str(),
2816 "bash resume: wait cleared and resume handled; stopping"
2817 );
2818 return;
2819 }
2820 }
2821 }
2822
2823 tracing::warn!(
2824 %session_id,
2825 attempts = MAX_RESUME_ATTEMPTS,
2826 "bash resume: exhausted clobber-retry budget without confirming resume; giving up"
2827 );
2828 }
2829}
2830
2831impl BashResumeHook for ChildCompletionCoordinator {
2832 fn arrange_bash_self_resume(&self, session_id: String, bash_ids: Vec<String>) {
2833 let coordinator = Arc::new(self.clone());
2834 tokio::spawn(async move {
2835 coordinator.bash_self_resume(session_id, bash_ids).await;
2836 });
2837 }
2838}
2839
2840fn bash_completion_injection_body(info: &BashCompletionInfo) -> String {
2845 let exit = match info.exit_code {
2846 Some(code) => code.to_string(),
2847 None => "none (signal/killed)".to_string(),
2848 };
2849 let mut body = format!(
2850 "Runtime notification: background shell `{}` (`{}`) finished — status {}, exit code {}.",
2851 info.bash_id, info.command, info.status, exit
2852 );
2853 if info.output_tail.trim().is_empty() {
2854 body.push_str(" It produced no captured output.");
2855 } else {
2856 body.push_str("\n\nOutput tail:\n");
2857 body.push_str(&info.output_tail);
2858 }
2859 body.push_str(&format!(
2860 "\n\nUse BashOutput with bash_id=\"{}\" for the full output, then continue the task.",
2861 info.bash_id
2862 ));
2863 body
2864}
2865
2866fn bash_completion_envelope(info: &BashCompletionInfo) -> bamboo_domain::SessionMessageEnvelope {
2867 let provider_message = bash_resume_message_from_info(info);
2868 let data = serde_json::json!({
2869 "session_id": info.session_id,
2870 "bash_id": info.bash_id,
2871 "command": info.command,
2872 "exit_code": info.exit_code,
2873 "status": info.status,
2874 "output_tail": info.output_tail,
2875 });
2876 let identity = data.clone();
2881 bamboo_domain::SessionMessageEnvelope {
2882 id: bamboo_domain::SessionMessageId::stable("background_bash_completion", &identity),
2883 source: bamboo_domain::SessionMessageSource::Runtime {
2884 subsystem: "background_bash".to_string(),
2885 },
2886 target_session_id: info.session_id.clone(),
2887 kind: bamboo_domain::SessionMessageKind::RuntimeInstruction,
2888 body: bamboo_domain::SessionMessageBody::RuntimeInstruction(
2889 bamboo_domain::SessionRuntimeInstruction {
2890 instruction: "background_bash_completed".to_string(),
2891 content: Some(bamboo_domain::SessionMessageContent::text(
2892 bash_completion_injection_body(info),
2893 )),
2894 data: Some(data),
2895 provider_message: Some(session_provider_message(&provider_message)),
2896 },
2897 ),
2898 created_at: Utc::now(),
2899 thread_id: None,
2900 in_reply_to: None,
2901 attempt: None,
2902 correlation_id: Some(info.bash_id.clone()),
2903 }
2904}
2905
2906async fn enqueue_bash_completion_injection(
2912 persistence: &LockedSessionStore,
2913 info: &BashCompletionInfo,
2914) -> std::io::Result<Option<Session>> {
2915 let body = bash_completion_injection_body(info);
2916 let queued = serde_json::json!({
2917 "content": body,
2918 "created_at": Utc::now(),
2919 });
2920 persistence
2921 .update_runtime_config(&info.session_id, move |session| {
2922 let mut pending = session.pending_injected_messages().unwrap_or_default();
2923 pending.push(queued);
2924 session.set_pending_injected_messages(pending);
2925 })
2926 .await
2927}
2928
2929fn bash_resume_message_from_info(info: &BashCompletionInfo) -> Message {
2937 let mut message = Message::user(bash_completion_injection_body(info));
2938 message.metadata = Some(serde_json::json!({
2939 RUNTIME_RESUME_MESSAGE_HIDDEN_KEY: true,
2940 RUNTIME_RESUME_MESSAGE_KIND_KEY: BASH_COMPLETION_RESUME_KIND,
2941 }));
2942 message.never_compress = false;
2943 message
2944}
2945
2946impl ChildCompletionCoordinator {
2947 async fn deliver_bash_completion(&self, mut info: BashCompletionInfo) {
2959 if let Some(hook_session) = self.load_session(&info.session_id).await {
2965 let config_snapshot = self.config.read().await.clone();
2966 let _ = run_background_bash_post_tool_hooks(
2967 &config_snapshot.lifecycle_hooks,
2968 Some(self.app_data_dir.clone()),
2969 &hook_session,
2970 &mut info,
2971 )
2972 .await;
2973 }
2974
2975 let guard = session_resume_lock(&info.session_id);
2976 let _held = guard.lock().await;
2977
2978 let Some(session) = self.load_session(&info.session_id).await else {
2979 tracing::warn!(
2980 session_id = %info.session_id,
2981 bash_id = %info.bash_id,
2982 "background bash completion: owning session not found; nothing to notify"
2983 );
2984 return;
2985 };
2986
2987 let waiting = read_runtime_state(&session).waiting_for_bash.is_some();
2988 let all_shells_done =
2994 bamboo_tools::tools::bash_runtime::running_shells_for_session(&info.session_id)
2995 .is_empty();
2996 let delivery_plan = bash_completion_delivery_plan(waiting, all_shells_done);
2997
2998 let messenger = self.agent.session_messenger().cloned();
3004 let admission = match messenger.as_ref() {
3005 Some(messenger) => match messenger.admit(bash_completion_envelope(&info)).await {
3006 Ok(admission) => Some(admission),
3007 Err(error) => {
3008 tracing::warn!(
3009 session_id = %info.session_id,
3010 bash_id = %info.bash_id,
3011 %error,
3012 "background bash completion: durable SessionInbox admission failed; leaving wait armed"
3013 );
3014 return;
3015 }
3016 },
3017 None => {
3018 tracing::warn!(
3019 session_id = %info.session_id,
3020 bash_id = %info.bash_id,
3021 "SessionMessenger unavailable; using compatibility injection before clearing wait"
3022 );
3023 match enqueue_bash_completion_injection(&self.persistence, &info).await {
3024 Ok(Some(_)) => None,
3025 Ok(None) => {
3026 tracing::warn!(
3027 session_id = %info.session_id,
3028 "background bash compatibility target disappeared"
3029 );
3030 return;
3031 }
3032 Err(error) => {
3033 tracing::warn!(
3034 session_id = %info.session_id,
3035 %error,
3036 "background bash compatibility admission failed; leaving wait armed"
3037 );
3038 return;
3039 }
3040 }
3041 }
3042 };
3043
3044 if delivery_plan == BashCompletionDeliveryPlan::DurableOnly {
3045 tracing::info!(
3046 session_id = %info.session_id,
3047 bash_id = %info.bash_id,
3048 "background bash completion is durable; sibling shells still run, so wait remains armed"
3049 );
3050 return;
3051 }
3052
3053 if let (Some(messenger), Some(admission)) = (messenger.as_ref(), admission.as_ref()) {
3054 if let Err(error) = messenger.prepare_activation(admission).await {
3055 tracing::warn!(
3056 session_id = %info.session_id,
3057 bash_id = %info.bash_id,
3058 %error,
3059 "background bash completion activation watermark failed; leaving wait armed"
3060 );
3061 return;
3062 }
3063 }
3064
3065 if delivery_plan == BashCompletionDeliveryPlan::ClearWaitThenActivate {
3066 tracing::info!(
3067 session_id = %info.session_id,
3068 bash_id = %info.bash_id,
3069 status = %info.status,
3070 "background bash completion: push-resuming suspended loop (event-driven)"
3071 );
3072 let mut resumable = session.clone();
3073 let mut runtime_state = read_runtime_state(&resumable);
3074 runtime_state.waiting_for_bash = None;
3075 runtime_state.status = AgentStatusState::Idle;
3076 runtime_state.suspension = None;
3077 write_runtime_state(&mut resumable, &runtime_state);
3078 resumable.metadata.remove("runtime.suspend_reason");
3079 resumable.updated_at = Utc::now();
3080 if let Err(error) = self.persistence.merge_save_runtime(&mut resumable).await {
3081 tracing::warn!(
3082 session_id = %info.session_id,
3083 %error,
3084 "background bash completion is durable but wait-state clear failed; leaving activation to the wait backstop"
3085 );
3086 return;
3087 }
3088 self.sessions.insert(
3089 resumable.id.clone(),
3090 Arc::new(crate::SessionSnapshot::new(resumable)),
3091 );
3092 }
3093
3094 let (Some(messenger), Some(admission)) = (messenger, admission) else {
3098 if delivery_plan == BashCompletionDeliveryPlan::ClearWaitThenActivate {
3099 let _ = self.resume_parent(info.session_id.clone()).await;
3100 }
3101 return;
3102 };
3103 match messenger.activate_prepared(&admission).await {
3104 Ok(receipt) => tracing::info!(
3105 session_id = %info.session_id,
3106 bash_id = %info.bash_id,
3107 status = %info.status,
3108 waiting,
3109 generation = receipt.delivery.generation,
3110 activation = ?receipt.activation,
3111 "background bash completion delivered through SessionMessenger"
3112 ),
3113 Err(error) => tracing::warn!(
3114 session_id = %info.session_id,
3115 bash_id = %info.bash_id,
3116 %error,
3117 "background bash completion: SessionMessenger delivery failed"
3118 ),
3119 }
3120 }
3121}
3122
3123impl BashCompletionSink for ChildCompletionCoordinator {
3124 fn on_bash_completed(&self, info: BashCompletionInfo) {
3125 let coordinator = Arc::new(self.clone());
3129 tokio::spawn(async move {
3130 coordinator.deliver_bash_completion(info).await;
3131 });
3132 }
3133}
3134
3135const CHILD_WAIT_SWEEP_INTERVAL_SECS: u64 = 30;
3141const CHILD_WAIT_REGISTRATION_GRACE_SECS: i64 = 60;
3145const DEAD_CHILD_GRACE_SECS: i64 = 120;
3148const STALE_RUNNER_SLACK_SECS: i64 = 600;
3154
3155fn is_dead_child_candidate_status(status: Option<&str>) -> bool {
3160 match status {
3161 Some(status) => !is_terminal_child_status(status) && status != "suspended",
3164 None => true,
3165 }
3166}
3167
3168fn completion_child_is_owned(reported_parent: &str, child_parent_linkage: Option<&str>) -> bool {
3176 child_parent_linkage == Some(reported_parent)
3177}
3178
3179fn child_wait_watchdog_resume_message(body: String) -> Message {
3180 let mut message = Message::user(body);
3181 message.metadata = Some(serde_json::json!({
3182 RUNTIME_RESUME_MESSAGE_HIDDEN_KEY: true,
3183 RUNTIME_RESUME_MESSAGE_KIND_KEY: "child_wait_watchdog_resume",
3184 }));
3185 message.never_compress = false;
3186 message
3187}
3188
3189fn empty_child_wait_message() -> Message {
3190 let inspection = serde_json::json!({"intent": "inspect", "message": "tree"});
3191 child_wait_watchdog_resume_message(format!(
3192 "Runtime notification: this session was suspended waiting for child sessions, but the \
3193 wait tracked no children (internal inconsistency). The session has been resumed; use \
3194 SubAgent with {inspection} to inspect child state and continue the task.",
3195 ))
3196}
3197
3198fn child_wait_lease_expired_message(child_ids: &[String]) -> Message {
3199 let inspection = serde_json::json!({"intent": "inspect", "message": "tree"});
3200 child_wait_watchdog_resume_message(format!(
3201 "Runtime notification: the wait lease for child session(s) [{}] expired before they all \
3202 reported completion. They were NOT cancelled and may still be running or already \
3203 finished — verify their actual status by calling SubAgent with {inspection} before assuming \
3204 anything, then continue the task.",
3205 child_ids.join(", ")
3206 ))
3207}
3208
3209impl ChildCompletionCoordinator {
3225 pub fn spawn_child_wait_watchdog(self: &Arc<Self>) {
3228 let coordinator = Arc::clone(self);
3229 tokio::spawn(async move {
3230 use futures::FutureExt;
3231 if std::panic::AssertUnwindSafe(coordinator.reconcile_orphans_at_boot())
3232 .catch_unwind()
3233 .await
3234 .is_err()
3235 {
3236 tracing::error!("child-wait watchdog: boot reconciliation panicked");
3237 }
3238 let mut ticker = tokio::time::interval(std::time::Duration::from_secs(
3239 CHILD_WAIT_SWEEP_INTERVAL_SECS,
3240 ));
3241 ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
3242 ticker.tick().await;
3244 loop {
3245 ticker.tick().await;
3246 if std::panic::AssertUnwindSafe(coordinator.sweep_child_waits())
3247 .catch_unwind()
3248 .await
3249 .is_err()
3250 {
3251 tracing::error!("child-wait watchdog: sweep panicked; continuing");
3252 }
3253 }
3254 });
3255 }
3256
3257 async fn reconcile_orphans_at_boot(&self) {
3262 let cutoff = Utc::now();
3263
3264 let running = self
3270 .storage
3271 .list_sessions_by_run_status("running")
3272 .await
3273 .unwrap_or_default();
3274 for (child_id, parent_id) in running {
3275 let Some(parent_id) = parent_id else { continue };
3276 if self.runner_is_running(&child_id).await {
3277 continue;
3278 }
3279 let Some(control_plane) = self.load_control_plane(&child_id).await else {
3280 continue;
3281 };
3282 if control_plane.updated_at >= cutoff {
3285 continue;
3286 }
3287 tracing::warn!(
3288 child_session_id = %child_id,
3289 parent_session_id = %parent_id,
3290 "boot reconciliation: child was running when the process died; \
3291 marking it error and waking the parent"
3292 );
3293 self.synthesize_child_completion(
3294 &parent_id,
3295 &child_id,
3296 "error",
3297 Some(
3298 "orphaned by server restart: the process died while this child session \
3299 was running"
3300 .to_string(),
3301 ),
3302 )
3303 .await;
3304 }
3305
3306 let suspended = self
3310 .storage
3311 .list_sessions_by_run_status("suspended")
3312 .await
3313 .unwrap_or_default();
3314 for (session_id, _) in suspended {
3315 let Some(control_plane) = self.load_control_plane(&session_id).await else {
3316 continue;
3317 };
3318 if control_plane
3319 .metadata
3320 .get("runtime.suspend_reason")
3321 .map(String::as_str)
3322 != Some("waiting_for_bash")
3323 {
3324 continue;
3325 }
3326 if let Some(wait) = read_runtime_state(&control_plane).waiting_for_bash {
3327 tracing::warn!(
3328 %session_id,
3329 "boot reconciliation: re-arming bash self-resume backstop lost in restart"
3330 );
3331 let coordinator = self.clone();
3332 tokio::spawn(async move {
3333 coordinator
3334 .bash_self_resume(session_id, wait.bash_ids)
3335 .await;
3336 });
3337 }
3338 }
3339 }
3340
3341 async fn runner_is_running(&self, session_id: &str) -> bool {
3342 let runners = self.agent_runners.read().await;
3343 runners
3344 .get(session_id)
3345 .is_some_and(|runner| matches!(runner.status, AgentStatus::Running))
3346 }
3347
3348 async fn load_control_plane(&self, session_id: &str) -> Option<Session> {
3349 match self.storage.load_runtime_control_plane(session_id).await {
3350 Ok(session) => session,
3351 Err(error) => {
3352 tracing::warn!(
3353 %session_id,
3354 %error,
3355 "child-wait watchdog: failed to load session control plane"
3356 );
3357 None
3358 }
3359 }
3360 }
3361
3362 async fn sweep_child_waits(&self) {
3366 let suspended = match self.storage.list_sessions_by_run_status("suspended").await {
3367 Ok(entries) => entries,
3368 Err(error) => {
3369 tracing::warn!(%error, "child-wait watchdog: failed to list suspended sessions");
3370 return;
3371 }
3372 };
3373 for (session_id, _) in suspended {
3374 self.sweep_one_suspended_session(&session_id).await;
3375 }
3376 }
3377
3378 async fn sweep_one_suspended_session(&self, session_id: &str) {
3379 if self.runner_is_running(session_id).await {
3382 return;
3383 }
3384 let Some(session) = self.load_control_plane(session_id).await else {
3385 return;
3386 };
3387 let runtime_state = read_runtime_state(&session);
3388 let suspend_reason = session
3389 .metadata
3390 .get("runtime.suspend_reason")
3391 .map(String::as_str)
3392 .unwrap_or_default()
3393 .to_string();
3394 match (
3395 suspend_reason.as_str(),
3396 runtime_state.waiting_for_children.clone(),
3397 ) {
3398 ("waiting_for_bash", _)
3400 | ("awaiting_clarification", _)
3401 | ("awaiting_parent_approval", _) => {}
3402 (_, Some(wait)) => self.sweep_child_wait(session_id, wait).await,
3403 ("waiting_for_children", None) | ("", None) => {
3406 self.rescue_stranded_resume(session_id).await;
3407 }
3408 _ => {}
3409 }
3410 }
3411
3412 async fn sweep_child_wait(&self, parent_session_id: &str, wait: WaitingForChildrenState) {
3417 let now = Utc::now();
3418
3419 if wait.child_session_ids.is_empty() {
3420 tracing::warn!(
3421 %parent_session_id,
3422 "child-wait watchdog: wait armed over an empty child set; force-resuming"
3423 );
3424 self.force_resume_child_wait(parent_session_id, empty_child_wait_message())
3425 .await;
3426 return;
3427 }
3428
3429 if wait.timeout_at.is_some_and(|deadline| now >= deadline) {
3433 tracing::warn!(
3434 %parent_session_id,
3435 "child-wait watchdog: wait lease expired; force-resuming parent"
3436 );
3437 self.force_resume_child_wait(
3438 parent_session_id,
3439 child_wait_lease_expired_message(&wait.child_session_ids),
3440 )
3441 .await;
3442 return;
3443 }
3444
3445 if now.signed_duration_since(wait.registered_at).num_seconds()
3447 < CHILD_WAIT_REGISTRATION_GRACE_SECS
3448 {
3449 return;
3450 }
3451
3452 let statuses: HashMap<String, Option<String>> = self
3453 .storage
3454 .list_child_run_statuses(parent_session_id)
3455 .await
3456 .unwrap_or_default()
3457 .into_iter()
3458 .collect();
3459
3460 struct DeadChild {
3461 child_id: String,
3462 status: String,
3463 reason: String,
3464 owned: bool,
3468 }
3469
3470 let mut terminal: Vec<(String, String)> = Vec::new();
3471 let mut dead: Vec<DeadChild> = Vec::new();
3472 for child_id in &wait.child_session_ids {
3473 let status = statuses.get(child_id).and_then(|status| status.as_deref());
3474 if let Some(status) = status {
3475 if is_terminal_child_status(status) {
3476 terminal.push((child_id.clone(), status.to_string()));
3477 continue;
3478 }
3479 }
3480 if !is_dead_child_candidate_status(status) {
3481 continue;
3482 }
3483
3484 let control_plane = self.load_control_plane(child_id).await;
3491 let owned = control_plane
3492 .as_ref()
3493 .is_some_and(|cp| cp.parent_session_id.as_deref() == Some(parent_session_id));
3494 if !owned {
3495 dead.push(DeadChild {
3496 child_id: child_id.clone(),
3497 status: "error".to_string(),
3498 reason: if control_plane.is_some() {
3499 "waited-on session id is not a child of this session; clearing it \
3500 from the wait without touching that session"
3501 .to_string()
3502 } else {
3503 "waited-on child session does not exist".to_string()
3504 },
3505 owned: false,
3506 });
3507 continue;
3508 }
3509
3510 let runner = { self.agent_runners.read().await.get(child_id).cloned() };
3511 match runner {
3512 Some(runner) if matches!(runner.status, AgentStatus::Running) => {
3513 let last_activity = runner.last_activity_at().unwrap_or(runner.started_at);
3519 let idle_secs = now.signed_duration_since(last_activity).num_seconds();
3520 let total_secs = now.signed_duration_since(runner.started_at).num_seconds();
3521 let policy = match &control_plane {
3522 Some(child) => {
3523 crate::runtime::execution::spawn::watchdog_policy_for_session(child)
3524 }
3525 None => Default::default(),
3526 };
3527 let idle_limit = policy.max_idle_secs.saturating_add(STALE_RUNNER_SLACK_SECS);
3528 let total_limit = policy
3529 .max_total_secs
3530 .saturating_add(STALE_RUNNER_SLACK_SECS);
3531 if idle_secs >= idle_limit || total_secs >= total_limit {
3532 runner.cancel_token.cancel();
3533 dead.push(DeadChild {
3534 child_id: child_id.clone(),
3535 status: "timeout".to_string(),
3536 reason: format!(
3537 "child runner stalled: no events for {idle_secs}s \
3538 (limit {idle_limit}s including watchdog slack); \
3539 force-finalized by the child-wait watchdog"
3540 ),
3541 owned: true,
3542 });
3543 }
3544 }
3545 _ => {
3546 let quiet_secs = control_plane
3550 .as_ref()
3551 .map(|child| now.signed_duration_since(child.updated_at).num_seconds())
3552 .unwrap_or(i64::MAX);
3553 if quiet_secs >= DEAD_CHILD_GRACE_SECS {
3554 dead.push(DeadChild {
3555 child_id: child_id.clone(),
3556 status: "error".to_string(),
3557 reason: format!(
3558 "child runner lost (crashed task, dropped spawn job, or \
3559 process restart): index status {status:?} with no live \
3560 runner driving it"
3561 ),
3562 owned: true,
3563 });
3564 }
3565 }
3566 }
3567 }
3568
3569 if !dead.is_empty() {
3570 for entry in dead {
3571 tracing::warn!(
3572 %parent_session_id,
3573 child_session_id = %entry.child_id,
3574 status = %entry.status,
3575 reason = %entry.reason,
3576 owned = entry.owned,
3577 "child-wait watchdog: synthesizing terminal completion for dead child"
3578 );
3579 if entry.owned {
3580 self.synthesize_child_completion(
3581 parent_session_id,
3582 &entry.child_id,
3583 &entry.status,
3584 Some(entry.reason),
3585 )
3586 .await;
3587 } else {
3588 self.publish_synthetic_completion(
3590 parent_session_id,
3591 &entry.child_id,
3592 &entry.status,
3593 Some(entry.reason),
3594 )
3595 .await;
3596 }
3597 }
3598 return;
3600 }
3601
3602 terminal.sort_by_key(|(_, status)| !is_error_like(status));
3607 for (child_id, status) in terminal {
3608 tracing::debug!(
3609 %parent_session_id,
3610 child_session_id = %child_id,
3611 "child-wait watchdog: replaying terminal child outcome"
3612 );
3613 let control_plane = self.load_control_plane(&child_id).await;
3614 let mut completion = ChildCompletion {
3615 parent_session_id: parent_session_id.into(),
3616 child_session_id: child_id.clone(),
3617 status,
3618 error: control_plane.as_ref().and_then(Session::last_run_error),
3619 completed_at: Utc::now(),
3620 source: None,
3621 };
3622 if control_plane
3625 .as_ref()
3626 .is_some_and(|child| child.parent_session_id.as_deref() == Some(parent_session_id))
3627 {
3628 let Ok(Some(child)) = self.storage.load_session(&child_id).await else {
3629 continue;
3630 };
3631 completion.source = ChildCompletionSource::from_committed_session(&child)
3632 .filter(|source| source.matches_completion(&completion));
3633 if ChildCompletionSource::has_source_record(&child) && completion.source.is_none() {
3634 tracing::warn!(%child_id,
3635 "unavailable committed replay source; leaving wait armed");
3636 continue;
3637 }
3638 completion.completed_at = child.updated_at;
3639 }
3640 self.publish_completion(completion).await;
3641 }
3642 }
3643
3644 async fn synthesize_child_completion(
3650 &self,
3651 parent_session_id: &str,
3652 child_session_id: &str,
3653 status: &str,
3654 error: Option<String>,
3655 ) {
3656 match self.storage.load_session(child_session_id).await {
3657 Ok(Some(mut child)) => {
3658 if child.parent_session_id.as_deref() != Some(parent_session_id) {
3663 tracing::warn!(
3664 %parent_session_id,
3665 child_session_id = %child.id,
3666 "child-wait watchdog: refusing to synthesize status onto a session \
3667 that is not a child of this parent"
3668 );
3669 self.publish_synthetic_completion(
3670 parent_session_id,
3671 child_session_id,
3672 status,
3673 error,
3674 )
3675 .await;
3676 return;
3677 }
3678 child.set_last_run_status(status);
3679 match &error {
3680 Some(message) => child.set_last_run_error(message.clone()),
3681 None => child.clear_last_run_error(),
3682 }
3683 child.updated_at = Utc::now();
3684 if let Err(save_error) = self.persistence.merge_save_runtime(&mut child).await {
3685 tracing::warn!(
3686 child_session_id = %child.id,
3687 %save_error,
3688 "child-wait watchdog: failed to persist synthesized terminal status"
3689 );
3690 }
3691 self.sessions.insert(
3692 child.id.clone(),
3693 Arc::new(crate::SessionSnapshot::new(child)),
3694 );
3695 }
3696 Ok(None) => {}
3697 Err(load_error) => {
3698 tracing::warn!(
3699 %child_session_id,
3700 %load_error,
3701 "child-wait watchdog: failed to load child for synthesized terminal status"
3702 );
3703 }
3704 }
3705 finalize_runner(
3706 &self.agent_runners,
3707 child_session_id,
3708 &Err(bamboo_agent_core::AgentError::LLM(
3709 error
3710 .clone()
3711 .unwrap_or_else(|| format!("synthesized {status}")),
3712 )),
3713 )
3714 .await;
3715 self.publish_synthetic_completion(parent_session_id, child_session_id, status, error)
3716 .await;
3717 }
3718
3719 async fn publish_synthetic_completion(
3720 &self,
3721 parent_session_id: &str,
3722 child_session_id: &str,
3723 status: &str,
3724 error: Option<String>,
3725 ) {
3726 self.publish_completion(ChildCompletion {
3727 parent_session_id: parent_session_id.into(),
3728 child_session_id: child_session_id.into(),
3729 status: status.into(),
3730 error,
3731 completed_at: Utc::now(),
3732 source: None,
3733 })
3734 .await;
3735 }
3736
3737 async fn publish_completion(&self, completion: ChildCompletion) {
3738 let publisher =
3739 crate::runtime::execution::session_events::ReplayableSessionEventPublisher::new(
3740 self.agent_runners.clone(),
3741 self.session_event_senders.clone(),
3742 self.account_feed_inbox.clone(),
3743 );
3744 let handler: Arc<dyn ChildCompletionHandler> = Arc::new(self.clone());
3745 crate::runtime::execution::spawn::publish_child_completion(
3746 &publisher,
3747 Some(handler),
3748 completion,
3749 )
3750 .await;
3751 }
3752
3753 async fn rescue_stranded_resume(&self, session_id: &str) {
3759 let Some(session) = self.load_session(session_id).await else {
3760 return;
3761 };
3762 let pending_runtime_resume = session.messages.last().is_some_and(|message| {
3763 matches!(message.role, Role::User)
3764 && message
3765 .metadata
3766 .as_ref()
3767 .is_some_and(|meta| meta.get(RUNTIME_RESUME_MESSAGE_KIND_KEY).is_some())
3768 });
3769 if !pending_runtime_resume {
3770 return;
3771 }
3772 tracing::warn!(
3773 %session_id,
3774 "child-wait watchdog: stranded resume detected (wait cleared, resume never \
3775 spawned); resuming"
3776 );
3777 self.resume_parent(session_id.to_string()).await;
3778 }
3779
3780 async fn force_resume_child_wait(&self, session_id: &str, resume_message: Message) {
3786 const MAX_ATTEMPTS: u8 = 5;
3787 let lock = session_resume_lock(session_id);
3788 for attempt in 0..MAX_ATTEMPTS {
3789 if attempt > 0 {
3790 tokio::time::sleep(Duration::from_millis(200)).await;
3791 }
3792 {
3793 let _held = lock.lock().await;
3794 let Some(mut session) = self.load_session(session_id).await else {
3795 return;
3796 };
3797 let mut runtime_state = read_runtime_state(&session);
3798 if runtime_state.waiting_for_children.is_none() {
3799 return;
3801 }
3802 runtime_state.waiting_for_children = None;
3803 runtime_state.status = AgentStatusState::Idle;
3804 runtime_state.suspension = None;
3805 write_runtime_state(&mut session, &runtime_state);
3806 session.metadata.remove("runtime.suspend_reason");
3807 session.add_message(resume_message.clone());
3808 session.updated_at = Utc::now();
3809 self.save_and_cache(&mut session).await;
3810 }
3811 let outcome = self.resume_parent(session_id.to_string()).await;
3812 match outcome {
3813 ResumeOutcome::Started { .. } | ResumeOutcome::NotFound => return,
3814 ResumeOutcome::Completed | ResumeOutcome::AlreadyRunning { .. } => {
3815 let clobbered = self
3819 .load_session(session_id)
3820 .await
3821 .map(|session| read_runtime_state(&session).waiting_for_children.is_some())
3822 .unwrap_or(false);
3823 if !clobbered {
3824 return;
3825 }
3826 }
3827 }
3828 }
3829 tracing::error!(
3830 %session_id,
3831 "child-wait watchdog: force-resume exhausted its clobber-retry budget"
3832 );
3833 }
3834}
3835
3836#[cfg(test)]
3837mod tests {
3838 use super::*;
3839 use bamboo_domain::session::runtime_state::SuspensionState;
3840
3841 #[test]
3842 fn skill_request_child_provider_constructor_keeps_data_absent_and_rejects_injection() {
3843 let provider = Message::user("rendered child outcome");
3844 let presentation = session_provider_message(&provider);
3845 assert_eq!(presentation.content.skill_request, None);
3846 assert_eq!(presentation.content.text, provider.content);
3847 let mut envelope = SessionMessageEnvelope::user_input("parent", "unused");
3848 envelope.source = SessionMessageSource::Runtime {
3849 subsystem: "child_completion_coordinator".into(),
3850 };
3851 envelope.kind = SessionMessageKind::ChildOutcome;
3852 envelope.body = SessionMessageBody::ChildOutcome(SessionChildOutcome {
3853 child_session_id: "child".into(),
3854 status: "completed".into(),
3855 result: Some("typed result".into()),
3856 error: None,
3857 provider_message: Some(presentation),
3858 });
3859 envelope.validate().unwrap();
3860 assert_eq!(
3861 envelope.to_provider_message().unwrap().content,
3862 "rendered child outcome"
3863 );
3864 if let SessionMessageBody::ChildOutcome(outcome) = &mut envelope.body {
3865 outcome
3866 .provider_message
3867 .as_mut()
3868 .unwrap()
3869 .content
3870 .skill_request = Some(bamboo_domain::SessionSkillRequest {
3871 mode: None,
3872 selections: vec![bamboo_domain::SessionSkillSelection {
3873 id: "review".into(),
3874 source: "builtin".into(),
3875 revision: 7,
3876 args: serde_json::Value::Null,
3877 }],
3878 });
3879 }
3880 assert_eq!(
3881 envelope.validate(),
3882 Err(bamboo_domain::SessionMessageValidationError::KindSourceBodyMismatch)
3883 );
3884 }
3885
3886 #[tokio::test]
3887 async fn cancelled_resume_waiters_do_not_retain_historical_parent_ids() {
3888 for index in 0..512 {
3889 let id = format!("resume-lock-reclaim-{index}");
3890 let owner = session_resume_lock(&id);
3891 let held = owner.lock().await;
3892 let waiter = session_resume_lock(&id);
3893 let mut waiting = Box::pin(waiter.lock());
3894 assert!(futures::poll!(waiting.as_mut()).is_pending());
3895 drop(held);
3896 drop(owner);
3897 drop(waiting);
3898 drop(waiter);
3899 assert!(!parent_locks().lock().recover_poison().contains_key(&id));
3900 }
3901 }
3902
3903 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
3904 async fn resume_lock_reclamation_preserves_exclusive_parent_wake_ownership() {
3905 use std::sync::atomic::{AtomicUsize, Ordering};
3906 let active = Arc::new(AtomicUsize::new(0));
3907 let mut tasks = Vec::new();
3908 for _ in 0..32 {
3909 let active = active.clone();
3910 tasks.push(tokio::spawn(async move {
3911 for _ in 0..16 {
3912 let lease = session_resume_lock("resume-lock-exclusive");
3913 let _guard = lease.lock().await;
3914 assert_eq!(active.fetch_add(1, Ordering::SeqCst), 0);
3915 tokio::task::yield_now().await;
3916 assert_eq!(active.fetch_sub(1, Ordering::SeqCst), 1);
3917 }
3918 }));
3919 }
3920 for task in tasks {
3921 task.await.unwrap();
3922 }
3923 assert!(!parent_locks()
3924 .lock()
3925 .recover_poison()
3926 .contains_key("resume-lock-exclusive"));
3927 }
3928 use bamboo_agent_core::Message;
3929 use bamboo_domain::SessionInboxPort;
3930 use futures::stream;
3931 use std::sync::atomic::{AtomicUsize, Ordering};
3932
3933 struct FailingRuntimeSaveStorage {
3934 inner: Arc<bamboo_storage::SessionStoreV2>,
3935 }
3936
3937 #[async_trait]
3938 impl Storage for FailingRuntimeSaveStorage {
3939 async fn save_session(&self, session: &Session) -> std::io::Result<()> {
3940 self.inner.save_session(session).await
3941 }
3942
3943 async fn load_session(&self, id: &str) -> std::io::Result<Option<Session>> {
3944 self.inner.load_session(id).await
3945 }
3946
3947 async fn delete_session(&self, id: &str) -> std::io::Result<bool> {
3948 self.inner.delete_session(id).await
3949 }
3950
3951 async fn load_runtime_control_plane(&self, id: &str) -> std::io::Result<Option<Session>> {
3952 self.inner.load_runtime_control_plane(id).await
3953 }
3954
3955 async fn save_runtime_state(&self, _session: &Session) -> std::io::Result<()> {
3956 Err(std::io::Error::other("injected sidecar save failure"))
3957 }
3958 }
3959
3960 struct EmptyTools;
3961
3962 #[async_trait]
3963 impl bamboo_agent_core::tools::ToolExecutor for EmptyTools {
3964 async fn execute(
3965 &self,
3966 _call: &bamboo_agent_core::tools::ToolCall,
3967 ) -> Result<bamboo_agent_core::tools::ToolResult, bamboo_agent_core::tools::ToolError>
3968 {
3969 Err(bamboo_agent_core::tools::ToolError::NotFound(
3970 "no tools".to_string(),
3971 ))
3972 }
3973
3974 fn list_tools(&self) -> Vec<bamboo_agent_core::tools::ToolSchema> {
3975 Vec::new()
3976 }
3977 }
3978
3979 struct CompletedTestProvider;
3980
3981 #[async_trait]
3982 impl bamboo_llm::LLMProvider for CompletedTestProvider {
3983 async fn chat_stream(
3984 &self,
3985 _messages: &[Message],
3986 _tools: &[bamboo_agent_core::tools::ToolSchema],
3987 _max_output_tokens: Option<u32>,
3988 _model: &str,
3989 ) -> Result<bamboo_llm::LLMStream, bamboo_llm::LLMError> {
3990 let chunks: Vec<bamboo_llm::provider::Result<bamboo_llm::LLMChunk>> = vec![
3991 Ok(bamboo_llm::LLMChunk::Token("done".to_string())),
3992 Ok(bamboo_llm::LLMChunk::Done),
3993 ];
3994 Ok(Box::pin(stream::iter(chunks)))
3995 }
3996 }
3997
3998 struct CountingActivationSpawner {
3999 reservations: Arc<AtomicUsize>,
4000 launches: Arc<AtomicUsize>,
4001 }
4002
4003 #[async_trait]
4004 impl SessionActivationSpawner for CountingActivationSpawner {
4005 async fn reserve_activation(
4006 &self,
4007 target_session_id: &str,
4008 inbox_generation: u64,
4009 ) -> Result<SessionActivationReserveOutcome, bamboo_domain::SessionActivationError>
4010 {
4011 self.reservations.fetch_add(1, Ordering::SeqCst);
4012 let launches = self.launches.clone();
4013 Ok(SessionActivationReserveOutcome::Reserved(
4014 SessionActivationLaunch::new(
4015 format!("{target_session_id}-{inbox_generation}"),
4016 move || {
4017 launches.fetch_add(1, Ordering::SeqCst);
4018 },
4019 ),
4020 ))
4021 }
4022 }
4023
4024 async fn save_sourced_test_child(store: &bamboo_storage::SessionStoreV2, child: &Session) {
4025 let mut saved = child.clone();
4026 ChildCompletionSource::prepare(
4027 &mut saved,
4028 &format!("test-run-{}", child.id),
4029 &Default::default(),
4030 );
4031 store.save_session(&saved).await.unwrap();
4032 }
4033
4034 async fn test_child_source(
4035 store: &bamboo_storage::SessionStoreV2,
4036 id: &str,
4037 ) -> Option<ChildCompletionSource> {
4038 ChildCompletionSource::from_committed_session(
4039 &store.load_session(id).await.unwrap().unwrap(),
4040 )
4041 }
4042
4043 fn typed_test_report() -> serde_json::Value {
4044 serde_json::json!({"version":1,"outcome":"blocked","summary":"Quoted \"claim\" \\ 用户🪷\n",
4045 "reported_evidence":[{"description":"reported only","reference":"/never/read/reported-ref","sha256":null}],
4046 "reported_verification":[{"check":"fixture","reported_status":"passed","details":"a model claim"}],
4047 "proposals":["Try the next step"],"blockers":["Need approval"],"open_decisions":["Which route?"]})
4048 }
4049
4050 fn typed_test_child(parent: &Session, id: &str, report: &str) -> Session {
4051 let mut child = Session::new_child(id, &parent.id, "model", "Child");
4052 let packet = bamboo_domain::ChildContextPacket {
4053 version: 1,
4054 objective: "bounded task".into(),
4055 constraints: vec![],
4056 acceptance: vec!["report only".into()],
4057 non_goals: vec![],
4058 necessary_user_instructions: vec![],
4059 recorded_decisions: vec![],
4060 source_user_message_ids: vec![parent.messages[0].id.clone()],
4061 background_message_ids: vec![],
4062 };
4063 let resolved = packet.resolve(parent, "bounded task").unwrap();
4064 let binding = bamboo_domain::ChildContextBinding::new(
4065 parent,
4066 id,
4067 resolved.required_brief.clone(),
4068 resolved,
4069 )
4070 .unwrap();
4071 binding.install(&mut child).unwrap();
4072 child.add_message(binding.assignment_message());
4073 child.add_message(Message::assistant(report, None));
4074 child.set_last_run_status("completed");
4075 ChildCompletionSource::prepare(&mut child, "sealed-report-run", &Default::default());
4076 child
4077 }
4078
4079 fn typed_test_completion(child: &Session) -> ChildCompletion {
4080 ChildCompletion {
4081 parent_session_id: child.parent_session_id.clone().unwrap(),
4082 child_session_id: child.id.clone(),
4083 status: child.last_run_status().unwrap(),
4084 error: child.last_run_error(),
4085 completed_at: child.updated_at,
4086 source: ChildCompletionSource::from_committed_session(child),
4087 }
4088 }
4089
4090 fn unusable_typed_test_child(
4091 parent: &Session,
4092 id: &str,
4093 case: &str,
4094 ) -> (Session, &'static str) {
4095 let mut child = typed_test_child(parent, id, &typed_test_report().to_string());
4096 let mut prior = Default::default();
4097 let reason = match case {
4098 "prose" => {
4099 child.messages.last_mut().unwrap().content = "UNUSABLE_RAW_PROSE".repeat(64);
4100 "report_malformed"
4101 }
4102 "malformed" => {
4103 child.messages.last_mut().unwrap().content =
4104 "{\"version\":1,UNUSABLE_RAW_JSON".into();
4105 "report_malformed"
4106 }
4107 "oversized" => {
4108 child
4109 .messages
4110 .last_mut()
4111 .unwrap()
4112 .content
4113 .push_str(&" ".repeat(8193));
4114 "result_budget_exceeded"
4115 }
4116 "multipart" => {
4117 child.messages.last_mut().unwrap().content_parts = Some(
4118 serde_json::from_value(
4119 serde_json::json!([{"type":"text","text":"UNUSABLE_RAW_PART"}]),
4120 )
4121 .unwrap(),
4122 );
4123 "report_not_current_final"
4124 }
4125 "commentary" => {
4126 child.messages.last_mut().unwrap().phase =
4127 Some(bamboo_domain::MessagePhase::Commentary);
4128 "report_not_current_final"
4129 }
4130 "blank" => {
4131 child.messages.last_mut().unwrap().content = " ".into();
4132 "report_not_current_final"
4133 }
4134 "old_final" => {
4135 prior = child.messages.iter().map(|m| m.id.clone()).collect();
4136 "report_not_current_final"
4137 }
4138 "error" => {
4139 prior = child.messages.iter().map(|m| m.id.clone()).collect();
4140 child.set_last_run_status("error");
4141 child.set_last_run_error("UNUSABLE_RAW_ERROR\"\\\n".repeat(4096));
4142 "report_not_current_final"
4143 }
4144 "absent" => {
4145 child.messages.pop();
4146 "report_absent"
4147 }
4148 _ => panic!("unknown unusable report case"),
4149 };
4150 ChildCompletionSource::prepare(&mut child, "unusable-report-run", &prior);
4151 (child, reason)
4152 }
4153
4154 #[test]
4155 fn sealed_unusable_reports_keep_current_source_boundary_and_escape_budget() {
4156 let mut parent = Session::new("unavailable-parent", "model");
4157 parent.add_message(Message::user("required source"));
4158 for case in [
4159 "prose",
4160 "malformed",
4161 "oversized",
4162 "multipart",
4163 "commentary",
4164 "blank",
4165 "old_final",
4166 "error",
4167 "absent",
4168 ] {
4169 let (child, reason) = unusable_typed_test_child(&parent, "unavailable-child", case);
4170 let completion = typed_test_completion(&child);
4171 assert!(completion.source.is_some(), "valid seal: {case}");
4172 let mut projection = sealed_child_report(&completion, &child, &parent).unwrap();
4173 assert_eq!(
4174 projection,
4175 unavailable_child_result("typed_result", reason),
4176 "{case}"
4177 );
4178 let mut message = typed_child_resume_message(&completion, 0, &projection);
4179 let raw = completion.source.as_ref().unwrap().result(&child, &parent);
4180 let mut envelope =
4181 child_completion_envelope(&completion, Utc::now(), raw.clone(), &message);
4182 assert!(bound_typed_child_delivery(
4183 &mut envelope,
4184 &mut message,
4185 &mut projection,
4186 &completion,
4187 0
4188 ));
4189 assert!(fits_child_result_budget(&envelope));
4190 assert!(fits_child_result_budget(&message));
4191 assert!(fits_child_result_budget(
4192 &envelope.to_provider_message().unwrap()
4193 ));
4194 assert!(message.content.contains("SubAgent"));
4195 assert_eq!(
4196 message.metadata.as_ref().unwrap()["child_final_response_included"],
4197 false
4198 );
4199 assert!(projection.get("child_report").is_none());
4200 assert!(projection.get("host_observation").is_none());
4201 let SessionMessageBody::ChildOutcome(outcome) = envelope.body else {
4202 panic!("child outcome")
4203 };
4204 assert!(outcome.result.is_none());
4205 assert!(outcome.error.is_none());
4206 if let Some(raw) = raw {
4207 assert!(!message.content.contains(&raw), "no raw report: {case}");
4208 }
4209 let mut stale = child.clone();
4212 stale
4213 .messages
4214 .last_mut()
4215 .unwrap()
4216 .content
4217 .push_str("changed");
4218 assert!(sealed_child_report(&completion, &stale, &parent).is_none());
4219 }
4220 }
4221
4222 #[test]
4223 fn unavailable_delivery_counts_escaped_wrappers_at_exact_limit() {
4224 let mut parent = Session::new("unavailable-parent", "model");
4225 parent.add_message(Message::user("required source"));
4226 let (child, _) = unusable_typed_test_child(&parent, "unavailable-child", "oversized");
4227 let completion = typed_test_completion(&child);
4228 let mut projection = sealed_child_report(&completion, &child, &parent).unwrap();
4229 let mut message = typed_child_resume_message(&completion, 0, &projection);
4230 let mut base = child_completion_envelope(&completion, Utc::now(), None, &message);
4231 assert!(bound_typed_child_delivery(
4232 &mut base,
4233 &mut message,
4234 &mut projection,
4235 &completion,
4236 0
4237 ));
4238 let bytes = serde_json::to_vec(&base).unwrap().len().max(
4239 serde_json::to_vec(&base.to_provider_message().unwrap())
4240 .unwrap()
4241 .len(),
4242 );
4243 for extra in [8192 - bytes, 8193 - bytes] {
4244 let mut envelope = base.clone();
4245 let padding = format!("{}{}", "\"".repeat(extra / 2), "x".repeat(extra % 2));
4247 envelope.correlation_id.as_mut().unwrap().push_str(&padding);
4248 let mut projection = projection.clone();
4249 let mut message = message.clone();
4250 assert_eq!(
4251 bound_typed_child_delivery(
4252 &mut envelope,
4253 &mut message,
4254 &mut projection,
4255 &completion,
4256 0
4257 ),
4258 bytes + extra == 8192
4259 );
4260 assert_eq!(
4261 serde_json::to_vec(&envelope).unwrap().len().max(
4262 serde_json::to_vec(&envelope.to_provider_message().unwrap())
4263 .unwrap()
4264 .len()
4265 ),
4266 bytes + extra
4267 );
4268 assert!(fits_child_result_budget(&message));
4269 }
4270 }
4271
4272 #[test]
4273 fn sealed_typed_projection_keeps_host_source_distinct_from_all_reported_claims() {
4274 let mut parent = Session::new("typed-parent", "model");
4275 parent.add_message(Message::user("required user constraint"));
4276 let report = typed_test_report();
4277 let child = typed_test_child(&parent, "typed-child", &report.to_string());
4278 let completion = typed_test_completion(&child);
4279 let projection = sealed_child_report(&completion, &child, &parent).unwrap();
4280 assert_eq!(projection["child_report"], report);
4281 assert_eq!(
4282 projection["host_observation"]["terminal_source"],
4283 serde_json::to_value(&completion.source).unwrap()
4284 );
4285 assert_eq!(
4286 projection["host_observation"]["kind"],
4287 "committed_terminal_source"
4288 );
4289 assert_eq!(
4290 projection["host_observation"]["last_run_status"],
4291 "completed"
4292 );
4293 assert_eq!(projection["child_report"]["outcome"], "blocked");
4294 assert_eq!(
4295 projection["child_report"]["reported_verification"][0]["reported_status"],
4296 "passed"
4297 );
4298 assert!(projection["host_observation"].get("verified").is_none());
4299 for excluded in ["guardian", "resident", "legacy", "missing_source"] {
4300 let mut excluded_child = child.clone();
4301 match excluded {
4302 "guardian" => {
4303 excluded_child
4304 .metadata
4305 .insert("subagent_type".into(), "guardian".into());
4306 }
4307 "resident" => {
4308 excluded_child
4309 .metadata
4310 .insert("lifecycle".into(), "resident".into());
4311 }
4312 "legacy" => {
4313 excluded_child =
4314 Session::new_child("typed-child", &parent.id, "model", "Child");
4315 excluded_child.add_message(Message::assistant(report.to_string(), None));
4316 excluded_child.set_last_run_status("completed");
4317 }
4318 _ => {}
4319 }
4320 ChildCompletionSource::prepare(
4321 &mut excluded_child,
4322 "excluded-run",
4323 &Default::default(),
4324 );
4325 let mut excluded_completion = typed_test_completion(&excluded_child);
4326 if excluded == "missing_source" {
4327 excluded_completion.source = None;
4328 }
4329 assert!(
4330 sealed_child_report(&excluded_completion, &excluded_child, &parent).is_none(),
4331 "{excluded}"
4332 );
4333 }
4334 }
4335
4336 #[tokio::test]
4337 async fn typed_delivery_counts_complete_escaped_envelope_and_final_correlation() {
4338 let mut parent = Session::new("typed-parent", "model");
4339 parent.add_message(Message::user("required source"));
4340 let child = typed_test_child(&parent, "typed-child", &typed_test_report().to_string());
4341 let completion = typed_test_completion(&child);
4342 let original = sealed_child_report(&completion, &child, &parent).unwrap();
4343 let presentation = typed_child_resume_message(&completion, 0, &original);
4344 let mut base = child_completion_envelope(
4345 &completion,
4346 Utc::now(),
4347 completion.source.as_ref().unwrap().result(&child, &parent),
4348 &presentation,
4349 );
4350 let mut projection = original.clone();
4351 let mut message = presentation.clone();
4352 assert!(bound_typed_child_delivery(
4353 &mut base,
4354 &mut message,
4355 &mut projection,
4356 &completion,
4357 0
4358 ));
4359 assert_eq!(projection, original);
4360 let temp = tempfile::tempdir().unwrap();
4361 let store = Arc::new(
4362 bamboo_storage::SessionStoreV2::new(temp.path().to_path_buf())
4363 .await
4364 .unwrap(),
4365 );
4366 store.save_session(&parent).await.unwrap();
4367 let inbox = bamboo_storage::FileSessionInbox::new(store, Default::default());
4368 let first = inbox.deliver(&base).await.unwrap();
4369 let stable_id = base.id.clone();
4370 let base_bytes = serde_json::to_vec(&base.to_provider_message().unwrap())
4371 .unwrap()
4372 .len();
4373 assert!(base_bytes < 8192);
4374 let correlation = base.correlation_id.as_ref().unwrap().clone();
4375 for (extra, available) in [(8192 - base_bytes, true), (8193 - base_bytes, false)] {
4376 let mut envelope = base.clone();
4377 envelope.correlation_id = Some(format!("{correlation}{}", "x".repeat(extra)));
4378 assert_eq!(
4379 serde_json::to_vec(&envelope.to_provider_message().unwrap())
4380 .unwrap()
4381 .len(),
4382 base_bytes + extra
4383 );
4384 let mut message = presentation.clone();
4385 let mut projection = original.clone();
4386 assert!(bound_typed_child_delivery(
4387 &mut envelope,
4388 &mut message,
4389 &mut projection,
4390 &completion,
4391 0
4392 ));
4393 assert_eq!(projection["available"], available);
4394 assert!(fits_child_result_budget(&envelope));
4395 assert!(fits_child_result_budget(&message));
4396 assert!(fits_child_result_budget(
4397 &envelope.to_provider_message().unwrap()
4398 ));
4399 assert_eq!(
4400 envelope.idempotency_semantics(),
4401 base.idempotency_semantics()
4402 );
4403 assert_eq!(envelope.id, stable_id);
4404 assert_eq!(
4405 inbox.deliver(&envelope).await.unwrap(),
4406 first,
4407 "presentation overflow cannot change delivery semantics"
4408 );
4409 if !available {
4410 assert_eq!(projection["reason"], "result_budget_exceeded");
4411 assert!(projection.get("child_report").is_none());
4412 assert!(message.content.contains("SubAgent"));
4413 assert!(!message.content.contains("Bounded tail"));
4414 }
4415 }
4416 let mut large = typed_test_report();
4417 large["summary"] = serde_json::json!("\"\\\n".repeat(500));
4418 large["proposals"] = serde_json::json!(["\"\\\n".repeat(250)]);
4419 let child = typed_test_child(&parent, "escaped-child", &large.to_string());
4420 assert!(decode_report(&large.to_string()).is_ok());
4421 let completion = typed_test_completion(&child);
4422 let mut projection = sealed_child_report(&completion, &child, &parent).unwrap();
4423 let mut message = typed_child_resume_message(&completion, 0, &projection);
4424 let mut envelope =
4425 child_completion_envelope(&completion, Utc::now(), Some(large.to_string()), &message);
4426 assert!(serde_json::to_vec(&envelope).unwrap().len() > 8192);
4427 assert!(bound_typed_child_delivery(
4428 &mut envelope,
4429 &mut message,
4430 &mut projection,
4431 &completion,
4432 0
4433 ));
4434 assert_eq!(projection["available"], false);
4435 assert!(fits_child_result_budget(&envelope));
4436 assert!(fits_child_result_budget(&message));
4437 assert!(fits_child_result_budget(
4438 &envelope.to_provider_message().unwrap()
4439 ));
4440 }
4441
4442 #[tokio::test]
4443 async fn typed_source_mutations_reject_before_admission_and_wait_clear() {
4444 for mutation in ["birth", "assignment", "input", "result", "parent_source"] {
4445 let (_temp, store, inbox, coordinator, _, launches) = completion_inbox_fixture().await;
4446 save_waiting_test_parent(
4447 &store,
4448 "typed-parent",
4449 vec!["typed-child".into()],
4450 ChildWaitPolicy::All,
4451 )
4452 .await;
4453 let mut parent = store.load_session("typed-parent").await.unwrap().unwrap();
4454 parent.add_message(Message::user("required source"));
4455 store.save_session(&parent).await.unwrap();
4456 let mut child =
4457 typed_test_child(&parent, "typed-child", &typed_test_report().to_string());
4458 store.save_session(&child).await.unwrap();
4459 let mut completion = typed_test_completion(&child);
4460 match mutation {
4461 "birth" => {
4462 completion.source.as_mut().unwrap().child_created_at +=
4463 chrono::Duration::seconds(1)
4464 }
4465 "assignment" => {
4466 child.messages[0].content.push_str("changed assignment");
4467 }
4468 "input" => child.add_message(Message::user("new user input")),
4469 "result" => child
4470 .messages
4471 .last_mut()
4472 .unwrap()
4473 .content
4474 .push_str("changed"),
4475 _ => {
4476 parent.messages[0].content.push_str("changed");
4477 store.save_session(&parent).await.unwrap();
4478 }
4479 }
4480 store.save_session(&child).await.unwrap();
4481 coordinator.on_child_completed(completion).await;
4482 assert_eq!(
4483 inbox.inspect(&parent.id).await.unwrap().pending,
4484 0,
4485 "{mutation}"
4486 );
4487 assert_eq!(launches.load(Ordering::SeqCst), 0, "{mutation}");
4488 assert!(
4489 read_runtime_state(&store.load_session(&parent.id).await.unwrap().unwrap())
4490 .waiting_for_children
4491 .is_some(),
4492 "{mutation}"
4493 );
4494 }
4495 }
4496
4497 #[tokio::test]
4498 async fn typed_cold_watchdog_and_terminal_siblings_admit_each_source_once() {
4499 for case in [
4500 "valid",
4501 "prose",
4502 "malformed",
4503 "oversized",
4504 "multipart",
4505 "commentary",
4506 "blank",
4507 "old_final",
4508 "error",
4509 "absent",
4510 ] {
4511 for cold_watchdog in [false, true] {
4512 let (_temp, store, inbox, coordinator, reservations, launches) =
4513 completion_inbox_fixture().await;
4514 let ids = vec!["typed-a".into(), "typed-b".into()];
4515 save_waiting_test_parent(&store, "typed-parent", ids.clone(), ChildWaitPolicy::All)
4516 .await;
4517 let mut parent = store.load_session("typed-parent").await.unwrap().unwrap();
4518 parent.add_message(Message::user("required source"));
4519 store.save_session(&parent).await.unwrap();
4520 let mut completions = Vec::new();
4521 for id in &ids {
4522 let child = if case == "valid" {
4523 typed_test_child(&parent, id, &typed_test_report().to_string())
4524 } else {
4525 unusable_typed_test_child(&parent, id, case).0
4526 };
4527 store.save_session(&child).await.unwrap();
4528 completions.push(typed_test_completion(&child));
4529 }
4530 if cold_watchdog {
4531 let mut wait = read_runtime_state(&parent).waiting_for_children.unwrap();
4533 wait.registered_at = Utc::now() - chrono::Duration::seconds(61);
4534 coordinator.sweep_child_wait(&parent.id, wait.clone()).await;
4535 coordinator.sweep_child_wait(&parent.id, wait).await;
4536 } else {
4537 coordinator.on_child_completed(completions[0].clone()).await;
4539 }
4540 for completion in completions {
4541 coordinator.on_child_completed(completion).await;
4542 }
4543 let claims = inbox.claim(&parent.id, 8).await.unwrap();
4544 assert_eq!(claims.len(), 2);
4545 assert_eq!(reservations.load(Ordering::SeqCst), 1);
4546 assert_eq!(launches.load(Ordering::SeqCst), 1);
4547 for claim in claims {
4548 assert!(fits_child_result_budget(&claim.envelope));
4549 assert!(fits_child_result_budget(
4550 &claim.envelope.to_provider_message().unwrap()
4551 ));
4552 let actual = claim.envelope.to_provider_message().unwrap();
4553 assert!(!actual.content.contains("UNUSABLE_RAW"));
4554 let SessionMessageBody::ChildOutcome(outcome) = claim.envelope.body else {
4555 panic!("child outcome");
4556 };
4557 let projection: serde_json::Value = serde_json::from_str(
4558 outcome
4559 .provider_message
4560 .as_ref()
4561 .unwrap()
4562 .content
4563 .text
4564 .split_once("Child typed result:\n")
4565 .unwrap()
4566 .1
4567 .lines()
4568 .next()
4569 .unwrap(),
4570 )
4571 .unwrap();
4572 if case != "valid" {
4573 let expected =
4574 unusable_typed_test_child(&parent, &outcome.child_session_id, case).1;
4575 assert_eq!(
4576 projection,
4577 unavailable_child_result("typed_result", expected)
4578 );
4579 assert!(outcome.result.is_none());
4580 assert!(outcome.error.is_none());
4581 assert!(fits_child_result_budget(&outcome.provider_message));
4582 continue;
4583 }
4584 assert_eq!(projection["child_report"], typed_test_report());
4585 assert_eq!(
4586 projection["host_observation"]["terminal_source"]["child_session_id"],
4587 outcome.child_session_id
4588 );
4589 assert_eq!(
4590 projection["host_observation"]["last_run_status"],
4591 "completed"
4592 );
4593 assert!(fits_child_result_budget(&outcome.provider_message));
4594 }
4595 }
4596 }
4597 }
4598
4599 async fn completion_inbox_fixture() -> (
4600 tempfile::TempDir,
4601 Arc<bamboo_storage::SessionStoreV2>,
4602 Arc<dyn SessionInboxPort>,
4603 Arc<ChildCompletionCoordinator>,
4604 Arc<AtomicUsize>,
4605 Arc<AtomicUsize>,
4606 ) {
4607 completion_inbox_fixture_with_limits(bamboo_domain::SessionInboxLimits::default()).await
4608 }
4609
4610 async fn completion_inbox_fixture_with_limits(
4611 limits: bamboo_domain::SessionInboxLimits,
4612 ) -> (
4613 tempfile::TempDir,
4614 Arc<bamboo_storage::SessionStoreV2>,
4615 Arc<dyn SessionInboxPort>,
4616 Arc<ChildCompletionCoordinator>,
4617 Arc<AtomicUsize>,
4618 Arc<AtomicUsize>,
4619 ) {
4620 let temp = tempfile::tempdir().unwrap();
4621 let store = Arc::new(
4622 bamboo_storage::SessionStoreV2::new(temp.path().to_path_buf())
4623 .await
4624 .unwrap(),
4625 );
4626 let storage: Arc<dyn Storage> = store.clone();
4627 let locked = Arc::new(LockedSessionStore::new(storage.clone()));
4628 let inbox: Arc<dyn SessionInboxPort> =
4629 Arc::new(bamboo_storage::FileSessionInbox::new(store.clone(), limits));
4630 let router = crate::SessionActivationRouter::new();
4631 let messenger = Arc::new(crate::SessionMessenger::new(
4632 storage.clone(),
4633 inbox.clone(),
4634 router.clone(),
4635 ));
4636 let reservations = Arc::new(AtomicUsize::new(0));
4637 let launches = Arc::new(AtomicUsize::new(0));
4638 router
4639 .set_spawner(Arc::new(CountingActivationSpawner {
4640 reservations: reservations.clone(),
4641 launches: launches.clone(),
4642 }))
4643 .await;
4644 let provider: Arc<dyn bamboo_llm::LLMProvider> = Arc::new(CompletedTestProvider);
4645 let config = Arc::new(RwLock::new(Config::default()));
4646 let metrics = bamboo_metrics::MetricsCollector::spawn(
4647 Arc::new(bamboo_metrics::SqliteMetricsStorage::new(
4648 temp.path().join("metrics.db"),
4649 )),
4650 7,
4651 );
4652 let tools: Arc<dyn ToolExecutor> = Arc::new(EmptyTools);
4653 let agent = Arc::new(
4654 Agent::builder()
4655 .storage(storage.clone())
4656 .persistence(locked.clone())
4657 .session_inbox(inbox.clone())
4658 .activation_router(router)
4659 .session_messenger(messenger)
4660 .attachment_reader(store.clone())
4661 .skill_manager(Arc::new(bamboo_skills::SkillManager::new()))
4662 .metrics_collector(metrics)
4663 .config(config.clone())
4664 .provider(provider.clone())
4665 .default_tools(tools)
4666 .build()
4667 .unwrap(),
4668 );
4669 let mut providers = HashMap::new();
4670 providers.insert("test".to_string(), provider);
4671 let registry = Arc::new(ProviderRegistry::new(providers, "test".to_string()));
4672 let provider_router = Arc::new(ProviderModelRouter::new(registry.clone()));
4673 let coordinator = Arc::new(ChildCompletionCoordinator::new(
4674 storage,
4675 locked,
4676 Arc::default(),
4677 Arc::new(RwLock::new(HashMap::new())),
4678 Arc::new(RwLock::new(HashMap::new())),
4679 agent,
4680 config,
4681 registry,
4682 provider_router,
4683 temp.path().to_path_buf(),
4684 None,
4685 ));
4686 (temp, store, inbox, coordinator, reservations, launches)
4687 }
4688
4689 async fn save_waiting_test_parent(
4690 store: &bamboo_storage::SessionStoreV2,
4691 parent_id: &str,
4692 ids: Vec<String>,
4693 policy: ChildWaitPolicy,
4694 ) {
4695 let mut parent = Session::new(parent_id, "model");
4696 let mut runtime = AgentRuntimeState::new("waiting-run");
4697 runtime.status = AgentStatusState::Suspended;
4698 runtime.waiting_for_children = Some(WaitingForChildrenState::for_children(
4699 ids,
4700 policy,
4701 Utc::now(),
4702 ));
4703 runtime.suspension = Some(SuspensionState {
4704 reason: "waiting_for_children".into(),
4705 suspended_at: Utc::now(),
4706 resumable: true,
4707 hook_point: Some("ChildCompletion".into()),
4708 });
4709 write_runtime_state(&mut parent, &runtime);
4710 parent.metadata.insert(
4711 "runtime.suspend_reason".into(),
4712 "waiting_for_children".into(),
4713 );
4714 store.save_session(&parent).await.unwrap();
4715 }
4716
4717 async fn save_completion_test_child(
4718 store: &bamboo_storage::SessionStoreV2,
4719 parent_id: &str,
4720 child_id: &str,
4721 status: &str,
4722 ) -> ChildCompletion {
4723 let mut child = store
4724 .load_session(child_id)
4725 .await
4726 .unwrap()
4727 .unwrap_or_else(|| Session::new_child(child_id, parent_id, "model", "Child"));
4728 child.add_message(Message::assistant(format!("result-{child_id}"), None));
4729 child.set_last_run_status(status);
4730 save_sourced_test_child(store, &child).await;
4731 ChildCompletion {
4732 parent_session_id: parent_id.into(),
4733 child_session_id: child_id.into(),
4734 status: status.into(),
4735 error: None,
4736 completed_at: child.updated_at,
4737 source: test_child_source(store, child_id).await,
4738 }
4739 }
4740
4741 #[tokio::test]
4742 async fn unrelated_invalid_pending_body_does_not_block_incremental_child_activation() {
4743 for queue in ["new", "cur"] {
4744 let (_temp, store, inbox, coordinator, reservations, launches) =
4745 completion_inbox_fixture().await;
4746 let parent_id = "invalid-pending-parent";
4747 save_waiting_test_parent(
4748 &store,
4749 parent_id,
4750 vec!["invalid-pending-fast".into(), "invalid-pending-slow".into()],
4751 ChildWaitPolicy::All,
4752 )
4753 .await;
4754 let before = read_runtime_state(&store.load_session(parent_id).await.unwrap().unwrap());
4755 let fast =
4756 save_completion_test_child(&store, parent_id, "invalid-pending-fast", "completed")
4757 .await;
4758 save_completion_test_child(&store, parent_id, "invalid-pending-slow", "running").await;
4759
4760 let mut unrelated = SessionMessageEnvelope::user_input(parent_id, "unrelated bash");
4761 unrelated.source = SessionMessageSource::Runtime {
4762 subsystem: "bash".into(),
4763 };
4764 unrelated.kind = SessionMessageKind::RuntimeInstruction;
4765 unrelated.body =
4766 SessionMessageBody::RuntimeInstruction(bamboo_domain::SessionRuntimeInstruction {
4767 instruction: "bash_completion".into(),
4768 content: Some(bamboo_domain::SessionMessageContent::text("unrelated bash")),
4769 data: None,
4770 provider_message: None,
4771 });
4772 inbox.deliver(&unrelated).await.unwrap();
4773 let dir = store
4774 .bamboo_home_dir()
4775 .join(store.resolve_rel_path(parent_id).await.unwrap())
4776 .join("inbox");
4777 let staged = tokio::fs::read_dir(dir.join("new"))
4778 .await
4779 .unwrap()
4780 .next_entry()
4781 .await
4782 .unwrap()
4783 .unwrap()
4784 .path();
4785 let mut transport: serde_json::Value =
4786 serde_json::from_slice(&tokio::fs::read(&staged).await.unwrap()).unwrap();
4787 transport["body"] = serde_json::json!({"not": "an envelope"});
4788 let invalid_bytes = serde_json::to_vec(&transport).unwrap();
4789 tokio::fs::write(&staged, &invalid_bytes).await.unwrap();
4790 let invalid_path = if queue == "cur" {
4791 let destination = dir.join("cur").join(staged.file_name().unwrap());
4792 tokio::fs::create_dir_all(dir.join("cur")).await.unwrap();
4793 tokio::fs::rename(&staged, &destination).await.unwrap();
4794 destination
4795 } else {
4796 staged
4797 };
4798
4799 coordinator.on_child_completed(fast.clone()).await;
4800 let outcomes = inbox
4801 .pending_child_outcomes(parent_id, "invalid-pending-fast")
4802 .await
4803 .unwrap();
4804 assert_eq!(outcomes.len(), 1);
4805 assert!(fits_child_result_budget(&outcomes[0]));
4806 let after = read_runtime_state(&store.load_session(parent_id).await.unwrap().unwrap());
4807 assert_eq!(after, before, "All wait and suspension remain armed");
4808 assert_eq!(reservations.load(Ordering::SeqCst), 1);
4809 assert_eq!(launches.load(Ordering::SeqCst), 1);
4810 let backlog = inbox.inspect(parent_id).await.unwrap();
4811 assert!(backlog.interrupt_pending());
4812 assert_eq!(backlog.coordinator_generation, 0);
4813 let ready = inbox
4814 .inspect_wake_readiness(parent_id, Utc::now())
4815 .await
4816 .unwrap()
4817 .ready
4818 .unwrap();
4819 assert_eq!(ready.id, outcomes[0].id);
4820 assert_eq!(
4821 ready.activation_policy,
4822 SessionActivationPolicy::InterruptSpecificWait
4823 );
4824 assert_eq!(tokio::fs::read(&invalid_path).await.unwrap(), invalid_bytes);
4825 assert!(!inbox.was_admitted(parent_id, &unrelated.id).await.unwrap());
4826
4827 let claims = inbox.claim(parent_id, 8).await.unwrap();
4828 assert_eq!(claims.len(), 1);
4829 assert_eq!(claims[0].envelope.id, outcomes[0].id);
4830 inbox.ack(parent_id, &claims[0]).await.unwrap();
4831 coordinator.on_child_completed(fast).await;
4832 assert_eq!(launches.load(Ordering::SeqCst), 1);
4833 assert!(!inbox.was_admitted(parent_id, &unrelated.id).await.unwrap());
4834 }
4835 }
4836
4837 #[tokio::test]
4838 async fn partial_all_outcome_preserves_wait_and_active_parent_control_state() {
4839 let (_temp, store, inbox, coordinator, _, launches) = completion_inbox_fixture().await;
4840 let parent_id = "incremental-parent";
4841 save_waiting_test_parent(
4842 &store,
4843 parent_id,
4844 vec!["ready-child".into(), "slow-child".into()],
4845 ChildWaitPolicy::All,
4846 )
4847 .await;
4848 let mut parent = store.load_session(parent_id).await.unwrap().unwrap();
4849 let mut runtime = read_runtime_state(&parent);
4850 runtime.status = AgentStatusState::Running;
4851 runtime.suspension = None;
4852 parent.metadata.remove("runtime.suspend_reason");
4853 write_runtime_state(&mut parent, &runtime);
4854 store.save_session(&parent).await.unwrap();
4855 let ready = save_completion_test_child(&store, parent_id, "ready-child", "completed").await;
4856 save_completion_test_child(&store, parent_id, "slow-child", "running").await;
4857
4858 coordinator.on_child_completed(ready.clone()).await;
4859 let after = store.load_session(parent_id).await.unwrap().unwrap();
4860 assert_eq!(read_runtime_state(&after), runtime);
4861 assert_eq!(after.updated_at, parent.updated_at);
4862 assert_eq!(launches.load(Ordering::SeqCst), 1);
4863 let backlog = inbox.inspect(parent_id).await.unwrap();
4864 assert!(backlog.interrupt_pending());
4865 assert_eq!(backlog.coordinator_generation, 0, "no blanket prefix grant");
4866 let claims = inbox.claim(parent_id, 8).await.unwrap();
4867 assert_eq!(claims.len(), 1);
4868 inbox.ack(parent_id, &claims[0]).await.unwrap();
4869
4870 coordinator.on_child_completed(ready).await;
4871 assert_eq!(inbox.inspect(parent_id).await.unwrap().pending, 0);
4872 let slow = save_completion_test_child(&store, parent_id, "slow-child", "completed").await;
4873 coordinator.on_child_completed(slow).await;
4874 assert!(
4875 read_runtime_state(&store.load_session(parent_id).await.unwrap().unwrap())
4876 .waiting_for_children
4877 .is_none()
4878 );
4879 let claims = inbox.claim(parent_id, 8).await.unwrap();
4880 assert_eq!(
4881 claims.len(),
4882 1,
4883 "the acknowledged first result is not repeated at wait release"
4884 );
4885 let SessionMessageBody::ChildOutcome(outcome) = &claims[0].envelope.body else {
4886 panic!("child outcome");
4887 };
4888 assert_eq!(outcome.child_session_id, "slow-child");
4889 }
4890
4891 #[tokio::test]
4892 async fn incremental_completions_during_owner_exit_share_one_successor() {
4893 let (_temp, store, inbox, coordinator, reservations, launches) =
4894 completion_inbox_fixture().await;
4895 let parent_id = "handoff-parent";
4896 save_waiting_test_parent(
4897 &store,
4898 parent_id,
4899 vec![
4900 "handoff-a".into(),
4901 "handoff-b".into(),
4902 "handoff-slow".into(),
4903 ],
4904 ChildWaitPolicy::All,
4905 )
4906 .await;
4907 let a = save_completion_test_child(&store, parent_id, "handoff-a", "completed").await;
4908 let b = save_completion_test_child(&store, parent_id, "handoff-b", "completed").await;
4909 save_completion_test_child(&store, parent_id, "handoff-slow", "running").await;
4910 let router = coordinator.agent.activation_router().unwrap();
4911 router.set_inbox(inbox.clone());
4912 let mut owner = router
4913 .register_run(parent_id, "old-parent-run")
4914 .await
4915 .unwrap();
4916
4917 coordinator.on_child_completed(a.clone()).await;
4918 assert!(inbox
4919 .claim_for_turn(parent_id, 8, Some("old-parent-run"))
4920 .await
4921 .unwrap()
4922 .is_empty());
4923 owner.begin_finalization().await;
4924 coordinator.on_child_completed(b.clone()).await;
4925 assert_eq!(reservations.load(Ordering::SeqCst), 0);
4926 assert_eq!(launches.load(Ordering::SeqCst), 0);
4927 owner.finish(0).await.unwrap();
4928 assert_eq!(reservations.load(Ordering::SeqCst), 1);
4929 assert_eq!(launches.load(Ordering::SeqCst), 1);
4930 let successor = router.current_run_id(parent_id).await.unwrap();
4931 let claims = inbox
4932 .claim_for_turn(parent_id, 8, Some(&successor))
4933 .await
4934 .unwrap();
4935 assert_eq!(claims.len(), 2);
4936 for claim in &claims {
4937 inbox.ack(parent_id, claim).await.unwrap();
4938 }
4939 tokio::join!(
4940 coordinator.on_child_completed(a),
4941 coordinator.on_child_completed(b)
4942 );
4943 assert_eq!(inbox.inspect(parent_id).await.unwrap().pending, 0);
4944 assert_eq!(reservations.load(Ordering::SeqCst), 1);
4945 assert!(
4946 read_runtime_state(&store.load_session(parent_id).await.unwrap().unwrap())
4947 .waiting_for_children
4948 .is_some()
4949 );
4950 }
4951
4952 #[tokio::test]
4953 async fn cold_watchdog_replays_each_ready_child_before_all_wait_is_satisfied() {
4954 let (_temp, store, inbox, coordinator, _, launches) = completion_inbox_fixture().await;
4955 let parent_id = "cold-incremental-parent";
4956 save_waiting_test_parent(
4957 &store,
4958 parent_id,
4959 vec!["cold-a".into(), "cold-b".into(), "cold-slow".into()],
4960 ChildWaitPolicy::All,
4961 )
4962 .await;
4963 let mut parent = store.load_session(parent_id).await.unwrap().unwrap();
4964 let mut runtime = read_runtime_state(&parent);
4965 runtime.waiting_for_children.as_mut().unwrap().registered_at -=
4966 chrono::Duration::seconds(61);
4967 write_runtime_state(&mut parent, &runtime);
4968 store.save_session(&parent).await.unwrap();
4969 save_completion_test_child(&store, parent_id, "cold-a", "completed").await;
4970 save_completion_test_child(&store, parent_id, "cold-b", "completed").await;
4971 save_completion_test_child(&store, parent_id, "cold-slow", "running").await;
4972
4973 let wait = runtime.waiting_for_children.unwrap();
4974 coordinator.sweep_child_wait(parent_id, wait.clone()).await;
4975 let reopened = bamboo_storage::FileSessionInbox::new(store.clone(), Default::default());
4976 let backlog = reopened.inspect(parent_id).await.unwrap();
4977 assert_eq!(backlog.pending, 2);
4978 assert!(backlog.interrupt_pending());
4979 assert_eq!(backlog.coordinator_generation, 0);
4980 assert_eq!(launches.load(Ordering::SeqCst), 1);
4981 let claims = reopened.claim(parent_id, 8).await.unwrap();
4982 assert_eq!(claims.len(), 2);
4983 for claim in &claims {
4984 reopened.ack(parent_id, claim).await.unwrap();
4985 }
4986 coordinator.sweep_child_wait(parent_id, wait).await;
4987 assert_eq!(inbox.inspect(parent_id).await.unwrap().pending, 0);
4988 assert!(
4989 read_runtime_state(&store.load_session(parent_id).await.unwrap().unwrap())
4990 .waiting_for_children
4991 .is_some()
4992 );
4993 }
4994
4995 #[tokio::test]
4996 async fn any_wait_delivers_a_late_direct_child_without_repeating_the_first_outcome() {
4997 let (_temp, store, inbox, coordinator, _, _) = completion_inbox_fixture().await;
4998 let parent_id = "any-incremental-parent";
4999 save_waiting_test_parent(
5000 &store,
5001 parent_id,
5002 vec!["any-first".into(), "any-late".into()],
5003 ChildWaitPolicy::Any,
5004 )
5005 .await;
5006 let first = save_completion_test_child(&store, parent_id, "any-first", "completed").await;
5007 save_completion_test_child(&store, parent_id, "any-late", "running").await;
5008 coordinator.on_child_completed(first.clone()).await;
5009 assert!(
5010 read_runtime_state(&store.load_session(parent_id).await.unwrap().unwrap())
5011 .waiting_for_children
5012 .is_none()
5013 );
5014 let first_claims = inbox.claim(parent_id, 8).await.unwrap();
5015 assert_eq!(first_claims.len(), 1);
5016 inbox.ack(parent_id, &first_claims[0]).await.unwrap();
5017
5018 let late = save_completion_test_child(&store, parent_id, "any-late", "completed").await;
5019 coordinator.on_child_completed(late.clone()).await;
5020 coordinator.on_child_completed(first).await;
5021 coordinator.on_child_completed(late).await;
5022 let claims = inbox.claim(parent_id, 8).await.unwrap();
5023 assert_eq!(claims.len(), 1);
5024 let SessionMessageBody::ChildOutcome(outcome) = &claims[0].envelope.body else {
5025 panic!("child outcome");
5026 };
5027 assert_eq!(outcome.child_session_id, "any-late");
5028 assert!(
5029 read_runtime_state(&store.load_session(parent_id).await.unwrap().unwrap())
5030 .waiting_for_children
5031 .is_none()
5032 );
5033 }
5034
5035 #[tokio::test]
5036 async fn terminal_batch_upgrades_lower_pending_outcome_below_an_acknowledged_sibling() {
5037 let (_temp, store, inbox, coordinator, _, _) = completion_inbox_fixture().await;
5038 let parent_id = "terminal-batch-parent";
5039 save_waiting_test_parent(
5040 &store,
5041 parent_id,
5042 vec!["batch-low".into(), "batch-high".into()],
5043 ChildWaitPolicy::All,
5044 )
5045 .await;
5046 let mut parent = store.load_session(parent_id).await.unwrap().unwrap();
5047 let wait = read_runtime_state(&parent).waiting_for_children.unwrap();
5048 let low = save_completion_test_child(&store, parent_id, "batch-low", "completed").await;
5049 let high = save_completion_test_child(&store, parent_id, "batch-high", "completed").await;
5050 let mut receipts = Vec::new();
5051 for completion in [&low, &high] {
5052 let result = format!("result-{}", completion.child_session_id);
5053 let presentation = runtime_resume_message(completion, 0, Some(&result));
5054 let envelope = child_completion_envelope(
5055 completion,
5056 wait.registered_at,
5057 Some(result),
5058 &presentation,
5059 );
5060 receipts.push(inbox.deliver(&envelope).await.unwrap());
5061 }
5062 inbox
5063 .mark_message_activation_eligible(
5064 parent_id,
5065 &receipts[1],
5066 SessionActivationPolicy::InterruptSpecificWait,
5067 )
5068 .await
5069 .unwrap();
5070 let claims = inbox.claim(parent_id, 8).await.unwrap();
5071 assert_eq!(claims.len(), 1);
5072 assert_eq!(claims[0].generation, receipts[1].generation);
5073 parent.add_message(claims[0].envelope.to_provider_message().unwrap());
5074 store.save_session(&parent).await.unwrap();
5075 inbox.ack(parent_id, &claims[0]).await.unwrap();
5076
5077 coordinator.on_child_completed(low).await;
5078 let mut rearmed = store.load_session(parent_id).await.unwrap().unwrap();
5079 let mut runtime = read_runtime_state(&rearmed);
5080 assert!(runtime.waiting_for_children.is_none());
5081 let new_wait = WaitingForChildrenState::for_children(
5082 vec!["next-slow-child".into()],
5083 ChildWaitPolicy::All,
5084 Utc::now(),
5085 );
5086 runtime.waiting_for_children = Some(new_wait.clone());
5087 runtime.status = AgentStatusState::Suspended;
5088 write_runtime_state(&mut rearmed, &runtime);
5089 store.save_session(&rearmed).await.unwrap();
5090 let backlog = inbox.inspect(parent_id).await.unwrap();
5091 assert_eq!(backlog.pending, 1);
5092 assert!(
5093 backlog.interrupt_pending(),
5094 "the low generation has its own grant"
5095 );
5096 let (prepared, ready) = prepare_session_inbox_activation(
5097 &coordinator.persistence,
5098 parent_id,
5099 backlog.interrupt_pending(),
5100 )
5101 .await
5102 .unwrap()
5103 .unwrap();
5104 assert!(ready);
5105 assert_eq!(
5106 read_runtime_state(&prepared).waiting_for_children,
5107 Some(new_wait)
5108 );
5109 let claims = inbox.claim(parent_id, 8).await.unwrap();
5110 assert_eq!(claims.len(), 1);
5111 assert_eq!(claims[0].generation, receipts[0].generation);
5112 }
5113
5114 #[tokio::test]
5115 async fn large_legal_typed_source_falls_back_within_the_delivery_budget() {
5116 let (_temp, store, inbox, coordinator, _, launches) = completion_inbox_fixture().await;
5117 let parent_id = "large-source-parent";
5118 let child_id = "large-source-child";
5119 save_waiting_test_parent(
5120 &store,
5121 parent_id,
5122 vec![child_id.into()],
5123 ChildWaitPolicy::All,
5124 )
5125 .await;
5126 let mut parent = store.load_session(parent_id).await.unwrap().unwrap();
5127 parent.add_message(Message::user("required original source"));
5128 store.save_session(&parent).await.unwrap();
5129 let mut child = typed_test_child(&parent, child_id, &typed_test_report().to_string());
5130 let source = ChildCompletionSource::from_committed_session(&child).unwrap();
5131 let run_length =
5132 3900 - serde_json::to_vec(&source).unwrap().len() + source.activation_run_id.len();
5133 ChildCompletionSource::prepare(&mut child, &"r".repeat(run_length), &Default::default());
5134 let completion = typed_test_completion(&child);
5135 let source_bytes = serde_json::to_vec(completion.source.as_ref().unwrap())
5136 .unwrap()
5137 .len();
5138 assert!((3800..=4096).contains(&source_bytes));
5139 store.save_session(&child).await.unwrap();
5140
5141 coordinator.on_child_completed(completion).await;
5142 let claims = inbox.claim(parent_id, 8).await.unwrap();
5143 assert_eq!(claims.len(), 1);
5144 assert_eq!(launches.load(Ordering::SeqCst), 1);
5145 assert!(fits_child_result_budget(&claims[0].envelope));
5146 let message = claims[0].envelope.to_provider_message().unwrap();
5147 assert!(fits_child_result_budget(&message));
5148 assert!(message.content.contains("result_budget_exceeded"));
5149 }
5150
5151 #[tokio::test]
5152 async fn upgrade_promotes_one_existing_legacy_partial_delivery_without_a_second_id() {
5153 let (_temp, store, inbox, coordinator, _, launches) = completion_inbox_fixture().await;
5154 let parent_id = "legacy-partial-parent";
5155 save_waiting_test_parent(
5156 &store,
5157 parent_id,
5158 vec!["legacy-ready".into(), "legacy-slow".into()],
5159 ChildWaitPolicy::All,
5160 )
5161 .await;
5162 let completion =
5163 save_completion_test_child(&store, parent_id, "legacy-ready", "completed").await;
5164 save_completion_test_child(&store, parent_id, "legacy-slow", "running").await;
5165 let parent = store.load_session(parent_id).await.unwrap().unwrap();
5166 let wait = read_runtime_state(&parent).waiting_for_children.unwrap();
5167 let result = "result-legacy-ready";
5168 let presentation = runtime_resume_message(&completion, 1, Some(result));
5169 let mut legacy = child_completion_envelope(
5170 &completion,
5171 wait.registered_at,
5172 Some(result.into()),
5173 &presentation,
5174 );
5175 legacy.id = child_completion_id(&completion, wait.registered_at, Some(result), true);
5176 let receipt = inbox.deliver(&legacy).await.unwrap();
5177 assert!(!inbox.inspect(parent_id).await.unwrap().activation_pending());
5178
5179 coordinator.on_child_completed(completion.clone()).await;
5180 coordinator.on_child_completed(completion).await;
5181 let claims = inbox.claim(parent_id, 8).await.unwrap();
5182 assert_eq!(claims.len(), 1);
5183 assert_eq!(claims[0].envelope.id, legacy.id);
5184 assert_eq!(claims[0].generation, receipt.generation);
5185 assert_eq!(launches.load(Ordering::SeqCst), 1);
5186 assert_eq!(
5187 read_runtime_state(&store.load_session(parent_id).await.unwrap().unwrap())
5188 .waiting_for_children
5189 .as_ref(),
5190 Some(&wait)
5191 );
5192 }
5193
5194 #[tokio::test]
5195 async fn upgrade_keeps_admitted_legacy_raw_guardian_and_typed_sources_deduplicated_without_a_wait(
5196 ) {
5197 for presentation_kind in ["raw", "guardian", "typed", "typed_unavailable_digest"] {
5198 let (_temp, store, inbox, coordinator, _, launches) = completion_inbox_fixture().await;
5199 let parent_id = "legacy-admitted-parent";
5200 let child_id = "legacy-admitted-child";
5201 let mut parent = Session::new(parent_id, "model");
5202 parent.add_message(Message::user("required original source"));
5203 store.save_session(&parent).await.unwrap();
5204 let completion = if presentation_kind.starts_with("typed") {
5205 let child = typed_test_child(&parent, child_id, &typed_test_report().to_string());
5206 store.save_session(&child).await.unwrap();
5207 typed_test_completion(&child)
5208 } else {
5209 save_completion_test_child(&store, parent_id, child_id, "completed").await
5210 };
5211 let child = store.load_session(child_id).await.unwrap().unwrap();
5212 let result = completion.source.as_ref().unwrap().result(&child, &parent);
5213 let mut projection = sealed_child_report(&completion, &child, &parent);
5214 if presentation_kind == "typed_unavailable_digest" {
5215 projection = Some(unavailable_child_result(
5216 "typed_result",
5217 "result_budget_exceeded",
5218 ));
5219 }
5220 let mut presentation = match presentation_kind {
5221 "guardian" => guardian_resume_message(&completion, &GuardianVerdict::approved()),
5222 "typed" | "typed_unavailable_digest" => {
5223 typed_child_resume_message(&completion, 0, projection.as_ref().unwrap())
5224 }
5225 _ => runtime_resume_message(&completion, 0, result.as_deref()),
5226 };
5227 if presentation_kind == "typed" {
5228 presentation
5229 .metadata
5230 .as_mut()
5231 .unwrap()
5232 .as_object_mut()
5233 .unwrap()
5234 .remove("child_completion_source_sha256");
5235 }
5236 let old_wait = Utc::now() - chrono::Duration::minutes(2);
5237 let mut legacy =
5238 child_completion_envelope(&completion, old_wait, result.clone(), &presentation);
5239 legacy.id = child_completion_id(&completion, old_wait, result.as_deref(), true);
5240 if let Some(projection) = projection.as_mut() {
5241 assert!(bound_typed_child_delivery(
5242 &mut legacy,
5243 &mut presentation,
5244 projection,
5245 &completion,
5246 0
5247 ));
5248 }
5249 let receipt = inbox.deliver(&legacy).await.unwrap();
5250 inbox
5251 .mark_message_activation_eligible(
5252 parent_id,
5253 &receipt,
5254 SessionActivationPolicy::InterruptSpecificWait,
5255 )
5256 .await
5257 .unwrap();
5258 let claims = inbox.claim(parent_id, 1).await.unwrap();
5259 parent.add_message(claims[0].envelope.to_provider_message().unwrap());
5260 store.save_session(&parent).await.unwrap();
5261 inbox.ack(parent_id, &claims[0]).await.unwrap();
5262
5263 coordinator.on_child_completed(completion).await;
5264 assert_eq!(
5265 inbox.inspect(parent_id).await.unwrap().generation,
5266 receipt.generation,
5267 "{presentation_kind}"
5268 );
5269 assert_eq!(
5270 inbox.inspect(parent_id).await.unwrap().pending,
5271 0,
5272 "{presentation_kind}"
5273 );
5274 assert_eq!(launches.load(Ordering::SeqCst), 0, "{presentation_kind}");
5275 }
5276 }
5277
5278 #[tokio::test]
5279 async fn upgrade_reuses_a_pending_legacy_source_after_its_wait_was_cleared() {
5280 for (arm, acked) in [
5281 ("none", false),
5282 ("partial", false),
5283 ("terminal", false),
5284 ("terminal", true),
5285 ] {
5286 let (_temp, store, inbox, coordinator, _, launches) = completion_inbox_fixture().await;
5287 let parent_id = "legacy-pending-parent";
5288 let child_id = "legacy-pending-child";
5289 let mut parent = Session::new(parent_id, "model");
5290 store.save_session(&parent).await.unwrap();
5291 let completion =
5292 save_completion_test_child(&store, parent_id, child_id, "completed").await;
5293 let result = "result-legacy-pending-child";
5294 let presentation = runtime_resume_message(&completion, 0, Some(result));
5295 let old_wait = Utc::now() - chrono::Duration::minutes(2);
5296 let mut legacy = child_completion_envelope(
5297 &completion,
5298 old_wait,
5299 Some(result.into()),
5300 &presentation,
5301 );
5302 legacy.id = child_completion_id(&completion, old_wait, Some(result), true);
5303 let receipt = inbox.deliver(&legacy).await.unwrap();
5304 inbox
5305 .mark_message_activation_eligible(
5306 parent_id,
5307 &receipt,
5308 SessionActivationPolicy::RespectSpecificWait,
5309 )
5310 .await
5311 .unwrap();
5312 if acked {
5313 let claims = inbox.claim(parent_id, 1).await.unwrap();
5314 parent.add_message(claims[0].envelope.to_provider_message().unwrap());
5315 store.save_session(&parent).await.unwrap();
5316 inbox.ack(parent_id, &claims[0]).await.unwrap();
5317 }
5318 let new_wait = if arm != "none" {
5319 let mut ids = vec![child_id.into()];
5320 if arm == "partial" {
5321 ids.push("legacy-pending-slow".into());
5322 save_completion_test_child(&store, parent_id, "legacy-pending-slow", "running")
5323 .await;
5324 }
5325 let wait =
5326 WaitingForChildrenState::for_children(ids, ChildWaitPolicy::All, Utc::now());
5327 let mut runtime = read_runtime_state(&parent);
5328 runtime.waiting_for_children = Some(wait.clone());
5329 runtime.status = AgentStatusState::Suspended;
5330 write_runtime_state(&mut parent, &runtime);
5331 store.save_session(&parent).await.unwrap();
5332 Some(wait)
5333 } else {
5334 None
5335 };
5336
5337 coordinator.on_child_completed(completion.clone()).await;
5338 coordinator.on_child_completed(completion).await;
5339 let backlog = inbox.inspect(parent_id).await.unwrap();
5340 assert_eq!(
5341 backlog.generation, receipt.generation,
5342 "{arm}, acked={acked}"
5343 );
5344 assert_eq!(backlog.pending, usize::from(!acked), "{arm}, acked={acked}");
5345 assert_eq!(backlog.interrupt_pending(), !acked, "{arm}, acked={acked}");
5346 assert_eq!(
5350 launches.load(Ordering::SeqCst),
5351 usize::from(!acked || arm == "terminal"),
5352 "{arm}, acked={acked}"
5353 );
5354 let wait = read_runtime_state(&store.load_session(parent_id).await.unwrap().unwrap())
5355 .waiting_for_children;
5356 assert_eq!(wait, if arm == "partial" { new_wait } else { None });
5357 let claims = inbox.claim(parent_id, 8).await.unwrap();
5358 assert_eq!(claims.len(), usize::from(!acked));
5359 if !acked {
5360 assert_eq!(claims[0].envelope.id, legacy.id);
5361 assert_eq!(claims[0].generation, receipt.generation);
5362 }
5363 }
5364 }
5365
5366 #[tokio::test]
5367 async fn admitted_legacy_guardian_source_still_commits_a_pending_verdict() {
5368 let (_temp, store, inbox, coordinator, _, launches) = completion_inbox_fixture().await;
5369 let parent_id = "legacy-pending-guardian-parent";
5370 let child_id = "legacy-pending-guardian-child";
5371 let mut parent = Session::new(parent_id, "model");
5372 let mut guardian = crate::runtime::guardian_state::ensure_guardian_state(&parent);
5373 guardian.record_spawn(child_id);
5374 write_guardian_state(&mut parent, guardian);
5375 store.save_session(&parent).await.unwrap();
5376 let verdict = GuardianVerdict::approved();
5377 let mut child = Session::new_child(child_id, parent_id, "model", "Guardian");
5378 child.set_subagent_type("guardian");
5379 child.add_message(Message::assistant(
5380 serde_json::to_string(&verdict).unwrap(),
5381 None,
5382 ));
5383 child.set_last_run_status("completed");
5384 save_sourced_test_child(&store, &child).await;
5385 let completion = ChildCompletion {
5386 parent_session_id: parent_id.into(),
5387 child_session_id: child_id.into(),
5388 status: "completed".into(),
5389 error: None,
5390 completed_at: child.updated_at,
5391 source: test_child_source(&store, child_id).await,
5392 };
5393 let result = serde_json::to_string(&verdict).unwrap();
5394 let old_wait = Utc::now() - chrono::Duration::minutes(2);
5395 let presentation = guardian_resume_message(&completion, &verdict);
5396 let mut legacy =
5397 child_completion_envelope(&completion, old_wait, Some(result.clone()), &presentation);
5398 legacy.id = child_completion_id(&completion, old_wait, Some(&result), true);
5399 let receipt = inbox.deliver(&legacy).await.unwrap();
5400 inbox
5401 .mark_message_activation_eligible(
5402 parent_id,
5403 &receipt,
5404 SessionActivationPolicy::InterruptSpecificWait,
5405 )
5406 .await
5407 .unwrap();
5408 let claims = inbox.claim(parent_id, 1).await.unwrap();
5409 parent.add_message(claims[0].envelope.to_provider_message().unwrap());
5410 store.save_session(&parent).await.unwrap();
5411 inbox.ack(parent_id, &claims[0]).await.unwrap();
5412
5413 coordinator.on_child_completed(completion.clone()).await;
5414 let saved = store.load_session(parent_id).await.unwrap().unwrap();
5415 let guardian = read_guardian_state(&saved).unwrap();
5416 assert!(guardian.phase.is_reviewed());
5417 assert!(guardian.last_approved());
5418 assert_eq!(guardian.review_count, 1);
5419 let saved_guardian = saved.metadata["guardian.state"].clone();
5420 coordinator.on_child_completed(completion).await;
5421 assert_eq!(
5422 store
5423 .load_session(parent_id)
5424 .await
5425 .unwrap()
5426 .unwrap()
5427 .metadata["guardian.state"],
5428 saved_guardian
5429 );
5430 let backlog = inbox.inspect(parent_id).await.unwrap();
5431 assert_eq!(backlog.generation, receipt.generation);
5432 assert_eq!(backlog.pending, 0);
5433 assert_eq!(launches.load(Ordering::SeqCst), 0);
5434 }
5435
5436 #[tokio::test]
5437 async fn terminal_index_ahead_of_callbacks_preserves_every_waited_child_outcome() {
5438 for partial_first in [false, true] {
5439 let (_temp, store, inbox, coordinator, reservations, launches) =
5440 completion_inbox_fixture().await;
5441 let parent_id = "parallel-parent";
5442 let ids: Vec<_> = (0..4)
5443 .map(|slot| format!("parallel-child-{slot}"))
5444 .collect();
5445 save_waiting_test_parent(&store, parent_id, ids.clone(), ChildWaitPolicy::All).await;
5446 for (slot, id) in ids.iter().enumerate() {
5447 let mut child = Session::new_child(id, parent_id, "model", "Child");
5448 child.add_message(Message::assistant(format!("result-{id}"), None));
5449 child.set_last_run_status(if partial_first && slot > 0 {
5450 "running"
5451 } else {
5452 "completed"
5453 });
5454 save_sourced_test_child(&store, &child).await;
5455 }
5456 let mut sources = HashMap::new();
5457 for id in &ids {
5458 sources.insert(id.clone(), test_child_source(&store, id).await);
5459 }
5460 let completion = |id: &str| ChildCompletion {
5461 parent_session_id: parent_id.into(),
5462 child_session_id: id.into(),
5463 status: "completed".into(),
5464 error: None,
5465 completed_at: Utc::now(),
5466 source: sources.get(id).cloned().flatten(),
5467 };
5468 if partial_first {
5469 ChildCompletionHandler::on_child_completed(
5470 coordinator.as_ref(),
5471 completion(&ids[0]),
5472 )
5473 .await;
5474 assert!(
5475 read_runtime_state(&store.load_session(parent_id).await.unwrap().unwrap())
5476 .waiting_for_children
5477 .is_some()
5478 );
5479 assert_eq!(inbox.inspect(parent_id).await.unwrap().pending, 1);
5480 assert_eq!(launches.load(Ordering::SeqCst), 1);
5481 for id in &ids[1..] {
5482 let mut child = store.load_session(id).await.unwrap().unwrap();
5483 child.set_last_run_status("completed");
5484 save_sourced_test_child(&store, &child).await;
5485 }
5486 }
5487 for id in &ids {
5490 sources.insert(id.clone(), test_child_source(&store, id).await);
5491 }
5492 let completion = |id: &str| ChildCompletion {
5493 parent_session_id: parent_id.into(),
5494 child_session_id: id.into(),
5495 status: "completed".into(),
5496 error: None,
5497 completed_at: Utc::now(),
5498 source: sources.get(id).cloned().flatten(),
5499 };
5500 ChildCompletionHandler::on_child_completed(coordinator.as_ref(), completion(&ids[1]))
5501 .await;
5502 assert!(
5503 read_runtime_state(&store.load_session(parent_id).await.unwrap().unwrap())
5504 .waiting_for_children
5505 .is_none()
5506 );
5507 assert_eq!(inbox.inspect(parent_id).await.unwrap().pending, 4);
5508 assert_eq!(reservations.load(Ordering::SeqCst), 1);
5509 assert_eq!(launches.load(Ordering::SeqCst), 1);
5510 for id in &ids {
5511 ChildCompletionHandler::on_child_completed(coordinator.as_ref(), completion(id))
5512 .await;
5513 }
5514 let claims = inbox.claim(parent_id, 8).await.unwrap();
5515 assert_eq!(claims.len(), 4, "late callbacks cannot duplicate outcomes");
5516 assert_eq!(reservations.load(Ordering::SeqCst), 1);
5517 for id in &ids {
5518 let matching: Vec<_> = claims
5519 .iter()
5520 .filter_map(|claim| match &claim.envelope.body {
5521 SessionMessageBody::ChildOutcome(outcome)
5522 if &outcome.child_session_id == id =>
5523 {
5524 Some(outcome)
5525 }
5526 _ => None,
5527 })
5528 .collect();
5529 assert_eq!(matching.len(), 1);
5530 assert_eq!(
5531 matching[0].result.as_deref(),
5532 Some(format!("result-{id}").as_str())
5533 );
5534 }
5535 }
5536 }
5537
5538 #[tokio::test]
5539 async fn saved_a_callback_barrier_never_attributes_completed_b_to_a() {
5540 for guardian in [false, true] {
5541 let (_temp, store, inbox, coordinator, _, launches) = completion_inbox_fixture().await;
5542 let parent_id = "source-barrier-parent";
5543 let child_id = "source-barrier-child";
5544 save_waiting_test_parent(
5545 &store,
5546 parent_id,
5547 vec![child_id.into()],
5548 ChildWaitPolicy::All,
5549 )
5550 .await;
5551 if guardian {
5552 let mut parent = store.load_session(parent_id).await.unwrap().unwrap();
5553 let mut state = crate::runtime::guardian_state::ensure_guardian_state(&parent);
5554 state.record_spawn(child_id);
5555 write_guardian_state(&mut parent, state);
5556 store.save_session(&parent).await.unwrap();
5557 }
5558 let mut child = Session::new_child(child_id, parent_id, "model", "Child");
5559 if guardian {
5560 child
5561 .metadata
5562 .insert("subagent_type".into(), "guardian".into());
5563 }
5564 child.add_message(Message::user("assignment A"));
5565 store.save_session(&child).await.unwrap();
5566 let prior = child.messages.iter().map(|m| m.id.clone()).collect();
5567 child.add_message(Message::assistant("answer A", None));
5568 child.set_last_run_status("completed");
5569 ChildCompletionSource::prepare(&mut child, "actual-run-A", &prior);
5570 coordinator
5571 .agent
5572 .persistence()
5573 .save_runtime_session(&mut child)
5574 .await
5575 .unwrap();
5576 let completion_a = ChildCompletion {
5577 parent_session_id: parent_id.into(),
5578 child_session_id: child_id.into(),
5579 status: "completed".into(),
5580 error: None,
5581 completed_at: Utc::now(),
5582 source: ChildCompletionSource::after_final_save(&child, true),
5583 };
5584 assert_eq!(
5585 completion_a.source.as_ref().unwrap().activation_run_id,
5586 "actual-run-A"
5587 );
5588 let (release, barrier) = tokio::sync::oneshot::channel();
5591 let target = coordinator.clone();
5592 let callback = tokio::spawn(async move {
5593 barrier.await.unwrap();
5594 target.on_child_completed(completion_a).await;
5595 });
5596 let prior = child.messages.iter().map(|m| m.id.clone()).collect();
5597 child.add_message(Message::user("assignment B"));
5598 let answer_b = if guardian {
5599 r#"{"approve":true,"summary":"B approved","findings":[]}"#
5600 } else {
5601 "answer B"
5602 };
5603 child.add_message(Message::assistant(answer_b, None));
5604 child.set_last_run_status("completed");
5605 ChildCompletionSource::prepare(&mut child, "actual-run-B", &prior);
5606 coordinator
5607 .agent
5608 .persistence()
5609 .save_runtime_session(&mut child)
5610 .await
5611 .unwrap();
5612 release.send(()).unwrap();
5613 callback.await.unwrap();
5614 assert_eq!(inbox.inspect(parent_id).await.unwrap().pending, 0);
5615 let parent = store.load_session(parent_id).await.unwrap().unwrap();
5616 assert!(read_runtime_state(&parent).waiting_for_children.is_some());
5617 assert_eq!(launches.load(Ordering::SeqCst), 0);
5618 if guardian {
5619 assert!(read_guardian_state(&parent).unwrap().phase.is_pending());
5620 }
5621
5622 let completion_b = ChildCompletion {
5623 parent_session_id: parent_id.into(),
5624 child_session_id: child_id.into(),
5625 status: "completed".into(),
5626 error: None,
5627 completed_at: Utc::now(),
5628 source: ChildCompletionSource::after_final_save(&child, true),
5629 };
5630 coordinator.on_child_completed(completion_b.clone()).await;
5631 coordinator.on_child_completed(completion_b).await;
5632 let claims = inbox.claim(parent_id, 8).await.unwrap();
5633 assert_eq!(
5634 claims.len(),
5635 1,
5636 "B is admitted once with B's own provenance"
5637 );
5638 let SessionMessageBody::ChildOutcome(outcome) = &claims[0].envelope.body else {
5639 panic!("expected ChildOutcome");
5640 };
5641 assert_eq!(outcome.result.as_deref(), Some(answer_b));
5642 assert_eq!(launches.load(Ordering::SeqCst), 1);
5643 }
5644 }
5645
5646 #[tokio::test]
5647 async fn missing_source_cannot_import_latest_guardian_approval() {
5648 let (_temp, store, inbox, coordinator, _, _) = completion_inbox_fixture().await;
5649 let parent_id = "unavailable-guardian-parent";
5650 let child_id = "unavailable-guardian-child";
5651 save_waiting_test_parent(
5652 &store,
5653 parent_id,
5654 vec![child_id.into()],
5655 ChildWaitPolicy::All,
5656 )
5657 .await;
5658 let mut parent = store.load_session(parent_id).await.unwrap().unwrap();
5659 let mut state = crate::runtime::guardian_state::ensure_guardian_state(&parent);
5660 state.record_spawn(child_id);
5661 write_guardian_state(&mut parent, state);
5662 store.save_session(&parent).await.unwrap();
5663 let mut child = Session::new_child(child_id, parent_id, "model", "Child");
5664 child
5665 .metadata
5666 .insert("subagent_type".into(), "guardian".into());
5667 child.add_message(Message::assistant(
5668 r#"{"approve":true,"summary":"unbound latest","findings":[]}"#,
5669 None,
5670 ));
5671 child.set_last_run_status("completed");
5672 store.save_session(&child).await.unwrap();
5673 coordinator
5674 .on_child_completed(ChildCompletion {
5675 parent_session_id: parent_id.into(),
5676 child_session_id: child_id.into(),
5677 status: "completed".into(),
5678 error: None,
5679 completed_at: Utc::now(),
5680 source: None,
5681 })
5682 .await;
5683 let parent = store.load_session(parent_id).await.unwrap().unwrap();
5684 let state = read_guardian_state(&parent).unwrap();
5685 assert!(state.phase.is_reviewed());
5686 assert!(
5687 !state.last_approved(),
5688 "source-less latest approval is never authoritative"
5689 );
5690 let claims = inbox.claim(parent_id, 8).await.unwrap();
5691 let SessionMessageBody::ChildOutcome(outcome) = &claims[0].envelope.body else {
5692 panic!("expected ChildOutcome");
5693 };
5694 assert!(outcome.result.is_none());
5695 }
5696
5697 #[tokio::test]
5698 async fn lost_callback_replay_uses_the_committed_seal() {
5699 for valid in [true, false] {
5700 let (_temp, store, inbox, coordinator, _, _) = completion_inbox_fixture().await;
5701 save_waiting_test_parent(
5702 &store,
5703 "replay-parent",
5704 vec!["replay-child".into()],
5705 ChildWaitPolicy::All,
5706 )
5707 .await;
5708 let mut child = Session::new_child("replay-child", "replay-parent", "model", "Child");
5709 child.add_message(Message::assistant("sealed replay result", None));
5710 child.set_last_run_status("completed");
5711 save_sourced_test_child(&store, &child).await;
5712 if !valid {
5713 let mut child = store.load_session("replay-child").await.unwrap().unwrap();
5714 child.messages[0].content = "unsealed successor result".into();
5715 store.save_session(&child).await.unwrap();
5716 }
5717 let parent = store.load_session("replay-parent").await.unwrap().unwrap();
5718 let mut wait = read_runtime_state(&parent).waiting_for_children.unwrap();
5719 wait.registered_at = Utc::now() - chrono::Duration::seconds(61);
5720 coordinator.sweep_child_wait("replay-parent", wait).await;
5721 let claims = inbox.claim("replay-parent", 1).await.unwrap();
5722 if !valid {
5723 assert!(claims.is_empty());
5724 let parent = store.load_session("replay-parent").await.unwrap().unwrap();
5725 assert!(read_runtime_state(&parent).waiting_for_children.is_some());
5726 continue;
5727 }
5728 let SessionMessageBody::ChildOutcome(outcome) = &claims[0].envelope.body else {
5729 panic!("expected ChildOutcome");
5730 };
5731 assert_eq!(outcome.result.as_deref(), Some("sealed replay result"));
5732 }
5733 }
5734
5735 #[tokio::test]
5736 async fn partial_admission_then_source_mutation_ignores_stale_retry() {
5737 for parent_changed in [false, true] {
5738 let (_temp, store, inbox, coordinator, _, launches) = completion_inbox_fixture().await;
5739 let parent_id = "partial-source-parent";
5740 let child_id = "partial-source-child";
5741 save_waiting_test_parent(
5742 &store,
5743 parent_id,
5744 vec![child_id.into(), "still-running-child".into()],
5745 ChildWaitPolicy::All,
5746 )
5747 .await;
5748 let mut child = Session::new_child(child_id, parent_id, "model", "Child");
5749 if parent_changed {
5750 let mut parent = store.load_session(parent_id).await.unwrap().unwrap();
5751 let mut source = Message::user("required parent constraint");
5752 source.id = "parent-source".into();
5753 parent.add_message(source);
5754 store.save_session(&parent).await.unwrap();
5755 let packet = bamboo_domain::ChildContextPacket {
5756 version: 1,
5757 objective: "task".into(),
5758 constraints: vec![],
5759 acceptance: vec!["source remains attributable".into()],
5760 non_goals: vec![],
5761 necessary_user_instructions: vec![],
5762 recorded_decisions: vec![],
5763 source_user_message_ids: vec!["parent-source".into()],
5764 background_message_ids: vec![],
5765 };
5766 let resolved = packet.resolve(&parent, "task").unwrap();
5767 let binding = bamboo_domain::ChildContextBinding::new(
5768 &parent,
5769 child_id,
5770 resolved.required_brief.clone(),
5771 resolved,
5772 )
5773 .unwrap();
5774 binding.install(&mut child).unwrap();
5775 child.add_message(binding.assignment_message());
5776 }
5777 child.add_message(Message::assistant("original source", None));
5778 child.set_last_run_status("completed");
5779 save_sourced_test_child(&store, &child).await;
5780 let completion = ChildCompletion {
5781 parent_session_id: parent_id.into(),
5782 child_session_id: child_id.into(),
5783 status: "completed".into(),
5784 error: None,
5785 completed_at: Utc::now(),
5786 source: test_child_source(&store, child_id).await,
5787 };
5788 coordinator.on_child_completed(completion.clone()).await;
5789 assert_eq!(inbox.inspect(parent_id).await.unwrap().pending, 1);
5790 if parent_changed {
5791 let mut parent = store.load_session(parent_id).await.unwrap().unwrap();
5792 parent.messages[0].content = "modified parent source".into();
5793 store.save_session(&parent).await.unwrap();
5794 } else {
5795 child = store.load_session(child_id).await.unwrap().unwrap();
5796 child.messages[0].content = "modified after admission".into();
5797 store.save_session(&child).await.unwrap();
5798 }
5799 coordinator.on_child_completed(completion).await;
5800 assert_eq!(inbox.inspect(parent_id).await.unwrap().pending, 1);
5801 assert_eq!(launches.load(Ordering::SeqCst), 1);
5802 assert!(
5803 read_runtime_state(&store.load_session(parent_id).await.unwrap().unwrap())
5804 .waiting_for_children
5805 .is_some()
5806 );
5807 let mut sibling =
5808 Session::new_child("still-running-child", parent_id, "model", "Child");
5809 sibling.add_message(Message::assistant("sibling result", None));
5810 sibling.set_last_run_status("completed");
5811 save_sourced_test_child(&store, &sibling).await;
5812 coordinator
5813 .on_child_completed(ChildCompletion {
5814 parent_session_id: parent_id.into(),
5815 child_session_id: sibling.id.clone(),
5816 status: "completed".into(),
5817 error: None,
5818 completed_at: Utc::now(),
5819 source: test_child_source(&store, &sibling.id).await,
5820 })
5821 .await;
5822 assert_eq!(
5823 inbox.inspect(parent_id).await.unwrap().pending,
5824 2,
5825 "sibling completion cannot restage mutated A as a neutral outcome"
5826 );
5827 assert_eq!(launches.load(Ordering::SeqCst), 1);
5828 assert!(
5829 read_runtime_state(&store.load_session(parent_id).await.unwrap().unwrap())
5830 .waiting_for_children
5831 .is_some()
5832 );
5833 assert_eq!(
5834 inbox.inspect(parent_id).await.unwrap().claimed,
5835 0,
5836 "stale replay does not create another claim or reasoning owner"
5837 );
5838 }
5839 }
5840
5841 #[tokio::test]
5842 async fn short_circuit_wait_stages_terminal_siblings_without_waiting_or_foreign_content() {
5843 for policy in [ChildWaitPolicy::Any, ChildWaitPolicy::FirstError] {
5844 let (_temp, store, inbox, coordinator, _, launches) = completion_inbox_fixture().await;
5845 let parent_id = "short-circuit-parent";
5846 let ids = vec![
5847 "failed-child".into(),
5848 "done-child".into(),
5849 "running-child".into(),
5850 "foreign-child".into(),
5851 ];
5852 save_waiting_test_parent(&store, parent_id, ids, policy).await;
5853 for (id, owner, status, result) in [
5854 ("failed-child", parent_id, "error", "failed-owned-result"),
5855 ("done-child", parent_id, "completed", "done-owned-result"),
5856 ("running-child", parent_id, "running", "not-terminal"),
5857 (
5858 "foreign-child",
5859 "another-parent",
5860 "completed",
5861 "foreign-secret",
5862 ),
5863 ] {
5864 let mut child = Session::new_child(id, owner, "model", "Child");
5865 child.add_message(Message::assistant(result, None));
5866 child.set_last_run_status(status);
5867 save_sourced_test_child(&store, &child).await;
5868 }
5869 ChildCompletionHandler::on_child_completed(
5870 coordinator.as_ref(),
5871 ChildCompletion {
5872 parent_session_id: parent_id.into(),
5873 child_session_id: "failed-child".into(),
5874 status: "error".into(),
5875 error: None,
5876 completed_at: Utc::now(),
5877 source: test_child_source(&store, "failed-child").await,
5878 },
5879 )
5880 .await;
5881 assert!(
5882 read_runtime_state(&store.load_session(parent_id).await.unwrap().unwrap())
5883 .waiting_for_children
5884 .is_none()
5885 );
5886 let claims = inbox.claim(parent_id, 8).await.unwrap();
5887 assert_eq!(claims.len(), 2);
5888 let content = serde_json::to_string(
5889 &claims
5890 .iter()
5891 .map(|claim| &claim.envelope)
5892 .collect::<Vec<_>>(),
5893 )
5894 .unwrap();
5895 assert!(content.contains("done-owned-result"));
5896 assert!(!content.contains("foreign-secret"));
5897 assert!(!content.contains("not-terminal"));
5898 assert_eq!(
5899 store
5900 .load_session("running-child")
5901 .await
5902 .unwrap()
5903 .unwrap()
5904 .last_run_status()
5905 .as_deref(),
5906 Some("running")
5907 );
5908 assert_eq!(launches.load(Ordering::SeqCst), 1);
5909 }
5910 }
5911
5912 #[tokio::test]
5913 async fn sibling_admission_failure_keeps_wait_armed_without_activation() {
5914 let limits = bamboo_domain::SessionInboxLimits {
5915 max_backlog: 1,
5916 ..Default::default()
5917 };
5918 let (_temp, store, inbox, coordinator, reservations, launches) =
5919 completion_inbox_fixture_with_limits(limits).await;
5920 let parent_id = "partial-stage-parent";
5921 let ids = vec!["first-child".into(), "second-child".into()];
5922 save_waiting_test_parent(&store, parent_id, ids, ChildWaitPolicy::All).await;
5923 for id in ["first-child", "second-child"] {
5924 let mut child = Session::new_child(id, parent_id, "model", "Child");
5925 child.add_message(Message::assistant(format!("result-{id}"), None));
5926 child.set_last_run_status("completed");
5927 save_sourced_test_child(&store, &child).await;
5928 }
5929 ChildCompletionHandler::on_child_completed(
5930 coordinator.as_ref(),
5931 ChildCompletion {
5932 parent_session_id: parent_id.into(),
5933 child_session_id: "first-child".into(),
5934 status: "completed".into(),
5935 error: None,
5936 completed_at: Utc::now(),
5937 source: test_child_source(&store, "first-child").await,
5938 },
5939 )
5940 .await;
5941 let durable = store.load_session(parent_id).await.unwrap().unwrap();
5942 assert!(read_runtime_state(&durable).waiting_for_children.is_some());
5943 assert_eq!(
5944 read_runtime_state(&durable).status,
5945 AgentStatusState::Suspended
5946 );
5947 assert_eq!(inbox.inspect(parent_id).await.unwrap().pending, 1);
5948 let backlog = inbox.inspect(parent_id).await.unwrap();
5949 assert!(backlog.activation_pending());
5950 assert!(
5951 !backlog.interrupt_pending(),
5952 "failed wait transition is still respected"
5953 );
5954 assert_eq!(reservations.load(Ordering::SeqCst), 0);
5955 assert_eq!(launches.load(Ordering::SeqCst), 0);
5956 }
5957
5958 #[tokio::test]
5959 async fn supervisor_resume_rejects_old_incarnation_without_replacing_cache() {
5960 let (_temp, store, _inbox, coordinator, _reservations, _launches) =
5961 completion_inbox_fixture().await;
5962 let original = store
5963 .get_or_create_default_supervisor("model")
5964 .await
5965 .unwrap();
5966 let mut stale = store
5967 .load_session(&original.session_id)
5968 .await
5969 .unwrap()
5970 .unwrap();
5971 store.delete_session(&original.session_id).await.unwrap();
5972 let recreated = store
5973 .get_or_create_default_supervisor("replacement")
5974 .await
5975 .unwrap();
5976 assert_ne!(original.incarnation_id, recreated.incarnation_id);
5977 let mut current = store
5978 .load_session(&recreated.session_id)
5979 .await
5980 .unwrap()
5981 .unwrap();
5982 coordinator.save_and_cache(&mut current).await;
5983 let cached = coordinator
5984 .sessions
5985 .get(¤t.id)
5986 .unwrap()
5987 .value()
5988 .clone();
5989 coordinator.save_and_cache(&mut stale).await;
5990 assert!(Arc::ptr_eq(
5991 &cached,
5992 coordinator.sessions.get(¤t.id).unwrap().value()
5993 ));
5994 assert_eq!(
5995 store
5996 .load_session(¤t.id)
5997 .await
5998 .unwrap()
5999 .unwrap()
6000 .authority_identity,
6001 current.authority_identity
6002 );
6003 }
6004
6005 #[test]
6008 fn dead_child_candidate_status_matrix() {
6009 assert!(is_dead_child_candidate_status(None));
6011 assert!(is_dead_child_candidate_status(Some("running")));
6014 assert!(is_dead_child_candidate_status(Some("pending")));
6015 assert!(!is_dead_child_candidate_status(Some("suspended")));
6017 for status in ["completed", "error", "timeout", "cancelled", "skipped"] {
6019 assert!(!is_dead_child_candidate_status(Some(status)), "{status}");
6020 }
6021 }
6022
6023 #[test]
6024 fn completion_child_ownership_gates_content_fold() {
6025 assert!(completion_child_is_owned("parent-1", Some("parent-1")));
6027 assert!(!completion_child_is_owned("parent-1", Some("parent-2")));
6030 assert!(!completion_child_is_owned("parent-1", None));
6032 }
6033
6034 #[test]
6035 fn watchdog_resume_messages_are_hidden_runtime_messages() {
6036 for message in [
6037 empty_child_wait_message(),
6038 child_wait_lease_expired_message(&["c-1".to_string(), "c-2".to_string()]),
6039 ] {
6040 assert!(matches!(message.role, Role::User));
6041 let meta = message.metadata.expect("hidden runtime metadata");
6042 assert_eq!(meta[RUNTIME_RESUME_MESSAGE_HIDDEN_KEY], true);
6043 assert_eq!(
6044 meta[RUNTIME_RESUME_MESSAGE_KIND_KEY],
6045 "child_wait_watchdog_resume"
6046 );
6047 }
6048 let lease = child_wait_lease_expired_message(&["c-1".to_string()]);
6049 assert!(lease.content.contains("NOT cancelled"));
6051 assert!(lease.content.contains("c-1"));
6052 }
6053
6054 #[test]
6055 fn runtime_child_guidance_uses_compact_inspection_calls() {
6056 let completion = make_completion("completed");
6057 let child_inspection = serde_json::json!({
6058 "intent": "inspect", "target": "child-1", "message": "messages",
6059 })
6060 .to_string();
6061 let tree_inspection =
6062 serde_json::json!({"intent": "inspect", "message": "tree"}).to_string();
6063 for (message, expected_call) in [
6064 (
6065 runtime_resume_message(&completion, 0, Some("evidence")),
6066 &child_inspection,
6067 ),
6068 (
6069 guardian_resume_message(&completion, &GuardianVerdict::approved()),
6070 &child_inspection,
6071 ),
6072 (empty_child_wait_message(), &tree_inspection),
6073 (
6074 child_wait_lease_expired_message(&["child-1".to_string()]),
6075 &tree_inspection,
6076 ),
6077 ] {
6078 assert!(message.content.contains(expected_call));
6079 for legacy in [
6080 "SubAgent.get",
6081 "SubAgent.list",
6082 "SubAgent.wait",
6083 "child_session_id=",
6084 ] {
6085 assert!(!message.content.contains(legacy), "stale hint: {legacy}");
6086 }
6087 }
6088 }
6089
6090 #[test]
6093 fn non_terminal_statuses_never_satisfy_wait_policies() {
6094 assert!(!is_terminal_child_status("suspended"));
6097 assert!(!is_terminal_child_status("running"));
6098 assert!(!is_terminal_child_status("pending"));
6099 }
6100
6101 fn make_completion(status: &str) -> ChildCompletion {
6102 ChildCompletion {
6103 parent_session_id: "parent-1".to_string(),
6104 child_session_id: "child-1".to_string(),
6105 status: status.to_string(),
6106 error: None,
6107 completed_at: Utc::now(),
6108 source: None,
6109 }
6110 }
6111
6112 #[test]
6113 fn oversized_child_outcome_is_bounded_and_keeps_full_content_identity() {
6114 let mut completion = make_completion("completed");
6115 let wait_registered_at = Utc::now();
6116 let huge = format!("prefix-A-{}", "x".repeat(300 * 1024));
6117 let presentation = runtime_resume_message(&completion, 0, Some(&huge));
6118 let first = child_completion_envelope(
6119 &completion,
6120 wait_registered_at,
6121 Some(huge.clone()),
6122 &presentation,
6123 );
6124 assert!(
6125 serde_json::to_vec(&first).unwrap().len()
6126 < bamboo_domain::SessionInboxLimits::default().max_payload_bytes
6127 );
6128 let SessionMessageBody::ChildOutcome(outcome) = &first.body else {
6129 panic!("typed child outcome");
6130 };
6131 let stored = outcome.result.as_deref().unwrap();
6132 assert!(stored.contains("sha256="));
6133 assert!(stored.contains("\"intent\":\"inspect\""));
6134 assert!(stored.contains("\"target\":\"child-1\""));
6135 assert!(stored.contains("\"message\":\"result\""));
6136 assert!(!stored.contains("SubAgent.get"));
6137 assert!(stored.len() < CHILD_COMPLETION_INLINE_FIELD_BYTES);
6138
6139 completion.completed_at += chrono::Duration::seconds(1);
6142 let exact_retry = child_completion_envelope(
6143 &completion,
6144 wait_registered_at,
6145 Some(huge.clone()),
6146 &runtime_resume_message(&completion, 9, Some(&huge)),
6147 );
6148 assert_eq!(exact_retry.id, first.id);
6149 let changed = format!("prefix-B-{}", "x".repeat(300 * 1024));
6150 let corrected = child_completion_envelope(
6151 &completion,
6152 wait_registered_at,
6153 Some(changed.clone()),
6154 &runtime_resume_message(&completion, 0, Some(&changed)),
6155 );
6156 assert_ne!(corrected.id, first.id);
6157 }
6158
6159 #[test]
6160 fn oversized_child_error_points_to_bounded_error_view() {
6161 let mut completion = make_completion("error");
6162 completion.error = Some("失败🙂".repeat(30_000));
6163 let presentation = runtime_resume_message(&completion, 0, None);
6164 let envelope = child_completion_envelope(&completion, Utc::now(), None, &presentation);
6165 assert!(
6166 serde_json::to_vec(&envelope).unwrap().len()
6167 < bamboo_domain::SessionInboxLimits::default().max_payload_bytes
6168 );
6169 let SessionMessageBody::ChildOutcome(outcome) = &envelope.body else {
6170 panic!("typed child outcome");
6171 };
6172 let stored = outcome.error.as_deref().expect("bounded error");
6173 assert!(stored.contains(r#""message":"diagnostics""#));
6174 assert!(stored.contains("authenticated inspector"));
6175 assert!(!stored.contains("bounded slices"));
6176 assert!(!stored.contains("cursors"));
6177 assert!(stored.len() < CHILD_COMPLETION_INLINE_FIELD_BYTES);
6178 let metadata_error = outcome
6179 .provider_message
6180 .as_ref()
6181 .and_then(|provider| provider.metadata.get("child_error"))
6182 .and_then(serde_json::Value::as_str)
6183 .expect("bounded metadata error");
6184 assert_eq!(metadata_error, stored);
6185 assert!(!metadata_error.contains(&"失败🙂".repeat(30_000)));
6186 }
6187
6188 #[test]
6189 fn escaped_and_mixed_child_fields_fit_the_serialized_inbox_budget() {
6190 let mut completion = make_completion("error");
6191 completion.error = Some("\u{0001}".repeat(CHILD_COMPLETION_INLINE_FIELD_BYTES));
6192 let result = "\n".repeat(CHILD_COMPLETION_INLINE_FIELD_BYTES);
6193 let wait_registered_at = Utc::now();
6194 let presentation = runtime_resume_message(&completion, 0, Some(&result));
6195 let envelope = child_completion_envelope(
6196 &completion,
6197 wait_registered_at,
6198 Some(result.clone()),
6199 &presentation,
6200 );
6201 assert!(
6202 serde_json::to_vec(&envelope).unwrap().len()
6203 < bamboo_domain::SessionInboxLimits::default().max_payload_bytes
6204 );
6205 let SessionMessageBody::ChildOutcome(outcome) = &envelope.body else {
6206 panic!("typed child outcome");
6207 };
6208 let error = outcome.error.as_deref().unwrap();
6209 let stored_result = outcome.result.as_deref().unwrap();
6210 assert!(error.contains(r#""message":"diagnostics""#));
6211 assert!(stored_result.contains(r#""message":"result""#));
6212 assert!(!error.contains("Bounded tail"));
6213 assert!(!stored_result.contains("Bounded tail"));
6214 assert_eq!(
6215 outcome
6216 .provider_message
6217 .as_ref()
6218 .and_then(|provider| provider.metadata.get("child_error"))
6219 .and_then(serde_json::Value::as_str),
6220 Some(error)
6221 );
6222 let retry = child_completion_envelope(
6223 &completion,
6224 wait_registered_at,
6225 Some(result),
6226 &runtime_resume_message(&completion, 8, None),
6227 );
6228 assert_eq!(envelope.id, retry.id);
6229 }
6230
6231 #[test]
6232 fn compact_guardian_outcome_keeps_rejected_verdict_and_result_retrieval() {
6233 let completion = make_completion("completed");
6234 let result = format!(
6235 "{}{{\"approve\":false,\"findings\":[\"needs fix\"]}}",
6236 "\u{0001}".repeat(CHILD_COMPLETION_INLINE_FIELD_BYTES - 80)
6237 );
6238 assert!(result.len() <= CHILD_COMPLETION_INLINE_FIELD_BYTES);
6239 let verdict = parse_guardian_verdict(&result).expect("trailing guardian verdict");
6240 let presentation = guardian_resume_message(&completion, &verdict);
6241 assert!(
6242 serde_json::to_vec(&presentation).unwrap().len() <= CHILD_COMPLETION_INLINE_FIELD_BYTES,
6243 "the original guardian presentation must fit the inline cap"
6244 );
6245 let wait_registered_at = Utc::now();
6246 let expanded = build_child_completion_envelope(
6247 &completion,
6248 wait_registered_at,
6249 Some(&result),
6250 &presentation,
6251 false,
6252 );
6253 assert!(
6254 serde_json::to_vec(&expanded).unwrap().len()
6255 > bamboo_domain::SessionInboxLimits::default().max_payload_bytes,
6256 "JSON escaping must trigger the compact fallback"
6257 );
6258
6259 let envelope =
6260 child_completion_envelope(&completion, wait_registered_at, Some(result), &presentation);
6261 assert!(
6262 serde_json::to_vec(&envelope).unwrap().len()
6263 <= bamboo_domain::SessionInboxLimits::default().max_payload_bytes
6264 );
6265 let SessionMessageBody::ChildOutcome(outcome) = &envelope.body else {
6266 panic!("typed child outcome");
6267 };
6268 let provider = outcome.provider_message.as_ref().unwrap();
6269 assert!(provider.content.text.contains("Guardian review REJECTED"));
6270 assert!(provider.content.text.contains(r#""message":"result""#));
6271 assert_eq!(
6272 provider.metadata.get("guardian_approved"),
6273 Some(&serde_json::json!(false))
6274 );
6275 assert!(envelope
6276 .to_provider_message()
6277 .unwrap()
6278 .content
6279 .contains("Guardian review REJECTED"));
6280 }
6281
6282 #[tokio::test]
6283 async fn oversized_multibyte_child_error_is_durable_and_replayable() {
6284 let home = tempfile::tempdir().unwrap();
6285 let store = Arc::new(
6286 bamboo_storage::SessionStoreV2::new(home.path().to_path_buf())
6287 .await
6288 .unwrap(),
6289 );
6290 let parent_id = "large-error-parent";
6291 store
6292 .save_session(&Session::new(parent_id, "model"))
6293 .await
6294 .unwrap();
6295 let inbox = bamboo_storage::FileSessionInbox::new(
6296 store.clone(),
6297 bamboo_domain::SessionInboxLimits::default(),
6298 );
6299 let mut completion = make_completion("error");
6300 completion.parent_session_id = parent_id.to_string();
6301 completion.error = Some("失败🙂".repeat(30_000));
6302 let presentation = runtime_resume_message(&completion, 0, None);
6303 let envelope = child_completion_envelope(&completion, Utc::now(), None, &presentation);
6304 inbox.deliver(&envelope).await.unwrap();
6305 completion.child_session_id = "escaped-error-child".to_string();
6306 completion.error = Some("\u{0001}".repeat(CHILD_COMPLETION_INLINE_FIELD_BYTES));
6307 let result = "\n".repeat(CHILD_COMPLETION_INLINE_FIELD_BYTES);
6308 let mixed = child_completion_envelope(
6309 &completion,
6310 Utc::now(),
6311 Some(result.clone()),
6312 &runtime_resume_message(&completion, 0, Some(&result)),
6313 );
6314 let receipt = inbox.deliver(&mixed).await.unwrap();
6315 inbox
6316 .mark_activation_eligible(
6317 parent_id,
6318 receipt.generation,
6319 bamboo_domain::SessionActivationPolicy::RespectSpecificWait,
6320 )
6321 .await
6322 .unwrap();
6323 drop(inbox);
6324 drop(store);
6325
6326 let reopened_store = Arc::new(
6327 bamboo_storage::SessionStoreV2::new(home.path().to_path_buf())
6328 .await
6329 .unwrap(),
6330 );
6331 let reopened = bamboo_storage::FileSessionInbox::new(
6332 reopened_store,
6333 bamboo_domain::SessionInboxLimits::default(),
6334 );
6335 let claims = reopened.claim(parent_id, 2).await.unwrap();
6336 assert_eq!(claims.len(), 2);
6337 assert_eq!(claims[0].envelope.id, envelope.id);
6338 assert_eq!(claims[1].envelope.id, mixed.id);
6339 for claim in claims {
6340 assert!(
6341 serde_json::to_vec(&claim.envelope).unwrap().len()
6342 < bamboo_domain::SessionInboxLimits::default().max_payload_bytes
6343 );
6344 }
6345 }
6346
6347 #[tokio::test]
6348 async fn failed_child_wait_sidecar_save_keeps_cache_wait_and_transcript() {
6349 let (_temp, store, inbox, mut coordinator, reservations, launches) =
6350 completion_inbox_fixture().await;
6351 let parent_id = "failed-sidecar-parent";
6352 let child_id = "failed-sidecar-child";
6353 let now = Utc::now();
6354 let mut parent = Session::new(parent_id, "model");
6355 parent.add_message(Message::user("durable root turn"));
6356 let mut runtime = AgentRuntimeState::new("waiting-run");
6357 runtime.status = AgentStatusState::Suspended;
6358 runtime.waiting_for_children = Some(WaitingForChildrenState::for_children(
6359 vec![child_id.to_string()],
6360 ChildWaitPolicy::All,
6361 now,
6362 ));
6363 write_runtime_state(&mut parent, &runtime);
6364 parent.metadata.insert(
6365 "runtime.suspend_reason".to_string(),
6366 "waiting_for_children".to_string(),
6367 );
6368 store.save_session(&parent).await.unwrap();
6369 let mut cached = parent.clone();
6370 cached.add_message(Message::assistant("newer cached root turn", None));
6371 coordinator.sessions.insert(
6372 parent_id.to_string(),
6373 Arc::new(crate::SessionSnapshot::new(cached)),
6374 );
6375
6376 let mut child = Session::new_child(child_id, parent_id, "model", "Child");
6377 child.set_last_run_status("completed");
6378 save_sourced_test_child(&store, &child).await;
6379 Arc::get_mut(&mut coordinator).unwrap().persistence = Arc::new(LockedSessionStore::new(
6380 Arc::new(FailingRuntimeSaveStorage {
6381 inner: store.clone(),
6382 }),
6383 ));
6384
6385 ChildCompletionHandler::on_child_completed(
6386 coordinator.as_ref(),
6387 ChildCompletion {
6388 parent_session_id: parent_id.to_string(),
6389 child_session_id: child_id.to_string(),
6390 status: "completed".to_string(),
6391 error: None,
6392 completed_at: Utc::now(),
6393 source: None,
6394 },
6395 )
6396 .await;
6397
6398 let durable = store.load_session(parent_id).await.unwrap().unwrap();
6399 assert!(read_runtime_state(&durable).waiting_for_children.is_some());
6400 assert_eq!(durable.messages.len(), 1);
6401 assert_eq!(durable.messages[0].content, "durable root turn");
6402 let cached = coordinator.sessions.get(parent_id).unwrap();
6403 let cached = cached.value().read();
6404 assert!(read_runtime_state(&cached).waiting_for_children.is_some());
6405 assert!(cached
6406 .messages
6407 .iter()
6408 .any(|message| message.content == "newer cached root turn"));
6409 let backlog = inbox.inspect(parent_id).await.unwrap();
6410 assert_eq!(backlog.pending + backlog.claimed, 1);
6411 assert_eq!(reservations.load(Ordering::SeqCst), 0);
6412 assert_eq!(launches.load(Ordering::SeqCst), 0);
6413 }
6414
6415 #[tokio::test]
6416 async fn oversized_child_completion_clears_wait_and_activates_exactly_once() {
6417 let (_temp, store, inbox, coordinator, reservations, launches) =
6418 completion_inbox_fixture().await;
6419 let parent_id = "oversized-parent";
6420 let child_id = "oversized-child";
6421 let now = Utc::now();
6422 let mut parent = Session::new(parent_id, "model");
6423 let mut parent_runtime = AgentRuntimeState::new("waiting-run");
6424 parent_runtime.status = AgentStatusState::Suspended;
6425 parent_runtime.waiting_for_children = Some(WaitingForChildrenState::for_children(
6426 vec![child_id.to_string()],
6427 ChildWaitPolicy::All,
6428 now,
6429 ));
6430 parent_runtime.suspension = Some(SuspensionState {
6431 reason: "waiting_for_children".to_string(),
6432 suspended_at: now,
6433 resumable: true,
6434 hook_point: Some("ChildCompletion".to_string()),
6435 });
6436 write_runtime_state(&mut parent, &parent_runtime);
6437 parent.metadata.insert(
6438 "runtime.suspend_reason".to_string(),
6439 "waiting_for_children".to_string(),
6440 );
6441 store.save_session(&parent).await.unwrap();
6442 let mut cached_parent = parent.clone();
6446 cached_parent.add_message(Message::assistant("concurrent root turn", None));
6447 coordinator.sessions.insert(
6448 parent_id.to_string(),
6449 Arc::new(crate::SessionSnapshot::new(cached_parent)),
6450 );
6451
6452 let mut child = Session::new_child(child_id, parent_id, "model", "Child");
6453 child.add_message(Message::assistant("z".repeat(300 * 1024), None));
6454 child.set_last_run_status("completed");
6455 save_sourced_test_child(&store, &child).await;
6456 let completion = ChildCompletion {
6457 parent_session_id: parent_id.to_string(),
6458 child_session_id: child_id.to_string(),
6459 status: "completed".to_string(),
6460 error: None,
6461 completed_at: Utc::now(),
6462 source: test_child_source(&store, child_id).await,
6463 };
6464
6465 ChildCompletionHandler::on_child_completed(coordinator.as_ref(), completion.clone()).await;
6466 let durable_parent = store.load_session(parent_id).await.unwrap().unwrap();
6467 let durable_runtime = read_runtime_state(&durable_parent);
6468 assert!(durable_runtime.waiting_for_children.is_none());
6469 assert_eq!(durable_runtime.status, AgentStatusState::Idle);
6470 assert!(!durable_parent
6471 .metadata
6472 .contains_key("runtime.suspend_reason"));
6473 let cached_parent = coordinator.sessions.get(parent_id).unwrap();
6474 let cached_parent = cached_parent.value().read();
6475 assert!(cached_parent
6476 .messages
6477 .iter()
6478 .any(|message| message.content == "concurrent root turn"));
6479 assert!(read_runtime_state(&cached_parent)
6480 .waiting_for_children
6481 .is_none());
6482 let backlog = inbox.inspect(parent_id).await.unwrap();
6483 assert_eq!(backlog.pending + backlog.claimed, 1);
6484 assert!(backlog.activation_pending());
6485 assert_eq!(reservations.load(Ordering::SeqCst), 1);
6486 assert_eq!(launches.load(Ordering::SeqCst), 1);
6487
6488 let claim = inbox.claim(parent_id, 1).await.unwrap().remove(0);
6489 assert!(
6490 serde_json::to_vec(&claim.envelope).unwrap().len()
6491 < bamboo_domain::SessionInboxLimits::default().max_payload_bytes
6492 );
6493 ChildCompletionHandler::on_child_completed(coordinator.as_ref(), completion).await;
6496 assert_eq!(reservations.load(Ordering::SeqCst), 1);
6497 assert_eq!(launches.load(Ordering::SeqCst), 1);
6498 assert_eq!(inbox.inspect(parent_id).await.unwrap().claimed, 1);
6499 }
6500
6501 #[tokio::test]
6502 async fn completion_before_parent_wait_does_not_checkpoint_stale_root() {
6503 let (_temp, store, inbox, coordinator, reservations, launches) =
6504 completion_inbox_fixture().await;
6505 let parent_id = "parent-without-wait";
6506 let child_id = "already-finished-child";
6507 let mut parent = Session::new(parent_id, "model");
6508 parent.add_message(Message::assistant("root is still reasoning", None));
6509 store.save_session(&parent).await.unwrap();
6510 let before = store.load_session(parent_id).await.unwrap().unwrap();
6511
6512 let mut child = Session::new_child(child_id, parent_id, "model", "Child");
6513 child.set_last_run_status("completed");
6514 save_sourced_test_child(&store, &child).await;
6515 ChildCompletionHandler::on_child_completed(
6516 coordinator.as_ref(),
6517 ChildCompletion {
6518 parent_session_id: parent_id.to_string(),
6519 child_session_id: child_id.to_string(),
6520 status: "completed".to_string(),
6521 error: None,
6522 completed_at: Utc::now(),
6523 source: None,
6524 },
6525 )
6526 .await;
6527
6528 let after = store.load_session(parent_id).await.unwrap().unwrap();
6529 assert_eq!(after.updated_at, before.updated_at);
6530 assert_eq!(
6531 serde_json::to_value(&after.messages).unwrap(),
6532 serde_json::to_value(&before.messages).unwrap()
6533 );
6534 assert!(read_runtime_state(&after).waiting_for_children.is_none());
6535 assert_eq!(inbox.inspect(parent_id).await.unwrap().pending, 0);
6536 assert_eq!(reservations.load(Ordering::SeqCst), 0);
6537 assert_eq!(launches.load(Ordering::SeqCst), 0);
6538 }
6539
6540 #[tokio::test]
6541 async fn latest_locked_activation_mutation_preserves_wait_armed_after_stale_load() {
6542 let temp = tempfile::tempdir().unwrap();
6543 let store = Arc::new(
6544 bamboo_storage::SessionStoreV2::new(temp.path().to_path_buf())
6545 .await
6546 .unwrap(),
6547 );
6548 let storage: Arc<dyn Storage> = store.clone();
6549 let locked = LockedSessionStore::new(storage);
6550 let mut session = Session::new("activation-stale-wait", "model");
6551 session.agent_runtime_state = Some(AgentRuntimeState::new("old-run"));
6552 store.save_session(&session).await.unwrap();
6553
6554 let stale = store.load_session(&session.id).await.unwrap().unwrap();
6556 assert!(read_runtime_state(&stale).waiting_for_children.is_none());
6557
6558 locked
6559 .update_runtime_config(&session.id, |latest| {
6560 let mut state = read_runtime_state(latest);
6561 state.status = AgentStatusState::Suspended;
6562 state.waiting_for_children = Some(WaitingForChildrenState::for_children(
6563 vec!["child-new".to_string()],
6564 ChildWaitPolicy::All,
6565 Utc::now(),
6566 ));
6567 state.suspension = Some(SuspensionState {
6568 reason: "waiting_for_children".to_string(),
6569 suspended_at: Utc::now(),
6570 resumable: true,
6571 hook_point: None,
6572 });
6573 write_runtime_state(latest, &state);
6574 latest.metadata.insert(
6575 "runtime.suspend_reason".to_string(),
6576 "waiting_for_children".to_string(),
6577 );
6578 })
6579 .await
6580 .unwrap();
6581
6582 let (prepared, ready) = prepare_session_inbox_activation(&locked, &session.id, false)
6583 .await
6584 .unwrap()
6585 .unwrap();
6586 assert!(!ready, "latest durable specific wait must block activation");
6587 let state = read_runtime_state(&prepared);
6588 assert!(state.waiting_for_children.is_some());
6589 assert_eq!(state.status, AgentStatusState::Suspended);
6590 assert_eq!(
6591 prepared
6592 .metadata
6593 .get("runtime.suspend_reason")
6594 .map(String::as_str),
6595 Some("waiting_for_children")
6596 );
6597 }
6598
6599 #[tokio::test]
6600 async fn partial_child_and_bash_backlogs_remain_inert_after_inbox_reopen() {
6601 let temp = tempfile::tempdir().unwrap();
6602 let store = Arc::new(
6603 bamboo_storage::SessionStoreV2::new(temp.path().to_path_buf())
6604 .await
6605 .unwrap(),
6606 );
6607 store
6608 .save_session(&Session::new("restart-child-parent", "model"))
6609 .await
6610 .unwrap();
6611 store
6612 .save_session(&Session::new("restart-bash-parent", "model"))
6613 .await
6614 .unwrap();
6615 let inbox = bamboo_storage::FileSessionInbox::new(
6616 store.clone(),
6617 bamboo_domain::SessionInboxLimits::default(),
6618 );
6619
6620 let completion = ChildCompletion {
6621 parent_session_id: "restart-child-parent".to_string(),
6622 child_session_id: "child-a".to_string(),
6623 status: "completed".to_string(),
6624 error: None,
6625 completed_at: Utc::now(),
6626 source: None,
6627 };
6628 let child_resume = runtime_resume_message(&completion, 1, Some("first child"));
6629 inbox
6630 .deliver(&child_completion_envelope(
6631 &completion,
6632 Utc::now(),
6633 Some("first child".to_string()),
6634 &child_resume,
6635 ))
6636 .await
6637 .unwrap();
6638 let bash = BashCompletionInfo {
6639 session_id: "restart-bash-parent".to_string(),
6640 bash_id: "bash-a".to_string(),
6641 command: "true".to_string(),
6642 exit_code: Some(0),
6643 status: "completed".to_string(),
6644 output_tail: String::new(),
6645 };
6646 inbox
6647 .deliver(&bash_completion_envelope(&bash))
6648 .await
6649 .unwrap();
6650
6651 let reopened = bamboo_storage::FileSessionInbox::new(
6652 store,
6653 bamboo_domain::SessionInboxLimits::default(),
6654 );
6655 for session_id in ["restart-child-parent", "restart-bash-parent"] {
6656 let backlog = reopened.inspect(session_id).await.unwrap();
6657 assert_eq!(backlog.pending, 1);
6658 assert_eq!(backlog.activation_generation, 0);
6659 assert!(
6660 !backlog.activation_pending(),
6661 "startup must not run a partial wait backlog for {session_id}"
6662 );
6663 }
6664 }
6665
6666 struct StubChildIndex {
6669 children: Vec<(String, Option<String>)>,
6670 }
6671
6672 #[async_trait]
6673 impl Storage for StubChildIndex {
6674 async fn save_session(&self, _session: &Session) -> std::io::Result<()> {
6675 Ok(())
6676 }
6677 async fn load_session(&self, _id: &str) -> std::io::Result<Option<Session>> {
6678 Ok(None)
6679 }
6680 async fn delete_session(&self, _id: &str) -> std::io::Result<bool> {
6681 Ok(false)
6682 }
6683 async fn list_child_run_statuses(
6684 &self,
6685 _parent_session_id: &str,
6686 ) -> std::io::Result<Vec<(String, Option<String>)>> {
6687 Ok(self.children.clone())
6688 }
6689 }
6690
6691 #[tokio::test]
6692 async fn derive_completed_only_includes_terminal_children() {
6693 let storage: Arc<dyn Storage> = Arc::new(StubChildIndex {
6694 children: vec![
6695 ("a".into(), Some("completed".into())),
6696 ("b".into(), Some("running".into())),
6697 ("c".into(), Some("error".into())),
6698 ("d".into(), None),
6699 ],
6700 });
6701 let completed = derive_completed_child_ids(&storage, "parent-1", "b").await;
6702 assert_eq!(
6704 completed,
6705 vec!["a".to_string(), "b".to_string(), "c".to_string()]
6706 );
6707 }
6708
6709 #[tokio::test]
6710 async fn derive_completed_folds_in_just_completed_when_index_lags() {
6711 let storage: Arc<dyn Storage> = Arc::new(StubChildIndex {
6713 children: vec![("only".into(), Some("running".into()))],
6714 });
6715 let completed = derive_completed_child_ids(&storage, "parent-1", "only").await;
6716 assert_eq!(completed, vec!["only".to_string()]);
6717 }
6718
6719 #[test]
6720 fn wait_policy_all_uses_derived_completed_set() {
6721 let waited = vec!["a".to_string(), "b".to_string()];
6722 assert!(!wait_policy_satisfied(
6723 ChildWaitPolicy::All,
6724 &waited,
6725 &["a".to_string()],
6726 "a",
6727 "completed"
6728 ));
6729 assert!(wait_policy_satisfied(
6730 ChildWaitPolicy::All,
6731 &waited,
6732 &["a".to_string(), "b".to_string()],
6733 "b",
6734 "completed"
6735 ));
6736 }
6737
6738 #[test]
6739 fn wait_policy_first_error_requires_tracked_membership() {
6740 let waited = vec!["a".to_string(), "b".to_string()];
6741 assert!(wait_policy_satisfied(
6743 ChildWaitPolicy::FirstError,
6744 &waited,
6745 &["a".to_string()],
6746 "a",
6747 "error"
6748 ));
6749 assert!(!wait_policy_satisfied(
6752 ChildWaitPolicy::FirstError,
6753 &waited,
6754 &["a".to_string()],
6755 "stray-child",
6756 "timeout"
6757 ));
6758 assert!(wait_policy_satisfied(
6760 ChildWaitPolicy::FirstError,
6761 &waited,
6762 &["a".to_string(), "b".to_string()],
6763 "stray-child",
6764 "completed"
6765 ));
6766 }
6767
6768 #[test]
6769 fn child_final_assistant_text_returns_last_assistant() {
6770 let mut session = Session::new("child-1", "gpt-4");
6771 session.messages.push(Message::user("hi"));
6772 session
6773 .messages
6774 .push(Message::assistant("first answer", None));
6775 session.messages.push(Message::user("again"));
6776 session
6777 .messages
6778 .push(Message::assistant("final answer", None));
6779
6780 assert_eq!(
6781 child_final_assistant_text(&session).as_deref(),
6782 Some("final answer")
6783 );
6784 }
6785
6786 #[test]
6787 fn child_final_assistant_text_returns_none_when_blank() {
6788 let mut session = Session::new("child-1", "gpt-4");
6789 session.messages.push(Message::assistant(" ", None));
6790 assert!(child_final_assistant_text(&session).is_none());
6791 }
6792
6793 #[test]
6794 fn child_final_assistant_text_returns_none_when_no_assistant() {
6795 let mut session = Session::new("child-1", "gpt-4");
6796 session.messages.push(Message::user("hi"));
6797 assert!(child_final_assistant_text(&session).is_none());
6798 }
6799
6800 #[test]
6801 fn runtime_resume_message_folds_full_response_without_truncation() {
6802 let completion = make_completion("completed");
6805 let long: String = "a".repeat(10_000);
6806 let message = runtime_resume_message(&completion, 0, Some(&long));
6807 assert!(message.content.contains(&long));
6808 assert!(!message.content.contains("truncated"));
6809 }
6810
6811 #[test]
6812 fn runtime_resume_message_includes_child_response_when_provided() {
6813 let completion = make_completion("completed");
6814 let message = runtime_resume_message(&completion, 0, Some("the answer is 42"));
6815
6816 assert!(matches!(message.role, Role::User));
6817 assert!(!message.never_compress);
6820 assert!(message.content.contains("Child final response:"));
6821 assert!(message.content.contains("the answer is 42"));
6822
6823 let metadata = message.metadata.expect("metadata present");
6824 assert_eq!(
6825 metadata.get("hidden_from_ui").and_then(|v| v.as_bool()),
6826 Some(true)
6827 );
6828 assert_eq!(
6829 metadata.get("runtime_kind").and_then(|v| v.as_str()),
6830 Some("child_completion_resume")
6831 );
6832 assert_eq!(
6833 metadata
6834 .get("child_final_response_included")
6835 .and_then(|v| v.as_bool()),
6836 Some(true)
6837 );
6838 }
6839
6840 #[test]
6841 fn runtime_resume_message_falls_back_to_error_when_no_response() {
6842 let mut completion = make_completion("error");
6843 completion.error = Some("boom".to_string());
6844
6845 let message = runtime_resume_message(&completion, 1, None);
6846 assert!(message.content.contains("Child error:"));
6847 assert!(message.content.contains("boom"));
6848 let metadata = message.metadata.expect("metadata present");
6849 assert_eq!(
6850 metadata
6851 .get("child_final_response_included")
6852 .and_then(|v| v.as_bool()),
6853 Some(false)
6854 );
6855 }
6856
6857 #[test]
6858 fn runtime_resume_message_minimal_when_no_response_and_no_error() {
6859 let completion = make_completion("completed");
6860 let message = runtime_resume_message(&completion, 2, None);
6861 assert!(!message.content.contains("Child final response:"));
6862 assert!(!message.content.contains("Child error:"));
6863 assert!(message.content.contains("Resume the parent task"));
6864 }
6865
6866 #[test]
6867 fn read_config_snapshot_refreshes_cached_snapshot_from_live_config() {
6868 let runtime = tokio::runtime::Runtime::new().expect("runtime");
6869
6870 runtime.block_on(async {
6871 let config = Arc::new(RwLock::new(Config::default()));
6872 config.write().await.provider = "copilot".to_string();
6873 let cached_config = StdRwLock::new(Config::default());
6874
6875 let snapshot = read_config_snapshot(&config, &cached_config);
6876
6877 assert_eq!(snapshot.provider, "copilot");
6878 assert_eq!(
6879 cached_config.read().expect("cached snapshot lock").provider,
6880 "copilot"
6881 );
6882 });
6883 }
6884
6885 #[test]
6886 fn read_config_snapshot_uses_cached_snapshot_when_live_lock_is_busy() {
6887 let runtime = tokio::runtime::Runtime::new().expect("runtime");
6888
6889 runtime.block_on(async {
6890 let mut cached_snapshot = Config::default();
6891 cached_snapshot.provider = "cached-provider".to_string();
6892
6893 let config = Arc::new(RwLock::new(Config::default()));
6894 let cached_config = StdRwLock::new(cached_snapshot);
6895 let _write_guard = config.write().await;
6896
6897 let snapshot = read_config_snapshot(&config, &cached_config);
6898
6899 assert_eq!(snapshot.provider, "cached-provider");
6900 });
6901 }
6902
6903 #[test]
6906 fn bash_completion_resume_message_normal_announces_completion() {
6907 let ids = vec!["bg-1".to_string(), "bg-2".to_string()];
6908 let message = bash_completion_resume_message(&ids, false);
6909 assert!(
6911 message.content.contains("have completed"),
6912 "normal resume message must announce completion: {}",
6913 message.content
6914 );
6915 let metadata = message.metadata.expect("metadata present");
6917 assert_eq!(
6918 metadata
6919 .get(RUNTIME_RESUME_MESSAGE_HIDDEN_KEY)
6920 .and_then(|v| v.as_bool()),
6921 Some(true),
6922 "resume message must be hidden from the UI"
6923 );
6924 assert_eq!(
6925 metadata
6926 .get(RUNTIME_RESUME_MESSAGE_KIND_KEY)
6927 .and_then(|v| v.as_str()),
6928 Some(BASH_COMPLETION_RESUME_KIND),
6929 "resume message must carry the bash-completion kind discriminant"
6930 );
6931 }
6932
6933 #[test]
6934 fn bash_completion_resume_message_deadline_does_not_claim_completion() {
6935 let ids = vec!["bg-long".to_string()];
6939 let message = bash_completion_resume_message(&ids, true);
6940 assert!(
6941 !message.content.contains("have completed"),
6942 "deadline resume message must NOT claim the shells completed: {}",
6943 message.content
6944 );
6945 assert!(
6946 message.content.contains("may still be running"),
6947 "deadline resume message must warn shells may still be running: {}",
6948 message.content
6949 );
6950 assert!(
6951 message.content.contains("BashOutput"),
6952 "deadline resume message must direct verification via BashOutput: {}",
6953 message.content
6954 );
6955 let metadata = message.metadata.expect("metadata present");
6957 assert_eq!(
6958 metadata
6959 .get(RUNTIME_RESUME_MESSAGE_KIND_KEY)
6960 .and_then(|v| v.as_str()),
6961 Some(BASH_COMPLETION_RESUME_KIND)
6962 );
6963 }
6964
6965 #[test]
6966 fn bash_resume_should_retry_matrix() {
6967 assert!(!bash_resume_should_retry(
6973 &ResumeOutcome::Started { run_id: "r".into() },
6974 true
6975 ));
6976 assert!(!bash_resume_should_retry(
6977 &ResumeOutcome::Started { run_id: "r".into() },
6978 false
6979 ));
6980
6981 assert!(!bash_resume_should_retry(&ResumeOutcome::NotFound, true));
6983 assert!(!bash_resume_should_retry(&ResumeOutcome::NotFound, false));
6984
6985 assert!(bash_resume_should_retry(&ResumeOutcome::Completed, true));
6987 assert!(!bash_resume_should_retry(&ResumeOutcome::Completed, false));
6990
6991 assert!(bash_resume_should_retry(
6994 &ResumeOutcome::AlreadyRunning { run_id: "r".into() },
6995 true
6996 ));
6997 assert!(!bash_resume_should_retry(
6999 &ResumeOutcome::AlreadyRunning { run_id: "r".into() },
7000 false
7001 ));
7002 }
7003
7004 #[test]
7007 fn injection_body_includes_status_exit_command_and_tail() {
7008 let info = BashCompletionInfo {
7009 session_id: "s".into(),
7010 bash_id: "abc123".into(),
7011 command: "make build".into(),
7012 exit_code: Some(0),
7013 status: "completed".into(),
7014 output_tail: "BUILD OK".into(),
7015 };
7016 let body = bash_completion_injection_body(&info);
7017 assert!(body.contains("abc123"), "body: {body}");
7018 assert!(body.contains("make build"), "body: {body}");
7019 assert!(body.contains("completed"), "body: {body}");
7020 assert!(body.contains("exit code 0"), "body: {body}");
7021 assert!(body.contains("BUILD OK"), "body: {body}");
7022 assert!(body.contains("BashOutput"), "body: {body}");
7024 assert!(body.contains("bash_id=\"abc123\""), "body: {body}");
7025 }
7026
7027 #[test]
7028 fn injection_body_handles_no_output_and_signal_kill() {
7029 let info = BashCompletionInfo {
7030 session_id: "s".into(),
7031 bash_id: "xyz".into(),
7032 command: "sleep 99".into(),
7033 exit_code: None,
7034 status: "killed".into(),
7035 output_tail: String::new(),
7036 };
7037 let body = bash_completion_injection_body(&info);
7038 assert!(body.contains("killed"), "body: {body}");
7039 assert!(body.contains("none (signal/killed)"), "body: {body}");
7040 assert!(body.contains("no captured output"), "body: {body}");
7041 assert!(!body.contains("Output tail:"), "body: {body}");
7043 }
7044
7045 #[test]
7046 fn background_completion_builds_post_tool_use_payload_and_feedback() {
7047 let mut info = BashCompletionInfo {
7048 session_id: "s".into(),
7049 bash_id: "bg-7".into(),
7050 command: "cargo test".into(),
7051 exit_code: Some(0),
7052 status: "completed".into(),
7053 output_tail: "test result: ok".into(),
7054 };
7055
7056 let payload = background_bash_post_tool_payload(&info);
7057 match payload {
7058 HookPayload::ToolResult {
7059 tool_name,
7060 tool_call_id,
7061 outcome,
7062 } => {
7063 assert_eq!(tool_name, "Bash");
7064 assert_eq!(tool_call_id, "bg-7");
7065 assert!(outcome.success);
7066 let response: serde_json::Value =
7067 serde_json::from_str(outcome.result.as_deref().unwrap()).unwrap();
7068 assert_eq!(response["command"], "cargo test");
7069 assert_eq!(response["exit_code"], 0);
7070 assert_eq!(response["status"], "completed");
7071 assert_eq!(response["output_tail"], "test result: ok");
7072 }
7073 other => panic!("expected PostToolUse payload, got {other:?}"),
7074 }
7075
7076 append_background_bash_hook_feedback(
7077 &mut info,
7078 vec!["Run the formatter before continuing".to_string()],
7079 );
7080 assert!(info.output_tail.contains("<post_tool_use_feedback>"));
7081 assert!(info
7082 .output_tail
7083 .contains("Run the formatter before continuing"));
7084 }
7085
7086 #[tokio::test]
7087 async fn bash_completion_payload_identity_matches_file_inbox_idempotency() {
7088 let baseline = BashCompletionInfo {
7089 session_id: "session".into(),
7090 bash_id: "bg-7".into(),
7091 command: "cargo test".into(),
7092 exit_code: Some(0),
7093 status: "completed".into(),
7094 output_tail: "first tail".into(),
7095 };
7096 let mut retried = baseline.clone();
7097 retried.output_tail = "first tail\nlater bytes\n<hook feedback>".into();
7098 retried.status = "completed-after-hook".into();
7099 let baseline_envelope = bash_completion_envelope(&baseline);
7100 let exact_retry = bash_completion_envelope(&baseline);
7101 let corrected_envelope = bash_completion_envelope(&retried);
7102 assert_eq!(baseline_envelope.id, exact_retry.id);
7103 assert_ne!(
7104 baseline_envelope.id, corrected_envelope.id,
7105 "changed payload semantics must receive a distinct id"
7106 );
7107
7108 let mut other_shell = baseline.clone();
7109 other_shell.bash_id = "bg-8".into();
7110 assert_ne!(
7111 bash_completion_envelope(&baseline).id,
7112 bash_completion_envelope(&other_shell).id
7113 );
7114
7115 let temp = tempfile::tempdir().unwrap();
7116 let store = Arc::new(
7117 bamboo_storage::SessionStoreV2::new(temp.path().to_path_buf())
7118 .await
7119 .unwrap(),
7120 );
7121 store
7122 .save_session(&Session::new("session", "model"))
7123 .await
7124 .unwrap();
7125 let inbox = bamboo_storage::FileSessionInbox::new(
7126 store,
7127 bamboo_domain::SessionInboxLimits::default(),
7128 );
7129 let first = inbox.deliver(&baseline_envelope).await.unwrap();
7130 let duplicate = inbox.deliver(&exact_retry).await.unwrap();
7131 let corrected = inbox.deliver(&corrected_envelope).await.unwrap();
7132 assert_eq!(duplicate, first, "exact payload retry is idempotent");
7133 assert_ne!(corrected.id, first.id);
7134 assert_eq!(corrected.generation, first.generation + 1);
7135 assert_eq!(inbox.inspect("session").await.unwrap().pending, 2);
7136 }
7137
7138 #[cfg(unix)]
7139 #[tokio::test]
7140 async fn background_completion_fires_configured_post_tool_use_command() {
7141 use bamboo_config::{
7142 LifecycleHookGroup, LifecycleHookHandler, LifecycleHooksConfig,
7143 DEFAULT_LIFECYCLE_HOOK_TIMEOUT_MS,
7144 };
7145
7146 let dir = tempfile::tempdir().unwrap();
7147 let output = dir.path().join("background-post-tool.json");
7148 let command = format!(
7149 "cat > '{}'; printf '%s' '{{\"additional_context\":\"inspect the completed build log\"}}'",
7150 output.display()
7151 );
7152 let config = LifecycleHooksConfig {
7153 enabled: true,
7154 post_tool_use: vec![LifecycleHookGroup {
7155 enabled: true,
7156 matcher: Some("^Bash$".to_string()),
7157 hooks: vec![LifecycleHookHandler::command(
7158 command,
7159 DEFAULT_LIFECYCLE_HOOK_TIMEOUT_MS,
7160 )],
7161 }],
7162 ..Default::default()
7163 };
7164 let mut session = Session::new("session-bg-hook", "test-model");
7165 session.workspace = Some(dir.path().to_string_lossy().into_owned());
7166 let mut info = BashCompletionInfo {
7167 session_id: session.id.clone(),
7168 bash_id: "bg-9".into(),
7169 command: "cargo test".into(),
7170 exit_code: Some(0),
7171 status: "completed".into(),
7172 output_tail: "test result: ok".into(),
7173 };
7174
7175 assert!(run_background_bash_post_tool_hooks(&config, None, &session, &mut info).await);
7176 let envelope: serde_json::Value =
7177 serde_json::from_str(&std::fs::read_to_string(output).unwrap()).unwrap();
7178 assert_eq!(envelope["hook_event_name"], "PostToolUse");
7179 assert_eq!(envelope["tool_name"], "Bash");
7180 assert_eq!(envelope["payload"]["tool_call_id"], "bg-9");
7181 let response = envelope["tool_response"]["result"]
7182 .as_str()
7183 .map(serde_json::from_str::<serde_json::Value>)
7184 .transpose()
7185 .unwrap()
7186 .unwrap();
7187 assert_eq!(response["command"], "cargo test");
7188 assert_eq!(response["status"], "completed");
7189 assert!(info.output_tail.contains("inspect the completed build log"));
7190 }
7191
7192 async fn temp_store() -> (tempfile::TempDir, Arc<dyn Storage>, LockedSessionStore) {
7193 let temp = tempfile::tempdir().unwrap();
7194 let storage: Arc<dyn Storage> = Arc::new(
7195 bamboo_storage::v2::SessionStoreV2::new(temp.path().to_path_buf())
7196 .await
7197 .expect("storage init"),
7198 );
7199 let persistence = LockedSessionStore::new(storage.clone());
7200 (temp, storage, persistence)
7201 }
7202
7203 #[tokio::test]
7204 async fn enqueue_writes_pending_injection_and_preserves_messages() {
7205 let (_temp, storage, persistence) = temp_store().await;
7206
7207 let mut session = Session::new("sess-enq", "test-model");
7208 session.add_message(Message::user("do the build"));
7209 storage.save_session(&session).await.unwrap();
7210
7211 let info = BashCompletionInfo {
7212 session_id: "sess-enq".into(),
7213 bash_id: "sh-1".into(),
7214 command: "make".into(),
7215 exit_code: Some(0),
7216 status: "completed".into(),
7217 output_tail: "done".into(),
7218 };
7219 let saved = enqueue_bash_completion_injection(&persistence, &info)
7220 .await
7221 .expect("enqueue io ok")
7222 .expect("session exists");
7223
7224 let pending = saved
7225 .pending_injected_messages()
7226 .expect("pending injection present");
7227 assert_eq!(pending.len(), 1);
7228 let content = pending[0].get("content").and_then(|v| v.as_str()).unwrap();
7229 assert!(content.contains("sh-1"), "content: {content}");
7230 assert!(content.contains("make"), "content: {content}");
7231 assert!(content.contains("done"), "content: {content}");
7232 assert_eq!(saved.messages.len(), 1);
7234 }
7235
7236 #[tokio::test]
7237 async fn enqueue_returns_none_for_missing_session() {
7238 let (_temp, _storage, persistence) = temp_store().await;
7239 let info = BashCompletionInfo {
7240 session_id: "does-not-exist".into(),
7241 bash_id: "x".into(),
7242 command: "true".into(),
7243 exit_code: Some(0),
7244 status: "completed".into(),
7245 output_tail: String::new(),
7246 };
7247 let result = enqueue_bash_completion_injection(&persistence, &info)
7248 .await
7249 .expect("io ok");
7250 assert!(result.is_none(), "no session → nothing enqueued");
7251 }
7252
7253 #[test]
7261 fn apply_bash_resume_transition_clears_wait_and_appends_message() {
7262 use bamboo_domain::session::runtime_state::WaitingForBashState;
7263
7264 let mut session = Session::new("sess-resume", "test-model");
7265 session.add_message(Message::user("kick off the build"));
7266 let mut rt = read_runtime_state(&session);
7267 rt.status = AgentStatusState::Running;
7268 rt.waiting_for_bash = Some(WaitingForBashState::for_bash(
7269 vec!["sh-1".into()],
7270 Utc::now(),
7271 ));
7272 write_runtime_state(&mut session, &rt);
7273 session.metadata.insert(
7274 "runtime.suspend_reason".to_string(),
7275 "waiting_for_bash".to_string(),
7276 );
7277
7278 let resume = bash_completion_resume_message(&["sh-1".to_string()], false);
7279 let did = apply_bash_resume_transition(&mut session, &resume);
7280
7281 assert!(did, "a suspended session must transition");
7282 let after = read_runtime_state(&session);
7283 assert!(
7284 after.waiting_for_bash.is_none(),
7285 "bash wait must be cleared"
7286 );
7287 assert_eq!(after.status, AgentStatusState::Idle, "runtime must be Idle");
7288 assert!(
7289 !session.metadata.contains_key("runtime.suspend_reason"),
7290 "suspend-reason marker must be removed"
7291 );
7292 assert_eq!(session.messages.len(), 2, "resume message must be appended");
7293 assert!(matches!(
7294 session.messages.last().map(|m| &m.role),
7295 Some(Role::User)
7296 ));
7297 }
7298
7299 #[test]
7303 fn apply_bash_resume_transition_noops_when_not_waiting() {
7304 let mut session = Session::new("sess-live", "test-model");
7305 session.add_message(Message::user("hi"));
7306
7307 let resume = bash_completion_resume_message(&["sh-1".to_string()], false);
7308 let did = apply_bash_resume_transition(&mut session, &resume);
7309
7310 assert!(!did, "a non-waiting session must not transition");
7311 assert_eq!(session.messages.len(), 1, "no resume message appended");
7312 }
7313
7314 #[test]
7318 fn bash_completion_should_resume_only_when_suspended_and_all_done() {
7319 assert!(bash_completion_should_resume(true, true));
7320 assert!(!bash_completion_should_resume(true, false)); assert!(!bash_completion_should_resume(false, true)); assert!(!bash_completion_should_resume(false, false));
7323 }
7324
7325 #[test]
7326 fn two_shell_delivery_stages_backlog_before_one_final_activation() {
7327 let mut waiting = true;
7328 let mut durable_backlog = 0;
7329 let mut reservations = 0;
7330
7331 durable_backlog += 1;
7333 let first = bash_completion_delivery_plan(waiting, false);
7334 assert_eq!(first, BashCompletionDeliveryPlan::DurableOnly);
7335 assert!(waiting);
7336 assert_eq!(durable_backlog, 1);
7337 assert_eq!(reservations, 0);
7338
7339 durable_backlog += 1;
7342 let last = bash_completion_delivery_plan(waiting, true);
7343 assert_eq!(last, BashCompletionDeliveryPlan::ClearWaitThenActivate);
7344 waiting = false;
7345 reservations += 1;
7346 assert!(!waiting);
7347 assert_eq!(durable_backlog, 2);
7348 assert_eq!(reservations, 1);
7349 }
7350
7351 #[test]
7355 fn bash_resume_message_from_info_carries_bashid_tail_and_kind() {
7356 let info = BashCompletionInfo {
7357 session_id: "s".into(),
7358 bash_id: "sh-42".into(),
7359 command: "cargo test".into(),
7360 exit_code: Some(0),
7361 status: "completed".into(),
7362 output_tail: "test result: ok".into(),
7363 };
7364 let msg = bash_resume_message_from_info(&info);
7365
7366 assert!(matches!(msg.role, Role::User));
7367 assert!(msg.content.contains("sh-42"), "content: {}", msg.content);
7368 assert!(
7369 msg.content.contains("cargo test"),
7370 "content: {}",
7371 msg.content
7372 );
7373 assert!(
7374 msg.content.contains("test result: ok"),
7375 "content: {}",
7376 msg.content
7377 );
7378 assert!(
7379 msg.content.contains("BashOutput"),
7380 "content: {}",
7381 msg.content
7382 );
7383 let meta = serde_json::to_string(&msg.metadata).unwrap();
7384 assert!(
7385 meta.contains(BASH_COMPLETION_RESUME_KIND),
7386 "resume message must be tagged as a bash-completion resume: {meta}"
7387 );
7388 }
7389}