1use std::collections::HashMap;
8use std::sync::{Arc, Mutex as StdMutex, OnceLock, RwLock as StdRwLock, Weak};
9use std::time::Duration;
10
11use bamboo_domain::poison::PoisonRecover;
12use bamboo_domain::{
13 AgentHookPoint, HookPayload, HookToolOutcome, SessionActivationPolicy, SessionChildOutcome,
14 SessionMessageBody, SessionMessageContent, SessionMessageEnvelope, SessionMessageId,
15 SessionMessageKind, SessionMessageSource, SessionProviderMessage,
16};
17
18use crate::execution::{
19 finalize_runner, reserve_runner_core, reserve_session_execution, spawn_session_execution,
20 AgentRunner, AgentStatus, ChildCompletion, ChildCompletionHandler, ChildCompletionSource,
21 ReserveOutcome, SessionExecutionArgs, SessionExecutionReservation,
22 SessionExecutionReserveOutcome, SpawnJob, SpawnScheduler,
23};
24use crate::runtime::config::{BashResumeHook, GuardianSpawner, BASH_COMPLETION_RESUME_KIND};
25use crate::runtime::guardian_state::{
26 parse_guardian_verdict, read_guardian_config, read_guardian_state, write_guardian_state,
27 GuardianVerdict,
28};
29use crate::Agent;
30use async_trait::async_trait;
31use bamboo_agent_core::storage::Storage;
32use bamboo_agent_core::tools::ToolExecutor;
33use bamboo_agent_core::{
34 AgentEvent, BashCompletionInfo, BashCompletionSink, Message, Role, Session, SessionKind,
35};
36use bamboo_domain::session::runtime_state::{
37 AgentRuntimeState, AgentStatusState, ChildWaitPolicy, WaitingForChildrenState,
38};
39use bamboo_llm::{Config, ProviderModelRouter, ProviderRegistry};
40use bamboo_storage::LockedSessionStore;
41use chrono::Utc;
42use sha2::{Digest, Sha256};
43use tokio::sync::{broadcast, RwLock};
44
45use crate::model_areas::resolve_global_area_models;
46use crate::model_config_helper::{
47 resolve_fast_model, resolve_gold_config, resolve_provider_routing_key, GOLD_CONFIG_METADATA_KEY,
48};
49use crate::session_activation::{
50 SessionActivationLaunch, SessionActivationReserveOutcome, SessionActivationSpawner,
51};
52use crate::session_app::child_session::result_projection::{
53 child_result_identity, decode_report, fits_child_result_budget, plain_child_report_message,
54 project_child_report, unavailable_child_result,
55};
56use crate::session_app::execute::consume_pending_clarification_resume;
57use crate::session_app::provider_model::{persist_model_ref, session_effective_model_ref};
58use crate::session_app::resume::{
59 resume_session_execution, ResumeExecutionPort, ResumeSpawnRequest,
60};
61use crate::session_app::types::{ResumeConfigSnapshot, ResumeOutcome};
62
63const AGENT_RUNTIME_STATE_METADATA_KEY: &str = "agent.runtime.state";
64const RUNTIME_RESUME_MESSAGE_HIDDEN_KEY: &str = "hidden_from_ui";
65const RUNTIME_RESUME_MESSAGE_KIND_KEY: &str = "runtime_kind";
66const CHILD_COMPLETION_INLINE_FIELD_BYTES: usize = 48 * 1024;
67const CHILD_COMPLETION_OVERSIZE_TAIL_BYTES: usize = 8 * 1024;
68
69fn read_runtime_state(session: &Session) -> AgentRuntimeState {
70 session
71 .agent_runtime_state
72 .clone()
73 .or_else(|| {
74 session
75 .metadata
76 .get(AGENT_RUNTIME_STATE_METADATA_KEY)
77 .and_then(|raw| serde_json::from_str::<AgentRuntimeState>(raw).ok())
78 })
79 .unwrap_or_else(|| AgentRuntimeState::new(format!("{}-child-wait", session.id)))
80}
81
82fn write_runtime_state(session: &mut Session, runtime_state: &AgentRuntimeState) {
83 session.agent_runtime_state = Some(runtime_state.clone());
84 if let Ok(serialized) = serde_json::to_string(runtime_state) {
85 session
86 .metadata
87 .insert(AGENT_RUNTIME_STATE_METADATA_KEY.to_string(), serialized);
88 }
89}
90
91async fn prepare_session_inbox_activation(
97 persistence: &LockedSessionStore,
98 session_id: &str,
99 interrupt_specific_wait: bool,
100) -> std::io::Result<Option<(Session, bool)>> {
101 let ready = Arc::new(std::sync::atomic::AtomicBool::new(false));
102 let ready_for_mutation = ready.clone();
103 let saved = persistence
104 .update_runtime_config(session_id, move |latest| {
105 if latest.has_pending_question() {
109 return;
110 }
111 let mut runtime_state = read_runtime_state(latest);
112 let specifically_waiting = runtime_state.waiting_for_children.is_some()
113 || runtime_state.waiting_for_bash.is_some();
114 if specifically_waiting && !interrupt_specific_wait {
115 return;
116 }
117 runtime_state.status = AgentStatusState::Idle;
123 runtime_state.suspension = None;
124 write_runtime_state(latest, &runtime_state);
125 latest.metadata.remove("runtime.suspend_reason");
126 latest.updated_at = Utc::now();
127 ready_for_mutation.store(true, std::sync::atomic::Ordering::Release);
128 })
129 .await?;
130 Ok(saved.map(|session| (session, ready.load(std::sync::atomic::Ordering::Acquire))))
131}
132
133fn is_error_like(status: &str) -> bool {
134 matches!(status, "error" | "timeout" | "cancelled")
135}
136
137fn is_terminal_child_status(status: &str) -> bool {
139 matches!(
140 status,
141 "completed" | "error" | "timeout" | "cancelled" | "skipped"
142 )
143}
144
145fn child_completion_envelope(
146 completion: &ChildCompletion,
147 wait_registered_at: chrono::DateTime<Utc>,
148 result: Option<String>,
149 provider_message: &Message,
150) -> SessionMessageEnvelope {
151 let envelope = build_child_completion_envelope(
152 completion,
153 wait_registered_at,
154 result.as_deref(),
155 provider_message,
156 false,
157 );
158 if serde_json::to_vec(&envelope).map_or(true, |bytes| {
159 bytes.len() > bamboo_domain::SessionInboxLimits::default().max_payload_bytes
160 }) {
161 return build_child_completion_envelope(
166 completion,
167 wait_registered_at,
168 result.as_deref(),
169 provider_message,
170 true,
171 );
172 }
173 envelope
174}
175
176fn child_completion_id(
177 completion: &ChildCompletion,
178 wait_registered_at: chrono::DateTime<Utc>,
179 result: Option<&str>,
180 legacy_wait_identity: bool,
181) -> SessionMessageId {
182 fn identity(value: Option<&str>) -> serde_json::Value {
183 match value {
184 Some(value) if value.len() > CHILD_COMPLETION_INLINE_FIELD_BYTES => serde_json::json!({
185 "oversized": true, "utf8_bytes": value.len(),
186 "sha256": hex::encode(Sha256::digest(value.as_bytes())),
187 }),
188 value => serde_json::json!(value),
189 }
190 }
191 SessionMessageId::stable(
192 "session_child_completion",
193 &serde_json::json!({
194 "parent_session_id": completion.parent_session_id,
195 "child_session_id": completion.child_session_id,
196 "status": completion.status,
197 "error": identity(completion.error.as_deref()), "result": identity(result),
198 "wait_registered_at": (legacy_wait_identity || completion.source.is_none()).then_some(wait_registered_at),
199 "terminal_source": completion.source,
200 }),
201 )
202}
203
204fn build_child_completion_envelope(
205 completion: &ChildCompletion,
206 wait_registered_at: chrono::DateTime<Utc>,
207 result: Option<&str>,
208 provider_message: &Message,
209 compact: bool,
210) -> SessionMessageEnvelope {
211 fn bounded_terminal_field(
212 label: &str,
213 child_session_id: &str,
214 value: Option<&str>,
215 compact: bool,
216 ) -> (Option<String>, serde_json::Value) {
217 let Some(value) = value else {
218 return (None, serde_json::Value::Null);
219 };
220 let oversized = value.len() > CHILD_COMPLETION_INLINE_FIELD_BYTES;
221 let identity = if oversized {
222 serde_json::json!({
223 "oversized": true,
224 "utf8_bytes": value.len(),
225 "sha256": hex::encode(Sha256::digest(value.as_bytes())),
226 })
227 } else {
228 serde_json::Value::String(value.to_string())
229 };
230 if !oversized && !compact {
231 return (Some(value.to_string()), identity);
232 }
233 let digest = hex::encode(Sha256::digest(value.as_bytes()));
234 let retrieval_view = if label == "error" {
235 "diagnostics"
236 } else {
237 "result"
238 };
239 let inspection = serde_json::json!({
240 "intent": "inspect", "target": child_session_id, "message": retrieval_view,
241 });
242 let recovery = if label == "error" {
243 format!(
244 "Check observed status and recovery diagnostics by calling SubAgent with {inspection}. \
245 Detailed execution errors remain available to the authenticated inspector."
246 )
247 } else {
248 format!(
249 "Read it in bounded slices by calling SubAgent with {inspection}; use message=\"messages\" for transcript previews. \
250 Follow returned cursors with a JSON-encoded view/cursor query in message, omitting unused query fields."
251 )
252 };
253 let mut summary = format!(
254 "Child {label} could not fit in the durable inline envelope ({} UTF-8 bytes, sha256={digest}). {recovery}",
255 value.len()
256 );
257 if !compact {
258 let mut tail_start = value
259 .len()
260 .saturating_sub(CHILD_COMPLETION_OVERSIZE_TAIL_BYTES);
261 while tail_start < value.len() && !value.is_char_boundary(tail_start) {
262 tail_start += 1;
263 }
264 summary.push_str("\n\nBounded tail:\n");
265 summary.push_str(&value[tail_start..]);
266 }
267 (Some(summary), identity)
268 }
269
270 let (stored_result, _) =
271 bounded_terminal_field("result", &completion.child_session_id, result, compact);
272 let (stored_error, _) = bounded_terminal_field(
273 "error",
274 &completion.child_session_id,
275 completion.error.as_deref(),
276 compact,
277 );
278 let mut bounded_provider_message = provider_message.clone();
279 let provider_oversized = serde_json::to_vec(provider_message)
280 .map(|bytes| bytes.len() > CHILD_COMPLETION_INLINE_FIELD_BYTES)
281 .unwrap_or(true);
282 if provider_oversized || compact {
283 let mut content = if provider_oversized {
287 format!(
288 "Runtime notification: child session `{}` finished with status `{}`.",
289 completion.child_session_id, completion.status
290 )
291 } else {
292 bounded_provider_message.content.clone()
293 };
294 if let Some(result) = stored_result.as_deref() {
295 content.push_str("\n\n");
296 content.push_str(result);
297 }
298 if let Some(error) = stored_error.as_deref() {
299 content.push_str("\n\n");
300 content.push_str(error);
301 }
302 bounded_provider_message.content = content;
303 if provider_oversized {
304 bounded_provider_message.content_parts = None;
305 }
306 if compact {
310 let original = bounded_provider_message
311 .metadata
312 .as_ref()
313 .and_then(serde_json::Value::as_object);
314 let mut metadata = serde_json::Map::new();
315 for key in [
316 RUNTIME_RESUME_MESSAGE_HIDDEN_KEY,
317 RUNTIME_RESUME_MESSAGE_KIND_KEY,
318 "child_session_id",
319 "child_status",
320 "child_final_response_included",
321 "child_completion_source",
322 "child_completion_source_sha256",
323 "guardian_approved",
324 ] {
325 if let Some(value) = original.and_then(|metadata| metadata.get(key)) {
326 metadata.insert(key.to_string(), value.clone());
327 }
328 }
329 if original.is_some_and(|metadata| metadata.contains_key("child_error")) {
330 metadata.insert(
331 "child_error".to_string(),
332 stored_error
333 .as_ref()
334 .map_or(serde_json::Value::Null, |error| {
335 serde_json::Value::String(error.clone())
336 }),
337 );
338 }
339 bounded_provider_message.metadata = Some(serde_json::Value::Object(metadata));
340 } else if let Some(child_error) = bounded_provider_message
341 .metadata
342 .as_mut()
343 .and_then(serde_json::Value::as_object_mut)
344 .and_then(|metadata| metadata.get_mut("child_error"))
345 {
346 *child_error = stored_error
347 .as_ref()
348 .map_or(serde_json::Value::Null, |error| {
349 serde_json::Value::String(error.clone())
350 });
351 }
352 }
353 let body = SessionMessageBody::ChildOutcome(SessionChildOutcome {
354 child_session_id: completion.child_session_id.clone(),
355 status: completion.status.clone(),
356 result: stored_result,
357 error: stored_error,
358 provider_message: Some(session_provider_message(&bounded_provider_message)),
359 });
360 SessionMessageEnvelope {
361 id: child_completion_id(completion, wait_registered_at, result, false),
364 source: SessionMessageSource::Runtime {
365 subsystem: "child_completion_coordinator".to_string(),
366 },
367 target_session_id: completion.parent_session_id.clone(),
368 kind: SessionMessageKind::ChildOutcome,
369 body,
370 created_at: completion.completed_at,
371 thread_id: None,
372 in_reply_to: None,
373 attempt: None,
374 correlation_id: Some(format!("child_completion:{}", completion.child_session_id)),
375 }
376}
377
378fn session_provider_message(message: &Message) -> SessionProviderMessage {
379 SessionProviderMessage {
380 content: SessionMessageContent {
381 text: message.content.clone(),
382 parts: message.content_parts.clone().unwrap_or_default(),
383 skill_request: None,
384 },
385 metadata: message
386 .metadata
387 .as_ref()
388 .and_then(serde_json::Value::as_object)
389 .cloned()
390 .unwrap_or_default(),
391 never_compress: message.never_compress,
392 }
393}
394
395async fn derive_completed_child_ids(
400 storage: &Arc<dyn Storage>,
401 parent_session_id: &str,
402 just_completed_child_id: &str,
403) -> Vec<String> {
404 let mut completed: Vec<String> = storage
405 .list_child_run_statuses(parent_session_id)
406 .await
407 .unwrap_or_default()
408 .into_iter()
409 .filter(|(_, status)| status.as_deref().is_some_and(is_terminal_child_status))
410 .map(|(id, _)| id)
411 .collect();
412 if !completed.iter().any(|id| id == just_completed_child_id) {
413 completed.push(just_completed_child_id.to_string());
414 }
415 completed.sort();
416 completed.dedup();
417 completed
418}
419
420fn read_config_snapshot(config: &Arc<RwLock<Config>>, cached_config: &StdRwLock<Config>) -> Config {
421 if let Ok(config_guard) = config.try_read() {
422 let snapshot = config_guard.clone();
423
424 if let Ok(mut cached_guard) = cached_config.try_write() {
425 *cached_guard = snapshot.clone();
426 }
427
428 snapshot
429 } else {
430 cached_config
431 .try_read()
432 .map(|guard| guard.clone())
433 .unwrap_or_default()
434 }
435}
436
437fn parent_locks() -> &'static std::sync::Mutex<HashMap<String, Arc<tokio::sync::Mutex<()>>>> {
453 static LOCKS: OnceLock<std::sync::Mutex<HashMap<String, Arc<tokio::sync::Mutex<()>>>>> =
454 OnceLock::new();
455 LOCKS.get_or_init(|| std::sync::Mutex::new(HashMap::new()))
456}
457
458pub(crate) fn session_resume_lock(session_id: &str) -> SessionResumeLock {
465 let mut map = parent_locks().lock().recover_poison();
466 let lock = map
467 .entry(session_id.to_string())
468 .or_insert_with(|| Arc::new(tokio::sync::Mutex::new(())))
469 .clone();
470 SessionResumeLock {
471 session_id: session_id.to_string(),
472 lock: Some(lock),
473 }
474}
475
476pub(crate) struct SessionResumeLock {
480 session_id: String,
481 lock: Option<Arc<tokio::sync::Mutex<()>>>,
482}
483
484impl std::ops::Deref for SessionResumeLock {
485 type Target = tokio::sync::Mutex<()>;
486 fn deref(&self) -> &Self::Target {
487 self.lock.as_ref().expect("live resume-lock lease")
488 }
489}
490
491impl Drop for SessionResumeLock {
492 fn drop(&mut self) {
493 self.lock.take();
494 let mut map = parent_locks().lock().recover_poison();
495 if map
496 .get(&self.session_id)
497 .is_some_and(|lock| Arc::strong_count(lock) == 1)
498 {
499 map.remove(&self.session_id);
500 }
501 }
502}
503
504fn wait_policy_satisfied(
505 policy: ChildWaitPolicy,
506 wait_child_ids: &[String],
507 completed_child_ids: &[String],
508 latest_child_id: &str,
509 latest_status: &str,
510) -> bool {
511 if wait_child_ids.is_empty() {
512 return false;
513 }
514
515 match policy {
516 ChildWaitPolicy::All => wait_child_ids
517 .iter()
518 .all(|id| completed_child_ids.iter().any(|completed| completed == id)),
519 ChildWaitPolicy::Any => completed_child_ids
520 .iter()
521 .any(|id| wait_child_ids.iter().any(|wait_id| wait_id == id)),
522 ChildWaitPolicy::FirstError => {
523 (is_error_like(latest_status) && wait_child_ids.iter().any(|id| id == latest_child_id))
529 || wait_child_ids
530 .iter()
531 .all(|id| completed_child_ids.iter().any(|completed| completed == id))
532 }
533 }
534}
535
536#[cfg(test)]
540fn child_final_assistant_text(child: &Session) -> Option<String> {
541 child
542 .messages
543 .iter()
544 .rev()
545 .find(|message| matches!(message.role, Role::Assistant))
546 .map(|message| message.content.clone())
547 .filter(|content| !content.trim().is_empty())
548}
549
550fn runtime_resume_message(
551 completion: &ChildCompletion,
552 remaining_children: usize,
553 child_final_response: Option<&str>,
554) -> Message {
555 let mut body = format!(
556 "Runtime notification: child session `{}` finished with status `{}`. Remaining child sessions: {}.",
557 completion.child_session_id, completion.status, remaining_children
558 );
559
560 let final_response = child_final_response.map(str::to_string);
566 if let Some(response) = final_response.as_deref() {
567 body.push_str("\n\nChild final response:\n");
568 body.push_str(response);
569 } else if let Some(error) = completion.error.as_deref() {
570 if !error.is_empty() {
571 body.push_str("\n\nChild error:\n");
572 body.push_str(error);
573 }
574 }
575
576 let inspection = serde_json::json!({
577 "intent": "inspect", "target": completion.child_session_id, "message": "messages",
578 });
579 body.push_str(&format!(
580 "\n\nResume the parent task using this child result and continue from the previous plan. \
581 If you need transcript evidence, call SubAgent with {inspection}. \
582 Follow returned cursors with a JSON-encoded view/cursor query in message; use view=\"message\" and message_id inside that query for a selected full message. Omit unused query fields.",
583 ));
584
585 let mut message = Message::user(body);
586 message.metadata = Some(serde_json::json!({
587 RUNTIME_RESUME_MESSAGE_HIDDEN_KEY: true,
588 RUNTIME_RESUME_MESSAGE_KIND_KEY: "child_completion_resume",
589 "child_session_id": completion.child_session_id,
590 "child_status": completion.status,
591 "child_error": completion.error,
592 "child_final_response_included": final_response.is_some(),
593 "child_completion_source": completion.source,
594 }));
595 message.never_compress = false;
599 message
600}
601
602fn sealed_child_report(
606 completion: &ChildCompletion,
607 child: &Session,
608 parent: &Session,
609) -> Option<serde_json::Value> {
610 if child.subagent_type().as_deref() == Some("guardian")
611 || child
612 .metadata
613 .get("lifecycle")
614 .is_some_and(|v| v == "resident")
615 {
616 return None;
617 }
618 let source = completion.source.as_ref()?;
619 if !source.matches_completion(completion)
620 || ChildCompletionSource::from_committed_session(child).as_ref() != Some(source)
621 || !source.matches_parent(child, parent)
622 {
623 return None;
624 }
625 let binding = bamboo_domain::ChildContextBinding::from_session(child).ok()??;
626 let Some(message) = child
627 .messages
628 .iter()
629 .find(|m| Some(&m.id) == source.result_message_id.as_ref())
630 else {
631 let reason = if source.last_assistant.is_some() {
632 "report_not_current_final"
633 } else {
634 "report_absent"
635 };
636 return Some(unavailable_child_result("typed_result", reason));
637 };
638 if !plain_child_report_message(message) {
639 return Some(unavailable_child_result(
640 "typed_result",
641 "report_not_current_final",
642 ));
643 }
644 let report = match decode_report(&message.content) {
645 Ok(report) => report,
646 Err(reason) => return Some(unavailable_child_result("typed_result", reason)),
647 };
648 let mut observation =
649 child_result_identity(parent, child, &binding, source.project_id.as_deref());
650 observation.as_object_mut()?.extend(
651 serde_json::json!({"kind":"committed_terminal_source","terminal_source":source,
652 "last_run_status":source.status})
653 .as_object()?
654 .clone(),
655 );
656 Some(project_child_report(report, observation))
657}
658
659fn child_completion_source_sha256(completion: &ChildCompletion) -> Option<String> {
660 completion.source.as_ref().map(|source| {
661 hex::encode(Sha256::digest(
662 serde_json::to_vec(source).expect("sealed Child source serializes"),
663 ))
664 })
665}
666
667fn typed_child_resume_message(
668 completion: &ChildCompletion,
669 remaining_children: usize,
670 projection: &serde_json::Value,
671) -> Message {
672 let inspection = serde_json::json!({
673 "intent":"inspect", "target":completion.child_session_id, "message":"result",
674 });
675 let mut message = Message::user(format!(
676 "Runtime notification: child session `{}` finished with status `{}`. Remaining child sessions: {}.\n\n\
677 Child typed result:\n{}\n\n\
678 Resume the parent task. Child-reported evidence and verification are claims, not host-certified facts. \
679 For unavailable results or transcript evidence, call SubAgent with {inspection} and follow returned cursors.",
680 completion.child_session_id, completion.status, remaining_children, projection,
681 ));
682 message.metadata = Some(serde_json::json!({
683 RUNTIME_RESUME_MESSAGE_HIDDEN_KEY:true,
684 RUNTIME_RESUME_MESSAGE_KIND_KEY:"child_completion_resume",
685 "child_session_id":completion.child_session_id,"child_status":completion.status,
686 "child_final_response_included":projection["available"] == true,
687 "child_typed_result_included":true,
688 "child_completion_source_sha256":child_completion_source_sha256(completion),
689 }));
690 message.never_compress = false;
691 message
692}
693
694fn bound_typed_child_delivery(
697 envelope: &mut SessionMessageEnvelope,
698 message: &mut Message,
699 projection: &mut serde_json::Value,
700 completion: &ChildCompletion,
701 remaining_children: usize,
702) -> bool {
703 fn install(envelope: &mut SessionMessageEnvelope, message: &Message) {
704 if let SessionMessageBody::ChildOutcome(outcome) = &mut envelope.body {
705 outcome.result = None;
710 outcome.error = None;
713 outcome.provider_message = Some(session_provider_message(message));
714 }
715 }
716 fn fits(envelope: &SessionMessageEnvelope, message: &Message) -> bool {
717 fits_child_result_budget(envelope)
718 && fits_child_result_budget(message)
719 && envelope
720 .to_provider_message()
721 .is_ok_and(|actual| fits_child_result_budget(&actual))
722 }
723 install(envelope, message);
724 if fits(envelope, message) {
725 return true;
726 }
727 *projection = unavailable_child_result("typed_result", "result_budget_exceeded");
728 *message = typed_child_resume_message(completion, remaining_children, projection);
729 install(envelope, message);
730 fits(envelope, message)
731}
732
733fn guardian_resume_message(completion: &ChildCompletion, verdict: &GuardianVerdict) -> Message {
738 let mut body = if verdict.approve {
739 String::from(
740 "Guardian review APPROVED: an independent reviewer verified the work and found no blocking issues. You may finalize the task.",
741 )
742 } else {
743 String::from(
744 "Guardian review REJECTED: an independent reviewer found issues. Address every finding below before completing — do NOT declare the task complete until they are resolved.",
745 )
746 };
747 if let Some(summary) = verdict.summary.as_deref().filter(|s| !s.trim().is_empty()) {
748 body.push_str("\n\nReviewer summary: ");
749 body.push_str(summary);
750 }
751 if !verdict.findings.is_empty() {
752 body.push_str("\n\nFindings:");
753 for (idx, finding) in verdict.findings.iter().enumerate() {
754 body.push_str(&format!("\n{}. {}", idx + 1, finding));
755 }
756 }
757 let inspection = serde_json::json!({
758 "intent": "inspect", "target": completion.child_session_id, "message": "messages",
759 });
760 body.push_str(&format!(
761 "\n\nIf you need guardian transcript evidence, call SubAgent with {inspection}. \
762 Follow returned cursors with a JSON-encoded view/cursor query in message, omitting unused query fields.",
763 ));
764
765 let mut message = Message::user(body);
766 message.metadata = Some(serde_json::json!({
767 RUNTIME_RESUME_MESSAGE_HIDDEN_KEY: true,
768 RUNTIME_RESUME_MESSAGE_KIND_KEY: "guardian_review_resume",
769 "child_session_id": completion.child_session_id,
770 "child_status": completion.status,
771 "guardian_approved": verdict.approve,
772 "child_completion_source": completion.source,
773 }));
774 message.never_compress = false;
775 message
776}
777
778pub type RootToolSurfaceObserver = Arc<
780 dyn Fn() -> futures::future::BoxFuture<'static, Option<Arc<dyn ToolExecutor>>> + Send + Sync,
781>;
782
783pub type ReservedRootExecutionAdapter = Arc<
784 dyn Fn(
785 Arc<Agent>,
786 &Session,
787 &SessionExecutionReservation,
788 Arc<dyn ToolExecutor>,
789 ) -> Result<(Arc<Agent>, Arc<dyn ToolExecutor>), bamboo_agent_core::AgentError>
790 + Send
791 + Sync,
792>;
793
794#[derive(Clone)]
795pub struct ChildCompletionCoordinator {
796 storage: Arc<dyn Storage>,
797 persistence: Arc<bamboo_storage::LockedSessionStore>,
798 sessions: crate::SessionCache,
799 agent_runners: Arc<RwLock<HashMap<String, AgentRunner>>>,
800 session_event_senders: Arc<RwLock<HashMap<String, broadcast::Sender<AgentEvent>>>>,
801 agent: Arc<Agent>,
802 config: Arc<RwLock<Config>>,
803 provider_registry: Arc<ProviderRegistry>,
804 provider_router: Arc<ProviderModelRouter>,
805 app_data_dir: std::path::PathBuf,
806 account_feed_inbox: Option<crate::execution::AccountFeedInbox>,
807 root_account_sink: Option<Arc<crate::events::AccountEventSink>>,
808 root_tools: Arc<RwLock<Option<Arc<dyn ToolExecutor>>>>,
809 reserved_root_adapter: Arc<StdRwLock<Option<ReservedRootExecutionAdapter>>>,
810 guardian_spawner: Arc<RwLock<Option<Arc<dyn GuardianSpawner>>>>,
814 spawn_scheduler: Arc<RwLock<Weak<SpawnScheduler>>>,
818}
819
820impl ChildCompletionCoordinator {
821 pub fn with_root_account_sink(mut self, sink: Arc<crate::events::AccountEventSink>) -> Self {
822 self.root_account_sink = Some(sink);
823 self
824 }
825
826 #[allow(clippy::too_many_arguments)]
827 pub fn new(
828 storage: Arc<dyn Storage>,
829 persistence: Arc<LockedSessionStore>,
830 sessions: crate::SessionCache,
831 agent_runners: Arc<RwLock<HashMap<String, AgentRunner>>>,
832 session_event_senders: Arc<RwLock<HashMap<String, broadcast::Sender<AgentEvent>>>>,
833 agent: Arc<Agent>,
834 config: Arc<RwLock<Config>>,
835 provider_registry: Arc<ProviderRegistry>,
836 provider_router: Arc<ProviderModelRouter>,
837 app_data_dir: std::path::PathBuf,
838 account_feed_inbox: Option<crate::execution::AccountFeedInbox>,
839 ) -> Self {
840 Self {
841 storage,
842 persistence,
843 sessions,
844 agent_runners,
845 session_event_senders,
846 agent,
847 config,
848 provider_registry,
849 provider_router,
850 app_data_dir,
851 account_feed_inbox,
852 root_account_sink: None,
853 root_tools: Arc::new(RwLock::new(None)),
854 reserved_root_adapter: Arc::new(StdRwLock::new(None)),
855 guardian_spawner: Arc::new(RwLock::new(None)),
856 spawn_scheduler: Arc::new(RwLock::new(Weak::new())),
857 }
858 }
859
860 pub fn set_reserved_root_execution_adapter(&self, adapter: ReservedRootExecutionAdapter) {
863 *self
864 .reserved_root_adapter
865 .write()
866 .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(adapter);
867 }
868
869 pub fn root_tool_surface_observer(&self) -> RootToolSurfaceObserver {
870 let source = Arc::downgrade(&self.root_tools);
871 Arc::new(move || {
872 let source = source.clone();
873 Box::pin(async move {
874 let source = source.upgrade()?;
875 let current = source.read().await.clone();
876 current
877 })
878 })
879 }
880
881 pub async fn set_root_tools(&self, tools: Arc<dyn ToolExecutor>) {
882 *self.root_tools.write().await = Some(tools);
883 }
884
885 pub async fn set_spawn_scheduler(&self, scheduler: &Arc<SpawnScheduler>) {
886 *self.spawn_scheduler.write().await = Arc::downgrade(scheduler);
887 }
888
889 pub async fn set_guardian_spawner(&self, spawner: Arc<dyn GuardianSpawner>) {
892 *self.guardian_spawner.write().await = Some(spawner);
893 }
894
895 fn build_resume_config(
896 &self,
897 session: &Session,
898 config_snapshot: &Config,
899 ) -> ResumeConfigSnapshot {
900 crate::session_app::resolution::resolve_resume_config_snapshot(
901 config_snapshot,
902 &self.provider_registry,
903 session,
904 None,
905 )
906 }
907
908 async fn resume_parent(&self, parent_session_id: String) -> ResumeOutcome {
916 for attempt in 0..=5u8 {
917 if attempt > 0 {
918 tokio::time::sleep(Duration::from_millis(250 * attempt as u64)).await;
919 }
920
921 let Some(session) = self.load_session(&parent_session_id).await else {
922 tracing::warn!(%parent_session_id, "cannot resume parent after child completion: session not found");
923 return ResumeOutcome::NotFound;
924 };
925 let config_snapshot = self.config.read().await.clone();
926 let resume_config = self.build_resume_config(&session, &config_snapshot);
927 let outcome = resume_session_execution(self, &parent_session_id, resume_config).await;
928 tracing::info!(
929 %parent_session_id,
930 attempt,
931 outcome = outcome.as_str(),
932 "child completion requested parent resume"
933 );
934
935 if !matches!(outcome, ResumeOutcome::AlreadyRunning { .. }) {
936 return outcome;
937 }
938 }
939 tracing::error!(
945 %parent_session_id,
946 "parent resume gave up after AlreadyRunning retry budget; \
947 relying on the child-wait watchdog backstop"
948 );
949 ResumeOutcome::AlreadyRunning {
950 run_id: String::new(),
951 }
952 }
953
954 async fn save_and_cache(&self, session: &mut Session) {
955 if let Err(error) = self
956 .persistence
957 .merge_save_runtime_and_publish(session, |saved, _| {
958 self.sessions.insert(
959 saved.id.clone(),
960 Arc::new(crate::SessionSnapshot::new(saved.clone())),
961 );
962 })
963 .await
964 {
965 tracing::warn!(session_id = %session.id, %error, "failed to persist session");
966 }
967 }
968
969 async fn reuse_legacy_child_delivery(
970 &self,
971 envelope: &mut SessionMessageEnvelope,
972 completion: &ChildCompletion,
973 wait: Option<&WaitingForChildrenState>,
974 result: Option<&str>,
975 ) -> Result<(), bamboo_domain::SessionInboxError> {
976 if let (Some(wait), Some(inbox), Some(_)) =
977 (wait, self.agent.session_inbox(), completion.source.as_ref())
978 {
979 let mut legacy = envelope.clone();
980 legacy.id = child_completion_id(completion, wait.registered_at, result, true);
981 if inbox.existing_staged_delivery(&legacy).await?.is_some() {
982 *envelope = legacy;
983 }
984 }
985 Ok(())
986 }
987
988 async fn legacy_child_delivery(
989 &self,
990 parent: &Session,
991 completion: &ChildCompletion,
992 ) -> Result<Option<(SessionMessageEnvelope, bool)>, bamboo_domain::SessionInboxError> {
993 let (Some(source), Some(inbox)) = (completion.source.as_ref(), self.agent.session_inbox())
994 else {
995 return Ok(None);
996 };
997 let source_sha256 = child_completion_source_sha256(completion);
998 let source = serde_json::to_value(source).expect("sealed Child source serializes");
999 let matches_source = |message: &Message, envelope: &SessionMessageEnvelope| {
1000 if !matches!(&envelope.source, SessionMessageSource::Runtime { subsystem }
1001 if subsystem == "child_completion_coordinator")
1002 || envelope.target_session_id != parent.id
1003 || !matches!(&envelope.body, SessionMessageBody::ChildOutcome(outcome)
1004 if outcome.child_session_id == completion.child_session_id && outcome.status == completion.status)
1005 || !bamboo_domain::is_matching_session_message(message, envelope)
1006 {
1007 return false;
1008 }
1009 let Some(metadata) = message.metadata.as_ref() else {
1010 return false;
1011 };
1012 let typed_source = message
1016 .content
1017 .split_once("\n\nChild typed result:\n")
1018 .and_then(|(_, text)| text.lines().next())
1019 .filter(|line| line.len() <= 8192)
1020 .and_then(|line| serde_json::from_str::<serde_json::Value>(line).ok())
1021 .and_then(|projection| {
1022 projection
1023 .get("host_observation")?
1024 .get("terminal_source")
1025 .cloned()
1026 });
1027 metadata.get("child_completion_source") == Some(&source)
1028 || source_sha256.as_ref().is_some_and(|digest| {
1029 metadata
1030 .get("child_completion_source_sha256")
1031 .and_then(serde_json::Value::as_str)
1032 == Some(digest.as_str())
1033 })
1034 || typed_source.as_ref() == Some(&source)
1035 };
1036 for message in &parent.messages {
1037 let Some(marker) = message
1038 .metadata
1039 .as_ref()
1040 .and_then(|m| m.get("session_message"))
1041 else {
1042 continue;
1043 };
1044 let Ok(envelope) = serde_json::from_value::<SessionMessageEnvelope>(marker.clone())
1045 else {
1046 continue;
1047 };
1048 if matches_source(message, &envelope)
1049 && inbox.was_admitted(&parent.id, &envelope.id).await?
1050 {
1051 return Ok(Some((envelope, true)));
1052 }
1053 }
1054 for envelope in inbox
1058 .pending_child_outcomes(&parent.id, &completion.child_session_id)
1059 .await?
1060 {
1061 if envelope
1062 .to_provider_message()
1063 .is_ok_and(|message| matches_source(&message, &envelope))
1064 {
1065 return Ok(Some((envelope, false)));
1066 }
1067 }
1068 Ok(None)
1069 }
1070}
1071
1072#[async_trait]
1073impl ChildCompletionHandler for ChildCompletionCoordinator {
1074 async fn on_child_completed(&self, completion: ChildCompletion) {
1075 if !is_terminal_child_status(&completion.status) {
1084 tracing::info!(
1085 parent_session_id = %completion.parent_session_id,
1086 child_session_id = %completion.child_session_id,
1087 status = %completion.status,
1088 "non-terminal child status; leaving the parent wait armed"
1089 );
1090 return;
1091 }
1092
1093 let per_parent = session_resume_lock(&completion.parent_session_id);
1098 let _per_parent_guard = per_parent.lock().await;
1099
1100 let (parent, should_resume, messenger, child_admission, child_admissions) = loop {
1101 let Some(mut parent) = self.load_session(&completion.parent_session_id).await else {
1102 tracing::warn!(
1103 parent_session_id = %completion.parent_session_id,
1104 child_session_id = %completion.child_session_id,
1105 "child completion received for missing parent"
1106 );
1107 return;
1108 };
1109
1110 let expected_parent = parent.clone();
1111
1112 let mut runtime_state = read_runtime_state(&parent);
1118
1119 let mut completed_child_ids = derive_completed_child_ids(
1122 &self.storage,
1123 &completion.parent_session_id,
1124 &completion.child_session_id,
1125 )
1126 .await;
1127
1128 let mut should_resume = false;
1129 let mut remaining_children = 0usize;
1130 let active_wait = runtime_state.waiting_for_children.clone();
1131 let tracked_wait = active_wait.as_ref().is_some_and(|wait| {
1132 wait.child_session_ids
1133 .contains(&completion.child_session_id)
1134 });
1135 if !tracked_wait {
1136 let mut unchanged = parent.clone();
1140 match self
1141 .persistence
1142 .compare_exchange_child_wait_and_publish(
1143 &parent,
1144 &mut unchanged,
1145 true,
1146 Arc::new(|_| {}),
1147 )
1148 .await
1149 {
1150 Ok(false) => continue,
1151 Ok(true) if completion.source.is_none() => return,
1152 Ok(true) => {}
1153 Err(error) => {
1154 tracing::warn!(parent_session_id = %parent.id, %error,
1155 "child completion could not settle no-wait observation");
1156 return;
1157 }
1158 }
1159 }
1160 if let Some(wait) = active_wait.as_ref() {
1163 for id in &wait.child_session_ids {
1164 if id == &completion.child_session_id {
1165 continue;
1166 }
1167 if let Ok(Some(child)) = self
1168 .storage
1169 .load_child_wait_session(&parent, id, false)
1170 .await
1171 {
1172 completed_child_ids.retain(|completed| completed != id);
1173 if completion_child_is_owned(&parent.id, child.parent_session_id.as_deref())
1174 && child
1175 .last_run_status()
1176 .as_deref()
1177 .is_some_and(is_terminal_child_status)
1178 {
1179 completed_child_ids.push(id.clone());
1180 }
1181 }
1182 }
1183 }
1184 if let Some(wait) = active_wait.as_ref() {
1185 remaining_children = wait
1186 .child_session_ids
1187 .iter()
1188 .filter(|id| !completed_child_ids.iter().any(|completed| completed == *id))
1189 .count();
1190 should_resume = tracked_wait
1191 && wait_policy_satisfied(
1192 wait.wait_for,
1193 &wait.child_session_ids,
1194 &completed_child_ids,
1195 &completion.child_session_id,
1196 &completion.status,
1197 );
1198 }
1199
1200 let reported_child_owned = match self
1212 .storage
1213 .load_child_wait_session(&parent, &completion.child_session_id, false)
1214 .await
1215 {
1216 Ok(Some(control_plane)) => completion_child_is_owned(
1217 &completion.parent_session_id,
1218 control_plane.parent_session_id.as_deref(),
1219 ),
1220 _ => false,
1221 };
1222
1223 let loaded_child = if reported_child_owned {
1228 match self
1229 .storage
1230 .load_child_wait_session(&parent, &completion.child_session_id, true)
1231 .await
1232 {
1233 Ok(child) => child,
1234 Err(error) => {
1235 tracing::warn!(
1236 child_session_id = %completion.child_session_id,
1237 %error,
1238 "failed to load child session for runtime resume message"
1239 );
1240 None
1241 }
1242 }
1243 } else {
1244 tracing::warn!(
1245 parent_session_id = %completion.parent_session_id,
1246 child_session_id = %completion.child_session_id,
1247 "completion child is not a child of this parent; resuming with a neutral \
1248 message and NOT folding its content"
1249 );
1250 None
1251 };
1252
1253 if let Some(source) = completion.source.as_ref() {
1257 if loaded_child
1258 .as_ref()
1259 .and_then(ChildCompletionSource::from_committed_session)
1260 .as_ref()
1261 != Some(source)
1262 || !source.matches_completion(&completion)
1263 || loaded_child
1264 .as_ref()
1265 .is_none_or(|child| !source.matches_parent(child, &parent))
1266 {
1267 tracing::warn!(child_session_id = %completion.child_session_id,
1268 "stale/unavailable committed child completion source; leaving wait armed");
1269 return;
1270 }
1271 }
1272 if !tracked_wait
1273 && crate::ticket_runtime::is_independent_ticket_child(
1274 self.storage.as_ref(),
1275 &parent,
1276 &completion.child_session_id,
1277 )
1278 .await
1279 {
1280 return;
1281 }
1282 let previous_delivery = match self.legacy_child_delivery(&parent, &completion).await {
1283 Ok(envelope) => envelope,
1284 Err(error) => {
1285 tracing::warn!(parent_session_id = %parent.id, %error,
1286 "cannot verify historical Child delivery; leaving replay recoverable");
1287 return;
1288 }
1289 };
1290 let previously_admitted = previous_delivery
1291 .as_ref()
1292 .is_some_and(|(_, admitted)| *admitted);
1293 let child_final_response = loaded_child
1294 .as_ref()
1295 .and_then(|child| completion.source.as_ref()?.result(child, &parent));
1296 let guardian_resume = if completion.source.is_some() || should_resume {
1301 let reviewed_round = runtime_state.round.current_round;
1302 loaded_child.as_ref().and_then(|child| {
1303 if child.subagent_type().as_deref() != Some("guardian") {
1304 return None;
1305 }
1306 let mut guardian_state = read_guardian_state(&parent)?;
1307 if guardian_state.guardian_child_id.as_deref()
1308 != Some(completion.child_session_id.as_str())
1309 {
1310 tracing::warn!(
1313 parent_session_id = %completion.parent_session_id,
1314 child_session_id = %completion.child_session_id,
1315 expected = ?guardian_state.guardian_child_id,
1316 "guardian completion does not match recorded guardian_child_id; using generic resume"
1317 );
1318 return None;
1319 }
1320 if guardian_state.phase.is_reviewed() {
1321 return guardian_state.last_verdict.as_ref()
1324 .map(|verdict| guardian_resume_message(&completion, verdict));
1325 }
1326 let verdict = child_final_response.clone()
1334 .and_then(|text| match parse_guardian_verdict(&text) {
1335 Ok(verdict) => Some(verdict),
1336 Err(error) => {
1337 tracing::warn!(
1338 child_session_id = %completion.child_session_id,
1339 %error,
1340 "guardian verdict unparseable; recording a synthetic reject"
1341 );
1342 None
1343 }
1344 })
1345 .unwrap_or_else(|| {
1346 GuardianVerdict::rejected(vec![
1347 "The guardian reviewer did not return a usable verdict (it errored or \
1348 emitted unparseable output); the work has NOT been independently \
1349 verified."
1350 .to_string(),
1351 ])
1352 });
1353 let approved = verdict.approve;
1354 let message = guardian_resume_message(&completion, &verdict);
1355 guardian_state.record_verdict(verdict, reviewed_round);
1356 write_guardian_state(&mut parent, guardian_state);
1357 tracing::info!(
1358 parent_session_id = %completion.parent_session_id,
1359 child_session_id = %completion.child_session_id,
1360 approved,
1361 "guardian verdict recorded; resuming parent"
1362 );
1363 Some(message)
1364 })
1365 } else {
1366 None
1367 };
1368 let guardian_control_changed = parent.metadata.get("guardian.state")
1369 != expected_parent.metadata.get("guardian.state");
1370 if previously_admitted && !should_resume && !guardian_control_changed {
1373 return;
1374 }
1375 let activation_policy = if should_resume || guardian_control_changed {
1376 SessionActivationPolicy::RespectSpecificWait
1377 } else {
1378 SessionActivationPolicy::InterruptSpecificWait
1379 };
1380 let mut typed_result = loaded_child
1381 .as_ref()
1382 .and_then(|child| sealed_child_report(&completion, child, &parent));
1383 let mut resume_message = if let Some(projection) = typed_result.as_mut() {
1384 let message =
1385 typed_child_resume_message(&completion, remaining_children, projection);
1386 if fits_child_result_budget(&message) {
1387 message
1388 } else {
1389 *projection =
1390 unavailable_child_result("typed_result", "result_budget_exceeded");
1391 typed_child_resume_message(&completion, remaining_children, projection)
1392 }
1393 } else {
1394 guardian_resume.unwrap_or_else(|| {
1395 runtime_resume_message(
1396 &completion,
1397 remaining_children,
1398 child_final_response.as_deref(),
1399 )
1400 })
1401 };
1402 if typed_result.is_some() && !fits_child_result_budget(&resume_message) {
1403 tracing::warn!(child_session_id = %completion.child_session_id,
1404 "typed child resume identity exceeds budget; leaving wait armed");
1405 return;
1406 }
1407
1408 let messenger = self.agent.session_messenger().cloned();
1412 let mut child_admission = if let Some(messenger) = messenger.as_ref() {
1413 let mut envelope = child_completion_envelope(
1414 &completion,
1415 active_wait
1416 .as_ref()
1417 .map_or(completion.completed_at, |wait| wait.registered_at),
1418 child_final_response.clone(),
1419 &resume_message,
1420 );
1421 if let Some(router) = self.agent.activation_router() {
1422 let previous_fenced_run = previous_delivery
1423 .as_ref()
1424 .and_then(|(envelope, _)| envelope.correlation_id.as_deref())
1425 .and_then(|value| value.strip_prefix("child_completion_after_run:"));
1426 if let Some(run_id) = router
1427 .child_completion_successor_run(
1428 &completion.parent_session_id,
1429 active_wait.is_some() || guardian_control_changed,
1430 previous_fenced_run,
1431 )
1432 .await
1433 {
1434 envelope.correlation_id =
1435 Some(format!("child_completion_after_run:{run_id}"));
1436 }
1437 }
1438 if let Some(projection) = typed_result.as_mut() {
1439 if !bound_typed_child_delivery(
1440 &mut envelope,
1441 &mut resume_message,
1442 projection,
1443 &completion,
1444 remaining_children,
1445 ) {
1446 tracing::warn!(child_session_id = %completion.child_session_id,
1447 "typed child outcome wrapper exceeds budget; leaving wait armed");
1448 return;
1449 }
1450 }
1451 if let Some((previous, _)) = previous_delivery.as_ref() {
1452 envelope = previous.clone();
1453 } else if let Err(error) = self
1454 .reuse_legacy_child_delivery(
1455 &mut envelope,
1456 &completion,
1457 active_wait.as_ref(),
1458 child_final_response.as_deref(),
1459 )
1460 .await
1461 {
1462 tracing::warn!(parent_session_id = %parent.id, %error,
1463 "cannot verify existing Child delivery; leaving wait armed");
1464 return;
1465 }
1466 match messenger.admit(envelope).await {
1467 Ok(admission) => {
1468 if !should_resume && !guardian_control_changed {
1469 let mut observed = expected_parent.clone();
1470 match self
1471 .persistence
1472 .compare_exchange_child_wait_and_publish(
1473 &expected_parent,
1474 &mut observed,
1475 true,
1476 Arc::new(|_| {}),
1477 )
1478 .await
1479 {
1480 Ok(false) => continue,
1481 Ok(true) => {}
1482 Err(error) => {
1483 tracing::warn!(parent_session_id = %parent.id, %error,
1484 "partial Child delivery lost its parent authority; leaving staged outcome inert");
1485 return;
1486 }
1487 }
1488 }
1489 if should_resume || !guardian_control_changed {
1493 if let Err(error) = messenger
1494 .prepare_message_activation(&admission, activation_policy)
1495 .await
1496 {
1497 tracing::warn!(parent_session_id = %completion.parent_session_id,
1498 child_session_id = %completion.child_session_id, %error,
1499 "child outcome notification permission failed; leaving wait armed");
1500 return;
1501 }
1502 }
1503 Some(admission)
1504 }
1505 Err(error) => {
1506 tracing::warn!(
1507 parent_session_id = %completion.parent_session_id,
1508 child_session_id = %completion.child_session_id,
1509 %error,
1510 "child outcome SessionInbox admission failed; leaving parent wait armed"
1511 );
1512 return;
1513 }
1514 }
1515 } else {
1516 None
1517 };
1518
1519 let mut child_admissions = child_admission.iter().cloned().collect::<Vec<_>>();
1520
1521 if should_resume {
1528 if let (Some(wait), Some(messenger)) = (active_wait.as_ref(), messenger.as_ref()) {
1529 let parent_run_id = if let Some(router) = self.agent.activation_router() {
1530 router
1531 .child_completion_successor_run(
1532 &completion.parent_session_id,
1533 true,
1534 None,
1535 )
1536 .await
1537 } else {
1538 None
1539 };
1540 for child_id in &wait.child_session_ids {
1541 if child_id == &completion.child_session_id
1542 || !completed_child_ids.contains(child_id)
1543 {
1544 continue;
1545 }
1546 let owned = match self
1550 .storage
1551 .load_child_wait_session(&parent, child_id, false)
1552 .await
1553 {
1554 Ok(Some(child)) => completion_child_is_owned(
1555 &completion.parent_session_id,
1556 child.parent_session_id.as_deref(),
1557 ),
1558 _ => false,
1559 };
1560 let sibling = if owned {
1561 self.storage
1562 .load_child_wait_session(&parent, child_id, true)
1563 .await
1564 } else {
1565 tracing::warn!(%child_id, "terminal sibling ownership unavailable; leaving parent wait armed");
1566 return;
1567 };
1568 let sibling = match sibling {
1569 Ok(Some(child))
1570 if completion_child_is_owned(
1571 &completion.parent_session_id,
1572 child.parent_session_id.as_deref(),
1573 ) && child
1574 .last_run_status()
1575 .as_deref()
1576 .is_some_and(is_terminal_child_status) =>
1577 {
1578 child
1579 }
1580 _ => {
1581 tracing::warn!(%child_id, "terminal sibling snapshot unavailable; leaving parent wait armed");
1582 return;
1583 }
1584 };
1585 if ChildCompletionSource::has_source_record(&sibling)
1586 && ChildCompletionSource::from_committed_session(&sibling).is_none()
1587 {
1588 tracing::warn!(%child_id, "terminal sibling source is invalid; leaving wait armed");
1589 return;
1590 }
1591 let sibling_completion = ChildCompletion {
1592 parent_session_id: completion.parent_session_id.clone(),
1593 child_session_id: child_id.clone(),
1594 status: sibling.last_run_status().expect("terminal status checked"),
1595 error: sibling.last_run_error(),
1596 completed_at: sibling.updated_at,
1597 source: ChildCompletionSource::from_committed_session(&sibling),
1598 };
1599 if sibling_completion
1600 .source
1601 .as_ref()
1602 .is_some_and(|source| !source.matches_parent(&sibling, &parent))
1603 {
1604 tracing::warn!(%child_id, "terminal sibling parent context is stale; leaving wait armed");
1605 return;
1606 }
1607 let result = sibling_completion
1608 .source
1609 .as_ref()
1610 .and_then(|source| source.result(&sibling, &parent));
1611 let mut typed_result =
1612 sealed_child_report(&sibling_completion, &sibling, &parent);
1613 let mut presentation = if let Some(projection) = typed_result.as_ref() {
1614 typed_child_resume_message(
1615 &sibling_completion,
1616 remaining_children,
1617 projection,
1618 )
1619 } else {
1620 runtime_resume_message(
1621 &sibling_completion,
1622 remaining_children,
1623 result.as_deref(),
1624 )
1625 };
1626 let mut envelope = child_completion_envelope(
1627 &sibling_completion,
1628 wait.registered_at,
1629 result.clone(),
1630 &presentation,
1631 );
1632 if let Some(run_id) = parent_run_id.as_ref() {
1633 envelope.correlation_id =
1634 Some(format!("child_completion_after_run:{run_id}"));
1635 }
1636 if let Some(projection) = typed_result.as_mut() {
1637 if !bound_typed_child_delivery(
1638 &mut envelope,
1639 &mut presentation,
1640 projection,
1641 &sibling_completion,
1642 remaining_children,
1643 ) {
1644 tracing::warn!(%child_id, "typed terminal sibling wrapper exceeds budget; leaving wait armed");
1645 return;
1646 }
1647 }
1648 let previous_delivery = match self
1649 .legacy_child_delivery(&parent, &sibling_completion)
1650 .await
1651 {
1652 Ok(delivery) => delivery,
1653 Err(error) => {
1654 tracing::warn!(%child_id, %error,
1655 "cannot verify historical sibling delivery; leaving wait armed");
1656 return;
1657 }
1658 };
1659 if let Some((previous, _)) = previous_delivery {
1660 envelope = previous;
1661 } else if let Err(error) = self
1662 .reuse_legacy_child_delivery(
1663 &mut envelope,
1664 &sibling_completion,
1665 Some(wait),
1666 result.as_deref(),
1667 )
1668 .await
1669 {
1670 tracing::warn!(%child_id, %error,
1671 "cannot verify existing sibling delivery; leaving wait armed");
1672 return;
1673 }
1674 match messenger.admit(envelope).await {
1675 Ok(admission) => {
1676 if let Err(error) = messenger
1677 .prepare_message_activation(
1678 &admission,
1679 SessionActivationPolicy::RespectSpecificWait,
1680 )
1681 .await
1682 {
1683 tracing::warn!(%child_id, %error,
1684 "terminal sibling notification permission failed; leaving wait armed");
1685 return;
1686 }
1687 child_admissions.push(admission.clone());
1688 if child_admission.as_ref().is_none_or(|current| {
1691 admission.delivery.generation > current.delivery.generation
1692 }) {
1693 child_admission = Some(admission);
1694 }
1695 }
1696 Err(error) => {
1697 tracing::warn!(%child_id, %error, "terminal sibling admission failed; leaving parent wait armed");
1698 return;
1699 }
1700 }
1701 }
1702 }
1703 }
1704
1705 if !should_resume && !guardian_control_changed && child_admission.is_some() {
1709 break (parent, true, messenger, child_admission, child_admissions);
1710 }
1711
1712 if should_resume {
1713 runtime_state.waiting_for_children = None;
1714 runtime_state.status = AgentStatusState::Idle;
1715 runtime_state.suspension = None;
1716 parent.metadata.remove("runtime.suspend_reason");
1717
1718 if child_admission.is_none() {
1719 parent.add_message(resume_message);
1723 }
1724 }
1725
1726 parent.updated_at = Utc::now();
1727 write_runtime_state(&mut parent, &runtime_state);
1728 let runtime_only = child_admission.is_some();
1735 let sessions = self.sessions.clone();
1736 let history = expected_parent.clone();
1737 let save = self
1738 .persistence
1739 .compare_exchange_child_wait_and_publish(
1740 &expected_parent,
1741 &mut parent,
1742 runtime_only,
1743 Arc::new(move |saved| {
1744 if runtime_only {
1745 if let Some(cached) = sessions.get(&saved.id) {
1746 cached.update(|current| {
1747 if current.created_at != saved.created_at {
1748 *current = saved.clone();
1749 current.messages = history.messages.clone();
1750 current.provider_transcript =
1751 history.provider_transcript.clone();
1752 current
1753 .runtime_metadata
1754 .get_or_insert_with(Default::default)
1755 .session_inbox_admission =
1756 history.session_inbox_admission().cloned();
1757 return;
1758 }
1759
1760 let messages = current.messages.clone();
1761 let provider_transcript = current.provider_transcript.clone();
1762 let admission =
1763 current.runtime_metadata.as_ref().and_then(|metadata| {
1764 metadata.session_inbox_admission.clone()
1765 });
1766 *current = saved.clone();
1767 current.messages = messages;
1768 current.provider_transcript = provider_transcript;
1769 current
1770 .runtime_metadata
1771 .get_or_insert_with(Default::default)
1772 .session_inbox_admission = admission;
1773 });
1774 } else {
1775 let mut snapshot = saved.clone();
1776 snapshot.messages = history.messages.clone();
1777 snapshot.provider_transcript = history.provider_transcript.clone();
1778 snapshot
1779 .runtime_metadata
1780 .get_or_insert_with(Default::default)
1781 .session_inbox_admission = history
1782 .runtime_metadata
1783 .as_ref()
1784 .and_then(|metadata| metadata.session_inbox_admission.clone());
1785 sessions.insert(
1786 snapshot.id.clone(),
1787 Arc::new(crate::SessionSnapshot::new(snapshot)),
1788 );
1789 }
1790 } else {
1791 sessions.insert(
1792 saved.id.clone(),
1793 Arc::new(crate::SessionSnapshot::new(saved.clone())),
1794 );
1795 }
1796 }),
1797 )
1798 .await;
1799 match save {
1800 Ok(false) => {
1801 continue;
1804 }
1805 Ok(true) => {
1806 break (
1807 parent,
1808 should_resume || (guardian_control_changed && !previously_admitted),
1809 messenger,
1810 child_admission,
1811 child_admissions,
1812 )
1813 }
1814 Err(error) => {
1815 tracing::warn!(parent_session_id = %completion.parent_session_id,
1816 child_session_id = %completion.child_session_id, %error,
1817 "child outcome is durable but parent wait transition failed; leaving activation deferred");
1818 return;
1819 }
1820 }
1821 };
1822
1823 let resume_parent_id = parent.id.clone();
1828 drop(_per_parent_guard);
1829
1830 if should_resume {
1831 if let (Some(messenger), Some(admission)) = (messenger, child_admission) {
1832 for outcome in &child_admissions {
1836 if let Err(error) = messenger
1837 .prepare_message_activation(
1838 outcome,
1839 SessionActivationPolicy::InterruptSpecificWait,
1840 )
1841 .await
1842 {
1843 tracing::warn!(parent_session_id = %resume_parent_id, %error,
1844 "child control transition committed but notification permission failed");
1845 return;
1846 }
1847 }
1848 if let Err(error) = messenger.activate_prepared(&admission).await {
1849 tracing::warn!(
1850 parent_session_id = %resume_parent_id,
1851 %error,
1852 "child outcome and wait transition are durable but activation failed"
1853 );
1854 }
1855 } else {
1856 self.resume_parent(resume_parent_id).await;
1857 }
1858 }
1859 }
1860}
1861
1862#[async_trait]
1863impl ResumeExecutionPort for ChildCompletionCoordinator {
1864 async fn load_session(&self, session_id: &str) -> Option<Session> {
1865 match self.storage.load_session(session_id).await {
1866 Ok(Some(session)) => Some(session),
1867 Ok(None) => self
1868 .sessions
1869 .get(session_id)
1870 .map(|e| e.value().clone())
1871 .map(|arc| arc.read().clone()),
1872 Err(error) => {
1873 tracing::warn!(%session_id, %error, "failed to load session from storage");
1874 self.sessions
1875 .get(session_id)
1876 .map(|e| e.value().clone())
1877 .map(|arc| arc.read().clone())
1878 }
1879 }
1880 }
1881
1882 async fn save_and_cache_session(&self, session: &mut Session) {
1883 self.save_and_cache(session).await;
1884 }
1885
1886 async fn reserve_session_execution(
1887 &self,
1888 session_id: &str,
1889 event_sender: &broadcast::Sender<AgentEvent>,
1890 ) -> SessionExecutionReserveOutcome {
1891 reserve_session_execution(
1892 &self.agent,
1893 &self.agent_runners,
1894 &self.session_event_senders,
1895 session_id,
1896 event_sender,
1897 )
1898 .await
1899 }
1900
1901 async fn get_or_create_event_sender(&self, session_id: &str) -> broadcast::Sender<AgentEvent> {
1902 crate::execution::session_events::get_or_create_event_sender(
1903 &self.session_event_senders,
1904 session_id,
1905 )
1906 .await
1907 }
1908
1909 fn dispatch_resume_execution(
1910 &self,
1911 request: ResumeSpawnRequest,
1912 ) -> Result<(), ResumeSpawnRequest> {
1913 let owner = self.clone();
1914 tokio::spawn(async move {
1915 ResumeExecutionPort::spawn_resume_execution(&owner, request).await;
1916 });
1917 Ok(())
1918 }
1919
1920 async fn spawn_resume_execution(&self, request: ResumeSpawnRequest) {
1921 let ResumeSpawnRequest {
1922 session_id,
1923 mut session,
1924 mut execution_reservation,
1925 event_sender,
1926 config,
1927 } = request;
1928 if let Err(error) = execution_reservation.ensure_registered().await {
1929 tracing::warn!(
1930 %session_id,
1931 run_id = %execution_reservation.run_id(),
1932 %error,
1933 "cannot resume after child completion without exact router ownership"
1934 );
1935 return;
1936 }
1937
1938 if self
1939 .agent
1940 .persistence()
1941 .root_actor_execution_required(&session)
1942 && self.root_account_sink.is_none()
1943 {
1944 tracing::warn!(%session_id, "Root resume has no actual account sink; rejecting before Actor claim");
1945 execution_reservation.abandon().await;
1946 return;
1947 }
1948 if let Err(error) = execution_reservation
1949 .bind_root_actor(&self.agent, &session)
1950 .await
1951 {
1952 tracing::warn!(%session_id, %error, "Root coordinator resume Actor binding rejected");
1953 execution_reservation.abandon().await;
1954 return;
1955 }
1956 let root_publication = execution_reservation.root_actor_writer().map(|owner| {
1957 crate::events::RootActorEventPublication::new(
1958 self.storage.clone(),
1959 self.root_account_sink
1960 .clone()
1961 .expect("required Root actual account sink checked before claim"),
1962 owner,
1963 )
1964 });
1965
1966 let Some(root_tools) = self.root_tools.read().await.clone() else {
1967 tracing::error!(%session_id, "cannot resume parent after child completion: root tool surface is not initialized");
1968 return;
1969 };
1970
1971 let adapter = self
1972 .reserved_root_adapter
1973 .read()
1974 .unwrap_or_else(std::sync::PoisonError::into_inner)
1975 .clone();
1976 let (agent, root_tools) = if let Some(adapter) = adapter {
1977 match adapter(
1978 self.agent.clone(),
1979 &session,
1980 &execution_reservation,
1981 root_tools,
1982 ) {
1983 Ok(bound) => bound,
1984 Err(error) => {
1985 tracing::error!(%session_id, %error, "registered Root execution producer rejected resume");
1986 execution_reservation.abandon().await;
1987 return;
1988 }
1989 }
1990 } else {
1991 (self.agent.clone(), root_tools)
1992 };
1993
1994 let config_snapshot = self.config.read().await.clone();
1995 let model = session.model.clone();
1996 let session_model_ref = session_effective_model_ref(&session);
1997 let requested_provider = session_model_ref
1998 .as_ref()
1999 .map(|model_ref| model_ref.provider.as_str())
2000 .unwrap_or(config.provider_name.as_str());
2001 let resolved_provider_name = match resolve_provider_routing_key(
2002 &config_snapshot,
2003 requested_provider,
2004 &self.provider_registry,
2005 ) {
2006 Ok(provider) => provider,
2007 Err(error) => {
2008 tracing::error!(
2009 session_id = %session_id,
2010 provider = requested_provider,
2011 %error,
2012 "child-completion resume provider is unavailable; refusing to fall back"
2013 );
2014 execution_reservation.abandon().await;
2015 return;
2016 }
2017 };
2018 let provider_override = if let Some(mut model_ref) = session_model_ref {
2019 model_ref.provider = resolved_provider_name.clone();
2020 persist_model_ref(&mut session, &model_ref);
2021 match self.provider_router.route(&model_ref) {
2022 Ok(provider) => Some(provider),
2023 Err(error) => {
2024 tracing::error!(
2025 session_id = %session_id,
2026 provider = %model_ref.provider,
2027 model = %model_ref.model,
2028 %error,
2029 "child-completion resume provider routing failed closed"
2030 );
2031 execution_reservation.abandon().await;
2032 return;
2033 }
2034 }
2035 } else {
2036 match self.provider_registry.get(&resolved_provider_name) {
2037 Some(provider) => Some(provider),
2038 None => {
2039 tracing::error!(
2040 session_id = %session_id,
2041 provider = %resolved_provider_name,
2042 "child-completion resume provider disappeared after resolution; refusing to fall back"
2043 );
2044 execution_reservation.abandon().await;
2045 return;
2046 }
2047 }
2048 };
2049 let resolved_fast_provider = resolve_fast_model(
2050 &config_snapshot,
2051 &resolved_provider_name,
2052 &self.provider_registry,
2053 )
2054 .map(|model| model.provider);
2055 let reasoning_effort = session.reasoning_effort;
2056 let reasoning_effort_source = session
2057 .metadata
2058 .get("reasoning_effort_source")
2059 .cloned()
2060 .unwrap_or_default();
2061 let gold_config = resolve_gold_config(
2062 &config_snapshot,
2063 session
2064 .metadata
2065 .get(GOLD_CONFIG_METADATA_KEY)
2066 .map(String::as_str),
2067 )
2068 .or(config.gold_config.clone());
2069
2070 let (mpsc_tx, _forwarder, history_commit_barrier) =
2071 crate::execution::create_event_forwarder_with_root_actor(
2072 session_id.clone(),
2073 execution_reservation.run_id().to_string(),
2074 event_sender,
2075 self.agent_runners.clone(),
2076 self.account_feed_inbox.clone(),
2077 root_publication,
2078 );
2079
2080 let config_handle = self.config.clone();
2081 let cached_config = Arc::new(StdRwLock::new(config_snapshot.clone()));
2082 let provider_registry = self.provider_registry.clone();
2083 let provider_name_for_aux = resolved_provider_name.clone();
2084 let auxiliary_model_resolver = std::sync::Arc::new(move || {
2085 let config_snapshot = read_config_snapshot(&config_handle, cached_config.as_ref());
2086 let areas = resolve_global_area_models(
2088 &config_snapshot,
2089 &provider_name_for_aux,
2090 &provider_registry,
2091 );
2092 crate::AuxiliaryModelConfig {
2093 fast_model_name: areas.fast.as_ref().map(|m| m.model_name.clone()),
2094 fast_model_provider: areas.fast.map(|m| m.provider),
2095 background_model_name: areas.background.as_ref().map(|m| m.model_name.clone()),
2096 planning_model_name: None,
2097 search_model_name: None,
2098 summarization_model_name: areas
2099 .summarization
2100 .as_ref()
2101 .map(|m| m.model_name.clone()),
2102 background_model_provider: areas.background.map(|m| m.provider),
2103 summarization_model_provider: areas.summarization.map(|m| m.provider),
2104 }
2105 });
2106 let model_roster = crate::ModelRoster {
2107 model: Some(model),
2108 provider_name: Some(resolved_provider_name),
2109 provider_type: config.provider_type.clone(),
2110 fast: crate::RoleModel::from_parts(config.fast_model, resolved_fast_provider),
2111 background: crate::RoleModel::from_parts(
2112 config.background_model,
2113 config.background_model_provider,
2114 ),
2115 summarization: crate::RoleModel::from_parts(
2116 config.summarization_model,
2117 config.summarization_model_provider,
2118 ),
2119 };
2120
2121 let guardian_config = read_guardian_config(&session);
2126 let guardian_spawner = self.guardian_spawner.read().await.clone();
2127
2128 consume_pending_clarification_resume(&mut session);
2129 spawn_session_execution(SessionExecutionArgs {
2130 agent,
2131 session_id,
2132 session,
2133 execution_reservation,
2134 tools_override: Some(root_tools),
2135 provider_override,
2136 model_roster,
2137 reasoning_effort,
2138 reasoning_effort_source,
2139 auxiliary_model_resolver: Some(auxiliary_model_resolver),
2140 disabled_filter_resolver: None,
2143 disabled_tools: Some(config.disabled_tools),
2144 disabled_skill_ids: Some(config.disabled_skill_ids),
2145 selected_skill_ids: None,
2146 selected_skill_mode: None,
2147 mpsc_tx,
2148 history_commit_barrier,
2149 image_fallback: config.image_fallback,
2150 gold_config,
2151 guardian_config,
2152 guardian_spawner,
2153 bash_resume_hook: {
2154 let hook: Arc<dyn BashResumeHook> = Arc::new(self.clone());
2155 Some(hook)
2156 },
2157 bash_completion_sink: {
2158 let sink: Arc<dyn BashCompletionSink> = Arc::new(self.clone());
2161 Some(sink)
2162 },
2163 app_data_dir: Some(self.app_data_dir.clone()),
2164 run_budget: None,
2167 runners: self.agent_runners.clone(),
2168 sessions_cache: self.sessions.clone(),
2169 on_complete: None,
2170 child_completion_handler: Some(Arc::new(self.clone())),
2174 });
2175 }
2176}
2177
2178#[async_trait]
2183impl SessionActivationSpawner for ChildCompletionCoordinator {
2184 async fn reserve_activation(
2185 &self,
2186 target_session_id: &str,
2187 _inbox_generation: u64,
2188 ) -> Result<SessionActivationReserveOutcome, bamboo_domain::SessionActivationError> {
2189 let Some(inbox) = self.agent.session_inbox() else {
2190 return Err(bamboo_domain::SessionActivationError::Internal(
2191 "agent runtime has no SessionInbox".to_string(),
2192 ));
2193 };
2194 let backlog = inbox
2195 .inspect(target_session_id)
2196 .await
2197 .map_err(|error| bamboo_domain::SessionActivationError::Internal(error.to_string()))?;
2198 if !backlog.activation_pending() {
2199 return Ok(SessionActivationReserveOutcome::NoWork);
2200 }
2201 let prepared = prepare_session_inbox_activation(
2204 &self.persistence,
2205 target_session_id,
2206 backlog.interrupt_pending(),
2207 )
2208 .await
2209 .map_err(|error| {
2210 bamboo_domain::SessionActivationError::Internal(format!(
2211 "persist resumable SessionInbox target: {error}"
2212 ))
2213 })?;
2214 let Some((session, ready)) = prepared else {
2215 return Ok(SessionActivationReserveOutcome::NotFound);
2216 };
2217 if !ready {
2218 tracing::info!(
2219 session_id = target_session_id,
2220 "SessionInbox backlog is activation-eligible but a specific durable wait remains armed"
2221 );
2222 return Ok(SessionActivationReserveOutcome::NoWork);
2223 }
2224
2225 enum LaunchPlan {
2226 Root(Box<ResumeConfigSnapshot>),
2227 Child {
2228 scheduler: Arc<SpawnScheduler>,
2229 parent_session_id: String,
2230 model: String,
2231 disabled_tools: Option<Vec<String>>,
2232 },
2233 }
2234
2235 let launch_plan = match session.kind {
2238 SessionKind::Root => {
2239 if self.root_tools.read().await.is_none() {
2240 return Err(bamboo_domain::SessionActivationError::Internal(
2241 "root tool surface is not initialized".to_string(),
2242 ));
2243 }
2244 let config_snapshot = self.config.read().await.clone();
2245 LaunchPlan::Root(Box::new(
2246 self.build_resume_config(&session, &config_snapshot),
2247 ))
2248 }
2249 SessionKind::Child => {
2250 let scheduler = self.spawn_scheduler.read().await.upgrade().ok_or_else(|| {
2251 bamboo_domain::SessionActivationError::Internal(
2252 "child spawn scheduler is not initialized".to_string(),
2253 )
2254 })?;
2255 let parent_session_id = session
2256 .parent_session_id
2257 .as_deref()
2258 .map(str::trim)
2259 .filter(|id| !id.is_empty())
2260 .map(ToOwned::to_owned)
2261 .ok_or_else(|| {
2262 bamboo_domain::SessionActivationError::Internal(format!(
2263 "child SessionInbox target {target_session_id} has no parent owner"
2264 ))
2265 })?;
2266 let parent = self
2267 .storage
2268 .load_session(&parent_session_id)
2269 .await
2270 .map_err(|error| {
2271 bamboo_domain::SessionActivationError::Internal(format!(
2272 "load parent owner {parent_session_id}: {error}"
2273 ))
2274 })?
2275 .ok_or_else(|| {
2276 bamboo_domain::SessionActivationError::Internal(format!(
2277 "child SessionInbox parent owner {parent_session_id} disappeared"
2278 ))
2279 })?;
2280 let child_root = if session.root_session_id.trim().is_empty() {
2281 parent_session_id.as_str()
2282 } else {
2283 session.root_session_id.as_str()
2284 };
2285 let parent_root = if parent.root_session_id.trim().is_empty() {
2286 parent.id.as_str()
2287 } else {
2288 parent.root_session_id.as_str()
2289 };
2290 if child_root != parent_root {
2291 return Err(bamboo_domain::SessionActivationError::Internal(format!(
2292 "child SessionInbox target {target_session_id} does not share its parent owner's root"
2293 )));
2294 }
2295 let model = if session.model.trim().is_empty() {
2296 parent.model.clone()
2297 } else {
2298 session.model.clone()
2299 };
2300 if model.trim().is_empty() {
2301 return Err(bamboo_domain::SessionActivationError::Internal(format!(
2302 "child SessionInbox target {target_session_id} has no executable model"
2303 )));
2304 }
2305 let disabled_tools = match session.metadata.get("disabled_tools") {
2306 None => None,
2307 Some(raw) => {
2308 let tools = serde_json::from_str::<std::collections::BTreeSet<String>>(raw)
2309 .map_err(|error| {
2310 bamboo_domain::SessionActivationError::Internal(format!(
2311 "child SessionInbox target {target_session_id} has malformed disabled_tools: {error}"
2312 ))
2313 })?;
2314 (!tools.is_empty()).then(|| tools.into_iter().collect())
2315 }
2316 };
2317 LaunchPlan::Child {
2318 scheduler,
2319 parent_session_id,
2320 model,
2321 disabled_tools,
2322 }
2323 }
2324 };
2325 let event_sender =
2326 ResumeExecutionPort::get_or_create_event_sender(self, target_session_id).await;
2327
2328 let reservation = match reserve_runner_core(
2329 &self.agent_runners,
2330 &self.session_event_senders,
2331 target_session_id,
2332 &event_sender,
2333 )
2334 .await
2335 {
2336 ReserveOutcome::Reserved(reservation) => reservation,
2337 ReserveOutcome::AlreadyRunning(run_id) => {
2338 return Ok(SessionActivationReserveOutcome::AlreadyRunning { run_id });
2339 }
2340 };
2341 let run_id = reservation.run_id.clone();
2342 let launch = match launch_plan {
2343 LaunchPlan::Root(config) => {
2344 let mut execution_reservation =
2345 SessionExecutionReservation::from_activation_placeholder(
2346 target_session_id,
2347 reservation,
2348 self.agent
2349 .activation_router()
2350 .expect("SessionInbox activation requires an activation router")
2351 .clone(),
2352 self.agent_runners.clone(),
2353 );
2354 if self
2358 .agent
2359 .persistence()
2360 .root_actor_execution_required(&session)
2361 && self.root_account_sink.is_none()
2362 {
2363 execution_reservation
2364 .rollback_unpublished_activation()
2365 .await;
2366 return Err(bamboo_domain::SessionActivationError::Internal(
2367 "Root activation has no actual account sink".into(),
2368 ));
2369 }
2370 if let Err(error) = execution_reservation
2371 .bind_root_actor(&self.agent, &session)
2372 .await
2373 {
2374 execution_reservation
2375 .rollback_unpublished_activation()
2376 .await;
2377 return Err(bamboo_domain::SessionActivationError::Internal(format!(
2378 "Root Actor reservation rejected: {error}"
2379 )));
2380 }
2381 let reservation_cell = Arc::new(StdMutex::new(Some(execution_reservation)));
2385 let launch_reservation = reservation_cell.clone();
2386 let rollback_reservation = reservation_cell;
2387 let coordinator = self.clone();
2388 let launch_sessions = self.sessions.clone();
2389 let launch_session_id = session.id.clone();
2390 let launch_session = session.clone();
2391 let request_session_id = target_session_id.to_string();
2392 SessionActivationLaunch::new_with_async_rollback(
2393 run_id,
2394 move || {
2395 let mut execution_reservation = launch_reservation
2396 .lock()
2397 .unwrap_or_else(std::sync::PoisonError::into_inner)
2398 .take()
2399 .expect("activation reservation launches or rolls back exactly once");
2400 execution_reservation.mark_activation_published();
2401 if execution_reservation.root_actor_writer().is_none() {
2405 launch_sessions.insert(
2406 launch_session_id,
2407 Arc::new(crate::SessionSnapshot::new(launch_session)),
2408 );
2409 }
2410 let request = ResumeSpawnRequest {
2411 session_id: request_session_id,
2412 session,
2413 execution_reservation,
2414 event_sender,
2415 config: *config,
2416 };
2417 tokio::spawn(async move {
2418 ResumeExecutionPort::spawn_resume_execution(&coordinator, request)
2419 .await;
2420 });
2421 },
2422 move || async move {
2423 let reservation = rollback_reservation
2424 .lock()
2425 .unwrap_or_else(std::sync::PoisonError::into_inner)
2426 .take();
2427 if let Some(reservation) = reservation {
2428 reservation.rollback_unpublished_activation().await;
2429 }
2430 },
2431 )
2432 }
2433 LaunchPlan::Child {
2434 scheduler,
2435 parent_session_id,
2436 model,
2437 disabled_tools,
2438 } => {
2439 let execution_reservation =
2440 SessionExecutionReservation::from_activation_placeholder(
2441 target_session_id,
2442 reservation,
2443 self.agent
2444 .activation_router()
2445 .expect("SessionInbox activation requires an activation router")
2446 .clone(),
2447 self.agent_runners.clone(),
2448 );
2449 let reservation_cell = Arc::new(StdMutex::new(Some(execution_reservation)));
2450 let launch_reservation = reservation_cell.clone();
2451 let rollback_reservation = reservation_cell;
2452 let job = SpawnJob {
2453 parent_session_id,
2454 child_session_id: target_session_id.to_string(),
2455 model,
2456 disabled_tools,
2457 };
2458 let launch_sessions = self.sessions.clone();
2459 let launch_session_id = session.id.clone();
2460 let launch_session = session;
2461 SessionActivationLaunch::new_with_async_rollback(
2462 run_id,
2463 move || {
2464 let mut execution_reservation = launch_reservation
2465 .lock()
2466 .unwrap_or_else(std::sync::PoisonError::into_inner)
2467 .take()
2468 .expect("activation reservation launches or rolls back exactly once");
2469 execution_reservation.mark_activation_published();
2470 launch_sessions.insert(
2473 launch_session_id,
2474 Arc::new(crate::SessionSnapshot::new(launch_session)),
2475 );
2476 drop(scheduler.launch_reserved(job, execution_reservation));
2480 },
2481 move || async move {
2482 let reservation = rollback_reservation
2483 .lock()
2484 .unwrap_or_else(std::sync::PoisonError::into_inner)
2485 .take();
2486 if let Some(reservation) = reservation {
2487 reservation.rollback_unpublished_activation().await;
2488 }
2489 },
2490 )
2491 }
2492 };
2493 Ok(SessionActivationReserveOutcome::Reserved(launch))
2494 }
2495}
2496
2497fn bash_completion_resume_message(bash_ids: &[String], timed_out: bool) -> Message {
2507 let body = if timed_out {
2508 format!(
2509 "Runtime notification: the background-Bash wait ceiling was reached while one or more \
2510 shell(s) ({}) may still be running. The session is being resumed so it is not \
2511 stranded; verify their actual status with BashOutput before assuming completion.",
2512 bash_ids.join(", ")
2513 )
2514 } else {
2515 format!(
2516 "Runtime notification: all background Bash shell(s) ({}) have completed. \
2517 Review their output with BashOutput and resume the task from where you left off.",
2518 bash_ids.join(", ")
2519 )
2520 };
2521 let mut message = Message::user(body);
2522 message.metadata = Some(serde_json::json!({
2523 RUNTIME_RESUME_MESSAGE_HIDDEN_KEY: true,
2524 RUNTIME_RESUME_MESSAGE_KIND_KEY: BASH_COMPLETION_RESUME_KIND,
2525 }));
2526 message.never_compress = false;
2527 message
2528}
2529
2530fn bash_resume_should_retry(outcome: &ResumeOutcome, persisted_waiting_for_bash: bool) -> bool {
2543 match outcome {
2544 ResumeOutcome::Started { .. } | ResumeOutcome::NotFound => false,
2545 ResumeOutcome::Completed | ResumeOutcome::AlreadyRunning { .. } => {
2546 persisted_waiting_for_bash
2547 }
2548 }
2549}
2550
2551fn bash_completion_should_resume(
2559 loop_suspended_on_bash: bool,
2560 all_waited_shells_done: bool,
2561) -> bool {
2562 loop_suspended_on_bash && all_waited_shells_done
2563}
2564
2565#[derive(Debug, Clone, Copy, PartialEq, Eq)]
2566enum BashCompletionDeliveryPlan {
2567 DurableOnly,
2571 Activate,
2573 ClearWaitThenActivate,
2575}
2576
2577fn bash_completion_delivery_plan(
2578 loop_suspended_on_bash: bool,
2579 all_waited_shells_done: bool,
2580) -> BashCompletionDeliveryPlan {
2581 if bash_completion_should_resume(loop_suspended_on_bash, all_waited_shells_done) {
2582 BashCompletionDeliveryPlan::ClearWaitThenActivate
2583 } else if loop_suspended_on_bash {
2584 BashCompletionDeliveryPlan::DurableOnly
2585 } else {
2586 BashCompletionDeliveryPlan::Activate
2587 }
2588}
2589
2590fn background_bash_post_tool_payload(info: &BashCompletionInfo) -> HookPayload {
2591 let success = info.status == "completed" && info.exit_code == Some(0);
2592 let response = serde_json::json!({
2593 "bash_id": info.bash_id,
2594 "command": info.command,
2595 "exit_code": info.exit_code,
2596 "status": info.status,
2597 "output_tail": info.output_tail,
2598 });
2599 HookPayload::ToolResult {
2600 tool_name: "Bash".to_string(),
2601 tool_call_id: info.bash_id.clone(),
2602 outcome: HookToolOutcome {
2603 success,
2604 result: success.then(|| response.to_string()),
2605 error: (!success).then(|| response.to_string()),
2606 needs_human: false,
2607 duration_ms: 0,
2608 },
2609 }
2610}
2611
2612fn append_background_bash_hook_feedback(info: &mut BashCompletionInfo, feedback: Vec<String>) {
2613 let feedback = feedback
2614 .into_iter()
2615 .map(|text| text.trim().to_string())
2616 .filter(|text| !text.is_empty())
2617 .collect::<Vec<_>>();
2618 if feedback.is_empty() {
2619 return;
2620 }
2621 if !info.output_tail.is_empty() {
2622 info.output_tail.push_str("\n\n");
2623 }
2624 info.output_tail.push_str("<post_tool_use_feedback>\n");
2625 info.output_tail.push_str(&feedback.join("\n"));
2626 info.output_tail.push_str("\n</post_tool_use_feedback>");
2627}
2628
2629async fn run_background_bash_post_tool_hooks(
2630 config: &bamboo_config::LifecycleHooksConfig,
2631 fallback_cwd: Option<std::path::PathBuf>,
2632 session: &Session,
2633 info: &mut BashCompletionInfo,
2634) -> bool {
2635 let runner = crate::HookRunner::new().with_lifecycle_config(config, fallback_cwd);
2636 if !runner.has_hooks_for(AgentHookPoint::AfterToolExecution) {
2637 return false;
2638 }
2639
2640 let mut runtime_state = session
2641 .agent_runtime_state
2642 .clone()
2643 .unwrap_or_else(|| AgentRuntimeState::new(&session.id));
2644 let outcome = runner
2645 .run_observer_hooks(
2646 AgentHookPoint::AfterToolExecution,
2647 &background_bash_post_tool_payload(info),
2648 session,
2649 &mut runtime_state,
2650 None,
2651 )
2652 .await;
2653 append_background_bash_hook_feedback(info, outcome.injected_contexts);
2654 true
2655}
2656
2657fn apply_bash_resume_transition(session: &mut Session, resume_message: &Message) -> bool {
2665 let mut runtime_state = read_runtime_state(session);
2666 if runtime_state.waiting_for_bash.is_none() {
2667 return false;
2668 }
2669 runtime_state.waiting_for_bash = None;
2670 runtime_state.status = AgentStatusState::Idle;
2671 runtime_state.suspension = None;
2672 write_runtime_state(session, &runtime_state);
2673 session.metadata.remove("runtime.suspend_reason");
2674 session.add_message(resume_message.clone());
2675 true
2676}
2677
2678impl ChildCompletionCoordinator {
2680 async fn bash_self_resume(&self, session_id: String, bash_ids: Vec<String>) {
2696 let mut delay = Duration::from_secs(1);
2697 let max_delay = Duration::from_secs(30);
2698 let max_poll = Duration::from_secs(6 * 3600 + 600);
2702 let deadline = tokio::time::Instant::now() + max_poll;
2703
2704 loop {
2705 tokio::time::sleep(delay).await;
2706
2707 let Some(session) = self.load_session(&session_id).await else {
2708 tracing::info!(%session_id, "bash self-resume backstop: session gone; nothing to do");
2709 return;
2710 };
2711 if read_runtime_state(&session).waiting_for_bash.is_none() {
2712 return;
2715 }
2716
2717 let still_running =
2718 bamboo_tools::tools::bash_runtime::running_shells_for_session(&session_id);
2719 let timed_out = tokio::time::Instant::now() >= deadline;
2720 if still_running.is_empty() || timed_out {
2721 let guard = session_resume_lock(&session_id);
2726 let _held = guard.lock().await;
2727 tracing::warn!(
2728 %session_id,
2729 shell_count = bash_ids.len(),
2730 timed_out,
2731 "bash self-resume backstop engaged (push lost or wait ceiling reached)"
2732 );
2733 self.perform_bash_resume(
2734 &session_id,
2735 bash_completion_resume_message(&bash_ids, timed_out),
2736 )
2737 .await;
2738 return;
2739 }
2740
2741 delay = (delay * 2).min(max_delay);
2742 }
2743 }
2744
2745 async fn perform_bash_resume(&self, session_id: &str, resume_message: Message) {
2766 let retry_backoff = Duration::from_millis(200);
2767 const MAX_RESUME_ATTEMPTS: u8 = 5;
2768 for attempt in 0..MAX_RESUME_ATTEMPTS {
2769 if attempt > 0 {
2770 tokio::time::sleep(retry_backoff).await;
2771 }
2772
2773 let Some(mut session) = self.load_session(session_id).await else {
2774 tracing::warn!(%session_id, "bash resume: session not found; nothing to resume");
2775 return;
2776 };
2777
2778 if !apply_bash_resume_transition(&mut session, &resume_message) {
2779 tracing::info!(
2783 %session_id, attempt,
2784 "bash resume: persisted bash wait already cleared; nothing to resume"
2785 );
2786 return;
2787 }
2788 session.updated_at = Utc::now();
2789 self.save_and_cache(&mut session).await;
2790 tracing::info!(
2791 %session_id, attempt,
2792 "bash resume: cleared bash wait and appended resume message"
2793 );
2794
2795 let outcome = self.resume_parent(session_id.to_string()).await;
2796 match outcome {
2797 ResumeOutcome::Started { .. } => {
2798 tracing::info!(%session_id, attempt, "bash resume: resume fired");
2799 return;
2800 }
2801 ResumeOutcome::NotFound => {
2802 tracing::warn!(%session_id, "bash resume: session vanished during resume");
2803 return;
2804 }
2805 _ => {
2806 let clobbered = match self.load_session(session_id).await {
2812 Some(reloaded) => read_runtime_state(&reloaded).waiting_for_bash.is_some(),
2813 None => {
2814 tracing::warn!(%session_id, "bash resume: session vanished after resume");
2815 return;
2816 }
2817 };
2818 if bash_resume_should_retry(&outcome, clobbered) {
2819 tracing::warn!(
2820 %session_id, attempt,
2821 outcome = outcome.as_str(),
2822 "bash resume: persisted wait still set after resume (finalize-clobber); retrying"
2823 );
2824 continue;
2825 }
2826 tracing::info!(
2827 %session_id, attempt,
2828 outcome = outcome.as_str(),
2829 "bash resume: wait cleared and resume handled; stopping"
2830 );
2831 return;
2832 }
2833 }
2834 }
2835
2836 tracing::warn!(
2837 %session_id,
2838 attempts = MAX_RESUME_ATTEMPTS,
2839 "bash resume: exhausted clobber-retry budget without confirming resume; giving up"
2840 );
2841 }
2842}
2843
2844impl BashResumeHook for ChildCompletionCoordinator {
2845 fn arrange_bash_self_resume(&self, session_id: String, bash_ids: Vec<String>) {
2846 let coordinator = Arc::new(self.clone());
2847 tokio::spawn(async move {
2848 coordinator.bash_self_resume(session_id, bash_ids).await;
2849 });
2850 }
2851}
2852
2853fn bash_completion_injection_body(info: &BashCompletionInfo) -> String {
2858 let exit = match info.exit_code {
2859 Some(code) => code.to_string(),
2860 None => "none (signal/killed)".to_string(),
2861 };
2862 let mut body = format!(
2863 "Runtime notification: background shell `{}` (`{}`) finished — status {}, exit code {}.",
2864 info.bash_id, info.command, info.status, exit
2865 );
2866 if info.output_tail.trim().is_empty() {
2867 body.push_str(" It produced no captured output.");
2868 } else {
2869 body.push_str("\n\nOutput tail:\n");
2870 body.push_str(&info.output_tail);
2871 }
2872 body.push_str(&format!(
2873 "\n\nUse BashOutput with bash_id=\"{}\" for the full output, then continue the task.",
2874 info.bash_id
2875 ));
2876 body
2877}
2878
2879fn bash_completion_envelope(info: &BashCompletionInfo) -> bamboo_domain::SessionMessageEnvelope {
2880 let provider_message = bash_resume_message_from_info(info);
2881 let data = serde_json::json!({
2882 "session_id": info.session_id,
2883 "bash_id": info.bash_id,
2884 "command": info.command,
2885 "exit_code": info.exit_code,
2886 "status": info.status,
2887 "output_tail": info.output_tail,
2888 });
2889 let identity = data.clone();
2894 bamboo_domain::SessionMessageEnvelope {
2895 id: bamboo_domain::SessionMessageId::stable("background_bash_completion", &identity),
2896 source: bamboo_domain::SessionMessageSource::Runtime {
2897 subsystem: "background_bash".to_string(),
2898 },
2899 target_session_id: info.session_id.clone(),
2900 kind: bamboo_domain::SessionMessageKind::RuntimeInstruction,
2901 body: bamboo_domain::SessionMessageBody::RuntimeInstruction(
2902 bamboo_domain::SessionRuntimeInstruction {
2903 instruction: "background_bash_completed".to_string(),
2904 content: Some(bamboo_domain::SessionMessageContent::text(
2905 bash_completion_injection_body(info),
2906 )),
2907 data: Some(data),
2908 provider_message: Some(session_provider_message(&provider_message)),
2909 },
2910 ),
2911 created_at: Utc::now(),
2912 thread_id: None,
2913 in_reply_to: None,
2914 attempt: None,
2915 correlation_id: Some(info.bash_id.clone()),
2916 }
2917}
2918
2919async fn enqueue_bash_completion_injection(
2925 persistence: &LockedSessionStore,
2926 info: &BashCompletionInfo,
2927) -> std::io::Result<Option<Session>> {
2928 let body = bash_completion_injection_body(info);
2929 let queued = serde_json::json!({
2930 "content": body,
2931 "created_at": Utc::now(),
2932 });
2933 persistence
2934 .update_runtime_config(&info.session_id, move |session| {
2935 let mut pending = session.pending_injected_messages().unwrap_or_default();
2936 pending.push(queued);
2937 session.set_pending_injected_messages(pending);
2938 })
2939 .await
2940}
2941
2942fn bash_resume_message_from_info(info: &BashCompletionInfo) -> Message {
2950 let mut message = Message::user(bash_completion_injection_body(info));
2951 message.metadata = Some(serde_json::json!({
2952 RUNTIME_RESUME_MESSAGE_HIDDEN_KEY: true,
2953 RUNTIME_RESUME_MESSAGE_KIND_KEY: BASH_COMPLETION_RESUME_KIND,
2954 }));
2955 message.never_compress = false;
2956 message
2957}
2958
2959impl ChildCompletionCoordinator {
2960 async fn deliver_bash_completion(&self, mut info: BashCompletionInfo) {
2972 if let Some(hook_session) = self.load_session(&info.session_id).await {
2978 let config_snapshot = self.config.read().await.clone();
2979 let _ = run_background_bash_post_tool_hooks(
2980 &config_snapshot.lifecycle_hooks,
2981 Some(self.app_data_dir.clone()),
2982 &hook_session,
2983 &mut info,
2984 )
2985 .await;
2986 }
2987
2988 let guard = session_resume_lock(&info.session_id);
2989 let _held = guard.lock().await;
2990
2991 let Some(session) = self.load_session(&info.session_id).await else {
2992 tracing::warn!(
2993 session_id = %info.session_id,
2994 bash_id = %info.bash_id,
2995 "background bash completion: owning session not found; nothing to notify"
2996 );
2997 return;
2998 };
2999
3000 let waiting = read_runtime_state(&session).waiting_for_bash.is_some();
3001 let all_shells_done =
3007 bamboo_tools::tools::bash_runtime::running_shells_for_session(&info.session_id)
3008 .is_empty();
3009 let delivery_plan = bash_completion_delivery_plan(waiting, all_shells_done);
3010
3011 let messenger = self.agent.session_messenger().cloned();
3017 let admission = match messenger.as_ref() {
3018 Some(messenger) => match messenger.admit(bash_completion_envelope(&info)).await {
3019 Ok(admission) => Some(admission),
3020 Err(error) => {
3021 tracing::warn!(
3022 session_id = %info.session_id,
3023 bash_id = %info.bash_id,
3024 %error,
3025 "background bash completion: durable SessionInbox admission failed; leaving wait armed"
3026 );
3027 return;
3028 }
3029 },
3030 None => {
3031 tracing::warn!(
3032 session_id = %info.session_id,
3033 bash_id = %info.bash_id,
3034 "SessionMessenger unavailable; using compatibility injection before clearing wait"
3035 );
3036 match enqueue_bash_completion_injection(&self.persistence, &info).await {
3037 Ok(Some(_)) => None,
3038 Ok(None) => {
3039 tracing::warn!(
3040 session_id = %info.session_id,
3041 "background bash compatibility target disappeared"
3042 );
3043 return;
3044 }
3045 Err(error) => {
3046 tracing::warn!(
3047 session_id = %info.session_id,
3048 %error,
3049 "background bash compatibility admission failed; leaving wait armed"
3050 );
3051 return;
3052 }
3053 }
3054 }
3055 };
3056
3057 if delivery_plan == BashCompletionDeliveryPlan::DurableOnly {
3058 tracing::info!(
3059 session_id = %info.session_id,
3060 bash_id = %info.bash_id,
3061 "background bash completion is durable; sibling shells still run, so wait remains armed"
3062 );
3063 return;
3064 }
3065
3066 if let (Some(messenger), Some(admission)) = (messenger.as_ref(), admission.as_ref()) {
3067 if let Err(error) = messenger.prepare_activation(admission).await {
3068 tracing::warn!(
3069 session_id = %info.session_id,
3070 bash_id = %info.bash_id,
3071 %error,
3072 "background bash completion activation watermark failed; leaving wait armed"
3073 );
3074 return;
3075 }
3076 }
3077
3078 if delivery_plan == BashCompletionDeliveryPlan::ClearWaitThenActivate {
3079 tracing::info!(
3080 session_id = %info.session_id,
3081 bash_id = %info.bash_id,
3082 status = %info.status,
3083 "background bash completion: push-resuming suspended loop (event-driven)"
3084 );
3085 let mut resumable = session.clone();
3086 let mut runtime_state = read_runtime_state(&resumable);
3087 runtime_state.waiting_for_bash = None;
3088 runtime_state.status = AgentStatusState::Idle;
3089 runtime_state.suspension = None;
3090 write_runtime_state(&mut resumable, &runtime_state);
3091 resumable.metadata.remove("runtime.suspend_reason");
3092 resumable.updated_at = Utc::now();
3093 if let Err(error) = self.persistence.merge_save_runtime(&mut resumable).await {
3094 tracing::warn!(
3095 session_id = %info.session_id,
3096 %error,
3097 "background bash completion is durable but wait-state clear failed; leaving activation to the wait backstop"
3098 );
3099 return;
3100 }
3101 self.sessions.insert(
3102 resumable.id.clone(),
3103 Arc::new(crate::SessionSnapshot::new(resumable)),
3104 );
3105 }
3106
3107 let (Some(messenger), Some(admission)) = (messenger, admission) else {
3111 if delivery_plan == BashCompletionDeliveryPlan::ClearWaitThenActivate {
3112 let _ = self.resume_parent(info.session_id.clone()).await;
3113 }
3114 return;
3115 };
3116 match messenger.activate_prepared(&admission).await {
3117 Ok(receipt) => tracing::info!(
3118 session_id = %info.session_id,
3119 bash_id = %info.bash_id,
3120 status = %info.status,
3121 waiting,
3122 generation = receipt.delivery.generation,
3123 activation = ?receipt.activation,
3124 "background bash completion delivered through SessionMessenger"
3125 ),
3126 Err(error) => tracing::warn!(
3127 session_id = %info.session_id,
3128 bash_id = %info.bash_id,
3129 %error,
3130 "background bash completion: SessionMessenger delivery failed"
3131 ),
3132 }
3133 }
3134}
3135
3136impl BashCompletionSink for ChildCompletionCoordinator {
3137 fn on_bash_completed(&self, info: BashCompletionInfo) {
3138 let coordinator = Arc::new(self.clone());
3142 tokio::spawn(async move {
3143 coordinator.deliver_bash_completion(info).await;
3144 });
3145 }
3146}
3147
3148const CHILD_WAIT_SWEEP_INTERVAL_SECS: u64 = 30;
3154const CHILD_WAIT_REGISTRATION_GRACE_SECS: i64 = 60;
3158const DEAD_CHILD_GRACE_SECS: i64 = 120;
3161const STALE_RUNNER_SLACK_SECS: i64 = 600;
3167
3168fn is_dead_child_candidate_status(status: Option<&str>) -> bool {
3173 match status {
3174 Some(status) => !is_terminal_child_status(status) && status != "suspended",
3177 None => true,
3178 }
3179}
3180
3181fn completion_child_is_owned(reported_parent: &str, child_parent_linkage: Option<&str>) -> bool {
3189 child_parent_linkage == Some(reported_parent)
3190}
3191
3192fn child_wait_watchdog_resume_message(body: String) -> Message {
3193 let mut message = Message::user(body);
3194 message.metadata = Some(serde_json::json!({
3195 RUNTIME_RESUME_MESSAGE_HIDDEN_KEY: true,
3196 RUNTIME_RESUME_MESSAGE_KIND_KEY: "child_wait_watchdog_resume",
3197 }));
3198 message.never_compress = false;
3199 message
3200}
3201
3202fn empty_child_wait_message() -> Message {
3203 let inspection = serde_json::json!({"intent": "inspect", "message": "tree"});
3204 child_wait_watchdog_resume_message(format!(
3205 "Runtime notification: this session was suspended waiting for child sessions, but the \
3206 wait tracked no children (internal inconsistency). The session has been resumed; use \
3207 SubAgent with {inspection} to inspect child state and continue the task.",
3208 ))
3209}
3210
3211fn child_wait_lease_expired_message(child_ids: &[String]) -> Message {
3212 let inspection = serde_json::json!({"intent": "inspect", "message": "tree"});
3213 child_wait_watchdog_resume_message(format!(
3214 "Runtime notification: the wait lease for child session(s) [{}] expired before they all \
3215 reported completion. They were NOT cancelled and may still be running or already \
3216 finished — verify their actual status by calling SubAgent with {inspection} before assuming \
3217 anything, then continue the task.",
3218 child_ids.join(", ")
3219 ))
3220}
3221
3222impl ChildCompletionCoordinator {
3238 pub fn spawn_child_wait_watchdog(self: &Arc<Self>) {
3241 let coordinator = Arc::clone(self);
3242 tokio::spawn(async move {
3243 use futures::FutureExt;
3244 if std::panic::AssertUnwindSafe(coordinator.reconcile_orphans_at_boot())
3245 .catch_unwind()
3246 .await
3247 .is_err()
3248 {
3249 tracing::error!("child-wait watchdog: boot reconciliation panicked");
3250 }
3251 let mut ticker = tokio::time::interval(std::time::Duration::from_secs(
3252 CHILD_WAIT_SWEEP_INTERVAL_SECS,
3253 ));
3254 ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
3255 ticker.tick().await;
3257 loop {
3258 ticker.tick().await;
3259 if std::panic::AssertUnwindSafe(coordinator.sweep_child_waits())
3260 .catch_unwind()
3261 .await
3262 .is_err()
3263 {
3264 tracing::error!("child-wait watchdog: sweep panicked; continuing");
3265 }
3266 }
3267 });
3268 }
3269
3270 async fn reconcile_orphans_at_boot(&self) {
3275 let cutoff = Utc::now();
3276
3277 let running = self
3283 .storage
3284 .list_sessions_by_run_status("running")
3285 .await
3286 .unwrap_or_default();
3287 for (child_id, parent_id) in running {
3288 let Some(parent_id) = parent_id else { continue };
3289 if self.runner_is_running(&child_id).await {
3290 continue;
3291 }
3292 let Some(control_plane) = self.load_control_plane(&child_id).await else {
3293 continue;
3294 };
3295 if control_plane.updated_at >= cutoff {
3298 continue;
3299 }
3300 tracing::warn!(
3301 child_session_id = %child_id,
3302 parent_session_id = %parent_id,
3303 "boot reconciliation: child was running when the process died; \
3304 marking it error and waking the parent"
3305 );
3306 self.synthesize_child_completion(
3307 &parent_id,
3308 &child_id,
3309 "error",
3310 Some(
3311 "orphaned by server restart: the process died while this child session \
3312 was running"
3313 .to_string(),
3314 ),
3315 )
3316 .await;
3317 }
3318
3319 let suspended = self
3323 .storage
3324 .list_sessions_by_run_status("suspended")
3325 .await
3326 .unwrap_or_default();
3327 for (session_id, _) in suspended {
3328 let Some(control_plane) = self.load_control_plane(&session_id).await else {
3329 continue;
3330 };
3331 if control_plane
3332 .metadata
3333 .get("runtime.suspend_reason")
3334 .map(String::as_str)
3335 != Some("waiting_for_bash")
3336 {
3337 continue;
3338 }
3339 if let Some(wait) = read_runtime_state(&control_plane).waiting_for_bash {
3340 tracing::warn!(
3341 %session_id,
3342 "boot reconciliation: re-arming bash self-resume backstop lost in restart"
3343 );
3344 let coordinator = self.clone();
3345 tokio::spawn(async move {
3346 coordinator
3347 .bash_self_resume(session_id, wait.bash_ids)
3348 .await;
3349 });
3350 }
3351 }
3352 }
3353
3354 async fn runner_is_running(&self, session_id: &str) -> bool {
3355 let runners = self.agent_runners.read().await;
3356 runners
3357 .get(session_id)
3358 .is_some_and(|runner| matches!(runner.status, AgentStatus::Running))
3359 }
3360
3361 async fn load_control_plane(&self, session_id: &str) -> Option<Session> {
3362 match self.storage.load_runtime_control_plane(session_id).await {
3363 Ok(session) => session,
3364 Err(error) => {
3365 tracing::warn!(
3366 %session_id,
3367 %error,
3368 "child-wait watchdog: failed to load session control plane"
3369 );
3370 None
3371 }
3372 }
3373 }
3374
3375 async fn sweep_child_waits(&self) {
3379 let suspended = match self.storage.list_sessions_by_run_status("suspended").await {
3380 Ok(entries) => entries,
3381 Err(error) => {
3382 tracing::warn!(%error, "child-wait watchdog: failed to list suspended sessions");
3383 return;
3384 }
3385 };
3386 for (session_id, _) in suspended {
3387 self.sweep_one_suspended_session(&session_id).await;
3388 }
3389 }
3390
3391 async fn sweep_one_suspended_session(&self, session_id: &str) {
3392 if self.runner_is_running(session_id).await {
3395 return;
3396 }
3397 let Some(session) = self.load_control_plane(session_id).await else {
3398 return;
3399 };
3400 let runtime_state = read_runtime_state(&session);
3401 let suspend_reason = session
3402 .metadata
3403 .get("runtime.suspend_reason")
3404 .map(String::as_str)
3405 .unwrap_or_default()
3406 .to_string();
3407 match (
3408 suspend_reason.as_str(),
3409 runtime_state.waiting_for_children.clone(),
3410 ) {
3411 ("waiting_for_bash", _)
3413 | ("awaiting_clarification", _)
3414 | ("awaiting_parent_approval", _) => {}
3415 (_, Some(wait)) => self.sweep_child_wait(session_id, wait).await,
3416 ("waiting_for_children", None) | ("", None) => {
3419 self.rescue_stranded_resume(session_id).await;
3420 }
3421 _ => {}
3422 }
3423 }
3424
3425 async fn sweep_child_wait(&self, parent_session_id: &str, wait: WaitingForChildrenState) {
3430 let now = Utc::now();
3431
3432 if wait.child_session_ids.is_empty() {
3433 tracing::warn!(
3434 %parent_session_id,
3435 "child-wait watchdog: wait armed over an empty child set; force-resuming"
3436 );
3437 self.force_resume_child_wait(parent_session_id, empty_child_wait_message())
3438 .await;
3439 return;
3440 }
3441
3442 if wait.timeout_at.is_some_and(|deadline| now >= deadline) {
3446 tracing::warn!(
3447 %parent_session_id,
3448 "child-wait watchdog: wait lease expired; force-resuming parent"
3449 );
3450 self.force_resume_child_wait(
3451 parent_session_id,
3452 child_wait_lease_expired_message(&wait.child_session_ids),
3453 )
3454 .await;
3455 return;
3456 }
3457
3458 if now.signed_duration_since(wait.registered_at).num_seconds()
3460 < CHILD_WAIT_REGISTRATION_GRACE_SECS
3461 {
3462 return;
3463 }
3464
3465 let statuses: HashMap<String, Option<String>> = self
3466 .storage
3467 .list_child_run_statuses(parent_session_id)
3468 .await
3469 .unwrap_or_default()
3470 .into_iter()
3471 .collect();
3472
3473 struct DeadChild {
3474 child_id: String,
3475 status: String,
3476 reason: String,
3477 owned: bool,
3481 }
3482
3483 let mut terminal: Vec<(String, String)> = Vec::new();
3484 let mut dead: Vec<DeadChild> = Vec::new();
3485 for child_id in &wait.child_session_ids {
3486 let status = statuses.get(child_id).and_then(|status| status.as_deref());
3487 if let Some(status) = status {
3488 if is_terminal_child_status(status) {
3489 terminal.push((child_id.clone(), status.to_string()));
3490 continue;
3491 }
3492 }
3493 if !is_dead_child_candidate_status(status) {
3494 continue;
3495 }
3496
3497 let control_plane = self.load_control_plane(child_id).await;
3504 let owned = control_plane
3505 .as_ref()
3506 .is_some_and(|cp| cp.parent_session_id.as_deref() == Some(parent_session_id));
3507 if !owned {
3508 dead.push(DeadChild {
3509 child_id: child_id.clone(),
3510 status: "error".to_string(),
3511 reason: if control_plane.is_some() {
3512 "waited-on session id is not a child of this session; clearing it \
3513 from the wait without touching that session"
3514 .to_string()
3515 } else {
3516 "waited-on child session does not exist".to_string()
3517 },
3518 owned: false,
3519 });
3520 continue;
3521 }
3522
3523 let runner = { self.agent_runners.read().await.get(child_id).cloned() };
3524 match runner {
3525 Some(runner) if matches!(runner.status, AgentStatus::Running) => {
3526 let last_activity = runner.last_activity_at().unwrap_or(runner.started_at);
3532 let idle_secs = now.signed_duration_since(last_activity).num_seconds();
3533 let total_secs = now.signed_duration_since(runner.started_at).num_seconds();
3534 let policy = match &control_plane {
3535 Some(child) => {
3536 crate::runtime::execution::spawn::watchdog_policy_for_session(child)
3537 }
3538 None => Default::default(),
3539 };
3540 let idle_limit = policy.max_idle_secs.saturating_add(STALE_RUNNER_SLACK_SECS);
3541 let total_limit = policy
3542 .max_total_secs
3543 .saturating_add(STALE_RUNNER_SLACK_SECS);
3544 if idle_secs >= idle_limit || total_secs >= total_limit {
3545 runner.cancel_token.cancel();
3546 dead.push(DeadChild {
3547 child_id: child_id.clone(),
3548 status: "timeout".to_string(),
3549 reason: format!(
3550 "child runner stalled: no events for {idle_secs}s \
3551 (limit {idle_limit}s including watchdog slack); \
3552 force-finalized by the child-wait watchdog"
3553 ),
3554 owned: true,
3555 });
3556 }
3557 }
3558 _ => {
3559 let quiet_secs = control_plane
3563 .as_ref()
3564 .map(|child| now.signed_duration_since(child.updated_at).num_seconds())
3565 .unwrap_or(i64::MAX);
3566 if quiet_secs >= DEAD_CHILD_GRACE_SECS {
3567 dead.push(DeadChild {
3568 child_id: child_id.clone(),
3569 status: "error".to_string(),
3570 reason: format!(
3571 "child runner lost (crashed task, dropped spawn job, or \
3572 process restart): index status {status:?} with no live \
3573 runner driving it"
3574 ),
3575 owned: true,
3576 });
3577 }
3578 }
3579 }
3580 }
3581
3582 if !dead.is_empty() {
3583 for entry in dead {
3584 tracing::warn!(
3585 %parent_session_id,
3586 child_session_id = %entry.child_id,
3587 status = %entry.status,
3588 reason = %entry.reason,
3589 owned = entry.owned,
3590 "child-wait watchdog: synthesizing terminal completion for dead child"
3591 );
3592 if entry.owned {
3593 self.synthesize_child_completion(
3594 parent_session_id,
3595 &entry.child_id,
3596 &entry.status,
3597 Some(entry.reason),
3598 )
3599 .await;
3600 } else {
3601 self.publish_synthetic_completion(
3603 parent_session_id,
3604 &entry.child_id,
3605 &entry.status,
3606 Some(entry.reason),
3607 )
3608 .await;
3609 }
3610 }
3611 return;
3613 }
3614
3615 terminal.sort_by_key(|(_, status)| !is_error_like(status));
3620 for (child_id, status) in terminal {
3621 tracing::debug!(
3622 %parent_session_id,
3623 child_session_id = %child_id,
3624 "child-wait watchdog: replaying terminal child outcome"
3625 );
3626 let control_plane = self.load_control_plane(&child_id).await;
3627 let mut completion = ChildCompletion {
3628 parent_session_id: parent_session_id.into(),
3629 child_session_id: child_id.clone(),
3630 status,
3631 error: control_plane.as_ref().and_then(Session::last_run_error),
3632 completed_at: Utc::now(),
3633 source: None,
3634 };
3635 if control_plane
3638 .as_ref()
3639 .is_some_and(|child| child.parent_session_id.as_deref() == Some(parent_session_id))
3640 {
3641 let Ok(Some(child)) = self.storage.load_session(&child_id).await else {
3642 continue;
3643 };
3644 completion.source = ChildCompletionSource::from_committed_session(&child)
3645 .filter(|source| source.matches_completion(&completion));
3646 if ChildCompletionSource::has_source_record(&child) && completion.source.is_none() {
3647 tracing::warn!(%child_id,
3648 "unavailable committed replay source; leaving wait armed");
3649 continue;
3650 }
3651 completion.completed_at = child.updated_at;
3652 }
3653 self.publish_completion(completion).await;
3654 }
3655 }
3656
3657 async fn synthesize_child_completion(
3663 &self,
3664 parent_session_id: &str,
3665 child_session_id: &str,
3666 status: &str,
3667 error: Option<String>,
3668 ) {
3669 match self.storage.load_session(child_session_id).await {
3670 Ok(Some(mut child)) => {
3671 if child.parent_session_id.as_deref() != Some(parent_session_id) {
3676 tracing::warn!(
3677 %parent_session_id,
3678 child_session_id = %child.id,
3679 "child-wait watchdog: refusing to synthesize status onto a session \
3680 that is not a child of this parent"
3681 );
3682 self.publish_synthetic_completion(
3683 parent_session_id,
3684 child_session_id,
3685 status,
3686 error,
3687 )
3688 .await;
3689 return;
3690 }
3691 child.set_last_run_status(status);
3692 match &error {
3693 Some(message) => child.set_last_run_error(message.clone()),
3694 None => child.clear_last_run_error(),
3695 }
3696 child.updated_at = Utc::now();
3697 if let Err(save_error) = self.persistence.merge_save_runtime(&mut child).await {
3698 tracing::warn!(
3699 child_session_id = %child.id,
3700 %save_error,
3701 "child-wait watchdog: failed to persist synthesized terminal status"
3702 );
3703 }
3704 self.sessions.insert(
3705 child.id.clone(),
3706 Arc::new(crate::SessionSnapshot::new(child)),
3707 );
3708 }
3709 Ok(None) => {}
3710 Err(load_error) => {
3711 tracing::warn!(
3712 %child_session_id,
3713 %load_error,
3714 "child-wait watchdog: failed to load child for synthesized terminal status"
3715 );
3716 }
3717 }
3718 finalize_runner(
3719 &self.agent_runners,
3720 child_session_id,
3721 &Err(bamboo_agent_core::AgentError::LLM(
3722 error
3723 .clone()
3724 .unwrap_or_else(|| format!("synthesized {status}")),
3725 )),
3726 )
3727 .await;
3728 self.publish_synthetic_completion(parent_session_id, child_session_id, status, error)
3729 .await;
3730 }
3731
3732 async fn publish_synthetic_completion(
3733 &self,
3734 parent_session_id: &str,
3735 child_session_id: &str,
3736 status: &str,
3737 error: Option<String>,
3738 ) {
3739 self.publish_completion(ChildCompletion {
3740 parent_session_id: parent_session_id.into(),
3741 child_session_id: child_session_id.into(),
3742 status: status.into(),
3743 error,
3744 completed_at: Utc::now(),
3745 source: None,
3746 })
3747 .await;
3748 }
3749
3750 async fn publish_completion(&self, completion: ChildCompletion) {
3751 let publisher =
3752 crate::runtime::execution::session_events::ReplayableSessionEventPublisher::new(
3753 self.agent_runners.clone(),
3754 self.session_event_senders.clone(),
3755 self.account_feed_inbox.clone(),
3756 );
3757 let handler: Arc<dyn ChildCompletionHandler> = Arc::new(self.clone());
3758 crate::runtime::execution::spawn::publish_child_completion(
3759 &publisher,
3760 Some(handler),
3761 completion,
3762 )
3763 .await;
3764 }
3765
3766 async fn rescue_stranded_resume(&self, session_id: &str) {
3772 let Some(session) = self.load_session(session_id).await else {
3773 return;
3774 };
3775 let pending_runtime_resume = session.messages.last().is_some_and(|message| {
3776 matches!(message.role, Role::User)
3777 && message
3778 .metadata
3779 .as_ref()
3780 .is_some_and(|meta| meta.get(RUNTIME_RESUME_MESSAGE_KIND_KEY).is_some())
3781 });
3782 if !pending_runtime_resume {
3783 return;
3784 }
3785 tracing::warn!(
3786 %session_id,
3787 "child-wait watchdog: stranded resume detected (wait cleared, resume never \
3788 spawned); resuming"
3789 );
3790 self.resume_parent(session_id.to_string()).await;
3791 }
3792
3793 async fn force_resume_child_wait(&self, session_id: &str, resume_message: Message) {
3799 const MAX_ATTEMPTS: u8 = 5;
3800 let lock = session_resume_lock(session_id);
3801 for attempt in 0..MAX_ATTEMPTS {
3802 if attempt > 0 {
3803 tokio::time::sleep(Duration::from_millis(200)).await;
3804 }
3805 {
3806 let _held = lock.lock().await;
3807 let Some(mut session) = self.load_session(session_id).await else {
3808 return;
3809 };
3810 let mut runtime_state = read_runtime_state(&session);
3811 if runtime_state.waiting_for_children.is_none() {
3812 return;
3814 }
3815 runtime_state.waiting_for_children = None;
3816 runtime_state.status = AgentStatusState::Idle;
3817 runtime_state.suspension = None;
3818 write_runtime_state(&mut session, &runtime_state);
3819 session.metadata.remove("runtime.suspend_reason");
3820 session.add_message(resume_message.clone());
3821 session.updated_at = Utc::now();
3822 self.save_and_cache(&mut session).await;
3823 }
3824 let outcome = self.resume_parent(session_id.to_string()).await;
3825 match outcome {
3826 ResumeOutcome::Started { .. } | ResumeOutcome::NotFound => return,
3827 ResumeOutcome::Completed | ResumeOutcome::AlreadyRunning { .. } => {
3828 let clobbered = self
3832 .load_session(session_id)
3833 .await
3834 .map(|session| read_runtime_state(&session).waiting_for_children.is_some())
3835 .unwrap_or(false);
3836 if !clobbered {
3837 return;
3838 }
3839 }
3840 }
3841 }
3842 tracing::error!(
3843 %session_id,
3844 "child-wait watchdog: force-resume exhausted its clobber-retry budget"
3845 );
3846 }
3847}
3848
3849#[cfg(test)]
3850mod tests {
3851 use super::*;
3852 include!("child_completion_background_tests.rs");
3853 use bamboo_domain::session::runtime_state::SuspensionState;
3854
3855 #[test]
3856 fn skill_request_child_provider_constructor_keeps_data_absent_and_rejects_injection() {
3857 let provider = Message::user("rendered child outcome");
3858 let presentation = session_provider_message(&provider);
3859 assert_eq!(presentation.content.skill_request, None);
3860 assert_eq!(presentation.content.text, provider.content);
3861 let mut envelope = SessionMessageEnvelope::user_input("parent", "unused");
3862 envelope.source = SessionMessageSource::Runtime {
3863 subsystem: "child_completion_coordinator".into(),
3864 };
3865 envelope.kind = SessionMessageKind::ChildOutcome;
3866 envelope.body = SessionMessageBody::ChildOutcome(SessionChildOutcome {
3867 child_session_id: "child".into(),
3868 status: "completed".into(),
3869 result: Some("typed result".into()),
3870 error: None,
3871 provider_message: Some(presentation),
3872 });
3873 envelope.validate().unwrap();
3874 assert_eq!(
3875 envelope.to_provider_message().unwrap().content,
3876 "rendered child outcome"
3877 );
3878 if let SessionMessageBody::ChildOutcome(outcome) = &mut envelope.body {
3879 outcome
3880 .provider_message
3881 .as_mut()
3882 .unwrap()
3883 .content
3884 .skill_request = Some(bamboo_domain::SessionSkillRequest {
3885 mode: None,
3886 selections: vec![bamboo_domain::SessionSkillSelection {
3887 id: "review".into(),
3888 source: "builtin".into(),
3889 revision: 7,
3890 args: serde_json::Value::Null,
3891 }],
3892 });
3893 }
3894 assert_eq!(
3895 envelope.validate(),
3896 Err(bamboo_domain::SessionMessageValidationError::KindSourceBodyMismatch)
3897 );
3898 }
3899
3900 #[tokio::test]
3901 async fn cancelled_resume_waiters_do_not_retain_historical_parent_ids() {
3902 for index in 0..512 {
3903 let id = format!("resume-lock-reclaim-{index}");
3904 let owner = session_resume_lock(&id);
3905 let held = owner.lock().await;
3906 let waiter = session_resume_lock(&id);
3907 let mut waiting = Box::pin(waiter.lock());
3908 assert!(futures::poll!(waiting.as_mut()).is_pending());
3909 drop(held);
3910 drop(owner);
3911 drop(waiting);
3912 drop(waiter);
3913 assert!(!parent_locks().lock().recover_poison().contains_key(&id));
3914 }
3915 }
3916
3917 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
3918 async fn resume_lock_reclamation_preserves_exclusive_parent_wake_ownership() {
3919 use std::sync::atomic::{AtomicUsize, Ordering};
3920 let active = Arc::new(AtomicUsize::new(0));
3921 let mut tasks = Vec::new();
3922 for _ in 0..32 {
3923 let active = active.clone();
3924 tasks.push(tokio::spawn(async move {
3925 for _ in 0..16 {
3926 let lease = session_resume_lock("resume-lock-exclusive");
3927 let _guard = lease.lock().await;
3928 assert_eq!(active.fetch_add(1, Ordering::SeqCst), 0);
3929 tokio::task::yield_now().await;
3930 assert_eq!(active.fetch_sub(1, Ordering::SeqCst), 1);
3931 }
3932 }));
3933 }
3934 for task in tasks {
3935 task.await.unwrap();
3936 }
3937 assert!(!parent_locks()
3938 .lock()
3939 .recover_poison()
3940 .contains_key("resume-lock-exclusive"));
3941 }
3942 use bamboo_agent_core::Message;
3943 use bamboo_domain::SessionInboxPort;
3944 use futures::stream;
3945 use std::sync::atomic::{AtomicUsize, Ordering};
3946
3947 struct FailingRuntimeSaveStorage {
3948 inner: Arc<bamboo_storage::SessionStoreV2>,
3949 }
3950
3951 #[async_trait]
3952 impl Storage for FailingRuntimeSaveStorage {
3953 async fn save_session(&self, session: &Session) -> std::io::Result<()> {
3954 self.inner.save_session(session).await
3955 }
3956
3957 async fn load_session(&self, id: &str) -> std::io::Result<Option<Session>> {
3958 self.inner.load_session(id).await
3959 }
3960
3961 async fn delete_session(&self, id: &str) -> std::io::Result<bool> {
3962 self.inner.delete_session(id).await
3963 }
3964
3965 async fn load_runtime_control_plane(&self, id: &str) -> std::io::Result<Option<Session>> {
3966 self.inner.load_runtime_control_plane(id).await
3967 }
3968
3969 async fn save_runtime_state(&self, _session: &Session) -> std::io::Result<()> {
3970 Err(std::io::Error::other("injected sidecar save failure"))
3971 }
3972 }
3973
3974 struct EmptyTools;
3975
3976 #[async_trait]
3977 impl bamboo_agent_core::tools::ToolExecutor for EmptyTools {
3978 async fn execute(
3979 &self,
3980 _call: &bamboo_agent_core::tools::ToolCall,
3981 ) -> Result<bamboo_agent_core::tools::ToolResult, bamboo_agent_core::tools::ToolError>
3982 {
3983 Err(bamboo_agent_core::tools::ToolError::NotFound(
3984 "no tools".to_string(),
3985 ))
3986 }
3987
3988 fn list_tools(&self) -> Vec<bamboo_agent_core::tools::ToolSchema> {
3989 Vec::new()
3990 }
3991 }
3992
3993 struct CompletedTestProvider;
3994
3995 #[async_trait]
3996 impl bamboo_llm::LLMProvider for CompletedTestProvider {
3997 async fn chat_stream(
3998 &self,
3999 _messages: &[Message],
4000 _tools: &[bamboo_agent_core::tools::ToolSchema],
4001 _max_output_tokens: Option<u32>,
4002 _model: &str,
4003 ) -> Result<bamboo_llm::LLMStream, bamboo_llm::LLMError> {
4004 let chunks: Vec<bamboo_llm::provider::Result<bamboo_llm::LLMChunk>> = vec![
4005 Ok(bamboo_llm::LLMChunk::Token("done".to_string())),
4006 Ok(bamboo_llm::LLMChunk::Done),
4007 ];
4008 Ok(Box::pin(stream::iter(chunks)))
4009 }
4010 }
4011
4012 struct CountingActivationSpawner {
4013 reservations: Arc<AtomicUsize>,
4014 launches: Arc<AtomicUsize>,
4015 }
4016
4017 #[async_trait]
4018 impl SessionActivationSpawner for CountingActivationSpawner {
4019 async fn reserve_activation(
4020 &self,
4021 target_session_id: &str,
4022 inbox_generation: u64,
4023 ) -> Result<SessionActivationReserveOutcome, bamboo_domain::SessionActivationError>
4024 {
4025 self.reservations.fetch_add(1, Ordering::SeqCst);
4026 let launches = self.launches.clone();
4027 Ok(SessionActivationReserveOutcome::Reserved(
4028 SessionActivationLaunch::new(
4029 format!("{target_session_id}-{inbox_generation}"),
4030 move || {
4031 launches.fetch_add(1, Ordering::SeqCst);
4032 },
4033 ),
4034 ))
4035 }
4036 }
4037
4038 async fn save_sourced_test_child(store: &bamboo_storage::SessionStoreV2, child: &Session) {
4039 let mut saved = child.clone();
4040 ChildCompletionSource::prepare(
4041 &mut saved,
4042 &format!("test-run-{}", child.id),
4043 &Default::default(),
4044 );
4045 store.save_session(&saved).await.unwrap();
4046 }
4047
4048 async fn test_child_source(
4049 store: &bamboo_storage::SessionStoreV2,
4050 id: &str,
4051 ) -> Option<ChildCompletionSource> {
4052 ChildCompletionSource::from_committed_session(
4053 &store.load_session(id).await.unwrap().unwrap(),
4054 )
4055 }
4056
4057 fn typed_test_report() -> serde_json::Value {
4058 serde_json::json!({"version":1,"outcome":"blocked","summary":"Quoted \"claim\" \\ 用户🪷\n",
4059 "reported_evidence":[{"description":"reported only","reference":"/never/read/reported-ref","sha256":null}],
4060 "reported_verification":[{"check":"fixture","reported_status":"passed","details":"a model claim"}],
4061 "proposals":["Try the next step"],"blockers":["Need approval"],"open_decisions":["Which route?"]})
4062 }
4063
4064 fn typed_test_child(parent: &Session, id: &str, report: &str) -> Session {
4065 let mut child = Session::new_child(id, &parent.id, "model", "Child");
4066 let packet = bamboo_domain::ChildContextPacket {
4067 version: 1,
4068 objective: "bounded task".into(),
4069 constraints: vec![],
4070 acceptance: vec!["report only".into()],
4071 non_goals: vec![],
4072 necessary_user_instructions: vec![],
4073 recorded_decisions: vec![],
4074 source_user_message_ids: vec![parent.messages[0].id.clone()],
4075 background_message_ids: vec![],
4076 };
4077 let resolved = packet.resolve(parent, "bounded task").unwrap();
4078 let binding = bamboo_domain::ChildContextBinding::new(
4079 parent,
4080 id,
4081 resolved.required_brief.clone(),
4082 resolved,
4083 )
4084 .unwrap();
4085 binding.install(&mut child).unwrap();
4086 child.add_message(binding.assignment_message());
4087 child.add_message(Message::assistant(report, None));
4088 child.set_last_run_status("completed");
4089 ChildCompletionSource::prepare(&mut child, "sealed-report-run", &Default::default());
4090 child
4091 }
4092
4093 fn typed_test_completion(child: &Session) -> ChildCompletion {
4094 ChildCompletion {
4095 parent_session_id: child.parent_session_id.clone().unwrap(),
4096 child_session_id: child.id.clone(),
4097 status: child.last_run_status().unwrap(),
4098 error: child.last_run_error(),
4099 completed_at: child.updated_at,
4100 source: ChildCompletionSource::from_committed_session(child),
4101 }
4102 }
4103
4104 fn unusable_typed_test_child(
4105 parent: &Session,
4106 id: &str,
4107 case: &str,
4108 ) -> (Session, &'static str) {
4109 let mut child = typed_test_child(parent, id, &typed_test_report().to_string());
4110 let mut prior = Default::default();
4111 let reason = match case {
4112 "prose" => {
4113 child.messages.last_mut().unwrap().content = "UNUSABLE_RAW_PROSE".repeat(64);
4114 "report_malformed"
4115 }
4116 "malformed" => {
4117 child.messages.last_mut().unwrap().content =
4118 "{\"version\":1,UNUSABLE_RAW_JSON".into();
4119 "report_malformed"
4120 }
4121 "oversized" => {
4122 child
4123 .messages
4124 .last_mut()
4125 .unwrap()
4126 .content
4127 .push_str(&" ".repeat(8193));
4128 "result_budget_exceeded"
4129 }
4130 "multipart" => {
4131 child.messages.last_mut().unwrap().content_parts = Some(
4132 serde_json::from_value(
4133 serde_json::json!([{"type":"text","text":"UNUSABLE_RAW_PART"}]),
4134 )
4135 .unwrap(),
4136 );
4137 "report_not_current_final"
4138 }
4139 "commentary" => {
4140 child.messages.last_mut().unwrap().phase =
4141 Some(bamboo_domain::MessagePhase::Commentary);
4142 "report_not_current_final"
4143 }
4144 "blank" => {
4145 child.messages.last_mut().unwrap().content = " ".into();
4146 "report_not_current_final"
4147 }
4148 "old_final" => {
4149 prior = child.messages.iter().map(|m| m.id.clone()).collect();
4150 "report_not_current_final"
4151 }
4152 "error" => {
4153 prior = child.messages.iter().map(|m| m.id.clone()).collect();
4154 child.set_last_run_status("error");
4155 child.set_last_run_error("UNUSABLE_RAW_ERROR\"\\\n".repeat(4096));
4156 "report_not_current_final"
4157 }
4158 "absent" => {
4159 child.messages.pop();
4160 "report_absent"
4161 }
4162 _ => panic!("unknown unusable report case"),
4163 };
4164 ChildCompletionSource::prepare(&mut child, "unusable-report-run", &prior);
4165 (child, reason)
4166 }
4167
4168 #[test]
4169 fn sealed_unusable_reports_keep_current_source_boundary_and_escape_budget() {
4170 let mut parent = Session::new("unavailable-parent", "model");
4171 parent.add_message(Message::user("required source"));
4172 for case in [
4173 "prose",
4174 "malformed",
4175 "oversized",
4176 "multipart",
4177 "commentary",
4178 "blank",
4179 "old_final",
4180 "error",
4181 "absent",
4182 ] {
4183 let (child, reason) = unusable_typed_test_child(&parent, "unavailable-child", case);
4184 let completion = typed_test_completion(&child);
4185 assert!(completion.source.is_some(), "valid seal: {case}");
4186 let mut projection = sealed_child_report(&completion, &child, &parent).unwrap();
4187 assert_eq!(
4188 projection,
4189 unavailable_child_result("typed_result", reason),
4190 "{case}"
4191 );
4192 let mut message = typed_child_resume_message(&completion, 0, &projection);
4193 let raw = completion.source.as_ref().unwrap().result(&child, &parent);
4194 let mut envelope =
4195 child_completion_envelope(&completion, Utc::now(), raw.clone(), &message);
4196 assert!(bound_typed_child_delivery(
4197 &mut envelope,
4198 &mut message,
4199 &mut projection,
4200 &completion,
4201 0
4202 ));
4203 assert!(fits_child_result_budget(&envelope));
4204 assert!(fits_child_result_budget(&message));
4205 assert!(fits_child_result_budget(
4206 &envelope.to_provider_message().unwrap()
4207 ));
4208 assert!(message.content.contains("SubAgent"));
4209 assert_eq!(
4210 message.metadata.as_ref().unwrap()["child_final_response_included"],
4211 false
4212 );
4213 assert!(projection.get("child_report").is_none());
4214 assert!(projection.get("host_observation").is_none());
4215 let SessionMessageBody::ChildOutcome(outcome) = envelope.body else {
4216 panic!("child outcome")
4217 };
4218 assert!(outcome.result.is_none());
4219 assert!(outcome.error.is_none());
4220 if let Some(raw) = raw {
4221 assert!(!message.content.contains(&raw), "no raw report: {case}");
4222 }
4223 let mut stale = child.clone();
4226 stale
4227 .messages
4228 .last_mut()
4229 .unwrap()
4230 .content
4231 .push_str("changed");
4232 assert!(sealed_child_report(&completion, &stale, &parent).is_none());
4233 }
4234 }
4235
4236 #[test]
4237 fn unavailable_delivery_counts_escaped_wrappers_at_exact_limit() {
4238 let mut parent = Session::new("unavailable-parent", "model");
4239 parent.add_message(Message::user("required source"));
4240 let (child, _) = unusable_typed_test_child(&parent, "unavailable-child", "oversized");
4241 let completion = typed_test_completion(&child);
4242 let mut projection = sealed_child_report(&completion, &child, &parent).unwrap();
4243 let mut message = typed_child_resume_message(&completion, 0, &projection);
4244 let mut base = child_completion_envelope(&completion, Utc::now(), None, &message);
4245 assert!(bound_typed_child_delivery(
4246 &mut base,
4247 &mut message,
4248 &mut projection,
4249 &completion,
4250 0
4251 ));
4252 let bytes = serde_json::to_vec(&base).unwrap().len().max(
4253 serde_json::to_vec(&base.to_provider_message().unwrap())
4254 .unwrap()
4255 .len(),
4256 );
4257 for extra in [8192 - bytes, 8193 - bytes] {
4258 let mut envelope = base.clone();
4259 let padding = format!("{}{}", "\"".repeat(extra / 2), "x".repeat(extra % 2));
4261 envelope.correlation_id.as_mut().unwrap().push_str(&padding);
4262 let mut projection = projection.clone();
4263 let mut message = message.clone();
4264 assert_eq!(
4265 bound_typed_child_delivery(
4266 &mut envelope,
4267 &mut message,
4268 &mut projection,
4269 &completion,
4270 0
4271 ),
4272 bytes + extra == 8192
4273 );
4274 assert_eq!(
4275 serde_json::to_vec(&envelope).unwrap().len().max(
4276 serde_json::to_vec(&envelope.to_provider_message().unwrap())
4277 .unwrap()
4278 .len()
4279 ),
4280 bytes + extra
4281 );
4282 assert!(fits_child_result_budget(&message));
4283 }
4284 }
4285
4286 #[test]
4287 fn sealed_typed_projection_keeps_host_source_distinct_from_all_reported_claims() {
4288 let mut parent = Session::new("typed-parent", "model");
4289 parent.add_message(Message::user("required user constraint"));
4290 let report = typed_test_report();
4291 let child = typed_test_child(&parent, "typed-child", &report.to_string());
4292 let completion = typed_test_completion(&child);
4293 let projection = sealed_child_report(&completion, &child, &parent).unwrap();
4294 assert_eq!(projection["child_report"], report);
4295 assert_eq!(
4296 projection["host_observation"]["terminal_source"],
4297 serde_json::to_value(&completion.source).unwrap()
4298 );
4299 assert_eq!(
4300 projection["host_observation"]["kind"],
4301 "committed_terminal_source"
4302 );
4303 assert_eq!(
4304 projection["host_observation"]["last_run_status"],
4305 "completed"
4306 );
4307 assert_eq!(projection["child_report"]["outcome"], "blocked");
4308 assert_eq!(
4309 projection["child_report"]["reported_verification"][0]["reported_status"],
4310 "passed"
4311 );
4312 assert!(projection["host_observation"].get("verified").is_none());
4313 for excluded in ["guardian", "resident", "legacy", "missing_source"] {
4314 let mut excluded_child = child.clone();
4315 match excluded {
4316 "guardian" => {
4317 excluded_child
4318 .metadata
4319 .insert("subagent_type".into(), "guardian".into());
4320 }
4321 "resident" => {
4322 excluded_child
4323 .metadata
4324 .insert("lifecycle".into(), "resident".into());
4325 }
4326 "legacy" => {
4327 excluded_child =
4328 Session::new_child("typed-child", &parent.id, "model", "Child");
4329 excluded_child.add_message(Message::assistant(report.to_string(), None));
4330 excluded_child.set_last_run_status("completed");
4331 }
4332 _ => {}
4333 }
4334 ChildCompletionSource::prepare(
4335 &mut excluded_child,
4336 "excluded-run",
4337 &Default::default(),
4338 );
4339 let mut excluded_completion = typed_test_completion(&excluded_child);
4340 if excluded == "missing_source" {
4341 excluded_completion.source = None;
4342 }
4343 assert!(
4344 sealed_child_report(&excluded_completion, &excluded_child, &parent).is_none(),
4345 "{excluded}"
4346 );
4347 }
4348 }
4349
4350 #[tokio::test]
4351 async fn typed_delivery_counts_complete_escaped_envelope_and_final_correlation() {
4352 let mut parent = Session::new("typed-parent", "model");
4353 parent.add_message(Message::user("required source"));
4354 let child = typed_test_child(&parent, "typed-child", &typed_test_report().to_string());
4355 let completion = typed_test_completion(&child);
4356 let original = sealed_child_report(&completion, &child, &parent).unwrap();
4357 let presentation = typed_child_resume_message(&completion, 0, &original);
4358 let mut base = child_completion_envelope(
4359 &completion,
4360 Utc::now(),
4361 completion.source.as_ref().unwrap().result(&child, &parent),
4362 &presentation,
4363 );
4364 let mut projection = original.clone();
4365 let mut message = presentation.clone();
4366 assert!(bound_typed_child_delivery(
4367 &mut base,
4368 &mut message,
4369 &mut projection,
4370 &completion,
4371 0
4372 ));
4373 assert_eq!(projection, original);
4374 let temp = tempfile::tempdir().unwrap();
4375 let store = Arc::new(
4376 bamboo_storage::SessionStoreV2::new(temp.path().to_path_buf())
4377 .await
4378 .unwrap(),
4379 );
4380 store.save_session(&parent).await.unwrap();
4381 let inbox = bamboo_storage::FileSessionInbox::new(store, Default::default());
4382 let first = inbox.deliver(&base).await.unwrap();
4383 let stable_id = base.id.clone();
4384 let base_bytes = serde_json::to_vec(&base.to_provider_message().unwrap())
4385 .unwrap()
4386 .len();
4387 assert!(base_bytes < 8192);
4388 let correlation = base.correlation_id.as_ref().unwrap().clone();
4389 for (extra, available) in [(8192 - base_bytes, true), (8193 - base_bytes, false)] {
4390 let mut envelope = base.clone();
4391 envelope.correlation_id = Some(format!("{correlation}{}", "x".repeat(extra)));
4392 assert_eq!(
4393 serde_json::to_vec(&envelope.to_provider_message().unwrap())
4394 .unwrap()
4395 .len(),
4396 base_bytes + extra
4397 );
4398 let mut message = presentation.clone();
4399 let mut projection = original.clone();
4400 assert!(bound_typed_child_delivery(
4401 &mut envelope,
4402 &mut message,
4403 &mut projection,
4404 &completion,
4405 0
4406 ));
4407 assert_eq!(projection["available"], available);
4408 assert!(fits_child_result_budget(&envelope));
4409 assert!(fits_child_result_budget(&message));
4410 assert!(fits_child_result_budget(
4411 &envelope.to_provider_message().unwrap()
4412 ));
4413 assert_eq!(
4414 envelope.idempotency_semantics(),
4415 base.idempotency_semantics()
4416 );
4417 assert_eq!(envelope.id, stable_id);
4418 assert_eq!(
4419 inbox.deliver(&envelope).await.unwrap(),
4420 first,
4421 "presentation overflow cannot change delivery semantics"
4422 );
4423 if !available {
4424 assert_eq!(projection["reason"], "result_budget_exceeded");
4425 assert!(projection.get("child_report").is_none());
4426 assert!(message.content.contains("SubAgent"));
4427 assert!(!message.content.contains("Bounded tail"));
4428 }
4429 }
4430 let mut large = typed_test_report();
4431 large["summary"] = serde_json::json!("\"\\\n".repeat(500));
4432 large["proposals"] = serde_json::json!(["\"\\\n".repeat(250)]);
4433 let child = typed_test_child(&parent, "escaped-child", &large.to_string());
4434 assert!(decode_report(&large.to_string()).is_ok());
4435 let completion = typed_test_completion(&child);
4436 let mut projection = sealed_child_report(&completion, &child, &parent).unwrap();
4437 let mut message = typed_child_resume_message(&completion, 0, &projection);
4438 let mut envelope =
4439 child_completion_envelope(&completion, Utc::now(), Some(large.to_string()), &message);
4440 assert!(serde_json::to_vec(&envelope).unwrap().len() > 8192);
4441 assert!(bound_typed_child_delivery(
4442 &mut envelope,
4443 &mut message,
4444 &mut projection,
4445 &completion,
4446 0
4447 ));
4448 assert_eq!(projection["available"], false);
4449 assert!(fits_child_result_budget(&envelope));
4450 assert!(fits_child_result_budget(&message));
4451 assert!(fits_child_result_budget(
4452 &envelope.to_provider_message().unwrap()
4453 ));
4454 }
4455
4456 #[tokio::test]
4457 async fn typed_source_mutations_reject_before_admission_and_wait_clear() {
4458 for mutation in ["birth", "assignment", "input", "result", "parent_source"] {
4459 let (_temp, store, inbox, coordinator, _, launches) = completion_inbox_fixture().await;
4460 save_waiting_test_parent(
4461 &store,
4462 "typed-parent",
4463 vec!["typed-child".into()],
4464 ChildWaitPolicy::All,
4465 )
4466 .await;
4467 let mut parent = store.load_session("typed-parent").await.unwrap().unwrap();
4468 parent.add_message(Message::user("required source"));
4469 store.save_session(&parent).await.unwrap();
4470 let mut child =
4471 typed_test_child(&parent, "typed-child", &typed_test_report().to_string());
4472 store.save_session(&child).await.unwrap();
4473 let mut completion = typed_test_completion(&child);
4474 match mutation {
4475 "birth" => {
4476 completion.source.as_mut().unwrap().child_created_at +=
4477 chrono::Duration::seconds(1)
4478 }
4479 "assignment" => {
4480 child.messages[0].content.push_str("changed assignment");
4481 }
4482 "input" => child.add_message(Message::user("new user input")),
4483 "result" => child
4484 .messages
4485 .last_mut()
4486 .unwrap()
4487 .content
4488 .push_str("changed"),
4489 _ => {
4490 parent.messages[0].content.push_str("changed");
4491 store.save_session(&parent).await.unwrap();
4492 }
4493 }
4494 store.save_session(&child).await.unwrap();
4495 coordinator.on_child_completed(completion).await;
4496 assert_eq!(
4497 inbox.inspect(&parent.id).await.unwrap().pending,
4498 0,
4499 "{mutation}"
4500 );
4501 assert_eq!(launches.load(Ordering::SeqCst), 0, "{mutation}");
4502 assert!(
4503 read_runtime_state(&store.load_session(&parent.id).await.unwrap().unwrap())
4504 .waiting_for_children
4505 .is_some(),
4506 "{mutation}"
4507 );
4508 }
4509 }
4510
4511 #[tokio::test]
4512 async fn typed_cold_watchdog_and_terminal_siblings_admit_each_source_once() {
4513 for case in [
4514 "valid",
4515 "prose",
4516 "malformed",
4517 "oversized",
4518 "multipart",
4519 "commentary",
4520 "blank",
4521 "old_final",
4522 "error",
4523 "absent",
4524 ] {
4525 for cold_watchdog in [false, true] {
4526 let (_temp, store, inbox, coordinator, reservations, launches) =
4527 completion_inbox_fixture().await;
4528 let ids = vec!["typed-a".into(), "typed-b".into()];
4529 save_waiting_test_parent(&store, "typed-parent", ids.clone(), ChildWaitPolicy::All)
4530 .await;
4531 let mut parent = store.load_session("typed-parent").await.unwrap().unwrap();
4532 parent.add_message(Message::user("required source"));
4533 store.save_session(&parent).await.unwrap();
4534 let mut completions = Vec::new();
4535 for id in &ids {
4536 let child = if case == "valid" {
4537 typed_test_child(&parent, id, &typed_test_report().to_string())
4538 } else {
4539 unusable_typed_test_child(&parent, id, case).0
4540 };
4541 store.save_session(&child).await.unwrap();
4542 completions.push(typed_test_completion(&child));
4543 }
4544 if cold_watchdog {
4545 let mut wait = read_runtime_state(&parent).waiting_for_children.unwrap();
4547 wait.registered_at = Utc::now() - chrono::Duration::seconds(61);
4548 coordinator.sweep_child_wait(&parent.id, wait.clone()).await;
4549 coordinator.sweep_child_wait(&parent.id, wait).await;
4550 } else {
4551 coordinator.on_child_completed(completions[0].clone()).await;
4553 }
4554 for completion in completions {
4555 coordinator.on_child_completed(completion).await;
4556 }
4557 let claims = inbox.claim(&parent.id, 8).await.unwrap();
4558 assert_eq!(claims.len(), 2);
4559 assert_eq!(reservations.load(Ordering::SeqCst), 1);
4560 assert_eq!(launches.load(Ordering::SeqCst), 1);
4561 for claim in claims {
4562 assert!(fits_child_result_budget(&claim.envelope));
4563 assert!(fits_child_result_budget(
4564 &claim.envelope.to_provider_message().unwrap()
4565 ));
4566 let actual = claim.envelope.to_provider_message().unwrap();
4567 assert!(!actual.content.contains("UNUSABLE_RAW"));
4568 let SessionMessageBody::ChildOutcome(outcome) = claim.envelope.body else {
4569 panic!("child outcome");
4570 };
4571 let projection: serde_json::Value = serde_json::from_str(
4572 outcome
4573 .provider_message
4574 .as_ref()
4575 .unwrap()
4576 .content
4577 .text
4578 .split_once("Child typed result:\n")
4579 .unwrap()
4580 .1
4581 .lines()
4582 .next()
4583 .unwrap(),
4584 )
4585 .unwrap();
4586 if case != "valid" {
4587 let expected =
4588 unusable_typed_test_child(&parent, &outcome.child_session_id, case).1;
4589 assert_eq!(
4590 projection,
4591 unavailable_child_result("typed_result", expected)
4592 );
4593 assert!(outcome.result.is_none());
4594 assert!(outcome.error.is_none());
4595 assert!(fits_child_result_budget(&outcome.provider_message));
4596 continue;
4597 }
4598 assert_eq!(projection["child_report"], typed_test_report());
4599 assert_eq!(
4600 projection["host_observation"]["terminal_source"]["child_session_id"],
4601 outcome.child_session_id
4602 );
4603 assert_eq!(
4604 projection["host_observation"]["last_run_status"],
4605 "completed"
4606 );
4607 assert!(fits_child_result_budget(&outcome.provider_message));
4608 }
4609 }
4610 }
4611 }
4612
4613 async fn completion_inbox_fixture() -> (
4614 tempfile::TempDir,
4615 Arc<bamboo_storage::SessionStoreV2>,
4616 Arc<dyn SessionInboxPort>,
4617 Arc<ChildCompletionCoordinator>,
4618 Arc<AtomicUsize>,
4619 Arc<AtomicUsize>,
4620 ) {
4621 completion_inbox_fixture_with_limits(bamboo_domain::SessionInboxLimits::default()).await
4622 }
4623
4624 async fn completion_inbox_fixture_with_limits(
4625 limits: bamboo_domain::SessionInboxLimits,
4626 ) -> (
4627 tempfile::TempDir,
4628 Arc<bamboo_storage::SessionStoreV2>,
4629 Arc<dyn SessionInboxPort>,
4630 Arc<ChildCompletionCoordinator>,
4631 Arc<AtomicUsize>,
4632 Arc<AtomicUsize>,
4633 ) {
4634 let temp = tempfile::tempdir().unwrap();
4635 let store = Arc::new(
4636 bamboo_storage::SessionStoreV2::new(temp.path().to_path_buf())
4637 .await
4638 .unwrap(),
4639 );
4640 let storage: Arc<dyn Storage> = store.clone();
4641 let locked = Arc::new(LockedSessionStore::new(storage.clone()));
4642 let inbox: Arc<dyn SessionInboxPort> =
4643 Arc::new(bamboo_storage::FileSessionInbox::new(store.clone(), limits));
4644 let router = crate::SessionActivationRouter::new();
4645 let messenger = Arc::new(crate::SessionMessenger::new(
4646 storage.clone(),
4647 inbox.clone(),
4648 router.clone(),
4649 ));
4650 let reservations = Arc::new(AtomicUsize::new(0));
4651 let launches = Arc::new(AtomicUsize::new(0));
4652 router
4653 .set_spawner(Arc::new(CountingActivationSpawner {
4654 reservations: reservations.clone(),
4655 launches: launches.clone(),
4656 }))
4657 .await;
4658 let provider: Arc<dyn bamboo_llm::LLMProvider> = Arc::new(CompletedTestProvider);
4659 let config = Arc::new(RwLock::new(Config::default()));
4660 let metrics = bamboo_metrics::MetricsCollector::spawn(
4661 Arc::new(bamboo_metrics::SqliteMetricsStorage::new(
4662 temp.path().join("metrics.db"),
4663 )),
4664 7,
4665 );
4666 let tools: Arc<dyn ToolExecutor> = Arc::new(EmptyTools);
4667 let agent = Arc::new(
4668 Agent::builder()
4669 .storage(storage.clone())
4670 .persistence(locked.clone())
4671 .session_inbox(inbox.clone())
4672 .activation_router(router)
4673 .session_messenger(messenger)
4674 .attachment_reader(store.clone())
4675 .skill_manager(Arc::new(bamboo_skills::SkillManager::new()))
4676 .metrics_collector(metrics)
4677 .config(config.clone())
4678 .provider(provider.clone())
4679 .default_tools(tools)
4680 .build()
4681 .unwrap(),
4682 );
4683 let mut providers = HashMap::new();
4684 providers.insert("test".to_string(), provider);
4685 let registry = Arc::new(ProviderRegistry::new(providers, "test".to_string()));
4686 let provider_router = Arc::new(ProviderModelRouter::new(registry.clone()));
4687 let coordinator = Arc::new(ChildCompletionCoordinator::new(
4688 storage,
4689 locked,
4690 Arc::default(),
4691 Arc::new(RwLock::new(HashMap::new())),
4692 Arc::new(RwLock::new(HashMap::new())),
4693 agent,
4694 config,
4695 registry,
4696 provider_router,
4697 temp.path().to_path_buf(),
4698 None,
4699 ));
4700 (temp, store, inbox, coordinator, reservations, launches)
4701 }
4702
4703 async fn save_waiting_test_parent(
4704 store: &bamboo_storage::SessionStoreV2,
4705 parent_id: &str,
4706 ids: Vec<String>,
4707 policy: ChildWaitPolicy,
4708 ) {
4709 let mut parent = Session::new(parent_id, "model");
4710 let mut runtime = AgentRuntimeState::new("waiting-run");
4711 runtime.status = AgentStatusState::Suspended;
4712 runtime.waiting_for_children = Some(WaitingForChildrenState::for_children(
4713 ids,
4714 policy,
4715 Utc::now(),
4716 ));
4717 runtime.suspension = Some(SuspensionState {
4718 reason: "waiting_for_children".into(),
4719 suspended_at: Utc::now(),
4720 resumable: true,
4721 hook_point: Some("ChildCompletion".into()),
4722 });
4723 write_runtime_state(&mut parent, &runtime);
4724 parent.metadata.insert(
4725 "runtime.suspend_reason".into(),
4726 "waiting_for_children".into(),
4727 );
4728 store.save_session(&parent).await.unwrap();
4729 }
4730
4731 async fn save_completion_test_child(
4732 store: &bamboo_storage::SessionStoreV2,
4733 parent_id: &str,
4734 child_id: &str,
4735 status: &str,
4736 ) -> ChildCompletion {
4737 let mut child = store
4738 .load_session(child_id)
4739 .await
4740 .unwrap()
4741 .unwrap_or_else(|| Session::new_child(child_id, parent_id, "model", "Child"));
4742 child.add_message(Message::assistant(format!("result-{child_id}"), None));
4743 child.set_last_run_status(status);
4744 save_sourced_test_child(store, &child).await;
4745 ChildCompletion {
4746 parent_session_id: parent_id.into(),
4747 child_session_id: child_id.into(),
4748 status: status.into(),
4749 error: None,
4750 completed_at: child.updated_at,
4751 source: test_child_source(store, child_id).await,
4752 }
4753 }
4754
4755 #[tokio::test]
4756 async fn unrelated_invalid_pending_body_does_not_block_incremental_child_activation() {
4757 for queue in ["new", "cur"] {
4758 let (_temp, store, inbox, coordinator, reservations, launches) =
4759 completion_inbox_fixture().await;
4760 let parent_id = "invalid-pending-parent";
4761 save_waiting_test_parent(
4762 &store,
4763 parent_id,
4764 vec!["invalid-pending-fast".into(), "invalid-pending-slow".into()],
4765 ChildWaitPolicy::All,
4766 )
4767 .await;
4768 let before = read_runtime_state(&store.load_session(parent_id).await.unwrap().unwrap());
4769 let fast =
4770 save_completion_test_child(&store, parent_id, "invalid-pending-fast", "completed")
4771 .await;
4772 save_completion_test_child(&store, parent_id, "invalid-pending-slow", "running").await;
4773
4774 let mut unrelated = SessionMessageEnvelope::user_input(parent_id, "unrelated bash");
4775 unrelated.source = SessionMessageSource::Runtime {
4776 subsystem: "bash".into(),
4777 };
4778 unrelated.kind = SessionMessageKind::RuntimeInstruction;
4779 unrelated.body =
4780 SessionMessageBody::RuntimeInstruction(bamboo_domain::SessionRuntimeInstruction {
4781 instruction: "bash_completion".into(),
4782 content: Some(bamboo_domain::SessionMessageContent::text("unrelated bash")),
4783 data: None,
4784 provider_message: None,
4785 });
4786 inbox.deliver(&unrelated).await.unwrap();
4787 let dir = store
4788 .bamboo_home_dir()
4789 .join(store.resolve_rel_path(parent_id).await.unwrap())
4790 .join("inbox");
4791 let staged = tokio::fs::read_dir(dir.join("new"))
4792 .await
4793 .unwrap()
4794 .next_entry()
4795 .await
4796 .unwrap()
4797 .unwrap()
4798 .path();
4799 let mut transport: serde_json::Value =
4800 serde_json::from_slice(&tokio::fs::read(&staged).await.unwrap()).unwrap();
4801 transport["body"] = serde_json::json!({"not": "an envelope"});
4802 let invalid_bytes = serde_json::to_vec(&transport).unwrap();
4803 tokio::fs::write(&staged, &invalid_bytes).await.unwrap();
4804 let invalid_path = if queue == "cur" {
4805 let destination = dir.join("cur").join(staged.file_name().unwrap());
4806 tokio::fs::create_dir_all(dir.join("cur")).await.unwrap();
4807 tokio::fs::rename(&staged, &destination).await.unwrap();
4808 destination
4809 } else {
4810 staged
4811 };
4812
4813 coordinator.on_child_completed(fast.clone()).await;
4814 let outcomes = inbox
4815 .pending_child_outcomes(parent_id, "invalid-pending-fast")
4816 .await
4817 .unwrap();
4818 assert_eq!(outcomes.len(), 1);
4819 assert!(fits_child_result_budget(&outcomes[0]));
4820 let after = read_runtime_state(&store.load_session(parent_id).await.unwrap().unwrap());
4821 assert_eq!(after, before, "All wait and suspension remain armed");
4822 assert_eq!(reservations.load(Ordering::SeqCst), 1);
4823 assert_eq!(launches.load(Ordering::SeqCst), 1);
4824 let backlog = inbox.inspect(parent_id).await.unwrap();
4825 assert!(backlog.interrupt_pending());
4826 assert_eq!(backlog.coordinator_generation, 0);
4827 let ready = inbox
4828 .inspect_wake_readiness(parent_id, Utc::now())
4829 .await
4830 .unwrap()
4831 .ready
4832 .unwrap();
4833 assert_eq!(ready.id, outcomes[0].id);
4834 assert_eq!(
4835 ready.activation_policy,
4836 SessionActivationPolicy::InterruptSpecificWait
4837 );
4838 assert_eq!(tokio::fs::read(&invalid_path).await.unwrap(), invalid_bytes);
4839 assert!(!inbox.was_admitted(parent_id, &unrelated.id).await.unwrap());
4840
4841 let claims = inbox.claim(parent_id, 8).await.unwrap();
4842 assert_eq!(claims.len(), 1);
4843 assert_eq!(claims[0].envelope.id, outcomes[0].id);
4844 inbox.ack(parent_id, &claims[0]).await.unwrap();
4845 coordinator.on_child_completed(fast).await;
4846 assert_eq!(launches.load(Ordering::SeqCst), 1);
4847 assert!(!inbox.was_admitted(parent_id, &unrelated.id).await.unwrap());
4848 }
4849 }
4850
4851 #[tokio::test]
4852 async fn partial_all_outcome_preserves_wait_and_active_parent_control_state() {
4853 let (_temp, store, inbox, coordinator, _, launches) = completion_inbox_fixture().await;
4854 let parent_id = "incremental-parent";
4855 save_waiting_test_parent(
4856 &store,
4857 parent_id,
4858 vec!["ready-child".into(), "slow-child".into()],
4859 ChildWaitPolicy::All,
4860 )
4861 .await;
4862 let mut parent = store.load_session(parent_id).await.unwrap().unwrap();
4863 let mut runtime = read_runtime_state(&parent);
4864 runtime.status = AgentStatusState::Running;
4865 runtime.suspension = None;
4866 parent.metadata.remove("runtime.suspend_reason");
4867 write_runtime_state(&mut parent, &runtime);
4868 store.save_session(&parent).await.unwrap();
4869 let ready = save_completion_test_child(&store, parent_id, "ready-child", "completed").await;
4870 save_completion_test_child(&store, parent_id, "slow-child", "running").await;
4871
4872 coordinator.on_child_completed(ready.clone()).await;
4873 let after = store.load_session(parent_id).await.unwrap().unwrap();
4874 assert_eq!(read_runtime_state(&after), runtime);
4875 assert_eq!(after.updated_at, parent.updated_at);
4876 assert_eq!(launches.load(Ordering::SeqCst), 1);
4877 let backlog = inbox.inspect(parent_id).await.unwrap();
4878 assert!(backlog.interrupt_pending());
4879 assert_eq!(backlog.coordinator_generation, 0, "no blanket prefix grant");
4880 let claims = inbox.claim(parent_id, 8).await.unwrap();
4881 assert_eq!(claims.len(), 1);
4882 inbox.ack(parent_id, &claims[0]).await.unwrap();
4883
4884 coordinator.on_child_completed(ready).await;
4885 assert_eq!(inbox.inspect(parent_id).await.unwrap().pending, 0);
4886 let slow = save_completion_test_child(&store, parent_id, "slow-child", "completed").await;
4887 coordinator.on_child_completed(slow).await;
4888 assert!(
4889 read_runtime_state(&store.load_session(parent_id).await.unwrap().unwrap())
4890 .waiting_for_children
4891 .is_none()
4892 );
4893 let claims = inbox.claim(parent_id, 8).await.unwrap();
4894 assert_eq!(
4895 claims.len(),
4896 1,
4897 "the acknowledged first result is not repeated at wait release"
4898 );
4899 let SessionMessageBody::ChildOutcome(outcome) = &claims[0].envelope.body else {
4900 panic!("child outcome");
4901 };
4902 assert_eq!(outcome.child_session_id, "slow-child");
4903 }
4904
4905 #[tokio::test]
4906 async fn incremental_completions_during_owner_exit_share_one_successor() {
4907 let (_temp, store, inbox, coordinator, reservations, launches) =
4908 completion_inbox_fixture().await;
4909 let parent_id = "handoff-parent";
4910 save_waiting_test_parent(
4911 &store,
4912 parent_id,
4913 vec![
4914 "handoff-a".into(),
4915 "handoff-b".into(),
4916 "handoff-slow".into(),
4917 ],
4918 ChildWaitPolicy::All,
4919 )
4920 .await;
4921 let a = save_completion_test_child(&store, parent_id, "handoff-a", "completed").await;
4922 let b = save_completion_test_child(&store, parent_id, "handoff-b", "completed").await;
4923 save_completion_test_child(&store, parent_id, "handoff-slow", "running").await;
4924 let router = coordinator.agent.activation_router().unwrap();
4925 router.set_inbox(inbox.clone());
4926 let mut owner = router
4927 .register_run(parent_id, "old-parent-run")
4928 .await
4929 .unwrap();
4930
4931 coordinator.on_child_completed(a.clone()).await;
4932 assert!(inbox
4933 .claim_for_turn(parent_id, 8, Some("old-parent-run"))
4934 .await
4935 .unwrap()
4936 .is_empty());
4937 owner.begin_finalization().await;
4938 coordinator.on_child_completed(b.clone()).await;
4939 assert_eq!(reservations.load(Ordering::SeqCst), 0);
4940 assert_eq!(launches.load(Ordering::SeqCst), 0);
4941 owner.finish(0).await.unwrap();
4942 assert_eq!(reservations.load(Ordering::SeqCst), 1);
4943 assert_eq!(launches.load(Ordering::SeqCst), 1);
4944 let successor = router.current_run_id(parent_id).await.unwrap();
4945 let claims = inbox
4946 .claim_for_turn(parent_id, 8, Some(&successor))
4947 .await
4948 .unwrap();
4949 assert_eq!(claims.len(), 2);
4950 for claim in &claims {
4951 inbox.ack(parent_id, claim).await.unwrap();
4952 }
4953 tokio::join!(
4954 coordinator.on_child_completed(a),
4955 coordinator.on_child_completed(b)
4956 );
4957 assert_eq!(inbox.inspect(parent_id).await.unwrap().pending, 0);
4958 assert_eq!(reservations.load(Ordering::SeqCst), 1);
4959 assert!(
4960 read_runtime_state(&store.load_session(parent_id).await.unwrap().unwrap())
4961 .waiting_for_children
4962 .is_some()
4963 );
4964 }
4965
4966 #[tokio::test]
4967 async fn cold_watchdog_replays_each_ready_child_before_all_wait_is_satisfied() {
4968 let (_temp, store, inbox, coordinator, _, launches) = completion_inbox_fixture().await;
4969 let parent_id = "cold-incremental-parent";
4970 save_waiting_test_parent(
4971 &store,
4972 parent_id,
4973 vec!["cold-a".into(), "cold-b".into(), "cold-slow".into()],
4974 ChildWaitPolicy::All,
4975 )
4976 .await;
4977 let mut parent = store.load_session(parent_id).await.unwrap().unwrap();
4978 let mut runtime = read_runtime_state(&parent);
4979 runtime.waiting_for_children.as_mut().unwrap().registered_at -=
4980 chrono::Duration::seconds(61);
4981 write_runtime_state(&mut parent, &runtime);
4982 store.save_session(&parent).await.unwrap();
4983 save_completion_test_child(&store, parent_id, "cold-a", "completed").await;
4984 save_completion_test_child(&store, parent_id, "cold-b", "completed").await;
4985 save_completion_test_child(&store, parent_id, "cold-slow", "running").await;
4986
4987 let wait = runtime.waiting_for_children.unwrap();
4988 coordinator.sweep_child_wait(parent_id, wait.clone()).await;
4989 let reopened = bamboo_storage::FileSessionInbox::new(store.clone(), Default::default());
4990 let backlog = reopened.inspect(parent_id).await.unwrap();
4991 assert_eq!(backlog.pending, 2);
4992 assert!(backlog.interrupt_pending());
4993 assert_eq!(backlog.coordinator_generation, 0);
4994 assert_eq!(launches.load(Ordering::SeqCst), 1);
4995 let claims = reopened.claim(parent_id, 8).await.unwrap();
4996 assert_eq!(claims.len(), 2);
4997 for claim in &claims {
4998 reopened.ack(parent_id, claim).await.unwrap();
4999 }
5000 coordinator.sweep_child_wait(parent_id, wait).await;
5001 assert_eq!(inbox.inspect(parent_id).await.unwrap().pending, 0);
5002 assert!(
5003 read_runtime_state(&store.load_session(parent_id).await.unwrap().unwrap())
5004 .waiting_for_children
5005 .is_some()
5006 );
5007 }
5008
5009 #[tokio::test]
5010 async fn any_wait_delivers_a_late_direct_child_without_repeating_the_first_outcome() {
5011 let (_temp, store, inbox, coordinator, _, _) = completion_inbox_fixture().await;
5012 let parent_id = "any-incremental-parent";
5013 save_waiting_test_parent(
5014 &store,
5015 parent_id,
5016 vec!["any-first".into(), "any-late".into()],
5017 ChildWaitPolicy::Any,
5018 )
5019 .await;
5020 let first = save_completion_test_child(&store, parent_id, "any-first", "completed").await;
5021 save_completion_test_child(&store, parent_id, "any-late", "running").await;
5022 coordinator.on_child_completed(first.clone()).await;
5023 assert!(
5024 read_runtime_state(&store.load_session(parent_id).await.unwrap().unwrap())
5025 .waiting_for_children
5026 .is_none()
5027 );
5028 let first_claims = inbox.claim(parent_id, 8).await.unwrap();
5029 assert_eq!(first_claims.len(), 1);
5030 inbox.ack(parent_id, &first_claims[0]).await.unwrap();
5031
5032 let late = save_completion_test_child(&store, parent_id, "any-late", "completed").await;
5033 coordinator.on_child_completed(late.clone()).await;
5034 coordinator.on_child_completed(first).await;
5035 coordinator.on_child_completed(late).await;
5036 let claims = inbox.claim(parent_id, 8).await.unwrap();
5037 assert_eq!(claims.len(), 1);
5038 let SessionMessageBody::ChildOutcome(outcome) = &claims[0].envelope.body else {
5039 panic!("child outcome");
5040 };
5041 assert_eq!(outcome.child_session_id, "any-late");
5042 assert!(
5043 read_runtime_state(&store.load_session(parent_id).await.unwrap().unwrap())
5044 .waiting_for_children
5045 .is_none()
5046 );
5047 }
5048
5049 #[tokio::test]
5050 async fn terminal_batch_upgrades_lower_pending_outcome_below_an_acknowledged_sibling() {
5051 let (_temp, store, inbox, coordinator, _, _) = completion_inbox_fixture().await;
5052 let parent_id = "terminal-batch-parent";
5053 save_waiting_test_parent(
5054 &store,
5055 parent_id,
5056 vec!["batch-low".into(), "batch-high".into()],
5057 ChildWaitPolicy::All,
5058 )
5059 .await;
5060 let mut parent = store.load_session(parent_id).await.unwrap().unwrap();
5061 let wait = read_runtime_state(&parent).waiting_for_children.unwrap();
5062 let low = save_completion_test_child(&store, parent_id, "batch-low", "completed").await;
5063 let high = save_completion_test_child(&store, parent_id, "batch-high", "completed").await;
5064 let mut receipts = Vec::new();
5065 for completion in [&low, &high] {
5066 let result = format!("result-{}", completion.child_session_id);
5067 let presentation = runtime_resume_message(completion, 0, Some(&result));
5068 let envelope = child_completion_envelope(
5069 completion,
5070 wait.registered_at,
5071 Some(result),
5072 &presentation,
5073 );
5074 receipts.push(inbox.deliver(&envelope).await.unwrap());
5075 }
5076 inbox
5077 .mark_message_activation_eligible(
5078 parent_id,
5079 &receipts[1],
5080 SessionActivationPolicy::InterruptSpecificWait,
5081 )
5082 .await
5083 .unwrap();
5084 let claims = inbox.claim(parent_id, 8).await.unwrap();
5085 assert_eq!(claims.len(), 1);
5086 assert_eq!(claims[0].generation, receipts[1].generation);
5087 parent.add_message(claims[0].envelope.to_provider_message().unwrap());
5088 store.save_session(&parent).await.unwrap();
5089 inbox.ack(parent_id, &claims[0]).await.unwrap();
5090
5091 coordinator.on_child_completed(low).await;
5092 let mut rearmed = store.load_session(parent_id).await.unwrap().unwrap();
5093 let mut runtime = read_runtime_state(&rearmed);
5094 assert!(runtime.waiting_for_children.is_none());
5095 let new_wait = WaitingForChildrenState::for_children(
5096 vec!["next-slow-child".into()],
5097 ChildWaitPolicy::All,
5098 Utc::now(),
5099 );
5100 runtime.waiting_for_children = Some(new_wait.clone());
5101 runtime.status = AgentStatusState::Suspended;
5102 write_runtime_state(&mut rearmed, &runtime);
5103 store.save_session(&rearmed).await.unwrap();
5104 let backlog = inbox.inspect(parent_id).await.unwrap();
5105 assert_eq!(backlog.pending, 1);
5106 assert!(
5107 backlog.interrupt_pending(),
5108 "the low generation has its own grant"
5109 );
5110 let (prepared, ready) = prepare_session_inbox_activation(
5111 &coordinator.persistence,
5112 parent_id,
5113 backlog.interrupt_pending(),
5114 )
5115 .await
5116 .unwrap()
5117 .unwrap();
5118 assert!(ready);
5119 assert_eq!(
5120 read_runtime_state(&prepared).waiting_for_children,
5121 Some(new_wait)
5122 );
5123 let claims = inbox.claim(parent_id, 8).await.unwrap();
5124 assert_eq!(claims.len(), 1);
5125 assert_eq!(claims[0].generation, receipts[0].generation);
5126 }
5127
5128 #[tokio::test]
5129 async fn large_legal_typed_source_falls_back_within_the_delivery_budget() {
5130 let (_temp, store, inbox, coordinator, _, launches) = completion_inbox_fixture().await;
5131 let parent_id = "large-source-parent";
5132 let child_id = "large-source-child";
5133 save_waiting_test_parent(
5134 &store,
5135 parent_id,
5136 vec![child_id.into()],
5137 ChildWaitPolicy::All,
5138 )
5139 .await;
5140 let mut parent = store.load_session(parent_id).await.unwrap().unwrap();
5141 parent.add_message(Message::user("required original source"));
5142 store.save_session(&parent).await.unwrap();
5143 let mut child = typed_test_child(&parent, child_id, &typed_test_report().to_string());
5144 let source = ChildCompletionSource::from_committed_session(&child).unwrap();
5145 let run_length =
5146 3900 - serde_json::to_vec(&source).unwrap().len() + source.activation_run_id.len();
5147 ChildCompletionSource::prepare(&mut child, &"r".repeat(run_length), &Default::default());
5148 let completion = typed_test_completion(&child);
5149 let source_bytes = serde_json::to_vec(completion.source.as_ref().unwrap())
5150 .unwrap()
5151 .len();
5152 assert!((3800..=4096).contains(&source_bytes));
5153 store.save_session(&child).await.unwrap();
5154
5155 coordinator.on_child_completed(completion).await;
5156 let claims = inbox.claim(parent_id, 8).await.unwrap();
5157 assert_eq!(claims.len(), 1);
5158 assert_eq!(launches.load(Ordering::SeqCst), 1);
5159 assert!(fits_child_result_budget(&claims[0].envelope));
5160 let message = claims[0].envelope.to_provider_message().unwrap();
5161 assert!(fits_child_result_budget(&message));
5162 assert!(message.content.contains("result_budget_exceeded"));
5163 }
5164
5165 #[tokio::test]
5166 async fn upgrade_promotes_one_existing_legacy_partial_delivery_without_a_second_id() {
5167 let (_temp, store, inbox, coordinator, _, launches) = completion_inbox_fixture().await;
5168 let parent_id = "legacy-partial-parent";
5169 save_waiting_test_parent(
5170 &store,
5171 parent_id,
5172 vec!["legacy-ready".into(), "legacy-slow".into()],
5173 ChildWaitPolicy::All,
5174 )
5175 .await;
5176 let completion =
5177 save_completion_test_child(&store, parent_id, "legacy-ready", "completed").await;
5178 save_completion_test_child(&store, parent_id, "legacy-slow", "running").await;
5179 let parent = store.load_session(parent_id).await.unwrap().unwrap();
5180 let wait = read_runtime_state(&parent).waiting_for_children.unwrap();
5181 let result = "result-legacy-ready";
5182 let presentation = runtime_resume_message(&completion, 1, Some(result));
5183 let mut legacy = child_completion_envelope(
5184 &completion,
5185 wait.registered_at,
5186 Some(result.into()),
5187 &presentation,
5188 );
5189 legacy.id = child_completion_id(&completion, wait.registered_at, Some(result), true);
5190 let receipt = inbox.deliver(&legacy).await.unwrap();
5191 assert!(!inbox.inspect(parent_id).await.unwrap().activation_pending());
5192
5193 coordinator.on_child_completed(completion.clone()).await;
5194 coordinator.on_child_completed(completion).await;
5195 let claims = inbox.claim(parent_id, 8).await.unwrap();
5196 assert_eq!(claims.len(), 1);
5197 assert_eq!(claims[0].envelope.id, legacy.id);
5198 assert_eq!(claims[0].generation, receipt.generation);
5199 assert_eq!(launches.load(Ordering::SeqCst), 1);
5200 assert_eq!(
5201 read_runtime_state(&store.load_session(parent_id).await.unwrap().unwrap())
5202 .waiting_for_children
5203 .as_ref(),
5204 Some(&wait)
5205 );
5206 }
5207
5208 #[tokio::test]
5209 async fn upgrade_keeps_admitted_legacy_raw_guardian_and_typed_sources_deduplicated_without_a_wait(
5210 ) {
5211 for presentation_kind in ["raw", "guardian", "typed", "typed_unavailable_digest"] {
5212 let (_temp, store, inbox, coordinator, _, launches) = completion_inbox_fixture().await;
5213 let parent_id = "legacy-admitted-parent";
5214 let child_id = "legacy-admitted-child";
5215 let mut parent = Session::new(parent_id, "model");
5216 parent.add_message(Message::user("required original source"));
5217 store.save_session(&parent).await.unwrap();
5218 let completion = if presentation_kind.starts_with("typed") {
5219 let child = typed_test_child(&parent, child_id, &typed_test_report().to_string());
5220 store.save_session(&child).await.unwrap();
5221 typed_test_completion(&child)
5222 } else {
5223 save_completion_test_child(&store, parent_id, child_id, "completed").await
5224 };
5225 let child = store.load_session(child_id).await.unwrap().unwrap();
5226 let result = completion.source.as_ref().unwrap().result(&child, &parent);
5227 let mut projection = sealed_child_report(&completion, &child, &parent);
5228 if presentation_kind == "typed_unavailable_digest" {
5229 projection = Some(unavailable_child_result(
5230 "typed_result",
5231 "result_budget_exceeded",
5232 ));
5233 }
5234 let mut presentation = match presentation_kind {
5235 "guardian" => guardian_resume_message(&completion, &GuardianVerdict::approved()),
5236 "typed" | "typed_unavailable_digest" => {
5237 typed_child_resume_message(&completion, 0, projection.as_ref().unwrap())
5238 }
5239 _ => runtime_resume_message(&completion, 0, result.as_deref()),
5240 };
5241 if presentation_kind == "typed" {
5242 presentation
5243 .metadata
5244 .as_mut()
5245 .unwrap()
5246 .as_object_mut()
5247 .unwrap()
5248 .remove("child_completion_source_sha256");
5249 }
5250 let old_wait = Utc::now() - chrono::Duration::minutes(2);
5251 let mut legacy =
5252 child_completion_envelope(&completion, old_wait, result.clone(), &presentation);
5253 legacy.id = child_completion_id(&completion, old_wait, result.as_deref(), true);
5254 if let Some(projection) = projection.as_mut() {
5255 assert!(bound_typed_child_delivery(
5256 &mut legacy,
5257 &mut presentation,
5258 projection,
5259 &completion,
5260 0
5261 ));
5262 }
5263 let receipt = inbox.deliver(&legacy).await.unwrap();
5264 inbox
5265 .mark_message_activation_eligible(
5266 parent_id,
5267 &receipt,
5268 SessionActivationPolicy::InterruptSpecificWait,
5269 )
5270 .await
5271 .unwrap();
5272 let claims = inbox.claim(parent_id, 1).await.unwrap();
5273 parent.add_message(claims[0].envelope.to_provider_message().unwrap());
5274 store.save_session(&parent).await.unwrap();
5275 inbox.ack(parent_id, &claims[0]).await.unwrap();
5276
5277 coordinator.on_child_completed(completion).await;
5278 assert_eq!(
5279 inbox.inspect(parent_id).await.unwrap().generation,
5280 receipt.generation,
5281 "{presentation_kind}"
5282 );
5283 assert_eq!(
5284 inbox.inspect(parent_id).await.unwrap().pending,
5285 0,
5286 "{presentation_kind}"
5287 );
5288 assert_eq!(launches.load(Ordering::SeqCst), 0, "{presentation_kind}");
5289 }
5290 }
5291
5292 #[tokio::test]
5293 async fn upgrade_reuses_a_pending_legacy_source_after_its_wait_was_cleared() {
5294 for (arm, acked) in [
5295 ("none", false),
5296 ("partial", false),
5297 ("terminal", false),
5298 ("terminal", true),
5299 ] {
5300 let (_temp, store, inbox, coordinator, _, launches) = completion_inbox_fixture().await;
5301 let parent_id = "legacy-pending-parent";
5302 let child_id = "legacy-pending-child";
5303 let mut parent = Session::new(parent_id, "model");
5304 store.save_session(&parent).await.unwrap();
5305 let completion =
5306 save_completion_test_child(&store, parent_id, child_id, "completed").await;
5307 let result = "result-legacy-pending-child";
5308 let presentation = runtime_resume_message(&completion, 0, Some(result));
5309 let old_wait = Utc::now() - chrono::Duration::minutes(2);
5310 let mut legacy = child_completion_envelope(
5311 &completion,
5312 old_wait,
5313 Some(result.into()),
5314 &presentation,
5315 );
5316 legacy.id = child_completion_id(&completion, old_wait, Some(result), true);
5317 let receipt = inbox.deliver(&legacy).await.unwrap();
5318 inbox
5319 .mark_message_activation_eligible(
5320 parent_id,
5321 &receipt,
5322 SessionActivationPolicy::RespectSpecificWait,
5323 )
5324 .await
5325 .unwrap();
5326 if acked {
5327 let claims = inbox.claim(parent_id, 1).await.unwrap();
5328 parent.add_message(claims[0].envelope.to_provider_message().unwrap());
5329 store.save_session(&parent).await.unwrap();
5330 inbox.ack(parent_id, &claims[0]).await.unwrap();
5331 }
5332 let new_wait = if arm != "none" {
5333 let mut ids = vec![child_id.into()];
5334 if arm == "partial" {
5335 ids.push("legacy-pending-slow".into());
5336 save_completion_test_child(&store, parent_id, "legacy-pending-slow", "running")
5337 .await;
5338 }
5339 let wait =
5340 WaitingForChildrenState::for_children(ids, ChildWaitPolicy::All, Utc::now());
5341 let mut runtime = read_runtime_state(&parent);
5342 runtime.waiting_for_children = Some(wait.clone());
5343 runtime.status = AgentStatusState::Suspended;
5344 write_runtime_state(&mut parent, &runtime);
5345 store.save_session(&parent).await.unwrap();
5346 Some(wait)
5347 } else {
5348 None
5349 };
5350
5351 coordinator.on_child_completed(completion.clone()).await;
5352 coordinator.on_child_completed(completion).await;
5353 let backlog = inbox.inspect(parent_id).await.unwrap();
5354 assert_eq!(
5355 backlog.generation, receipt.generation,
5356 "{arm}, acked={acked}"
5357 );
5358 assert_eq!(backlog.pending, usize::from(!acked), "{arm}, acked={acked}");
5359 assert_eq!(backlog.interrupt_pending(), !acked, "{arm}, acked={acked}");
5360 assert_eq!(
5364 launches.load(Ordering::SeqCst),
5365 usize::from(!acked || arm == "terminal"),
5366 "{arm}, acked={acked}"
5367 );
5368 let wait = read_runtime_state(&store.load_session(parent_id).await.unwrap().unwrap())
5369 .waiting_for_children;
5370 assert_eq!(wait, if arm == "partial" { new_wait } else { None });
5371 let claims = inbox.claim(parent_id, 8).await.unwrap();
5372 assert_eq!(claims.len(), usize::from(!acked));
5373 if !acked {
5374 assert_eq!(claims[0].envelope.id, legacy.id);
5375 assert_eq!(claims[0].generation, receipt.generation);
5376 }
5377 }
5378 }
5379
5380 #[tokio::test]
5381 async fn admitted_legacy_guardian_source_still_commits_a_pending_verdict() {
5382 let (_temp, store, inbox, coordinator, _, launches) = completion_inbox_fixture().await;
5383 let parent_id = "legacy-pending-guardian-parent";
5384 let child_id = "legacy-pending-guardian-child";
5385 let mut parent = Session::new(parent_id, "model");
5386 let mut guardian = crate::runtime::guardian_state::ensure_guardian_state(&parent);
5387 guardian.record_spawn(child_id);
5388 write_guardian_state(&mut parent, guardian);
5389 store.save_session(&parent).await.unwrap();
5390 let verdict = GuardianVerdict::approved();
5391 let mut child = Session::new_child(child_id, parent_id, "model", "Guardian");
5392 child.set_subagent_type("guardian");
5393 child.add_message(Message::assistant(
5394 serde_json::to_string(&verdict).unwrap(),
5395 None,
5396 ));
5397 child.set_last_run_status("completed");
5398 save_sourced_test_child(&store, &child).await;
5399 let completion = ChildCompletion {
5400 parent_session_id: parent_id.into(),
5401 child_session_id: child_id.into(),
5402 status: "completed".into(),
5403 error: None,
5404 completed_at: child.updated_at,
5405 source: test_child_source(&store, child_id).await,
5406 };
5407 let result = serde_json::to_string(&verdict).unwrap();
5408 let old_wait = Utc::now() - chrono::Duration::minutes(2);
5409 let presentation = guardian_resume_message(&completion, &verdict);
5410 let mut legacy =
5411 child_completion_envelope(&completion, old_wait, Some(result.clone()), &presentation);
5412 legacy.id = child_completion_id(&completion, old_wait, Some(&result), true);
5413 let receipt = inbox.deliver(&legacy).await.unwrap();
5414 inbox
5415 .mark_message_activation_eligible(
5416 parent_id,
5417 &receipt,
5418 SessionActivationPolicy::InterruptSpecificWait,
5419 )
5420 .await
5421 .unwrap();
5422 let claims = inbox.claim(parent_id, 1).await.unwrap();
5423 parent.add_message(claims[0].envelope.to_provider_message().unwrap());
5424 store.save_session(&parent).await.unwrap();
5425 inbox.ack(parent_id, &claims[0]).await.unwrap();
5426
5427 coordinator.on_child_completed(completion.clone()).await;
5428 let saved = store.load_session(parent_id).await.unwrap().unwrap();
5429 let guardian = read_guardian_state(&saved).unwrap();
5430 assert!(guardian.phase.is_reviewed());
5431 assert!(guardian.last_approved());
5432 assert_eq!(guardian.review_count, 1);
5433 let saved_guardian = saved.metadata["guardian.state"].clone();
5434 coordinator.on_child_completed(completion).await;
5435 assert_eq!(
5436 store
5437 .load_session(parent_id)
5438 .await
5439 .unwrap()
5440 .unwrap()
5441 .metadata["guardian.state"],
5442 saved_guardian
5443 );
5444 let backlog = inbox.inspect(parent_id).await.unwrap();
5445 assert_eq!(backlog.generation, receipt.generation);
5446 assert_eq!(backlog.pending, 0);
5447 assert_eq!(launches.load(Ordering::SeqCst), 0);
5448 }
5449
5450 #[tokio::test]
5451 async fn terminal_index_ahead_of_callbacks_preserves_every_waited_child_outcome() {
5452 for partial_first in [false, true] {
5453 let (_temp, store, inbox, coordinator, reservations, launches) =
5454 completion_inbox_fixture().await;
5455 let parent_id = "parallel-parent";
5456 let ids: Vec<_> = (0..4)
5457 .map(|slot| format!("parallel-child-{slot}"))
5458 .collect();
5459 save_waiting_test_parent(&store, parent_id, ids.clone(), ChildWaitPolicy::All).await;
5460 for (slot, id) in ids.iter().enumerate() {
5461 let mut child = Session::new_child(id, parent_id, "model", "Child");
5462 child.add_message(Message::assistant(format!("result-{id}"), None));
5463 child.set_last_run_status(if partial_first && slot > 0 {
5464 "running"
5465 } else {
5466 "completed"
5467 });
5468 save_sourced_test_child(&store, &child).await;
5469 }
5470 let mut sources = HashMap::new();
5471 for id in &ids {
5472 sources.insert(id.clone(), test_child_source(&store, id).await);
5473 }
5474 let completion = |id: &str| ChildCompletion {
5475 parent_session_id: parent_id.into(),
5476 child_session_id: id.into(),
5477 status: "completed".into(),
5478 error: None,
5479 completed_at: Utc::now(),
5480 source: sources.get(id).cloned().flatten(),
5481 };
5482 if partial_first {
5483 ChildCompletionHandler::on_child_completed(
5484 coordinator.as_ref(),
5485 completion(&ids[0]),
5486 )
5487 .await;
5488 assert!(
5489 read_runtime_state(&store.load_session(parent_id).await.unwrap().unwrap())
5490 .waiting_for_children
5491 .is_some()
5492 );
5493 assert_eq!(inbox.inspect(parent_id).await.unwrap().pending, 1);
5494 assert_eq!(launches.load(Ordering::SeqCst), 1);
5495 for id in &ids[1..] {
5496 let mut child = store.load_session(id).await.unwrap().unwrap();
5497 child.set_last_run_status("completed");
5498 save_sourced_test_child(&store, &child).await;
5499 }
5500 }
5501 for id in &ids {
5504 sources.insert(id.clone(), test_child_source(&store, id).await);
5505 }
5506 let completion = |id: &str| ChildCompletion {
5507 parent_session_id: parent_id.into(),
5508 child_session_id: id.into(),
5509 status: "completed".into(),
5510 error: None,
5511 completed_at: Utc::now(),
5512 source: sources.get(id).cloned().flatten(),
5513 };
5514 ChildCompletionHandler::on_child_completed(coordinator.as_ref(), completion(&ids[1]))
5515 .await;
5516 assert!(
5517 read_runtime_state(&store.load_session(parent_id).await.unwrap().unwrap())
5518 .waiting_for_children
5519 .is_none()
5520 );
5521 assert_eq!(inbox.inspect(parent_id).await.unwrap().pending, 4);
5522 assert_eq!(reservations.load(Ordering::SeqCst), 1);
5523 assert_eq!(launches.load(Ordering::SeqCst), 1);
5524 for id in &ids {
5525 ChildCompletionHandler::on_child_completed(coordinator.as_ref(), completion(id))
5526 .await;
5527 }
5528 let claims = inbox.claim(parent_id, 8).await.unwrap();
5529 assert_eq!(claims.len(), 4, "late callbacks cannot duplicate outcomes");
5530 assert_eq!(reservations.load(Ordering::SeqCst), 1);
5531 for id in &ids {
5532 let matching: Vec<_> = claims
5533 .iter()
5534 .filter_map(|claim| match &claim.envelope.body {
5535 SessionMessageBody::ChildOutcome(outcome)
5536 if &outcome.child_session_id == id =>
5537 {
5538 Some(outcome)
5539 }
5540 _ => None,
5541 })
5542 .collect();
5543 assert_eq!(matching.len(), 1);
5544 assert_eq!(
5545 matching[0].result.as_deref(),
5546 Some(format!("result-{id}").as_str())
5547 );
5548 }
5549 }
5550 }
5551
5552 #[tokio::test]
5553 async fn saved_a_callback_barrier_never_attributes_completed_b_to_a() {
5554 for guardian in [false, true] {
5555 let (_temp, store, inbox, coordinator, _, launches) = completion_inbox_fixture().await;
5556 let parent_id = "source-barrier-parent";
5557 let child_id = "source-barrier-child";
5558 save_waiting_test_parent(
5559 &store,
5560 parent_id,
5561 vec![child_id.into()],
5562 ChildWaitPolicy::All,
5563 )
5564 .await;
5565 if guardian {
5566 let mut parent = store.load_session(parent_id).await.unwrap().unwrap();
5567 let mut state = crate::runtime::guardian_state::ensure_guardian_state(&parent);
5568 state.record_spawn(child_id);
5569 write_guardian_state(&mut parent, state);
5570 store.save_session(&parent).await.unwrap();
5571 }
5572 let mut child = Session::new_child(child_id, parent_id, "model", "Child");
5573 if guardian {
5574 child
5575 .metadata
5576 .insert("subagent_type".into(), "guardian".into());
5577 }
5578 child.add_message(Message::user("assignment A"));
5579 store.save_session(&child).await.unwrap();
5580 let prior = child.messages.iter().map(|m| m.id.clone()).collect();
5581 child.add_message(Message::assistant("answer A", None));
5582 child.set_last_run_status("completed");
5583 ChildCompletionSource::prepare(&mut child, "actual-run-A", &prior);
5584 coordinator
5585 .agent
5586 .persistence()
5587 .save_runtime_session(&mut child)
5588 .await
5589 .unwrap();
5590 let completion_a = ChildCompletion {
5591 parent_session_id: parent_id.into(),
5592 child_session_id: child_id.into(),
5593 status: "completed".into(),
5594 error: None,
5595 completed_at: Utc::now(),
5596 source: ChildCompletionSource::after_final_save(&child, true),
5597 };
5598 assert_eq!(
5599 completion_a.source.as_ref().unwrap().activation_run_id,
5600 "actual-run-A"
5601 );
5602 let (release, barrier) = tokio::sync::oneshot::channel();
5605 let target = coordinator.clone();
5606 let callback = tokio::spawn(async move {
5607 barrier.await.unwrap();
5608 target.on_child_completed(completion_a).await;
5609 });
5610 let prior = child.messages.iter().map(|m| m.id.clone()).collect();
5611 child.add_message(Message::user("assignment B"));
5612 let answer_b = if guardian {
5613 r#"{"approve":true,"summary":"B approved","findings":[]}"#
5614 } else {
5615 "answer B"
5616 };
5617 child.add_message(Message::assistant(answer_b, None));
5618 child.set_last_run_status("completed");
5619 ChildCompletionSource::prepare(&mut child, "actual-run-B", &prior);
5620 coordinator
5621 .agent
5622 .persistence()
5623 .save_runtime_session(&mut child)
5624 .await
5625 .unwrap();
5626 release.send(()).unwrap();
5627 callback.await.unwrap();
5628 assert_eq!(inbox.inspect(parent_id).await.unwrap().pending, 0);
5629 let parent = store.load_session(parent_id).await.unwrap().unwrap();
5630 assert!(read_runtime_state(&parent).waiting_for_children.is_some());
5631 assert_eq!(launches.load(Ordering::SeqCst), 0);
5632 if guardian {
5633 assert!(read_guardian_state(&parent).unwrap().phase.is_pending());
5634 }
5635
5636 let completion_b = ChildCompletion {
5637 parent_session_id: parent_id.into(),
5638 child_session_id: child_id.into(),
5639 status: "completed".into(),
5640 error: None,
5641 completed_at: Utc::now(),
5642 source: ChildCompletionSource::after_final_save(&child, true),
5643 };
5644 coordinator.on_child_completed(completion_b.clone()).await;
5645 coordinator.on_child_completed(completion_b).await;
5646 let claims = inbox.claim(parent_id, 8).await.unwrap();
5647 assert_eq!(
5648 claims.len(),
5649 1,
5650 "B is admitted once with B's own provenance"
5651 );
5652 let SessionMessageBody::ChildOutcome(outcome) = &claims[0].envelope.body else {
5653 panic!("expected ChildOutcome");
5654 };
5655 assert_eq!(outcome.result.as_deref(), Some(answer_b));
5656 assert_eq!(launches.load(Ordering::SeqCst), 1);
5657 }
5658 }
5659
5660 #[tokio::test]
5661 async fn missing_source_cannot_import_latest_guardian_approval() {
5662 let (_temp, store, inbox, coordinator, _, _) = completion_inbox_fixture().await;
5663 let parent_id = "unavailable-guardian-parent";
5664 let child_id = "unavailable-guardian-child";
5665 save_waiting_test_parent(
5666 &store,
5667 parent_id,
5668 vec![child_id.into()],
5669 ChildWaitPolicy::All,
5670 )
5671 .await;
5672 let mut parent = store.load_session(parent_id).await.unwrap().unwrap();
5673 let mut state = crate::runtime::guardian_state::ensure_guardian_state(&parent);
5674 state.record_spawn(child_id);
5675 write_guardian_state(&mut parent, state);
5676 store.save_session(&parent).await.unwrap();
5677 let mut child = Session::new_child(child_id, parent_id, "model", "Child");
5678 child
5679 .metadata
5680 .insert("subagent_type".into(), "guardian".into());
5681 child.add_message(Message::assistant(
5682 r#"{"approve":true,"summary":"unbound latest","findings":[]}"#,
5683 None,
5684 ));
5685 child.set_last_run_status("completed");
5686 store.save_session(&child).await.unwrap();
5687 coordinator
5688 .on_child_completed(ChildCompletion {
5689 parent_session_id: parent_id.into(),
5690 child_session_id: child_id.into(),
5691 status: "completed".into(),
5692 error: None,
5693 completed_at: Utc::now(),
5694 source: None,
5695 })
5696 .await;
5697 let parent = store.load_session(parent_id).await.unwrap().unwrap();
5698 let state = read_guardian_state(&parent).unwrap();
5699 assert!(state.phase.is_reviewed());
5700 assert!(
5701 !state.last_approved(),
5702 "source-less latest approval is never authoritative"
5703 );
5704 let claims = inbox.claim(parent_id, 8).await.unwrap();
5705 let SessionMessageBody::ChildOutcome(outcome) = &claims[0].envelope.body else {
5706 panic!("expected ChildOutcome");
5707 };
5708 assert!(outcome.result.is_none());
5709 }
5710
5711 #[tokio::test]
5712 async fn lost_callback_replay_uses_the_committed_seal() {
5713 for valid in [true, false] {
5714 let (_temp, store, inbox, coordinator, _, _) = completion_inbox_fixture().await;
5715 save_waiting_test_parent(
5716 &store,
5717 "replay-parent",
5718 vec!["replay-child".into()],
5719 ChildWaitPolicy::All,
5720 )
5721 .await;
5722 let mut child = Session::new_child("replay-child", "replay-parent", "model", "Child");
5723 child.add_message(Message::assistant("sealed replay result", None));
5724 child.set_last_run_status("completed");
5725 save_sourced_test_child(&store, &child).await;
5726 if !valid {
5727 let mut child = store.load_session("replay-child").await.unwrap().unwrap();
5728 child.messages[0].content = "unsealed successor result".into();
5729 store.save_session(&child).await.unwrap();
5730 }
5731 let parent = store.load_session("replay-parent").await.unwrap().unwrap();
5732 let mut wait = read_runtime_state(&parent).waiting_for_children.unwrap();
5733 wait.registered_at = Utc::now() - chrono::Duration::seconds(61);
5734 coordinator.sweep_child_wait("replay-parent", wait).await;
5735 let claims = inbox.claim("replay-parent", 1).await.unwrap();
5736 if !valid {
5737 assert!(claims.is_empty());
5738 let parent = store.load_session("replay-parent").await.unwrap().unwrap();
5739 assert!(read_runtime_state(&parent).waiting_for_children.is_some());
5740 continue;
5741 }
5742 let SessionMessageBody::ChildOutcome(outcome) = &claims[0].envelope.body else {
5743 panic!("expected ChildOutcome");
5744 };
5745 assert_eq!(outcome.result.as_deref(), Some("sealed replay result"));
5746 }
5747 }
5748
5749 #[tokio::test]
5750 async fn partial_admission_then_source_mutation_ignores_stale_retry() {
5751 for parent_changed in [false, true] {
5752 let (_temp, store, inbox, coordinator, _, launches) = completion_inbox_fixture().await;
5753 let parent_id = "partial-source-parent";
5754 let child_id = "partial-source-child";
5755 save_waiting_test_parent(
5756 &store,
5757 parent_id,
5758 vec![child_id.into(), "still-running-child".into()],
5759 ChildWaitPolicy::All,
5760 )
5761 .await;
5762 let mut child = Session::new_child(child_id, parent_id, "model", "Child");
5763 if parent_changed {
5764 let mut parent = store.load_session(parent_id).await.unwrap().unwrap();
5765 let mut source = Message::user("required parent constraint");
5766 source.id = "parent-source".into();
5767 parent.add_message(source);
5768 store.save_session(&parent).await.unwrap();
5769 let packet = bamboo_domain::ChildContextPacket {
5770 version: 1,
5771 objective: "task".into(),
5772 constraints: vec![],
5773 acceptance: vec!["source remains attributable".into()],
5774 non_goals: vec![],
5775 necessary_user_instructions: vec![],
5776 recorded_decisions: vec![],
5777 source_user_message_ids: vec!["parent-source".into()],
5778 background_message_ids: vec![],
5779 };
5780 let resolved = packet.resolve(&parent, "task").unwrap();
5781 let binding = bamboo_domain::ChildContextBinding::new(
5782 &parent,
5783 child_id,
5784 resolved.required_brief.clone(),
5785 resolved,
5786 )
5787 .unwrap();
5788 binding.install(&mut child).unwrap();
5789 child.add_message(binding.assignment_message());
5790 }
5791 child.add_message(Message::assistant("original source", None));
5792 child.set_last_run_status("completed");
5793 save_sourced_test_child(&store, &child).await;
5794 let completion = ChildCompletion {
5795 parent_session_id: parent_id.into(),
5796 child_session_id: child_id.into(),
5797 status: "completed".into(),
5798 error: None,
5799 completed_at: Utc::now(),
5800 source: test_child_source(&store, child_id).await,
5801 };
5802 coordinator.on_child_completed(completion.clone()).await;
5803 assert_eq!(inbox.inspect(parent_id).await.unwrap().pending, 1);
5804 if parent_changed {
5805 let mut parent = store.load_session(parent_id).await.unwrap().unwrap();
5806 parent.messages[0].content = "modified parent source".into();
5807 store.save_session(&parent).await.unwrap();
5808 } else {
5809 child = store.load_session(child_id).await.unwrap().unwrap();
5810 child.messages[0].content = "modified after admission".into();
5811 store.save_session(&child).await.unwrap();
5812 }
5813 coordinator.on_child_completed(completion).await;
5814 assert_eq!(inbox.inspect(parent_id).await.unwrap().pending, 1);
5815 assert_eq!(launches.load(Ordering::SeqCst), 1);
5816 assert!(
5817 read_runtime_state(&store.load_session(parent_id).await.unwrap().unwrap())
5818 .waiting_for_children
5819 .is_some()
5820 );
5821 let mut sibling =
5822 Session::new_child("still-running-child", parent_id, "model", "Child");
5823 sibling.add_message(Message::assistant("sibling result", None));
5824 sibling.set_last_run_status("completed");
5825 save_sourced_test_child(&store, &sibling).await;
5826 coordinator
5827 .on_child_completed(ChildCompletion {
5828 parent_session_id: parent_id.into(),
5829 child_session_id: sibling.id.clone(),
5830 status: "completed".into(),
5831 error: None,
5832 completed_at: Utc::now(),
5833 source: test_child_source(&store, &sibling.id).await,
5834 })
5835 .await;
5836 assert_eq!(
5837 inbox.inspect(parent_id).await.unwrap().pending,
5838 2,
5839 "sibling completion cannot restage mutated A as a neutral outcome"
5840 );
5841 assert_eq!(launches.load(Ordering::SeqCst), 1);
5842 assert!(
5843 read_runtime_state(&store.load_session(parent_id).await.unwrap().unwrap())
5844 .waiting_for_children
5845 .is_some()
5846 );
5847 assert_eq!(
5848 inbox.inspect(parent_id).await.unwrap().claimed,
5849 0,
5850 "stale replay does not create another claim or reasoning owner"
5851 );
5852 }
5853 }
5854
5855 #[tokio::test]
5856 async fn short_circuit_wait_stages_terminal_siblings_without_waiting_or_foreign_content() {
5857 for policy in [ChildWaitPolicy::Any, ChildWaitPolicy::FirstError] {
5858 let (_temp, store, inbox, coordinator, _, launches) = completion_inbox_fixture().await;
5859 let parent_id = "short-circuit-parent";
5860 let ids = vec![
5861 "failed-child".into(),
5862 "done-child".into(),
5863 "running-child".into(),
5864 "foreign-child".into(),
5865 ];
5866 save_waiting_test_parent(&store, parent_id, ids, policy).await;
5867 for (id, owner, status, result) in [
5868 ("failed-child", parent_id, "error", "failed-owned-result"),
5869 ("done-child", parent_id, "completed", "done-owned-result"),
5870 ("running-child", parent_id, "running", "not-terminal"),
5871 (
5872 "foreign-child",
5873 "another-parent",
5874 "completed",
5875 "foreign-secret",
5876 ),
5877 ] {
5878 let mut child = Session::new_child(id, owner, "model", "Child");
5879 child.add_message(Message::assistant(result, None));
5880 child.set_last_run_status(status);
5881 save_sourced_test_child(&store, &child).await;
5882 }
5883 ChildCompletionHandler::on_child_completed(
5884 coordinator.as_ref(),
5885 ChildCompletion {
5886 parent_session_id: parent_id.into(),
5887 child_session_id: "failed-child".into(),
5888 status: "error".into(),
5889 error: None,
5890 completed_at: Utc::now(),
5891 source: test_child_source(&store, "failed-child").await,
5892 },
5893 )
5894 .await;
5895 assert!(
5896 read_runtime_state(&store.load_session(parent_id).await.unwrap().unwrap())
5897 .waiting_for_children
5898 .is_none()
5899 );
5900 let claims = inbox.claim(parent_id, 8).await.unwrap();
5901 assert_eq!(claims.len(), 2);
5902 let content = serde_json::to_string(
5903 &claims
5904 .iter()
5905 .map(|claim| &claim.envelope)
5906 .collect::<Vec<_>>(),
5907 )
5908 .unwrap();
5909 assert!(content.contains("done-owned-result"));
5910 assert!(!content.contains("foreign-secret"));
5911 assert!(!content.contains("not-terminal"));
5912 assert_eq!(
5913 store
5914 .load_session("running-child")
5915 .await
5916 .unwrap()
5917 .unwrap()
5918 .last_run_status()
5919 .as_deref(),
5920 Some("running")
5921 );
5922 assert_eq!(launches.load(Ordering::SeqCst), 1);
5923 }
5924 }
5925
5926 #[tokio::test]
5927 async fn sibling_admission_failure_keeps_wait_armed_without_activation() {
5928 let limits = bamboo_domain::SessionInboxLimits {
5929 max_backlog: 1,
5930 ..Default::default()
5931 };
5932 let (_temp, store, inbox, coordinator, reservations, launches) =
5933 completion_inbox_fixture_with_limits(limits).await;
5934 let parent_id = "partial-stage-parent";
5935 let ids = vec!["first-child".into(), "second-child".into()];
5936 save_waiting_test_parent(&store, parent_id, ids, ChildWaitPolicy::All).await;
5937 for id in ["first-child", "second-child"] {
5938 let mut child = Session::new_child(id, parent_id, "model", "Child");
5939 child.add_message(Message::assistant(format!("result-{id}"), None));
5940 child.set_last_run_status("completed");
5941 save_sourced_test_child(&store, &child).await;
5942 }
5943 ChildCompletionHandler::on_child_completed(
5944 coordinator.as_ref(),
5945 ChildCompletion {
5946 parent_session_id: parent_id.into(),
5947 child_session_id: "first-child".into(),
5948 status: "completed".into(),
5949 error: None,
5950 completed_at: Utc::now(),
5951 source: test_child_source(&store, "first-child").await,
5952 },
5953 )
5954 .await;
5955 let durable = store.load_session(parent_id).await.unwrap().unwrap();
5956 assert!(read_runtime_state(&durable).waiting_for_children.is_some());
5957 assert_eq!(
5958 read_runtime_state(&durable).status,
5959 AgentStatusState::Suspended
5960 );
5961 assert_eq!(inbox.inspect(parent_id).await.unwrap().pending, 1);
5962 let backlog = inbox.inspect(parent_id).await.unwrap();
5963 assert!(backlog.activation_pending());
5964 assert!(
5965 !backlog.interrupt_pending(),
5966 "failed wait transition is still respected"
5967 );
5968 assert_eq!(reservations.load(Ordering::SeqCst), 0);
5969 assert_eq!(launches.load(Ordering::SeqCst), 0);
5970 }
5971
5972 #[tokio::test]
5973 async fn supervisor_resume_rejects_old_incarnation_without_replacing_cache() {
5974 let (_temp, store, _inbox, coordinator, _reservations, _launches) =
5975 completion_inbox_fixture().await;
5976 let original = store
5977 .get_or_create_default_supervisor("model")
5978 .await
5979 .unwrap();
5980 let mut stale = store
5981 .load_session(&original.session_id)
5982 .await
5983 .unwrap()
5984 .unwrap();
5985 store.delete_session(&original.session_id).await.unwrap();
5986 let recreated = store
5987 .get_or_create_default_supervisor("replacement")
5988 .await
5989 .unwrap();
5990 assert_ne!(original.incarnation_id, recreated.incarnation_id);
5991 let mut current = store
5992 .load_session(&recreated.session_id)
5993 .await
5994 .unwrap()
5995 .unwrap();
5996 coordinator.save_and_cache(&mut current).await;
5997 let cached = coordinator
5998 .sessions
5999 .get(¤t.id)
6000 .unwrap()
6001 .value()
6002 .clone();
6003 coordinator.save_and_cache(&mut stale).await;
6004 assert!(Arc::ptr_eq(
6005 &cached,
6006 coordinator.sessions.get(¤t.id).unwrap().value()
6007 ));
6008 assert_eq!(
6009 store
6010 .load_session(¤t.id)
6011 .await
6012 .unwrap()
6013 .unwrap()
6014 .authority_identity,
6015 current.authority_identity
6016 );
6017 }
6018
6019 #[test]
6022 fn dead_child_candidate_status_matrix() {
6023 assert!(is_dead_child_candidate_status(None));
6025 assert!(is_dead_child_candidate_status(Some("running")));
6028 assert!(is_dead_child_candidate_status(Some("pending")));
6029 assert!(!is_dead_child_candidate_status(Some("suspended")));
6031 for status in ["completed", "error", "timeout", "cancelled", "skipped"] {
6033 assert!(!is_dead_child_candidate_status(Some(status)), "{status}");
6034 }
6035 }
6036
6037 #[test]
6038 fn completion_child_ownership_gates_content_fold() {
6039 assert!(completion_child_is_owned("parent-1", Some("parent-1")));
6041 assert!(!completion_child_is_owned("parent-1", Some("parent-2")));
6044 assert!(!completion_child_is_owned("parent-1", None));
6046 }
6047
6048 #[test]
6049 fn watchdog_resume_messages_are_hidden_runtime_messages() {
6050 for message in [
6051 empty_child_wait_message(),
6052 child_wait_lease_expired_message(&["c-1".to_string(), "c-2".to_string()]),
6053 ] {
6054 assert!(matches!(message.role, Role::User));
6055 let meta = message.metadata.expect("hidden runtime metadata");
6056 assert_eq!(meta[RUNTIME_RESUME_MESSAGE_HIDDEN_KEY], true);
6057 assert_eq!(
6058 meta[RUNTIME_RESUME_MESSAGE_KIND_KEY],
6059 "child_wait_watchdog_resume"
6060 );
6061 }
6062 let lease = child_wait_lease_expired_message(&["c-1".to_string()]);
6063 assert!(lease.content.contains("NOT cancelled"));
6065 assert!(lease.content.contains("c-1"));
6066 }
6067
6068 #[test]
6069 fn runtime_child_guidance_uses_compact_inspection_calls() {
6070 let completion = make_completion("completed");
6071 let child_inspection = serde_json::json!({
6072 "intent": "inspect", "target": "child-1", "message": "messages",
6073 })
6074 .to_string();
6075 let tree_inspection =
6076 serde_json::json!({"intent": "inspect", "message": "tree"}).to_string();
6077 for (message, expected_call) in [
6078 (
6079 runtime_resume_message(&completion, 0, Some("evidence")),
6080 &child_inspection,
6081 ),
6082 (
6083 guardian_resume_message(&completion, &GuardianVerdict::approved()),
6084 &child_inspection,
6085 ),
6086 (empty_child_wait_message(), &tree_inspection),
6087 (
6088 child_wait_lease_expired_message(&["child-1".to_string()]),
6089 &tree_inspection,
6090 ),
6091 ] {
6092 assert!(message.content.contains(expected_call));
6093 for legacy in [
6094 "SubAgent.get",
6095 "SubAgent.list",
6096 "SubAgent.wait",
6097 "child_session_id=",
6098 ] {
6099 assert!(!message.content.contains(legacy), "stale hint: {legacy}");
6100 }
6101 }
6102 }
6103
6104 #[test]
6107 fn non_terminal_statuses_never_satisfy_wait_policies() {
6108 assert!(!is_terminal_child_status("suspended"));
6111 assert!(!is_terminal_child_status("running"));
6112 assert!(!is_terminal_child_status("pending"));
6113 }
6114
6115 fn make_completion(status: &str) -> ChildCompletion {
6116 ChildCompletion {
6117 parent_session_id: "parent-1".to_string(),
6118 child_session_id: "child-1".to_string(),
6119 status: status.to_string(),
6120 error: None,
6121 completed_at: Utc::now(),
6122 source: None,
6123 }
6124 }
6125
6126 #[test]
6127 fn oversized_child_outcome_is_bounded_and_keeps_full_content_identity() {
6128 let mut completion = make_completion("completed");
6129 let wait_registered_at = Utc::now();
6130 let huge = format!("prefix-A-{}", "x".repeat(300 * 1024));
6131 let presentation = runtime_resume_message(&completion, 0, Some(&huge));
6132 let first = child_completion_envelope(
6133 &completion,
6134 wait_registered_at,
6135 Some(huge.clone()),
6136 &presentation,
6137 );
6138 assert!(
6139 serde_json::to_vec(&first).unwrap().len()
6140 < bamboo_domain::SessionInboxLimits::default().max_payload_bytes
6141 );
6142 let SessionMessageBody::ChildOutcome(outcome) = &first.body else {
6143 panic!("typed child outcome");
6144 };
6145 let stored = outcome.result.as_deref().unwrap();
6146 assert!(stored.contains("sha256="));
6147 assert!(stored.contains("\"intent\":\"inspect\""));
6148 assert!(stored.contains("\"target\":\"child-1\""));
6149 assert!(stored.contains("\"message\":\"result\""));
6150 assert!(!stored.contains("SubAgent.get"));
6151 assert!(stored.len() < CHILD_COMPLETION_INLINE_FIELD_BYTES);
6152
6153 completion.completed_at += chrono::Duration::seconds(1);
6156 let exact_retry = child_completion_envelope(
6157 &completion,
6158 wait_registered_at,
6159 Some(huge.clone()),
6160 &runtime_resume_message(&completion, 9, Some(&huge)),
6161 );
6162 assert_eq!(exact_retry.id, first.id);
6163 let changed = format!("prefix-B-{}", "x".repeat(300 * 1024));
6164 let corrected = child_completion_envelope(
6165 &completion,
6166 wait_registered_at,
6167 Some(changed.clone()),
6168 &runtime_resume_message(&completion, 0, Some(&changed)),
6169 );
6170 assert_ne!(corrected.id, first.id);
6171 }
6172
6173 #[test]
6174 fn oversized_child_error_points_to_bounded_error_view() {
6175 let mut completion = make_completion("error");
6176 completion.error = Some("失败🙂".repeat(30_000));
6177 let presentation = runtime_resume_message(&completion, 0, None);
6178 let envelope = child_completion_envelope(&completion, Utc::now(), None, &presentation);
6179 assert!(
6180 serde_json::to_vec(&envelope).unwrap().len()
6181 < bamboo_domain::SessionInboxLimits::default().max_payload_bytes
6182 );
6183 let SessionMessageBody::ChildOutcome(outcome) = &envelope.body else {
6184 panic!("typed child outcome");
6185 };
6186 let stored = outcome.error.as_deref().expect("bounded error");
6187 assert!(stored.contains(r#""message":"diagnostics""#));
6188 assert!(stored.contains("authenticated inspector"));
6189 assert!(!stored.contains("bounded slices"));
6190 assert!(!stored.contains("cursors"));
6191 assert!(stored.len() < CHILD_COMPLETION_INLINE_FIELD_BYTES);
6192 let metadata_error = outcome
6193 .provider_message
6194 .as_ref()
6195 .and_then(|provider| provider.metadata.get("child_error"))
6196 .and_then(serde_json::Value::as_str)
6197 .expect("bounded metadata error");
6198 assert_eq!(metadata_error, stored);
6199 assert!(!metadata_error.contains(&"失败🙂".repeat(30_000)));
6200 }
6201
6202 #[test]
6203 fn escaped_and_mixed_child_fields_fit_the_serialized_inbox_budget() {
6204 let mut completion = make_completion("error");
6205 completion.error = Some("\u{0001}".repeat(CHILD_COMPLETION_INLINE_FIELD_BYTES));
6206 let result = "\n".repeat(CHILD_COMPLETION_INLINE_FIELD_BYTES);
6207 let wait_registered_at = Utc::now();
6208 let presentation = runtime_resume_message(&completion, 0, Some(&result));
6209 let envelope = child_completion_envelope(
6210 &completion,
6211 wait_registered_at,
6212 Some(result.clone()),
6213 &presentation,
6214 );
6215 assert!(
6216 serde_json::to_vec(&envelope).unwrap().len()
6217 < bamboo_domain::SessionInboxLimits::default().max_payload_bytes
6218 );
6219 let SessionMessageBody::ChildOutcome(outcome) = &envelope.body else {
6220 panic!("typed child outcome");
6221 };
6222 let error = outcome.error.as_deref().unwrap();
6223 let stored_result = outcome.result.as_deref().unwrap();
6224 assert!(error.contains(r#""message":"diagnostics""#));
6225 assert!(stored_result.contains(r#""message":"result""#));
6226 assert!(!error.contains("Bounded tail"));
6227 assert!(!stored_result.contains("Bounded tail"));
6228 assert_eq!(
6229 outcome
6230 .provider_message
6231 .as_ref()
6232 .and_then(|provider| provider.metadata.get("child_error"))
6233 .and_then(serde_json::Value::as_str),
6234 Some(error)
6235 );
6236 let retry = child_completion_envelope(
6237 &completion,
6238 wait_registered_at,
6239 Some(result),
6240 &runtime_resume_message(&completion, 8, None),
6241 );
6242 assert_eq!(envelope.id, retry.id);
6243 }
6244
6245 #[test]
6246 fn compact_guardian_outcome_keeps_rejected_verdict_and_result_retrieval() {
6247 let completion = make_completion("completed");
6248 let result = format!(
6249 "{}{{\"approve\":false,\"findings\":[\"needs fix\"]}}",
6250 "\u{0001}".repeat(CHILD_COMPLETION_INLINE_FIELD_BYTES - 80)
6251 );
6252 assert!(result.len() <= CHILD_COMPLETION_INLINE_FIELD_BYTES);
6253 let verdict = parse_guardian_verdict(&result).expect("trailing guardian verdict");
6254 let presentation = guardian_resume_message(&completion, &verdict);
6255 assert!(
6256 serde_json::to_vec(&presentation).unwrap().len() <= CHILD_COMPLETION_INLINE_FIELD_BYTES,
6257 "the original guardian presentation must fit the inline cap"
6258 );
6259 let wait_registered_at = Utc::now();
6260 let expanded = build_child_completion_envelope(
6261 &completion,
6262 wait_registered_at,
6263 Some(&result),
6264 &presentation,
6265 false,
6266 );
6267 assert!(
6268 serde_json::to_vec(&expanded).unwrap().len()
6269 > bamboo_domain::SessionInboxLimits::default().max_payload_bytes,
6270 "JSON escaping must trigger the compact fallback"
6271 );
6272
6273 let envelope =
6274 child_completion_envelope(&completion, wait_registered_at, Some(result), &presentation);
6275 assert!(
6276 serde_json::to_vec(&envelope).unwrap().len()
6277 <= bamboo_domain::SessionInboxLimits::default().max_payload_bytes
6278 );
6279 let SessionMessageBody::ChildOutcome(outcome) = &envelope.body else {
6280 panic!("typed child outcome");
6281 };
6282 let provider = outcome.provider_message.as_ref().unwrap();
6283 assert!(provider.content.text.contains("Guardian review REJECTED"));
6284 assert!(provider.content.text.contains(r#""message":"result""#));
6285 assert_eq!(
6286 provider.metadata.get("guardian_approved"),
6287 Some(&serde_json::json!(false))
6288 );
6289 assert!(envelope
6290 .to_provider_message()
6291 .unwrap()
6292 .content
6293 .contains("Guardian review REJECTED"));
6294 }
6295
6296 #[tokio::test]
6297 async fn oversized_multibyte_child_error_is_durable_and_replayable() {
6298 let home = tempfile::tempdir().unwrap();
6299 let store = Arc::new(
6300 bamboo_storage::SessionStoreV2::new(home.path().to_path_buf())
6301 .await
6302 .unwrap(),
6303 );
6304 let parent_id = "large-error-parent";
6305 store
6306 .save_session(&Session::new(parent_id, "model"))
6307 .await
6308 .unwrap();
6309 let inbox = bamboo_storage::FileSessionInbox::new(
6310 store.clone(),
6311 bamboo_domain::SessionInboxLimits::default(),
6312 );
6313 let mut completion = make_completion("error");
6314 completion.parent_session_id = parent_id.to_string();
6315 completion.error = Some("失败🙂".repeat(30_000));
6316 let presentation = runtime_resume_message(&completion, 0, None);
6317 let envelope = child_completion_envelope(&completion, Utc::now(), None, &presentation);
6318 inbox.deliver(&envelope).await.unwrap();
6319 completion.child_session_id = "escaped-error-child".to_string();
6320 completion.error = Some("\u{0001}".repeat(CHILD_COMPLETION_INLINE_FIELD_BYTES));
6321 let result = "\n".repeat(CHILD_COMPLETION_INLINE_FIELD_BYTES);
6322 let mixed = child_completion_envelope(
6323 &completion,
6324 Utc::now(),
6325 Some(result.clone()),
6326 &runtime_resume_message(&completion, 0, Some(&result)),
6327 );
6328 let receipt = inbox.deliver(&mixed).await.unwrap();
6329 inbox
6330 .mark_activation_eligible(
6331 parent_id,
6332 receipt.generation,
6333 bamboo_domain::SessionActivationPolicy::RespectSpecificWait,
6334 )
6335 .await
6336 .unwrap();
6337 drop(inbox);
6338 drop(store);
6339
6340 let reopened_store = Arc::new(
6341 bamboo_storage::SessionStoreV2::new(home.path().to_path_buf())
6342 .await
6343 .unwrap(),
6344 );
6345 let reopened = bamboo_storage::FileSessionInbox::new(
6346 reopened_store,
6347 bamboo_domain::SessionInboxLimits::default(),
6348 );
6349 let claims = reopened.claim(parent_id, 2).await.unwrap();
6350 assert_eq!(claims.len(), 2);
6351 assert_eq!(claims[0].envelope.id, envelope.id);
6352 assert_eq!(claims[1].envelope.id, mixed.id);
6353 for claim in claims {
6354 assert!(
6355 serde_json::to_vec(&claim.envelope).unwrap().len()
6356 < bamboo_domain::SessionInboxLimits::default().max_payload_bytes
6357 );
6358 }
6359 }
6360
6361 #[tokio::test]
6362 async fn failed_child_wait_sidecar_save_keeps_cache_wait_and_transcript() {
6363 let (_temp, store, inbox, mut coordinator, reservations, launches) =
6364 completion_inbox_fixture().await;
6365 let parent_id = "failed-sidecar-parent";
6366 let child_id = "failed-sidecar-child";
6367 let now = Utc::now();
6368 let mut parent = Session::new(parent_id, "model");
6369 parent.add_message(Message::user("durable root turn"));
6370 let mut runtime = AgentRuntimeState::new("waiting-run");
6371 runtime.status = AgentStatusState::Suspended;
6372 runtime.waiting_for_children = Some(WaitingForChildrenState::for_children(
6373 vec![child_id.to_string()],
6374 ChildWaitPolicy::All,
6375 now,
6376 ));
6377 write_runtime_state(&mut parent, &runtime);
6378 parent.metadata.insert(
6379 "runtime.suspend_reason".to_string(),
6380 "waiting_for_children".to_string(),
6381 );
6382 store.save_session(&parent).await.unwrap();
6383 let mut cached = parent.clone();
6384 cached.add_message(Message::assistant("newer cached root turn", None));
6385 coordinator.sessions.insert(
6386 parent_id.to_string(),
6387 Arc::new(crate::SessionSnapshot::new(cached)),
6388 );
6389
6390 let mut child = Session::new_child(child_id, parent_id, "model", "Child");
6391 child.set_last_run_status("completed");
6392 save_sourced_test_child(&store, &child).await;
6393 Arc::get_mut(&mut coordinator).unwrap().persistence = Arc::new(LockedSessionStore::new(
6394 Arc::new(FailingRuntimeSaveStorage {
6395 inner: store.clone(),
6396 }),
6397 ));
6398
6399 ChildCompletionHandler::on_child_completed(
6400 coordinator.as_ref(),
6401 ChildCompletion {
6402 parent_session_id: parent_id.to_string(),
6403 child_session_id: child_id.to_string(),
6404 status: "completed".to_string(),
6405 error: None,
6406 completed_at: Utc::now(),
6407 source: None,
6408 },
6409 )
6410 .await;
6411
6412 let durable = store.load_session(parent_id).await.unwrap().unwrap();
6413 assert!(read_runtime_state(&durable).waiting_for_children.is_some());
6414 assert_eq!(durable.messages.len(), 1);
6415 assert_eq!(durable.messages[0].content, "durable root turn");
6416 let cached = coordinator.sessions.get(parent_id).unwrap();
6417 let cached = cached.value().read();
6418 assert!(read_runtime_state(&cached).waiting_for_children.is_some());
6419 assert!(cached
6420 .messages
6421 .iter()
6422 .any(|message| message.content == "newer cached root turn"));
6423 let backlog = inbox.inspect(parent_id).await.unwrap();
6424 assert_eq!(backlog.pending + backlog.claimed, 1);
6425 assert_eq!(reservations.load(Ordering::SeqCst), 0);
6426 assert_eq!(launches.load(Ordering::SeqCst), 0);
6427 }
6428
6429 #[tokio::test]
6430 async fn oversized_child_completion_clears_wait_and_activates_exactly_once() {
6431 let (_temp, store, inbox, coordinator, reservations, launches) =
6432 completion_inbox_fixture().await;
6433 let parent_id = "oversized-parent";
6434 let child_id = "oversized-child";
6435 let now = Utc::now();
6436 let mut parent = Session::new(parent_id, "model");
6437 let mut parent_runtime = AgentRuntimeState::new("waiting-run");
6438 parent_runtime.status = AgentStatusState::Suspended;
6439 parent_runtime.waiting_for_children = Some(WaitingForChildrenState::for_children(
6440 vec![child_id.to_string()],
6441 ChildWaitPolicy::All,
6442 now,
6443 ));
6444 parent_runtime.suspension = Some(SuspensionState {
6445 reason: "waiting_for_children".to_string(),
6446 suspended_at: now,
6447 resumable: true,
6448 hook_point: Some("ChildCompletion".to_string()),
6449 });
6450 write_runtime_state(&mut parent, &parent_runtime);
6451 parent.metadata.insert(
6452 "runtime.suspend_reason".to_string(),
6453 "waiting_for_children".to_string(),
6454 );
6455 store.save_session(&parent).await.unwrap();
6456 let mut cached_parent = parent.clone();
6460 cached_parent.add_message(Message::assistant("concurrent root turn", None));
6461 coordinator.sessions.insert(
6462 parent_id.to_string(),
6463 Arc::new(crate::SessionSnapshot::new(cached_parent)),
6464 );
6465
6466 let mut child = Session::new_child(child_id, parent_id, "model", "Child");
6467 child.add_message(Message::assistant("z".repeat(300 * 1024), None));
6468 child.set_last_run_status("completed");
6469 save_sourced_test_child(&store, &child).await;
6470 let completion = ChildCompletion {
6471 parent_session_id: parent_id.to_string(),
6472 child_session_id: child_id.to_string(),
6473 status: "completed".to_string(),
6474 error: None,
6475 completed_at: Utc::now(),
6476 source: test_child_source(&store, child_id).await,
6477 };
6478
6479 ChildCompletionHandler::on_child_completed(coordinator.as_ref(), completion.clone()).await;
6480 let durable_parent = store.load_session(parent_id).await.unwrap().unwrap();
6481 let durable_runtime = read_runtime_state(&durable_parent);
6482 assert!(durable_runtime.waiting_for_children.is_none());
6483 assert_eq!(durable_runtime.status, AgentStatusState::Idle);
6484 assert!(!durable_parent
6485 .metadata
6486 .contains_key("runtime.suspend_reason"));
6487 let cached_parent = coordinator.sessions.get(parent_id).unwrap();
6488 let cached_parent = cached_parent.value().read();
6489 assert!(cached_parent
6490 .messages
6491 .iter()
6492 .any(|message| message.content == "concurrent root turn"));
6493 assert!(read_runtime_state(&cached_parent)
6494 .waiting_for_children
6495 .is_none());
6496 let backlog = inbox.inspect(parent_id).await.unwrap();
6497 assert_eq!(backlog.pending + backlog.claimed, 1);
6498 assert!(backlog.activation_pending());
6499 assert_eq!(reservations.load(Ordering::SeqCst), 1);
6500 assert_eq!(launches.load(Ordering::SeqCst), 1);
6501
6502 let claim = inbox.claim(parent_id, 1).await.unwrap().remove(0);
6503 assert!(
6504 serde_json::to_vec(&claim.envelope).unwrap().len()
6505 < bamboo_domain::SessionInboxLimits::default().max_payload_bytes
6506 );
6507 ChildCompletionHandler::on_child_completed(coordinator.as_ref(), completion).await;
6510 assert_eq!(reservations.load(Ordering::SeqCst), 1);
6511 assert_eq!(launches.load(Ordering::SeqCst), 1);
6512 assert_eq!(inbox.inspect(parent_id).await.unwrap().claimed, 1);
6513 }
6514
6515 #[tokio::test]
6516 async fn completion_before_parent_wait_does_not_checkpoint_stale_root() {
6517 let (_temp, store, inbox, coordinator, reservations, launches) =
6518 completion_inbox_fixture().await;
6519 let parent_id = "parent-without-wait";
6520 let child_id = "already-finished-child";
6521 let mut parent = Session::new(parent_id, "model");
6522 parent.add_message(Message::assistant("root is still reasoning", None));
6523 store.save_session(&parent).await.unwrap();
6524 let before = store.load_session(parent_id).await.unwrap().unwrap();
6525
6526 let mut child = Session::new_child(child_id, parent_id, "model", "Child");
6527 child.set_last_run_status("completed");
6528 save_sourced_test_child(&store, &child).await;
6529 ChildCompletionHandler::on_child_completed(
6530 coordinator.as_ref(),
6531 ChildCompletion {
6532 parent_session_id: parent_id.to_string(),
6533 child_session_id: child_id.to_string(),
6534 status: "completed".to_string(),
6535 error: None,
6536 completed_at: Utc::now(),
6537 source: None,
6538 },
6539 )
6540 .await;
6541
6542 let after = store.load_session(parent_id).await.unwrap().unwrap();
6543 assert_eq!(after.updated_at, before.updated_at);
6544 assert_eq!(
6545 serde_json::to_value(&after.messages).unwrap(),
6546 serde_json::to_value(&before.messages).unwrap()
6547 );
6548 assert!(read_runtime_state(&after).waiting_for_children.is_none());
6549 assert_eq!(inbox.inspect(parent_id).await.unwrap().pending, 0);
6550 assert_eq!(reservations.load(Ordering::SeqCst), 0);
6551 assert_eq!(launches.load(Ordering::SeqCst), 0);
6552 }
6553
6554 #[tokio::test]
6555 async fn latest_locked_activation_mutation_preserves_wait_armed_after_stale_load() {
6556 let temp = tempfile::tempdir().unwrap();
6557 let store = Arc::new(
6558 bamboo_storage::SessionStoreV2::new(temp.path().to_path_buf())
6559 .await
6560 .unwrap(),
6561 );
6562 let storage: Arc<dyn Storage> = store.clone();
6563 let locked = LockedSessionStore::new(storage);
6564 let mut session = Session::new("activation-stale-wait", "model");
6565 session.agent_runtime_state = Some(AgentRuntimeState::new("old-run"));
6566 store.save_session(&session).await.unwrap();
6567
6568 let stale = store.load_session(&session.id).await.unwrap().unwrap();
6570 assert!(read_runtime_state(&stale).waiting_for_children.is_none());
6571
6572 locked
6573 .update_runtime_config(&session.id, |latest| {
6574 let mut state = read_runtime_state(latest);
6575 state.status = AgentStatusState::Suspended;
6576 state.waiting_for_children = Some(WaitingForChildrenState::for_children(
6577 vec!["child-new".to_string()],
6578 ChildWaitPolicy::All,
6579 Utc::now(),
6580 ));
6581 state.suspension = Some(SuspensionState {
6582 reason: "waiting_for_children".to_string(),
6583 suspended_at: Utc::now(),
6584 resumable: true,
6585 hook_point: None,
6586 });
6587 write_runtime_state(latest, &state);
6588 latest.metadata.insert(
6589 "runtime.suspend_reason".to_string(),
6590 "waiting_for_children".to_string(),
6591 );
6592 })
6593 .await
6594 .unwrap();
6595
6596 let (prepared, ready) = prepare_session_inbox_activation(&locked, &session.id, false)
6597 .await
6598 .unwrap()
6599 .unwrap();
6600 assert!(!ready, "latest durable specific wait must block activation");
6601 let state = read_runtime_state(&prepared);
6602 assert!(state.waiting_for_children.is_some());
6603 assert_eq!(state.status, AgentStatusState::Suspended);
6604 assert_eq!(
6605 prepared
6606 .metadata
6607 .get("runtime.suspend_reason")
6608 .map(String::as_str),
6609 Some("waiting_for_children")
6610 );
6611 }
6612
6613 #[tokio::test]
6614 async fn partial_child_and_bash_backlogs_remain_inert_after_inbox_reopen() {
6615 let temp = tempfile::tempdir().unwrap();
6616 let store = Arc::new(
6617 bamboo_storage::SessionStoreV2::new(temp.path().to_path_buf())
6618 .await
6619 .unwrap(),
6620 );
6621 store
6622 .save_session(&Session::new("restart-child-parent", "model"))
6623 .await
6624 .unwrap();
6625 store
6626 .save_session(&Session::new("restart-bash-parent", "model"))
6627 .await
6628 .unwrap();
6629 let inbox = bamboo_storage::FileSessionInbox::new(
6630 store.clone(),
6631 bamboo_domain::SessionInboxLimits::default(),
6632 );
6633
6634 let completion = ChildCompletion {
6635 parent_session_id: "restart-child-parent".to_string(),
6636 child_session_id: "child-a".to_string(),
6637 status: "completed".to_string(),
6638 error: None,
6639 completed_at: Utc::now(),
6640 source: None,
6641 };
6642 let child_resume = runtime_resume_message(&completion, 1, Some("first child"));
6643 inbox
6644 .deliver(&child_completion_envelope(
6645 &completion,
6646 Utc::now(),
6647 Some("first child".to_string()),
6648 &child_resume,
6649 ))
6650 .await
6651 .unwrap();
6652 let bash = BashCompletionInfo {
6653 session_id: "restart-bash-parent".to_string(),
6654 bash_id: "bash-a".to_string(),
6655 command: "true".to_string(),
6656 exit_code: Some(0),
6657 status: "completed".to_string(),
6658 output_tail: String::new(),
6659 };
6660 inbox
6661 .deliver(&bash_completion_envelope(&bash))
6662 .await
6663 .unwrap();
6664
6665 let reopened = bamboo_storage::FileSessionInbox::new(
6666 store,
6667 bamboo_domain::SessionInboxLimits::default(),
6668 );
6669 for session_id in ["restart-child-parent", "restart-bash-parent"] {
6670 let backlog = reopened.inspect(session_id).await.unwrap();
6671 assert_eq!(backlog.pending, 1);
6672 assert_eq!(backlog.activation_generation, 0);
6673 assert!(
6674 !backlog.activation_pending(),
6675 "startup must not run a partial wait backlog for {session_id}"
6676 );
6677 }
6678 }
6679
6680 struct StubChildIndex {
6683 children: Vec<(String, Option<String>)>,
6684 }
6685
6686 #[async_trait]
6687 impl Storage for StubChildIndex {
6688 async fn save_session(&self, _session: &Session) -> std::io::Result<()> {
6689 Ok(())
6690 }
6691 async fn load_session(&self, _id: &str) -> std::io::Result<Option<Session>> {
6692 Ok(None)
6693 }
6694 async fn delete_session(&self, _id: &str) -> std::io::Result<bool> {
6695 Ok(false)
6696 }
6697 async fn list_child_run_statuses(
6698 &self,
6699 _parent_session_id: &str,
6700 ) -> std::io::Result<Vec<(String, Option<String>)>> {
6701 Ok(self.children.clone())
6702 }
6703 }
6704
6705 #[tokio::test]
6706 async fn derive_completed_only_includes_terminal_children() {
6707 let storage: Arc<dyn Storage> = Arc::new(StubChildIndex {
6708 children: vec![
6709 ("a".into(), Some("completed".into())),
6710 ("b".into(), Some("running".into())),
6711 ("c".into(), Some("error".into())),
6712 ("d".into(), None),
6713 ],
6714 });
6715 let completed = derive_completed_child_ids(&storage, "parent-1", "b").await;
6716 assert_eq!(
6718 completed,
6719 vec!["a".to_string(), "b".to_string(), "c".to_string()]
6720 );
6721 }
6722
6723 #[tokio::test]
6724 async fn derive_completed_folds_in_just_completed_when_index_lags() {
6725 let storage: Arc<dyn Storage> = Arc::new(StubChildIndex {
6727 children: vec![("only".into(), Some("running".into()))],
6728 });
6729 let completed = derive_completed_child_ids(&storage, "parent-1", "only").await;
6730 assert_eq!(completed, vec!["only".to_string()]);
6731 }
6732
6733 #[test]
6734 fn wait_policy_all_uses_derived_completed_set() {
6735 let waited = vec!["a".to_string(), "b".to_string()];
6736 assert!(!wait_policy_satisfied(
6737 ChildWaitPolicy::All,
6738 &waited,
6739 &["a".to_string()],
6740 "a",
6741 "completed"
6742 ));
6743 assert!(wait_policy_satisfied(
6744 ChildWaitPolicy::All,
6745 &waited,
6746 &["a".to_string(), "b".to_string()],
6747 "b",
6748 "completed"
6749 ));
6750 }
6751
6752 #[test]
6753 fn wait_policy_first_error_requires_tracked_membership() {
6754 let waited = vec!["a".to_string(), "b".to_string()];
6755 assert!(wait_policy_satisfied(
6757 ChildWaitPolicy::FirstError,
6758 &waited,
6759 &["a".to_string()],
6760 "a",
6761 "error"
6762 ));
6763 assert!(!wait_policy_satisfied(
6766 ChildWaitPolicy::FirstError,
6767 &waited,
6768 &["a".to_string()],
6769 "stray-child",
6770 "timeout"
6771 ));
6772 assert!(wait_policy_satisfied(
6774 ChildWaitPolicy::FirstError,
6775 &waited,
6776 &["a".to_string(), "b".to_string()],
6777 "stray-child",
6778 "completed"
6779 ));
6780 }
6781
6782 #[test]
6783 fn child_final_assistant_text_returns_last_assistant() {
6784 let mut session = Session::new("child-1", "gpt-4");
6785 session.messages.push(Message::user("hi"));
6786 session
6787 .messages
6788 .push(Message::assistant("first answer", None));
6789 session.messages.push(Message::user("again"));
6790 session
6791 .messages
6792 .push(Message::assistant("final answer", None));
6793
6794 assert_eq!(
6795 child_final_assistant_text(&session).as_deref(),
6796 Some("final answer")
6797 );
6798 }
6799
6800 #[test]
6801 fn child_final_assistant_text_returns_none_when_blank() {
6802 let mut session = Session::new("child-1", "gpt-4");
6803 session.messages.push(Message::assistant(" ", None));
6804 assert!(child_final_assistant_text(&session).is_none());
6805 }
6806
6807 #[test]
6808 fn child_final_assistant_text_returns_none_when_no_assistant() {
6809 let mut session = Session::new("child-1", "gpt-4");
6810 session.messages.push(Message::user("hi"));
6811 assert!(child_final_assistant_text(&session).is_none());
6812 }
6813
6814 #[test]
6815 fn runtime_resume_message_folds_full_response_without_truncation() {
6816 let completion = make_completion("completed");
6819 let long: String = "a".repeat(10_000);
6820 let message = runtime_resume_message(&completion, 0, Some(&long));
6821 assert!(message.content.contains(&long));
6822 assert!(!message.content.contains("truncated"));
6823 }
6824
6825 #[test]
6826 fn runtime_resume_message_includes_child_response_when_provided() {
6827 let completion = make_completion("completed");
6828 let message = runtime_resume_message(&completion, 0, Some("the answer is 42"));
6829
6830 assert!(matches!(message.role, Role::User));
6831 assert!(!message.never_compress);
6834 assert!(message.content.contains("Child final response:"));
6835 assert!(message.content.contains("the answer is 42"));
6836
6837 let metadata = message.metadata.expect("metadata present");
6838 assert_eq!(
6839 metadata.get("hidden_from_ui").and_then(|v| v.as_bool()),
6840 Some(true)
6841 );
6842 assert_eq!(
6843 metadata.get("runtime_kind").and_then(|v| v.as_str()),
6844 Some("child_completion_resume")
6845 );
6846 assert_eq!(
6847 metadata
6848 .get("child_final_response_included")
6849 .and_then(|v| v.as_bool()),
6850 Some(true)
6851 );
6852 }
6853
6854 #[test]
6855 fn runtime_resume_message_falls_back_to_error_when_no_response() {
6856 let mut completion = make_completion("error");
6857 completion.error = Some("boom".to_string());
6858
6859 let message = runtime_resume_message(&completion, 1, None);
6860 assert!(message.content.contains("Child error:"));
6861 assert!(message.content.contains("boom"));
6862 let metadata = message.metadata.expect("metadata present");
6863 assert_eq!(
6864 metadata
6865 .get("child_final_response_included")
6866 .and_then(|v| v.as_bool()),
6867 Some(false)
6868 );
6869 }
6870
6871 #[test]
6872 fn runtime_resume_message_minimal_when_no_response_and_no_error() {
6873 let completion = make_completion("completed");
6874 let message = runtime_resume_message(&completion, 2, None);
6875 assert!(!message.content.contains("Child final response:"));
6876 assert!(!message.content.contains("Child error:"));
6877 assert!(message.content.contains("Resume the parent task"));
6878 }
6879
6880 #[test]
6881 fn read_config_snapshot_refreshes_cached_snapshot_from_live_config() {
6882 let runtime = tokio::runtime::Runtime::new().expect("runtime");
6883
6884 runtime.block_on(async {
6885 let config = Arc::new(RwLock::new(Config::default()));
6886 config.write().await.provider = "copilot".to_string();
6887 let cached_config = StdRwLock::new(Config::default());
6888
6889 let snapshot = read_config_snapshot(&config, &cached_config);
6890
6891 assert_eq!(snapshot.provider, "copilot");
6892 assert_eq!(
6893 cached_config.read().expect("cached snapshot lock").provider,
6894 "copilot"
6895 );
6896 });
6897 }
6898
6899 #[test]
6900 fn read_config_snapshot_uses_cached_snapshot_when_live_lock_is_busy() {
6901 let runtime = tokio::runtime::Runtime::new().expect("runtime");
6902
6903 runtime.block_on(async {
6904 let mut cached_snapshot = Config::default();
6905 cached_snapshot.provider = "cached-provider".to_string();
6906
6907 let config = Arc::new(RwLock::new(Config::default()));
6908 let cached_config = StdRwLock::new(cached_snapshot);
6909 let _write_guard = config.write().await;
6910
6911 let snapshot = read_config_snapshot(&config, &cached_config);
6912
6913 assert_eq!(snapshot.provider, "cached-provider");
6914 });
6915 }
6916
6917 #[test]
6920 fn bash_completion_resume_message_normal_announces_completion() {
6921 let ids = vec!["bg-1".to_string(), "bg-2".to_string()];
6922 let message = bash_completion_resume_message(&ids, false);
6923 assert!(
6925 message.content.contains("have completed"),
6926 "normal resume message must announce completion: {}",
6927 message.content
6928 );
6929 let metadata = message.metadata.expect("metadata present");
6931 assert_eq!(
6932 metadata
6933 .get(RUNTIME_RESUME_MESSAGE_HIDDEN_KEY)
6934 .and_then(|v| v.as_bool()),
6935 Some(true),
6936 "resume message must be hidden from the UI"
6937 );
6938 assert_eq!(
6939 metadata
6940 .get(RUNTIME_RESUME_MESSAGE_KIND_KEY)
6941 .and_then(|v| v.as_str()),
6942 Some(BASH_COMPLETION_RESUME_KIND),
6943 "resume message must carry the bash-completion kind discriminant"
6944 );
6945 }
6946
6947 #[test]
6948 fn bash_completion_resume_message_deadline_does_not_claim_completion() {
6949 let ids = vec!["bg-long".to_string()];
6953 let message = bash_completion_resume_message(&ids, true);
6954 assert!(
6955 !message.content.contains("have completed"),
6956 "deadline resume message must NOT claim the shells completed: {}",
6957 message.content
6958 );
6959 assert!(
6960 message.content.contains("may still be running"),
6961 "deadline resume message must warn shells may still be running: {}",
6962 message.content
6963 );
6964 assert!(
6965 message.content.contains("BashOutput"),
6966 "deadline resume message must direct verification via BashOutput: {}",
6967 message.content
6968 );
6969 let metadata = message.metadata.expect("metadata present");
6971 assert_eq!(
6972 metadata
6973 .get(RUNTIME_RESUME_MESSAGE_KIND_KEY)
6974 .and_then(|v| v.as_str()),
6975 Some(BASH_COMPLETION_RESUME_KIND)
6976 );
6977 }
6978
6979 #[test]
6980 fn bash_resume_should_retry_matrix() {
6981 assert!(!bash_resume_should_retry(
6987 &ResumeOutcome::Started { run_id: "r".into() },
6988 true
6989 ));
6990 assert!(!bash_resume_should_retry(
6991 &ResumeOutcome::Started { run_id: "r".into() },
6992 false
6993 ));
6994
6995 assert!(!bash_resume_should_retry(&ResumeOutcome::NotFound, true));
6997 assert!(!bash_resume_should_retry(&ResumeOutcome::NotFound, false));
6998
6999 assert!(bash_resume_should_retry(&ResumeOutcome::Completed, true));
7001 assert!(!bash_resume_should_retry(&ResumeOutcome::Completed, false));
7004
7005 assert!(bash_resume_should_retry(
7008 &ResumeOutcome::AlreadyRunning { run_id: "r".into() },
7009 true
7010 ));
7011 assert!(!bash_resume_should_retry(
7013 &ResumeOutcome::AlreadyRunning { run_id: "r".into() },
7014 false
7015 ));
7016 }
7017
7018 #[test]
7021 fn injection_body_includes_status_exit_command_and_tail() {
7022 let info = BashCompletionInfo {
7023 session_id: "s".into(),
7024 bash_id: "abc123".into(),
7025 command: "make build".into(),
7026 exit_code: Some(0),
7027 status: "completed".into(),
7028 output_tail: "BUILD OK".into(),
7029 };
7030 let body = bash_completion_injection_body(&info);
7031 assert!(body.contains("abc123"), "body: {body}");
7032 assert!(body.contains("make build"), "body: {body}");
7033 assert!(body.contains("completed"), "body: {body}");
7034 assert!(body.contains("exit code 0"), "body: {body}");
7035 assert!(body.contains("BUILD OK"), "body: {body}");
7036 assert!(body.contains("BashOutput"), "body: {body}");
7038 assert!(body.contains("bash_id=\"abc123\""), "body: {body}");
7039 }
7040
7041 #[test]
7042 fn injection_body_handles_no_output_and_signal_kill() {
7043 let info = BashCompletionInfo {
7044 session_id: "s".into(),
7045 bash_id: "xyz".into(),
7046 command: "sleep 99".into(),
7047 exit_code: None,
7048 status: "killed".into(),
7049 output_tail: String::new(),
7050 };
7051 let body = bash_completion_injection_body(&info);
7052 assert!(body.contains("killed"), "body: {body}");
7053 assert!(body.contains("none (signal/killed)"), "body: {body}");
7054 assert!(body.contains("no captured output"), "body: {body}");
7055 assert!(!body.contains("Output tail:"), "body: {body}");
7057 }
7058
7059 #[test]
7060 fn background_completion_builds_post_tool_use_payload_and_feedback() {
7061 let mut info = BashCompletionInfo {
7062 session_id: "s".into(),
7063 bash_id: "bg-7".into(),
7064 command: "cargo test".into(),
7065 exit_code: Some(0),
7066 status: "completed".into(),
7067 output_tail: "test result: ok".into(),
7068 };
7069
7070 let payload = background_bash_post_tool_payload(&info);
7071 match payload {
7072 HookPayload::ToolResult {
7073 tool_name,
7074 tool_call_id,
7075 outcome,
7076 } => {
7077 assert_eq!(tool_name, "Bash");
7078 assert_eq!(tool_call_id, "bg-7");
7079 assert!(outcome.success);
7080 let response: serde_json::Value =
7081 serde_json::from_str(outcome.result.as_deref().unwrap()).unwrap();
7082 assert_eq!(response["command"], "cargo test");
7083 assert_eq!(response["exit_code"], 0);
7084 assert_eq!(response["status"], "completed");
7085 assert_eq!(response["output_tail"], "test result: ok");
7086 }
7087 other => panic!("expected PostToolUse payload, got {other:?}"),
7088 }
7089
7090 append_background_bash_hook_feedback(
7091 &mut info,
7092 vec!["Run the formatter before continuing".to_string()],
7093 );
7094 assert!(info.output_tail.contains("<post_tool_use_feedback>"));
7095 assert!(info
7096 .output_tail
7097 .contains("Run the formatter before continuing"));
7098 }
7099
7100 #[tokio::test]
7101 async fn bash_completion_payload_identity_matches_file_inbox_idempotency() {
7102 let baseline = BashCompletionInfo {
7103 session_id: "session".into(),
7104 bash_id: "bg-7".into(),
7105 command: "cargo test".into(),
7106 exit_code: Some(0),
7107 status: "completed".into(),
7108 output_tail: "first tail".into(),
7109 };
7110 let mut retried = baseline.clone();
7111 retried.output_tail = "first tail\nlater bytes\n<hook feedback>".into();
7112 retried.status = "completed-after-hook".into();
7113 let baseline_envelope = bash_completion_envelope(&baseline);
7114 let exact_retry = bash_completion_envelope(&baseline);
7115 let corrected_envelope = bash_completion_envelope(&retried);
7116 assert_eq!(baseline_envelope.id, exact_retry.id);
7117 assert_ne!(
7118 baseline_envelope.id, corrected_envelope.id,
7119 "changed payload semantics must receive a distinct id"
7120 );
7121
7122 let mut other_shell = baseline.clone();
7123 other_shell.bash_id = "bg-8".into();
7124 assert_ne!(
7125 bash_completion_envelope(&baseline).id,
7126 bash_completion_envelope(&other_shell).id
7127 );
7128
7129 let temp = tempfile::tempdir().unwrap();
7130 let store = Arc::new(
7131 bamboo_storage::SessionStoreV2::new(temp.path().to_path_buf())
7132 .await
7133 .unwrap(),
7134 );
7135 store
7136 .save_session(&Session::new("session", "model"))
7137 .await
7138 .unwrap();
7139 let inbox = bamboo_storage::FileSessionInbox::new(
7140 store,
7141 bamboo_domain::SessionInboxLimits::default(),
7142 );
7143 let first = inbox.deliver(&baseline_envelope).await.unwrap();
7144 let duplicate = inbox.deliver(&exact_retry).await.unwrap();
7145 let corrected = inbox.deliver(&corrected_envelope).await.unwrap();
7146 assert_eq!(duplicate, first, "exact payload retry is idempotent");
7147 assert_ne!(corrected.id, first.id);
7148 assert_eq!(corrected.generation, first.generation + 1);
7149 assert_eq!(inbox.inspect("session").await.unwrap().pending, 2);
7150 }
7151
7152 #[cfg(unix)]
7153 #[tokio::test]
7154 async fn background_completion_fires_configured_post_tool_use_command() {
7155 use bamboo_config::{
7156 LifecycleHookGroup, LifecycleHookHandler, LifecycleHooksConfig,
7157 DEFAULT_LIFECYCLE_HOOK_TIMEOUT_MS,
7158 };
7159
7160 let dir = tempfile::tempdir().unwrap();
7161 let output = dir.path().join("background-post-tool.json");
7162 let command = format!(
7163 "cat > '{}'; printf '%s' '{{\"additional_context\":\"inspect the completed build log\"}}'",
7164 output.display()
7165 );
7166 let config = LifecycleHooksConfig {
7167 enabled: true,
7168 post_tool_use: vec![LifecycleHookGroup {
7169 enabled: true,
7170 matcher: Some("^Bash$".to_string()),
7171 hooks: vec![LifecycleHookHandler::command(
7172 command,
7173 DEFAULT_LIFECYCLE_HOOK_TIMEOUT_MS,
7174 )],
7175 }],
7176 ..Default::default()
7177 };
7178 let mut session = Session::new("session-bg-hook", "test-model");
7179 session.workspace = Some(dir.path().to_string_lossy().into_owned());
7180 let mut info = BashCompletionInfo {
7181 session_id: session.id.clone(),
7182 bash_id: "bg-9".into(),
7183 command: "cargo test".into(),
7184 exit_code: Some(0),
7185 status: "completed".into(),
7186 output_tail: "test result: ok".into(),
7187 };
7188
7189 assert!(run_background_bash_post_tool_hooks(&config, None, &session, &mut info).await);
7190 let envelope: serde_json::Value =
7191 serde_json::from_str(&std::fs::read_to_string(output).unwrap()).unwrap();
7192 assert_eq!(envelope["hook_event_name"], "PostToolUse");
7193 assert_eq!(envelope["tool_name"], "Bash");
7194 assert_eq!(envelope["payload"]["tool_call_id"], "bg-9");
7195 let response = envelope["tool_response"]["result"]
7196 .as_str()
7197 .map(serde_json::from_str::<serde_json::Value>)
7198 .transpose()
7199 .unwrap()
7200 .unwrap();
7201 assert_eq!(response["command"], "cargo test");
7202 assert_eq!(response["status"], "completed");
7203 assert!(info.output_tail.contains("inspect the completed build log"));
7204 }
7205
7206 async fn temp_store() -> (tempfile::TempDir, Arc<dyn Storage>, LockedSessionStore) {
7207 let temp = tempfile::tempdir().unwrap();
7208 let storage: Arc<dyn Storage> = Arc::new(
7209 bamboo_storage::v2::SessionStoreV2::new(temp.path().to_path_buf())
7210 .await
7211 .expect("storage init"),
7212 );
7213 let persistence = LockedSessionStore::new(storage.clone());
7214 (temp, storage, persistence)
7215 }
7216
7217 #[tokio::test]
7218 async fn enqueue_writes_pending_injection_and_preserves_messages() {
7219 let (_temp, storage, persistence) = temp_store().await;
7220
7221 let mut session = Session::new("sess-enq", "test-model");
7222 session.add_message(Message::user("do the build"));
7223 storage.save_session(&session).await.unwrap();
7224
7225 let info = BashCompletionInfo {
7226 session_id: "sess-enq".into(),
7227 bash_id: "sh-1".into(),
7228 command: "make".into(),
7229 exit_code: Some(0),
7230 status: "completed".into(),
7231 output_tail: "done".into(),
7232 };
7233 let saved = enqueue_bash_completion_injection(&persistence, &info)
7234 .await
7235 .expect("enqueue io ok")
7236 .expect("session exists");
7237
7238 let pending = saved
7239 .pending_injected_messages()
7240 .expect("pending injection present");
7241 assert_eq!(pending.len(), 1);
7242 let content = pending[0].get("content").and_then(|v| v.as_str()).unwrap();
7243 assert!(content.contains("sh-1"), "content: {content}");
7244 assert!(content.contains("make"), "content: {content}");
7245 assert!(content.contains("done"), "content: {content}");
7246 assert_eq!(saved.messages.len(), 1);
7248 }
7249
7250 #[tokio::test]
7251 async fn enqueue_returns_none_for_missing_session() {
7252 let (_temp, _storage, persistence) = temp_store().await;
7253 let info = BashCompletionInfo {
7254 session_id: "does-not-exist".into(),
7255 bash_id: "x".into(),
7256 command: "true".into(),
7257 exit_code: Some(0),
7258 status: "completed".into(),
7259 output_tail: String::new(),
7260 };
7261 let result = enqueue_bash_completion_injection(&persistence, &info)
7262 .await
7263 .expect("io ok");
7264 assert!(result.is_none(), "no session → nothing enqueued");
7265 }
7266
7267 #[test]
7275 fn apply_bash_resume_transition_clears_wait_and_appends_message() {
7276 use bamboo_domain::session::runtime_state::WaitingForBashState;
7277
7278 let mut session = Session::new("sess-resume", "test-model");
7279 session.add_message(Message::user("kick off the build"));
7280 let mut rt = read_runtime_state(&session);
7281 rt.status = AgentStatusState::Running;
7282 rt.waiting_for_bash = Some(WaitingForBashState::for_bash(
7283 vec!["sh-1".into()],
7284 Utc::now(),
7285 ));
7286 write_runtime_state(&mut session, &rt);
7287 session.metadata.insert(
7288 "runtime.suspend_reason".to_string(),
7289 "waiting_for_bash".to_string(),
7290 );
7291
7292 let resume = bash_completion_resume_message(&["sh-1".to_string()], false);
7293 let did = apply_bash_resume_transition(&mut session, &resume);
7294
7295 assert!(did, "a suspended session must transition");
7296 let after = read_runtime_state(&session);
7297 assert!(
7298 after.waiting_for_bash.is_none(),
7299 "bash wait must be cleared"
7300 );
7301 assert_eq!(after.status, AgentStatusState::Idle, "runtime must be Idle");
7302 assert!(
7303 !session.metadata.contains_key("runtime.suspend_reason"),
7304 "suspend-reason marker must be removed"
7305 );
7306 assert_eq!(session.messages.len(), 2, "resume message must be appended");
7307 assert!(matches!(
7308 session.messages.last().map(|m| &m.role),
7309 Some(Role::User)
7310 ));
7311 }
7312
7313 #[test]
7317 fn apply_bash_resume_transition_noops_when_not_waiting() {
7318 let mut session = Session::new("sess-live", "test-model");
7319 session.add_message(Message::user("hi"));
7320
7321 let resume = bash_completion_resume_message(&["sh-1".to_string()], false);
7322 let did = apply_bash_resume_transition(&mut session, &resume);
7323
7324 assert!(!did, "a non-waiting session must not transition");
7325 assert_eq!(session.messages.len(), 1, "no resume message appended");
7326 }
7327
7328 #[test]
7332 fn bash_completion_should_resume_only_when_suspended_and_all_done() {
7333 assert!(bash_completion_should_resume(true, true));
7334 assert!(!bash_completion_should_resume(true, false)); assert!(!bash_completion_should_resume(false, true)); assert!(!bash_completion_should_resume(false, false));
7337 }
7338
7339 #[test]
7340 fn two_shell_delivery_stages_backlog_before_one_final_activation() {
7341 let mut waiting = true;
7342 let mut durable_backlog = 0;
7343 let mut reservations = 0;
7344
7345 durable_backlog += 1;
7347 let first = bash_completion_delivery_plan(waiting, false);
7348 assert_eq!(first, BashCompletionDeliveryPlan::DurableOnly);
7349 assert!(waiting);
7350 assert_eq!(durable_backlog, 1);
7351 assert_eq!(reservations, 0);
7352
7353 durable_backlog += 1;
7356 let last = bash_completion_delivery_plan(waiting, true);
7357 assert_eq!(last, BashCompletionDeliveryPlan::ClearWaitThenActivate);
7358 waiting = false;
7359 reservations += 1;
7360 assert!(!waiting);
7361 assert_eq!(durable_backlog, 2);
7362 assert_eq!(reservations, 1);
7363 }
7364
7365 #[test]
7369 fn bash_resume_message_from_info_carries_bashid_tail_and_kind() {
7370 let info = BashCompletionInfo {
7371 session_id: "s".into(),
7372 bash_id: "sh-42".into(),
7373 command: "cargo test".into(),
7374 exit_code: Some(0),
7375 status: "completed".into(),
7376 output_tail: "test result: ok".into(),
7377 };
7378 let msg = bash_resume_message_from_info(&info);
7379
7380 assert!(matches!(msg.role, Role::User));
7381 assert!(msg.content.contains("sh-42"), "content: {}", msg.content);
7382 assert!(
7383 msg.content.contains("cargo test"),
7384 "content: {}",
7385 msg.content
7386 );
7387 assert!(
7388 msg.content.contains("test result: ok"),
7389 "content: {}",
7390 msg.content
7391 );
7392 assert!(
7393 msg.content.contains("BashOutput"),
7394 "content: {}",
7395 msg.content
7396 );
7397 let meta = serde_json::to_string(&msg.metadata).unwrap();
7398 assert!(
7399 meta.contains(BASH_COMPLETION_RESUME_KIND),
7400 "resume message must be tagged as a bash-completion resume: {meta}"
7401 );
7402 }
7403}