1use std::collections::HashMap;
8use std::sync::{Arc, Mutex as StdMutex, OnceLock, RwLock as StdRwLock, Weak};
9use std::time::Duration;
10
11use bamboo_domain::poison::PoisonRecover;
12use bamboo_domain::{
13 AgentHookPoint, HookPayload, HookToolOutcome, SessionActivationPolicy, SessionChildOutcome,
14 SessionMessageBody, SessionMessageContent, SessionMessageEnvelope, SessionMessageId,
15 SessionMessageKind, SessionMessageSource, SessionProviderMessage,
16};
17
18use crate::execution::{
19 finalize_runner, reserve_runner_core, reserve_session_execution, spawn_session_execution,
20 AgentRunner, AgentStatus, ChildCompletion, ChildCompletionHandler, ChildCompletionSource,
21 ReserveOutcome, SessionExecutionArgs, SessionExecutionReservation,
22 SessionExecutionReserveOutcome, SpawnJob, SpawnScheduler,
23};
24use crate::runtime::config::{BashResumeHook, GuardianSpawner, BASH_COMPLETION_RESUME_KIND};
25use crate::runtime::guardian_state::{
26 parse_guardian_verdict, read_guardian_config, read_guardian_state, write_guardian_state,
27 GuardianVerdict,
28};
29use crate::Agent;
30use async_trait::async_trait;
31use bamboo_agent_core::storage::Storage;
32use bamboo_agent_core::tools::ToolExecutor;
33use bamboo_agent_core::{
34 AgentEvent, BashCompletionInfo, BashCompletionSink, Message, Role, Session, SessionKind,
35};
36use bamboo_domain::session::runtime_state::{
37 AgentRuntimeState, AgentStatusState, ChildWaitPolicy, WaitingForChildrenState,
38};
39use bamboo_llm::{Config, ProviderModelRouter, ProviderRegistry};
40use bamboo_storage::LockedSessionStore;
41use chrono::Utc;
42use sha2::{Digest, Sha256};
43use tokio::sync::{broadcast, RwLock};
44
45use crate::model_areas::resolve_global_area_models;
46use crate::model_config_helper::{
47 resolve_fast_model, resolve_gold_config, resolve_provider_routing_key, GOLD_CONFIG_METADATA_KEY,
48};
49use crate::session_activation::{
50 SessionActivationLaunch, SessionActivationReserveOutcome, SessionActivationSpawner,
51};
52use crate::session_app::child_session::result_projection::{
53 child_result_identity, decode_report, fits_child_result_budget, plain_child_report_message,
54 project_child_report, unavailable_child_result,
55};
56use crate::session_app::execute::consume_pending_clarification_resume;
57use crate::session_app::provider_model::{persist_model_ref, session_effective_model_ref};
58use crate::session_app::resume::{
59 resume_session_execution, ResumeExecutionPort, ResumeSpawnRequest,
60};
61use crate::session_app::types::{ResumeConfigSnapshot, ResumeOutcome};
62
63const AGENT_RUNTIME_STATE_METADATA_KEY: &str = "agent.runtime.state";
64const RUNTIME_RESUME_MESSAGE_HIDDEN_KEY: &str = "hidden_from_ui";
65const RUNTIME_RESUME_MESSAGE_KIND_KEY: &str = "runtime_kind";
66const CHILD_COMPLETION_INLINE_FIELD_BYTES: usize = 48 * 1024;
67const CHILD_COMPLETION_OVERSIZE_TAIL_BYTES: usize = 8 * 1024;
68
69fn read_runtime_state(session: &Session) -> AgentRuntimeState {
70 session
71 .agent_runtime_state
72 .clone()
73 .or_else(|| {
74 session
75 .metadata
76 .get(AGENT_RUNTIME_STATE_METADATA_KEY)
77 .and_then(|raw| serde_json::from_str::<AgentRuntimeState>(raw).ok())
78 })
79 .unwrap_or_else(|| AgentRuntimeState::new(format!("{}-child-wait", session.id)))
80}
81
82fn write_runtime_state(session: &mut Session, runtime_state: &AgentRuntimeState) {
83 session.agent_runtime_state = Some(runtime_state.clone());
84 if let Ok(serialized) = serde_json::to_string(runtime_state) {
85 session
86 .metadata
87 .insert(AGENT_RUNTIME_STATE_METADATA_KEY.to_string(), serialized);
88 }
89}
90
91async fn prepare_session_inbox_activation(
97 persistence: &LockedSessionStore,
98 session_id: &str,
99 interrupt_specific_wait: bool,
100) -> std::io::Result<Option<(Session, bool)>> {
101 let ready = Arc::new(std::sync::atomic::AtomicBool::new(false));
102 let ready_for_mutation = ready.clone();
103 let saved = persistence
104 .update_runtime_config(session_id, move |latest| {
105 if latest.has_pending_question() {
109 return;
110 }
111 let mut runtime_state = read_runtime_state(latest);
112 let specifically_waiting = runtime_state.waiting_for_children.is_some()
113 || runtime_state.waiting_for_bash.is_some();
114 if specifically_waiting && !interrupt_specific_wait {
115 return;
116 }
117 runtime_state.status = AgentStatusState::Idle;
123 runtime_state.suspension = None;
124 write_runtime_state(latest, &runtime_state);
125 latest.metadata.remove("runtime.suspend_reason");
126 latest.updated_at = Utc::now();
127 ready_for_mutation.store(true, std::sync::atomic::Ordering::Release);
128 })
129 .await?;
130 Ok(saved.map(|session| (session, ready.load(std::sync::atomic::Ordering::Acquire))))
131}
132
133fn is_error_like(status: &str) -> bool {
134 matches!(status, "error" | "timeout" | "cancelled")
135}
136
137fn is_terminal_child_status(status: &str) -> bool {
139 matches!(
140 status,
141 "completed" | "error" | "timeout" | "cancelled" | "skipped"
142 )
143}
144
145fn child_completion_envelope(
146 completion: &ChildCompletion,
147 wait_registered_at: chrono::DateTime<Utc>,
148 result: Option<String>,
149 provider_message: &Message,
150) -> SessionMessageEnvelope {
151 let envelope = build_child_completion_envelope(
152 completion,
153 wait_registered_at,
154 result.as_deref(),
155 provider_message,
156 false,
157 );
158 if serde_json::to_vec(&envelope).map_or(true, |bytes| {
159 bytes.len() > bamboo_domain::SessionInboxLimits::default().max_payload_bytes
160 }) {
161 return build_child_completion_envelope(
166 completion,
167 wait_registered_at,
168 result.as_deref(),
169 provider_message,
170 true,
171 );
172 }
173 envelope
174}
175
176fn child_completion_id(
177 completion: &ChildCompletion,
178 wait_registered_at: chrono::DateTime<Utc>,
179 result: Option<&str>,
180 legacy_wait_identity: bool,
181) -> SessionMessageId {
182 fn identity(value: Option<&str>) -> serde_json::Value {
183 match value {
184 Some(value) if value.len() > CHILD_COMPLETION_INLINE_FIELD_BYTES => serde_json::json!({
185 "oversized": true, "utf8_bytes": value.len(),
186 "sha256": hex::encode(Sha256::digest(value.as_bytes())),
187 }),
188 value => serde_json::json!(value),
189 }
190 }
191 SessionMessageId::stable(
192 "session_child_completion",
193 &serde_json::json!({
194 "parent_session_id": completion.parent_session_id,
195 "child_session_id": completion.child_session_id,
196 "status": completion.status,
197 "error": identity(completion.error.as_deref()), "result": identity(result),
198 "wait_registered_at": (legacy_wait_identity || completion.source.is_none()).then_some(wait_registered_at),
199 "terminal_source": completion.source,
200 }),
201 )
202}
203
204fn build_child_completion_envelope(
205 completion: &ChildCompletion,
206 wait_registered_at: chrono::DateTime<Utc>,
207 result: Option<&str>,
208 provider_message: &Message,
209 compact: bool,
210) -> SessionMessageEnvelope {
211 fn bounded_terminal_field(
212 label: &str,
213 child_session_id: &str,
214 value: Option<&str>,
215 compact: bool,
216 ) -> (Option<String>, serde_json::Value) {
217 let Some(value) = value else {
218 return (None, serde_json::Value::Null);
219 };
220 let oversized = value.len() > CHILD_COMPLETION_INLINE_FIELD_BYTES;
221 let identity = if oversized {
222 serde_json::json!({
223 "oversized": true,
224 "utf8_bytes": value.len(),
225 "sha256": hex::encode(Sha256::digest(value.as_bytes())),
226 })
227 } else {
228 serde_json::Value::String(value.to_string())
229 };
230 if !oversized && !compact {
231 return (Some(value.to_string()), identity);
232 }
233 let digest = hex::encode(Sha256::digest(value.as_bytes()));
234 let retrieval_view = if label == "error" {
235 "diagnostics"
236 } else {
237 "result"
238 };
239 let inspection = serde_json::json!({
240 "intent": "inspect", "target": child_session_id, "message": retrieval_view,
241 });
242 let recovery = if label == "error" {
243 format!(
244 "Check observed status and recovery diagnostics by calling SubAgent with {inspection}. \
245 Detailed execution errors remain available to the authenticated inspector."
246 )
247 } else {
248 format!(
249 "Read it in bounded slices by calling SubAgent with {inspection}; use message=\"messages\" for transcript previews. \
250 Follow returned cursors with a JSON-encoded view/cursor query in message, omitting unused query fields."
251 )
252 };
253 let mut summary = format!(
254 "Child {label} could not fit in the durable inline envelope ({} UTF-8 bytes, sha256={digest}). {recovery}",
255 value.len()
256 );
257 if !compact {
258 let mut tail_start = value
259 .len()
260 .saturating_sub(CHILD_COMPLETION_OVERSIZE_TAIL_BYTES);
261 while tail_start < value.len() && !value.is_char_boundary(tail_start) {
262 tail_start += 1;
263 }
264 summary.push_str("\n\nBounded tail:\n");
265 summary.push_str(&value[tail_start..]);
266 }
267 (Some(summary), identity)
268 }
269
270 let (stored_result, _) =
271 bounded_terminal_field("result", &completion.child_session_id, result, compact);
272 let (stored_error, _) = bounded_terminal_field(
273 "error",
274 &completion.child_session_id,
275 completion.error.as_deref(),
276 compact,
277 );
278 let mut bounded_provider_message = provider_message.clone();
279 let provider_oversized = serde_json::to_vec(provider_message)
280 .map(|bytes| bytes.len() > CHILD_COMPLETION_INLINE_FIELD_BYTES)
281 .unwrap_or(true);
282 if provider_oversized || compact {
283 let mut content = if provider_oversized {
287 format!(
288 "Runtime notification: child session `{}` finished with status `{}`.",
289 completion.child_session_id, completion.status
290 )
291 } else {
292 bounded_provider_message.content.clone()
293 };
294 if let Some(result) = stored_result.as_deref() {
295 content.push_str("\n\n");
296 content.push_str(result);
297 }
298 if let Some(error) = stored_error.as_deref() {
299 content.push_str("\n\n");
300 content.push_str(error);
301 }
302 bounded_provider_message.content = content;
303 if provider_oversized {
304 bounded_provider_message.content_parts = None;
305 }
306 if compact {
310 let original = bounded_provider_message
311 .metadata
312 .as_ref()
313 .and_then(serde_json::Value::as_object);
314 let mut metadata = serde_json::Map::new();
315 for key in [
316 RUNTIME_RESUME_MESSAGE_HIDDEN_KEY,
317 RUNTIME_RESUME_MESSAGE_KIND_KEY,
318 "child_session_id",
319 "child_status",
320 "child_final_response_included",
321 "child_completion_source",
322 "child_completion_source_sha256",
323 "guardian_approved",
324 ] {
325 if let Some(value) = original.and_then(|metadata| metadata.get(key)) {
326 metadata.insert(key.to_string(), value.clone());
327 }
328 }
329 if original.is_some_and(|metadata| metadata.contains_key("child_error")) {
330 metadata.insert(
331 "child_error".to_string(),
332 stored_error
333 .as_ref()
334 .map_or(serde_json::Value::Null, |error| {
335 serde_json::Value::String(error.clone())
336 }),
337 );
338 }
339 bounded_provider_message.metadata = Some(serde_json::Value::Object(metadata));
340 } else if let Some(child_error) = bounded_provider_message
341 .metadata
342 .as_mut()
343 .and_then(serde_json::Value::as_object_mut)
344 .and_then(|metadata| metadata.get_mut("child_error"))
345 {
346 *child_error = stored_error
347 .as_ref()
348 .map_or(serde_json::Value::Null, |error| {
349 serde_json::Value::String(error.clone())
350 });
351 }
352 }
353 let body = SessionMessageBody::ChildOutcome(SessionChildOutcome {
354 child_session_id: completion.child_session_id.clone(),
355 status: completion.status.clone(),
356 result: stored_result,
357 error: stored_error,
358 provider_message: Some(session_provider_message(&bounded_provider_message)),
359 });
360 SessionMessageEnvelope {
361 id: child_completion_id(completion, wait_registered_at, result, false),
364 source: SessionMessageSource::Runtime {
365 subsystem: "child_completion_coordinator".to_string(),
366 },
367 target_session_id: completion.parent_session_id.clone(),
368 kind: SessionMessageKind::ChildOutcome,
369 body,
370 created_at: completion.completed_at,
371 thread_id: None,
372 in_reply_to: None,
373 attempt: None,
374 correlation_id: Some(format!("child_completion:{}", completion.child_session_id)),
375 }
376}
377
378fn session_provider_message(message: &Message) -> SessionProviderMessage {
379 SessionProviderMessage {
380 content: SessionMessageContent {
381 text: message.content.clone(),
382 parts: message.content_parts.clone().unwrap_or_default(),
383 skill_request: None,
384 },
385 metadata: message
386 .metadata
387 .as_ref()
388 .and_then(serde_json::Value::as_object)
389 .cloned()
390 .unwrap_or_default(),
391 never_compress: message.never_compress,
392 }
393}
394
395async fn derive_completed_child_ids(
400 storage: &Arc<dyn Storage>,
401 parent_session_id: &str,
402 just_completed_child_id: &str,
403) -> Vec<String> {
404 let mut completed: Vec<String> = storage
405 .list_child_run_statuses(parent_session_id)
406 .await
407 .unwrap_or_default()
408 .into_iter()
409 .filter(|(_, status)| status.as_deref().is_some_and(is_terminal_child_status))
410 .map(|(id, _)| id)
411 .collect();
412 if !completed.iter().any(|id| id == just_completed_child_id) {
413 completed.push(just_completed_child_id.to_string());
414 }
415 completed.sort();
416 completed.dedup();
417 completed
418}
419
420fn read_config_snapshot(config: &Arc<RwLock<Config>>, cached_config: &StdRwLock<Config>) -> Config {
421 if let Ok(config_guard) = config.try_read() {
422 let snapshot = config_guard.clone();
423
424 if let Ok(mut cached_guard) = cached_config.try_write() {
425 *cached_guard = snapshot.clone();
426 }
427
428 snapshot
429 } else {
430 cached_config
431 .try_read()
432 .map(|guard| guard.clone())
433 .unwrap_or_default()
434 }
435}
436
437fn parent_locks() -> &'static std::sync::Mutex<HashMap<String, Arc<tokio::sync::Mutex<()>>>> {
453 static LOCKS: OnceLock<std::sync::Mutex<HashMap<String, Arc<tokio::sync::Mutex<()>>>>> =
454 OnceLock::new();
455 LOCKS.get_or_init(|| std::sync::Mutex::new(HashMap::new()))
456}
457
458pub(crate) fn session_resume_lock(session_id: &str) -> SessionResumeLock {
465 let mut map = parent_locks().lock().recover_poison();
466 let lock = map
467 .entry(session_id.to_string())
468 .or_insert_with(|| Arc::new(tokio::sync::Mutex::new(())))
469 .clone();
470 SessionResumeLock {
471 session_id: session_id.to_string(),
472 lock: Some(lock),
473 }
474}
475
476pub(crate) struct SessionResumeLock {
480 session_id: String,
481 lock: Option<Arc<tokio::sync::Mutex<()>>>,
482}
483
484impl std::ops::Deref for SessionResumeLock {
485 type Target = tokio::sync::Mutex<()>;
486 fn deref(&self) -> &Self::Target {
487 self.lock.as_ref().expect("live resume-lock lease")
488 }
489}
490
491impl Drop for SessionResumeLock {
492 fn drop(&mut self) {
493 self.lock.take();
494 let mut map = parent_locks().lock().recover_poison();
495 if map
496 .get(&self.session_id)
497 .is_some_and(|lock| Arc::strong_count(lock) == 1)
498 {
499 map.remove(&self.session_id);
500 }
501 }
502}
503
504fn wait_policy_satisfied(
505 policy: ChildWaitPolicy,
506 wait_child_ids: &[String],
507 completed_child_ids: &[String],
508 latest_child_id: &str,
509 latest_status: &str,
510) -> bool {
511 if wait_child_ids.is_empty() {
512 return false;
513 }
514
515 match policy {
516 ChildWaitPolicy::All => wait_child_ids
517 .iter()
518 .all(|id| completed_child_ids.iter().any(|completed| completed == id)),
519 ChildWaitPolicy::Any => completed_child_ids
520 .iter()
521 .any(|id| wait_child_ids.iter().any(|wait_id| wait_id == id)),
522 ChildWaitPolicy::FirstError => {
523 (is_error_like(latest_status) && wait_child_ids.iter().any(|id| id == latest_child_id))
529 || wait_child_ids
530 .iter()
531 .all(|id| completed_child_ids.iter().any(|completed| completed == id))
532 }
533 }
534}
535
536#[cfg(test)]
540fn child_final_assistant_text(child: &Session) -> Option<String> {
541 child
542 .messages
543 .iter()
544 .rev()
545 .find(|message| matches!(message.role, Role::Assistant))
546 .map(|message| message.content.clone())
547 .filter(|content| !content.trim().is_empty())
548}
549
550fn runtime_resume_message(
551 completion: &ChildCompletion,
552 remaining_children: usize,
553 child_final_response: Option<&str>,
554) -> Message {
555 let mut body = format!(
556 "Runtime notification: child session `{}` finished with status `{}`. Remaining child sessions: {}.",
557 completion.child_session_id, completion.status, remaining_children
558 );
559
560 let final_response = child_final_response.map(str::to_string);
566 if let Some(response) = final_response.as_deref() {
567 body.push_str("\n\nChild final response:\n");
568 body.push_str(response);
569 } else if let Some(error) = completion.error.as_deref() {
570 if !error.is_empty() {
571 body.push_str("\n\nChild error:\n");
572 body.push_str(error);
573 }
574 }
575
576 let inspection = serde_json::json!({
577 "intent": "inspect", "target": completion.child_session_id, "message": "messages",
578 });
579 body.push_str(&format!(
580 "\n\nResume the parent task using this child result and continue from the previous plan. \
581 If you need transcript evidence, call SubAgent with {inspection}. \
582 Follow returned cursors with a JSON-encoded view/cursor query in message; use view=\"message\" and message_id inside that query for a selected full message. Omit unused query fields.",
583 ));
584
585 let mut message = Message::user(body);
586 message.metadata = Some(serde_json::json!({
587 RUNTIME_RESUME_MESSAGE_HIDDEN_KEY: true,
588 RUNTIME_RESUME_MESSAGE_KIND_KEY: "child_completion_resume",
589 "child_session_id": completion.child_session_id,
590 "child_status": completion.status,
591 "child_error": completion.error,
592 "child_final_response_included": final_response.is_some(),
593 "child_completion_source": completion.source,
594 }));
595 message.never_compress = false;
599 message
600}
601
602fn sealed_child_report(
606 completion: &ChildCompletion,
607 child: &Session,
608 parent: &Session,
609) -> Option<serde_json::Value> {
610 if child.subagent_type().as_deref() == Some("guardian")
611 || child
612 .metadata
613 .get("lifecycle")
614 .is_some_and(|v| v == "resident")
615 {
616 return None;
617 }
618 let source = completion.source.as_ref()?;
619 if !source.matches_completion(completion)
620 || ChildCompletionSource::from_committed_session(child).as_ref() != Some(source)
621 || !source.matches_parent(child, parent)
622 {
623 return None;
624 }
625 let binding = bamboo_domain::ChildContextBinding::from_session(child).ok()??;
626 let Some(message) = child
627 .messages
628 .iter()
629 .find(|m| Some(&m.id) == source.result_message_id.as_ref())
630 else {
631 let reason = if source.last_assistant.is_some() {
632 "report_not_current_final"
633 } else {
634 "report_absent"
635 };
636 return Some(unavailable_child_result("typed_result", reason));
637 };
638 if !plain_child_report_message(message) {
639 return Some(unavailable_child_result(
640 "typed_result",
641 "report_not_current_final",
642 ));
643 }
644 let report = match decode_report(&message.content) {
645 Ok(report) => report,
646 Err(reason) => return Some(unavailable_child_result("typed_result", reason)),
647 };
648 let mut observation =
649 child_result_identity(parent, child, &binding, source.project_id.as_deref());
650 observation.as_object_mut()?.extend(
651 serde_json::json!({"kind":"committed_terminal_source","terminal_source":source,
652 "last_run_status":source.status})
653 .as_object()?
654 .clone(),
655 );
656 Some(project_child_report(report, observation))
657}
658
659fn child_completion_source_sha256(completion: &ChildCompletion) -> Option<String> {
660 completion.source.as_ref().map(|source| {
661 hex::encode(Sha256::digest(
662 serde_json::to_vec(source).expect("sealed Child source serializes"),
663 ))
664 })
665}
666
667fn typed_child_resume_message(
668 completion: &ChildCompletion,
669 remaining_children: usize,
670 projection: &serde_json::Value,
671) -> Message {
672 let inspection = serde_json::json!({
673 "intent":"inspect", "target":completion.child_session_id, "message":"result",
674 });
675 let mut message = Message::user(format!(
676 "Runtime notification: child session `{}` finished with status `{}`. Remaining child sessions: {}.\n\n\
677 Child typed result:\n{}\n\n\
678 Resume the parent task. Child-reported evidence and verification are claims, not host-certified facts. \
679 For unavailable results or transcript evidence, call SubAgent with {inspection} and follow returned cursors.",
680 completion.child_session_id, completion.status, remaining_children, projection,
681 ));
682 message.metadata = Some(serde_json::json!({
683 RUNTIME_RESUME_MESSAGE_HIDDEN_KEY:true,
684 RUNTIME_RESUME_MESSAGE_KIND_KEY:"child_completion_resume",
685 "child_session_id":completion.child_session_id,"child_status":completion.status,
686 "child_final_response_included":projection["available"] == true,
687 "child_typed_result_included":true,
688 "child_completion_source_sha256":child_completion_source_sha256(completion),
689 }));
690 message.never_compress = false;
691 message
692}
693
694fn bound_typed_child_delivery(
697 envelope: &mut SessionMessageEnvelope,
698 message: &mut Message,
699 projection: &mut serde_json::Value,
700 completion: &ChildCompletion,
701 remaining_children: usize,
702) -> bool {
703 fn install(envelope: &mut SessionMessageEnvelope, message: &Message) {
704 if let SessionMessageBody::ChildOutcome(outcome) = &mut envelope.body {
705 outcome.result = None;
710 outcome.error = None;
713 outcome.provider_message = Some(session_provider_message(message));
714 }
715 }
716 fn fits(envelope: &SessionMessageEnvelope, message: &Message) -> bool {
717 fits_child_result_budget(envelope)
718 && fits_child_result_budget(message)
719 && envelope
720 .to_provider_message()
721 .is_ok_and(|actual| fits_child_result_budget(&actual))
722 }
723 install(envelope, message);
724 if fits(envelope, message) {
725 return true;
726 }
727 *projection = unavailable_child_result("typed_result", "result_budget_exceeded");
728 *message = typed_child_resume_message(completion, remaining_children, projection);
729 install(envelope, message);
730 fits(envelope, message)
731}
732
733fn guardian_resume_message(completion: &ChildCompletion, verdict: &GuardianVerdict) -> Message {
738 let mut body = if verdict.approve {
739 String::from(
740 "Guardian review APPROVED: an independent reviewer verified the work and found no blocking issues. You may finalize the task.",
741 )
742 } else {
743 String::from(
744 "Guardian review REJECTED: an independent reviewer found issues. Address every finding below before completing — do NOT declare the task complete until they are resolved.",
745 )
746 };
747 if let Some(summary) = verdict.summary.as_deref().filter(|s| !s.trim().is_empty()) {
748 body.push_str("\n\nReviewer summary: ");
749 body.push_str(summary);
750 }
751 if !verdict.findings.is_empty() {
752 body.push_str("\n\nFindings:");
753 for (idx, finding) in verdict.findings.iter().enumerate() {
754 body.push_str(&format!("\n{}. {}", idx + 1, finding));
755 }
756 }
757 let inspection = serde_json::json!({
758 "intent": "inspect", "target": completion.child_session_id, "message": "messages",
759 });
760 body.push_str(&format!(
761 "\n\nIf you need guardian transcript evidence, call SubAgent with {inspection}. \
762 Follow returned cursors with a JSON-encoded view/cursor query in message, omitting unused query fields.",
763 ));
764
765 let mut message = Message::user(body);
766 message.metadata = Some(serde_json::json!({
767 RUNTIME_RESUME_MESSAGE_HIDDEN_KEY: true,
768 RUNTIME_RESUME_MESSAGE_KIND_KEY: "guardian_review_resume",
769 "child_session_id": completion.child_session_id,
770 "child_status": completion.status,
771 "guardian_approved": verdict.approve,
772 "child_completion_source": completion.source,
773 }));
774 message.never_compress = false;
775 message
776}
777
778pub type RootToolSurfaceObserver = Arc<
780 dyn Fn() -> futures::future::BoxFuture<'static, Option<Arc<dyn ToolExecutor>>> + Send + Sync,
781>;
782
783pub type ReservedRootExecutionAdapter = Arc<
784 dyn Fn(
785 Arc<Agent>,
786 &Session,
787 &SessionExecutionReservation,
788 Arc<dyn ToolExecutor>,
789 ) -> Result<(Arc<Agent>, Arc<dyn ToolExecutor>), bamboo_agent_core::AgentError>
790 + Send
791 + Sync,
792>;
793
794#[derive(Clone)]
795pub struct ChildCompletionCoordinator {
796 storage: Arc<dyn Storage>,
797 persistence: Arc<bamboo_storage::LockedSessionStore>,
798 sessions: crate::SessionCache,
799 agent_runners: Arc<RwLock<HashMap<String, AgentRunner>>>,
800 session_event_senders: Arc<RwLock<HashMap<String, broadcast::Sender<AgentEvent>>>>,
801 agent: Arc<Agent>,
802 config: Arc<RwLock<Config>>,
803 provider_registry: Arc<ProviderRegistry>,
804 provider_router: Arc<ProviderModelRouter>,
805 app_data_dir: std::path::PathBuf,
806 account_feed_inbox: Option<crate::execution::AccountFeedInbox>,
807 root_account_sink: Option<Arc<crate::events::AccountEventSink>>,
808 root_tools: Arc<RwLock<Option<Arc<dyn ToolExecutor>>>>,
809 reserved_root_adapter: Arc<StdRwLock<Option<ReservedRootExecutionAdapter>>>,
810 guardian_spawner: Arc<RwLock<Option<Arc<dyn GuardianSpawner>>>>,
814 spawn_scheduler: Arc<RwLock<Weak<SpawnScheduler>>>,
818}
819
820impl ChildCompletionCoordinator {
821 pub fn with_root_account_sink(mut self, sink: Arc<crate::events::AccountEventSink>) -> Self {
822 self.root_account_sink = Some(sink);
823 self
824 }
825
826 #[allow(clippy::too_many_arguments)]
827 pub fn new(
828 storage: Arc<dyn Storage>,
829 persistence: Arc<LockedSessionStore>,
830 sessions: crate::SessionCache,
831 agent_runners: Arc<RwLock<HashMap<String, AgentRunner>>>,
832 session_event_senders: Arc<RwLock<HashMap<String, broadcast::Sender<AgentEvent>>>>,
833 agent: Arc<Agent>,
834 config: Arc<RwLock<Config>>,
835 provider_registry: Arc<ProviderRegistry>,
836 provider_router: Arc<ProviderModelRouter>,
837 app_data_dir: std::path::PathBuf,
838 account_feed_inbox: Option<crate::execution::AccountFeedInbox>,
839 ) -> Self {
840 Self {
841 storage,
842 persistence,
843 sessions,
844 agent_runners,
845 session_event_senders,
846 agent,
847 config,
848 provider_registry,
849 provider_router,
850 app_data_dir,
851 account_feed_inbox,
852 root_account_sink: None,
853 root_tools: Arc::new(RwLock::new(None)),
854 reserved_root_adapter: Arc::new(StdRwLock::new(None)),
855 guardian_spawner: Arc::new(RwLock::new(None)),
856 spawn_scheduler: Arc::new(RwLock::new(Weak::new())),
857 }
858 }
859
860 pub fn set_reserved_root_execution_adapter(&self, adapter: ReservedRootExecutionAdapter) {
863 *self
864 .reserved_root_adapter
865 .write()
866 .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(adapter);
867 }
868
869 pub fn root_tool_surface_observer(&self) -> RootToolSurfaceObserver {
870 let source = Arc::downgrade(&self.root_tools);
871 Arc::new(move || {
872 let source = source.clone();
873 Box::pin(async move {
874 let source = source.upgrade()?;
875 let current = source.read().await.clone();
876 current
877 })
878 })
879 }
880
881 pub async fn set_root_tools(&self, tools: Arc<dyn ToolExecutor>) {
882 *self.root_tools.write().await = Some(tools);
883 }
884
885 pub async fn set_spawn_scheduler(&self, scheduler: &Arc<SpawnScheduler>) {
886 *self.spawn_scheduler.write().await = Arc::downgrade(scheduler);
887 }
888
889 pub async fn set_guardian_spawner(&self, spawner: Arc<dyn GuardianSpawner>) {
892 *self.guardian_spawner.write().await = Some(spawner);
893 }
894
895 fn build_resume_config(
896 &self,
897 session: &Session,
898 config_snapshot: &Config,
899 ) -> ResumeConfigSnapshot {
900 crate::session_app::resolution::resolve_resume_config_snapshot(
901 config_snapshot,
902 &self.provider_registry,
903 session,
904 None,
905 )
906 }
907
908 async fn resume_parent(&self, parent_session_id: String) -> ResumeOutcome {
916 for attempt in 0..=5u8 {
917 if attempt > 0 {
918 tokio::time::sleep(Duration::from_millis(250 * attempt as u64)).await;
919 }
920
921 let Some(session) = self.load_session(&parent_session_id).await else {
922 tracing::warn!(%parent_session_id, "cannot resume parent after child completion: session not found");
923 return ResumeOutcome::NotFound;
924 };
925 let config_snapshot = self.config.read().await.clone();
926 let resume_config = self.build_resume_config(&session, &config_snapshot);
927 let outcome = resume_session_execution(self, &parent_session_id, resume_config).await;
928 tracing::info!(
929 %parent_session_id,
930 attempt,
931 outcome = outcome.as_str(),
932 "child completion requested parent resume"
933 );
934
935 if !matches!(outcome, ResumeOutcome::AlreadyRunning { .. }) {
936 return outcome;
937 }
938 }
939 tracing::error!(
945 %parent_session_id,
946 "parent resume gave up after AlreadyRunning retry budget; \
947 relying on the child-wait watchdog backstop"
948 );
949 ResumeOutcome::AlreadyRunning {
950 run_id: String::new(),
951 }
952 }
953
954 async fn save_and_cache(&self, session: &mut Session) {
955 if let Err(error) = self
956 .persistence
957 .merge_save_runtime_and_publish(session, |saved, _| {
958 self.sessions.insert(
959 saved.id.clone(),
960 Arc::new(crate::SessionSnapshot::new(saved.clone())),
961 );
962 })
963 .await
964 {
965 tracing::warn!(session_id = %session.id, %error, "failed to persist session");
966 }
967 }
968
969 async fn reuse_legacy_child_delivery(
970 &self,
971 envelope: &mut SessionMessageEnvelope,
972 completion: &ChildCompletion,
973 wait: Option<&WaitingForChildrenState>,
974 result: Option<&str>,
975 ) -> Result<(), bamboo_domain::SessionInboxError> {
976 if let (Some(wait), Some(inbox), Some(_)) =
977 (wait, self.agent.session_inbox(), completion.source.as_ref())
978 {
979 let mut legacy = envelope.clone();
980 legacy.id = child_completion_id(completion, wait.registered_at, result, true);
981 if inbox.existing_staged_delivery(&legacy).await?.is_some() {
982 *envelope = legacy;
983 }
984 }
985 Ok(())
986 }
987
988 async fn legacy_child_delivery(
989 &self,
990 parent: &Session,
991 completion: &ChildCompletion,
992 ) -> Result<Option<(SessionMessageEnvelope, bool)>, bamboo_domain::SessionInboxError> {
993 let (Some(source), Some(inbox)) = (completion.source.as_ref(), self.agent.session_inbox())
994 else {
995 return Ok(None);
996 };
997 let source_sha256 = child_completion_source_sha256(completion);
998 let source = serde_json::to_value(source).expect("sealed Child source serializes");
999 let matches_source = |message: &Message, envelope: &SessionMessageEnvelope| {
1000 if !matches!(&envelope.source, SessionMessageSource::Runtime { subsystem }
1001 if subsystem == "child_completion_coordinator")
1002 || envelope.target_session_id != parent.id
1003 || !matches!(&envelope.body, SessionMessageBody::ChildOutcome(outcome)
1004 if outcome.child_session_id == completion.child_session_id && outcome.status == completion.status)
1005 || !bamboo_domain::is_matching_session_message(message, envelope)
1006 {
1007 return false;
1008 }
1009 let Some(metadata) = message.metadata.as_ref() else {
1010 return false;
1011 };
1012 let typed_source = message
1016 .content
1017 .split_once("\n\nChild typed result:\n")
1018 .and_then(|(_, text)| text.lines().next())
1019 .filter(|line| line.len() <= 8192)
1020 .and_then(|line| serde_json::from_str::<serde_json::Value>(line).ok())
1021 .and_then(|projection| {
1022 projection
1023 .get("host_observation")?
1024 .get("terminal_source")
1025 .cloned()
1026 });
1027 metadata.get("child_completion_source") == Some(&source)
1028 || source_sha256.as_ref().is_some_and(|digest| {
1029 metadata
1030 .get("child_completion_source_sha256")
1031 .and_then(serde_json::Value::as_str)
1032 == Some(digest.as_str())
1033 })
1034 || typed_source.as_ref() == Some(&source)
1035 };
1036 for message in &parent.messages {
1037 let Some(marker) = message
1038 .metadata
1039 .as_ref()
1040 .and_then(|m| m.get("session_message"))
1041 else {
1042 continue;
1043 };
1044 let Ok(envelope) = serde_json::from_value::<SessionMessageEnvelope>(marker.clone())
1045 else {
1046 continue;
1047 };
1048 if matches_source(message, &envelope)
1049 && inbox.was_admitted(&parent.id, &envelope.id).await?
1050 {
1051 return Ok(Some((envelope, true)));
1052 }
1053 }
1054 for envelope in inbox
1058 .pending_child_outcomes(&parent.id, &completion.child_session_id)
1059 .await?
1060 {
1061 if envelope
1062 .to_provider_message()
1063 .is_ok_and(|message| matches_source(&message, &envelope))
1064 {
1065 return Ok(Some((envelope, false)));
1066 }
1067 }
1068 Ok(None)
1069 }
1070}
1071
1072#[async_trait]
1073impl ChildCompletionHandler for ChildCompletionCoordinator {
1074 async fn on_child_completed(&self, completion: ChildCompletion) {
1075 if !is_terminal_child_status(&completion.status) {
1084 tracing::info!(
1085 parent_session_id = %completion.parent_session_id,
1086 child_session_id = %completion.child_session_id,
1087 status = %completion.status,
1088 "non-terminal child status; leaving the parent wait armed"
1089 );
1090 return;
1091 }
1092
1093 let per_parent = session_resume_lock(&completion.parent_session_id);
1098 let _per_parent_guard = per_parent.lock().await;
1099
1100 let (parent, should_resume, messenger, child_admission, child_admissions) = loop {
1101 let Some(mut parent) = self.load_session(&completion.parent_session_id).await else {
1102 tracing::warn!(
1103 parent_session_id = %completion.parent_session_id,
1104 child_session_id = %completion.child_session_id,
1105 "child completion received for missing parent"
1106 );
1107 return;
1108 };
1109
1110 let expected_parent = parent.clone();
1111
1112 let mut runtime_state = read_runtime_state(&parent);
1118
1119 let mut completed_child_ids = derive_completed_child_ids(
1122 &self.storage,
1123 &completion.parent_session_id,
1124 &completion.child_session_id,
1125 )
1126 .await;
1127
1128 let mut should_resume = false;
1129 let mut remaining_children = 0usize;
1130 let active_wait = runtime_state.waiting_for_children.clone();
1131 let tracked_wait = active_wait.as_ref().is_some_and(|wait| {
1132 wait.child_session_ids
1133 .contains(&completion.child_session_id)
1134 });
1135 if !tracked_wait {
1136 let mut unchanged = parent.clone();
1140 match self
1141 .persistence
1142 .compare_exchange_child_wait_and_publish(
1143 &parent,
1144 &mut unchanged,
1145 true,
1146 Arc::new(|_| {}),
1147 )
1148 .await
1149 {
1150 Ok(false) => continue,
1151 Ok(true) if completion.source.is_none() => return,
1152 Ok(true) => {}
1153 Err(error) => {
1154 tracing::warn!(parent_session_id = %parent.id, %error,
1155 "child completion could not settle no-wait observation");
1156 return;
1157 }
1158 }
1159 }
1160 if let Some(wait) = active_wait.as_ref() {
1163 for id in &wait.child_session_ids {
1164 if id == &completion.child_session_id {
1165 continue;
1166 }
1167 if let Ok(Some(child)) = self
1168 .storage
1169 .load_child_wait_session(&parent, id, false)
1170 .await
1171 {
1172 completed_child_ids.retain(|completed| completed != id);
1173 if completion_child_is_owned(&parent.id, child.parent_session_id.as_deref())
1174 && child
1175 .last_run_status()
1176 .as_deref()
1177 .is_some_and(is_terminal_child_status)
1178 {
1179 completed_child_ids.push(id.clone());
1180 }
1181 }
1182 }
1183 }
1184 if let Some(wait) = active_wait.as_ref() {
1185 remaining_children = wait
1186 .child_session_ids
1187 .iter()
1188 .filter(|id| !completed_child_ids.iter().any(|completed| completed == *id))
1189 .count();
1190 should_resume = tracked_wait
1191 && wait_policy_satisfied(
1192 wait.wait_for,
1193 &wait.child_session_ids,
1194 &completed_child_ids,
1195 &completion.child_session_id,
1196 &completion.status,
1197 );
1198 }
1199
1200 let reported_child_owned = match self
1212 .storage
1213 .load_child_wait_session(&parent, &completion.child_session_id, false)
1214 .await
1215 {
1216 Ok(Some(control_plane)) => completion_child_is_owned(
1217 &completion.parent_session_id,
1218 control_plane.parent_session_id.as_deref(),
1219 ),
1220 _ => false,
1221 };
1222
1223 let loaded_child = if reported_child_owned {
1228 match self
1229 .storage
1230 .load_child_wait_session(&parent, &completion.child_session_id, true)
1231 .await
1232 {
1233 Ok(child) => child,
1234 Err(error) => {
1235 tracing::warn!(
1236 child_session_id = %completion.child_session_id,
1237 %error,
1238 "failed to load child session for runtime resume message"
1239 );
1240 None
1241 }
1242 }
1243 } else {
1244 tracing::warn!(
1245 parent_session_id = %completion.parent_session_id,
1246 child_session_id = %completion.child_session_id,
1247 "completion child is not a child of this parent; resuming with a neutral \
1248 message and NOT folding its content"
1249 );
1250 None
1251 };
1252
1253 if let Some(source) = completion.source.as_ref() {
1257 if loaded_child
1258 .as_ref()
1259 .and_then(ChildCompletionSource::from_committed_session)
1260 .as_ref()
1261 != Some(source)
1262 || !source.matches_completion(&completion)
1263 || loaded_child
1264 .as_ref()
1265 .is_none_or(|child| !source.matches_parent(child, &parent))
1266 {
1267 tracing::warn!(child_session_id = %completion.child_session_id,
1268 "stale/unavailable committed child completion source; leaving wait armed");
1269 return;
1270 }
1271 }
1272 if !tracked_wait
1273 && crate::ticket_runtime::is_independent_ticket_child(
1274 self.storage.as_ref(),
1275 &parent,
1276 &completion.child_session_id,
1277 )
1278 .await
1279 {
1280 return;
1281 }
1282 let previous_delivery = match self.legacy_child_delivery(&parent, &completion).await {
1283 Ok(envelope) => envelope,
1284 Err(error) => {
1285 tracing::warn!(parent_session_id = %parent.id, %error,
1286 "cannot verify historical Child delivery; leaving replay recoverable");
1287 return;
1288 }
1289 };
1290 let previously_admitted = previous_delivery
1291 .as_ref()
1292 .is_some_and(|(_, admitted)| *admitted);
1293 let child_final_response = loaded_child
1294 .as_ref()
1295 .and_then(|child| completion.source.as_ref()?.result(child, &parent));
1296 let guardian_resume = if completion.source.is_some() || should_resume {
1301 let reviewed_round = runtime_state.round.current_round;
1302 loaded_child.as_ref().and_then(|child| {
1303 if child.subagent_type().as_deref() != Some("guardian") {
1304 return None;
1305 }
1306 let mut guardian_state = read_guardian_state(&parent)?;
1307 if guardian_state.guardian_child_id.as_deref()
1308 != Some(completion.child_session_id.as_str())
1309 {
1310 tracing::warn!(
1313 parent_session_id = %completion.parent_session_id,
1314 child_session_id = %completion.child_session_id,
1315 expected = ?guardian_state.guardian_child_id,
1316 "guardian completion does not match recorded guardian_child_id; using generic resume"
1317 );
1318 return None;
1319 }
1320 if guardian_state.phase.is_reviewed() {
1321 return guardian_state.last_verdict.as_ref()
1324 .map(|verdict| guardian_resume_message(&completion, verdict));
1325 }
1326 let verdict = child_final_response.clone()
1334 .and_then(|text| match parse_guardian_verdict(&text) {
1335 Ok(verdict) => Some(verdict),
1336 Err(error) => {
1337 tracing::warn!(
1338 child_session_id = %completion.child_session_id,
1339 %error,
1340 "guardian verdict unparseable; recording a synthetic reject"
1341 );
1342 None
1343 }
1344 })
1345 .unwrap_or_else(|| {
1346 GuardianVerdict::rejected(vec![
1347 "The guardian reviewer did not return a usable verdict (it errored or \
1348 emitted unparseable output); the work has NOT been independently \
1349 verified."
1350 .to_string(),
1351 ])
1352 });
1353 let approved = verdict.approve;
1354 let message = guardian_resume_message(&completion, &verdict);
1355 guardian_state.record_verdict(verdict, reviewed_round);
1356 write_guardian_state(&mut parent, guardian_state);
1357 tracing::info!(
1358 parent_session_id = %completion.parent_session_id,
1359 child_session_id = %completion.child_session_id,
1360 approved,
1361 "guardian verdict recorded; resuming parent"
1362 );
1363 Some(message)
1364 })
1365 } else {
1366 None
1367 };
1368 let guardian_control_changed = parent.metadata.get("guardian.state")
1369 != expected_parent.metadata.get("guardian.state");
1370 if previously_admitted && !should_resume && !guardian_control_changed {
1373 return;
1374 }
1375 let activation_policy = if should_resume || guardian_control_changed {
1376 SessionActivationPolicy::RespectSpecificWait
1377 } else {
1378 SessionActivationPolicy::InterruptSpecificWait
1379 };
1380 let mut typed_result = loaded_child
1381 .as_ref()
1382 .and_then(|child| sealed_child_report(&completion, child, &parent));
1383 let mut resume_message = if let Some(projection) = typed_result.as_mut() {
1384 let message =
1385 typed_child_resume_message(&completion, remaining_children, projection);
1386 if fits_child_result_budget(&message) {
1387 message
1388 } else {
1389 *projection =
1390 unavailable_child_result("typed_result", "result_budget_exceeded");
1391 typed_child_resume_message(&completion, remaining_children, projection)
1392 }
1393 } else {
1394 guardian_resume.unwrap_or_else(|| {
1395 runtime_resume_message(
1396 &completion,
1397 remaining_children,
1398 child_final_response.as_deref(),
1399 )
1400 })
1401 };
1402 if typed_result.is_some() && !fits_child_result_budget(&resume_message) {
1403 tracing::warn!(child_session_id = %completion.child_session_id,
1404 "typed child resume identity exceeds budget; leaving wait armed");
1405 return;
1406 }
1407
1408 let messenger = self.agent.session_messenger().cloned();
1412 let mut child_admission = if let Some(messenger) = messenger.as_ref() {
1413 let mut envelope = child_completion_envelope(
1414 &completion,
1415 active_wait
1416 .as_ref()
1417 .map_or(completion.completed_at, |wait| wait.registered_at),
1418 child_final_response.clone(),
1419 &resume_message,
1420 );
1421 if let Some(router) = self.agent.activation_router() {
1422 let previous_fenced_run = previous_delivery
1423 .as_ref()
1424 .and_then(|(envelope, _)| envelope.correlation_id.as_deref())
1425 .and_then(|value| value.strip_prefix("child_completion_after_run:"));
1426 if let Some(run_id) = router
1427 .child_completion_successor_run(
1428 &completion.parent_session_id,
1429 active_wait.is_some() || guardian_control_changed,
1430 previous_fenced_run,
1431 )
1432 .await
1433 {
1434 envelope.correlation_id =
1435 Some(format!("child_completion_after_run:{run_id}"));
1436 }
1437 }
1438 if let Some(projection) = typed_result.as_mut() {
1439 if !bound_typed_child_delivery(
1440 &mut envelope,
1441 &mut resume_message,
1442 projection,
1443 &completion,
1444 remaining_children,
1445 ) {
1446 tracing::warn!(child_session_id = %completion.child_session_id,
1447 "typed child outcome wrapper exceeds budget; leaving wait armed");
1448 return;
1449 }
1450 }
1451 if let Some((previous, _)) = previous_delivery.as_ref() {
1452 envelope = previous.clone();
1453 } else if let Err(error) = self
1454 .reuse_legacy_child_delivery(
1455 &mut envelope,
1456 &completion,
1457 active_wait.as_ref(),
1458 child_final_response.as_deref(),
1459 )
1460 .await
1461 {
1462 tracing::warn!(parent_session_id = %parent.id, %error,
1463 "cannot verify existing Child delivery; leaving wait armed");
1464 return;
1465 }
1466 match messenger.admit(envelope).await {
1467 Ok(admission) => {
1468 if !should_resume && !guardian_control_changed {
1469 let mut observed = expected_parent.clone();
1470 match self
1471 .persistence
1472 .compare_exchange_child_wait_and_publish(
1473 &expected_parent,
1474 &mut observed,
1475 true,
1476 Arc::new(|_| {}),
1477 )
1478 .await
1479 {
1480 Ok(false) => continue,
1481 Ok(true) => {}
1482 Err(error) => {
1483 tracing::warn!(parent_session_id = %parent.id, %error,
1484 "partial Child delivery lost its parent authority; leaving staged outcome inert");
1485 return;
1486 }
1487 }
1488 }
1489 if should_resume || !guardian_control_changed {
1493 if let Err(error) = messenger
1494 .prepare_message_activation(&admission, activation_policy)
1495 .await
1496 {
1497 tracing::warn!(parent_session_id = %completion.parent_session_id,
1498 child_session_id = %completion.child_session_id, %error,
1499 "child outcome notification permission failed; leaving wait armed");
1500 return;
1501 }
1502 }
1503 Some(admission)
1504 }
1505 Err(error) => {
1506 tracing::warn!(
1507 parent_session_id = %completion.parent_session_id,
1508 child_session_id = %completion.child_session_id,
1509 %error,
1510 "child outcome SessionInbox admission failed; leaving parent wait armed"
1511 );
1512 return;
1513 }
1514 }
1515 } else {
1516 None
1517 };
1518
1519 let mut child_admissions = child_admission.iter().cloned().collect::<Vec<_>>();
1520
1521 if should_resume {
1528 if let (Some(wait), Some(messenger)) = (active_wait.as_ref(), messenger.as_ref()) {
1529 let parent_run_id = if let Some(router) = self.agent.activation_router() {
1530 router
1531 .child_completion_successor_run(
1532 &completion.parent_session_id,
1533 true,
1534 None,
1535 )
1536 .await
1537 } else {
1538 None
1539 };
1540 for child_id in &wait.child_session_ids {
1541 if child_id == &completion.child_session_id
1542 || !completed_child_ids.contains(child_id)
1543 {
1544 continue;
1545 }
1546 let owned = match self
1550 .storage
1551 .load_child_wait_session(&parent, child_id, false)
1552 .await
1553 {
1554 Ok(Some(child)) => completion_child_is_owned(
1555 &completion.parent_session_id,
1556 child.parent_session_id.as_deref(),
1557 ),
1558 _ => false,
1559 };
1560 let sibling = if owned {
1561 self.storage
1562 .load_child_wait_session(&parent, child_id, true)
1563 .await
1564 } else {
1565 tracing::warn!(%child_id, "terminal sibling ownership unavailable; leaving parent wait armed");
1566 return;
1567 };
1568 let sibling = match sibling {
1569 Ok(Some(child))
1570 if completion_child_is_owned(
1571 &completion.parent_session_id,
1572 child.parent_session_id.as_deref(),
1573 ) && child
1574 .last_run_status()
1575 .as_deref()
1576 .is_some_and(is_terminal_child_status) =>
1577 {
1578 child
1579 }
1580 _ => {
1581 tracing::warn!(%child_id, "terminal sibling snapshot unavailable; leaving parent wait armed");
1582 return;
1583 }
1584 };
1585 if ChildCompletionSource::has_source_record(&sibling)
1586 && ChildCompletionSource::from_committed_session(&sibling).is_none()
1587 {
1588 tracing::warn!(%child_id, "terminal sibling source is invalid; leaving wait armed");
1589 return;
1590 }
1591 let sibling_completion = ChildCompletion {
1592 parent_session_id: completion.parent_session_id.clone(),
1593 child_session_id: child_id.clone(),
1594 status: sibling.last_run_status().expect("terminal status checked"),
1595 error: sibling.last_run_error(),
1596 completed_at: sibling.updated_at,
1597 source: ChildCompletionSource::from_committed_session(&sibling),
1598 };
1599 if sibling_completion
1600 .source
1601 .as_ref()
1602 .is_some_and(|source| !source.matches_parent(&sibling, &parent))
1603 {
1604 tracing::warn!(%child_id, "terminal sibling parent context is stale; leaving wait armed");
1605 return;
1606 }
1607 let result = sibling_completion
1608 .source
1609 .as_ref()
1610 .and_then(|source| source.result(&sibling, &parent));
1611 let mut typed_result =
1612 sealed_child_report(&sibling_completion, &sibling, &parent);
1613 let mut presentation = if let Some(projection) = typed_result.as_ref() {
1614 typed_child_resume_message(
1615 &sibling_completion,
1616 remaining_children,
1617 projection,
1618 )
1619 } else {
1620 runtime_resume_message(
1621 &sibling_completion,
1622 remaining_children,
1623 result.as_deref(),
1624 )
1625 };
1626 let mut envelope = child_completion_envelope(
1627 &sibling_completion,
1628 wait.registered_at,
1629 result.clone(),
1630 &presentation,
1631 );
1632 if let Some(run_id) = parent_run_id.as_ref() {
1633 envelope.correlation_id =
1634 Some(format!("child_completion_after_run:{run_id}"));
1635 }
1636 if let Some(projection) = typed_result.as_mut() {
1637 if !bound_typed_child_delivery(
1638 &mut envelope,
1639 &mut presentation,
1640 projection,
1641 &sibling_completion,
1642 remaining_children,
1643 ) {
1644 tracing::warn!(%child_id, "typed terminal sibling wrapper exceeds budget; leaving wait armed");
1645 return;
1646 }
1647 }
1648 let previous_delivery = match self
1649 .legacy_child_delivery(&parent, &sibling_completion)
1650 .await
1651 {
1652 Ok(delivery) => delivery,
1653 Err(error) => {
1654 tracing::warn!(%child_id, %error,
1655 "cannot verify historical sibling delivery; leaving wait armed");
1656 return;
1657 }
1658 };
1659 if let Some((previous, _)) = previous_delivery {
1660 envelope = previous;
1661 } else if let Err(error) = self
1662 .reuse_legacy_child_delivery(
1663 &mut envelope,
1664 &sibling_completion,
1665 Some(wait),
1666 result.as_deref(),
1667 )
1668 .await
1669 {
1670 tracing::warn!(%child_id, %error,
1671 "cannot verify existing sibling delivery; leaving wait armed");
1672 return;
1673 }
1674 match messenger.admit(envelope).await {
1675 Ok(admission) => {
1676 if let Err(error) = messenger
1677 .prepare_message_activation(
1678 &admission,
1679 SessionActivationPolicy::RespectSpecificWait,
1680 )
1681 .await
1682 {
1683 tracing::warn!(%child_id, %error,
1684 "terminal sibling notification permission failed; leaving wait armed");
1685 return;
1686 }
1687 child_admissions.push(admission.clone());
1688 if child_admission.as_ref().is_none_or(|current| {
1691 admission.delivery.generation > current.delivery.generation
1692 }) {
1693 child_admission = Some(admission);
1694 }
1695 }
1696 Err(error) => {
1697 tracing::warn!(%child_id, %error, "terminal sibling admission failed; leaving parent wait armed");
1698 return;
1699 }
1700 }
1701 }
1702 }
1703 }
1704
1705 if !should_resume && !guardian_control_changed && child_admission.is_some() {
1709 break (parent, true, messenger, child_admission, child_admissions);
1710 }
1711
1712 if should_resume {
1713 runtime_state.waiting_for_children = None;
1714 runtime_state.status = AgentStatusState::Idle;
1715 runtime_state.suspension = None;
1716 parent.metadata.remove("runtime.suspend_reason");
1717
1718 if child_admission.is_none() {
1719 parent.add_message(resume_message);
1723 }
1724 }
1725
1726 parent.updated_at = Utc::now();
1727 write_runtime_state(&mut parent, &runtime_state);
1728 let runtime_only = child_admission.is_some();
1735 let sessions = self.sessions.clone();
1736 let history = expected_parent.clone();
1737 let save = self
1738 .persistence
1739 .compare_exchange_child_wait_and_publish(
1740 &expected_parent,
1741 &mut parent,
1742 runtime_only,
1743 Arc::new(move |saved| {
1744 if runtime_only {
1745 if let Some(cached) = sessions.get(&saved.id) {
1746 cached.update(|current| {
1747 if current.created_at != saved.created_at {
1748 *current = saved.clone();
1749 current.messages = history.messages.clone();
1750 current.provider_transcript =
1751 history.provider_transcript.clone();
1752 current
1753 .runtime_metadata
1754 .get_or_insert_with(Default::default)
1755 .session_inbox_admission =
1756 history.session_inbox_admission().cloned();
1757 return;
1758 }
1759
1760 let messages = current.messages.clone();
1761 let provider_transcript = current.provider_transcript.clone();
1762 let admission =
1763 current.runtime_metadata.as_ref().and_then(|metadata| {
1764 metadata.session_inbox_admission.clone()
1765 });
1766 *current = saved.clone();
1767 current.messages = messages;
1768 current.provider_transcript = provider_transcript;
1769 current
1770 .runtime_metadata
1771 .get_or_insert_with(Default::default)
1772 .session_inbox_admission = admission;
1773 });
1774 } else {
1775 let mut snapshot = saved.clone();
1776 snapshot.messages = history.messages.clone();
1777 snapshot.provider_transcript = history.provider_transcript.clone();
1778 snapshot
1779 .runtime_metadata
1780 .get_or_insert_with(Default::default)
1781 .session_inbox_admission = history
1782 .runtime_metadata
1783 .as_ref()
1784 .and_then(|metadata| metadata.session_inbox_admission.clone());
1785 sessions.insert(
1786 snapshot.id.clone(),
1787 Arc::new(crate::SessionSnapshot::new(snapshot)),
1788 );
1789 }
1790 } else {
1791 sessions.insert(
1792 saved.id.clone(),
1793 Arc::new(crate::SessionSnapshot::new(saved.clone())),
1794 );
1795 }
1796 }),
1797 )
1798 .await;
1799 match save {
1800 Ok(false) => {
1801 continue;
1804 }
1805 Ok(true) => {
1806 break (
1807 parent,
1808 should_resume || (guardian_control_changed && !previously_admitted),
1809 messenger,
1810 child_admission,
1811 child_admissions,
1812 )
1813 }
1814 Err(error) => {
1815 tracing::warn!(parent_session_id = %completion.parent_session_id,
1816 child_session_id = %completion.child_session_id, %error,
1817 "child outcome is durable but parent wait transition failed; leaving activation deferred");
1818 return;
1819 }
1820 }
1821 };
1822
1823 let resume_parent_id = parent.id.clone();
1828 drop(_per_parent_guard);
1829
1830 if should_resume {
1831 if let (Some(messenger), Some(admission)) = (messenger, child_admission) {
1832 for outcome in &child_admissions {
1836 if let Err(error) = messenger
1837 .prepare_message_activation(
1838 outcome,
1839 SessionActivationPolicy::InterruptSpecificWait,
1840 )
1841 .await
1842 {
1843 tracing::warn!(parent_session_id = %resume_parent_id, %error,
1844 "child control transition committed but notification permission failed");
1845 return;
1846 }
1847 }
1848 if let Err(error) = messenger.activate_prepared(&admission).await {
1849 tracing::warn!(
1850 parent_session_id = %resume_parent_id,
1851 %error,
1852 "child outcome and wait transition are durable but activation failed"
1853 );
1854 }
1855 } else {
1856 self.resume_parent(resume_parent_id).await;
1857 }
1858 }
1859 }
1860}
1861
1862#[async_trait]
1863impl ResumeExecutionPort for ChildCompletionCoordinator {
1864 async fn load_session(&self, session_id: &str) -> Option<Session> {
1865 match self.storage.load_session(session_id).await {
1866 Ok(Some(session)) => Some(session),
1867 Ok(None) => self
1868 .sessions
1869 .get(session_id)
1870 .map(|e| e.value().clone())
1871 .map(|arc| arc.read().clone()),
1872 Err(error) => {
1873 tracing::warn!(%session_id, %error, "failed to load session from storage");
1874 self.sessions
1875 .get(session_id)
1876 .map(|e| e.value().clone())
1877 .map(|arc| arc.read().clone())
1878 }
1879 }
1880 }
1881
1882 async fn save_and_cache_session(&self, session: &mut Session) {
1883 self.save_and_cache(session).await;
1884 }
1885
1886 async fn reserve_session_execution(
1887 &self,
1888 session_id: &str,
1889 event_sender: &broadcast::Sender<AgentEvent>,
1890 ) -> SessionExecutionReserveOutcome {
1891 reserve_session_execution(
1892 &self.agent,
1893 &self.agent_runners,
1894 &self.session_event_senders,
1895 session_id,
1896 event_sender,
1897 )
1898 .await
1899 }
1900
1901 async fn get_or_create_event_sender(&self, session_id: &str) -> broadcast::Sender<AgentEvent> {
1902 crate::execution::session_events::get_or_create_event_sender(
1903 &self.session_event_senders,
1904 session_id,
1905 )
1906 .await
1907 }
1908
1909 fn dispatch_resume_execution(
1910 &self,
1911 request: ResumeSpawnRequest,
1912 ) -> Result<(), ResumeSpawnRequest> {
1913 let owner = self.clone();
1914 tokio::spawn(async move {
1915 ResumeExecutionPort::spawn_resume_execution(&owner, request).await;
1916 });
1917 Ok(())
1918 }
1919
1920 async fn spawn_resume_execution(&self, request: ResumeSpawnRequest) {
1921 let ResumeSpawnRequest {
1922 session_id,
1923 mut session,
1924 mut execution_reservation,
1925 event_sender,
1926 config,
1927 } = request;
1928 if let Err(error) = execution_reservation.ensure_registered().await {
1929 tracing::warn!(
1930 %session_id,
1931 run_id = %execution_reservation.run_id(),
1932 %error,
1933 "cannot resume after child completion without exact router ownership"
1934 );
1935 return;
1936 }
1937
1938 if self
1939 .agent
1940 .persistence()
1941 .root_actor_execution_required(&session)
1942 && self.root_account_sink.is_none()
1943 {
1944 tracing::warn!(%session_id, "Root resume has no actual account sink; rejecting before Actor claim");
1945 execution_reservation.abandon().await;
1946 return;
1947 }
1948 if let Err(error) = execution_reservation
1949 .bind_root_actor(&self.agent, &session)
1950 .await
1951 {
1952 tracing::warn!(%session_id, %error, "Root coordinator resume Actor binding rejected");
1953 execution_reservation.abandon().await;
1954 return;
1955 }
1956 let root_publication = execution_reservation.root_actor_writer().map(|owner| {
1957 crate::events::RootActorEventPublication::new(
1958 self.storage.clone(),
1959 self.root_account_sink
1960 .clone()
1961 .expect("required Root actual account sink checked before claim"),
1962 owner,
1963 )
1964 });
1965
1966 let Some(root_tools) = self.root_tools.read().await.clone() else {
1967 tracing::error!(%session_id, "cannot resume parent after child completion: root tool surface is not initialized");
1968 return;
1969 };
1970
1971 let adapter = self
1972 .reserved_root_adapter
1973 .read()
1974 .unwrap_or_else(std::sync::PoisonError::into_inner)
1975 .clone();
1976 let (agent, root_tools) = if let Some(adapter) = adapter {
1977 if session.kind == SessionKind::Root {
1985 if let Err(error) = self
1986 .agent
1987 .prepare_external_session_for_execution(&mut session)
1988 .await
1989 {
1990 tracing::error!(%session_id, %error, "Root resume context preparation rejected before Source binding");
1991 execution_reservation.abandon().await;
1992 return;
1993 }
1994 }
1995 match adapter(
1996 self.agent.clone(),
1997 &session,
1998 &execution_reservation,
1999 root_tools,
2000 ) {
2001 Ok(bound) => bound,
2002 Err(error) => {
2003 tracing::error!(%session_id, %error, "registered Root execution producer rejected resume");
2004 execution_reservation.abandon().await;
2005 return;
2006 }
2007 }
2008 } else {
2009 (self.agent.clone(), root_tools)
2010 };
2011
2012 let config_snapshot = self.config.read().await.clone();
2013 let model = session.model.clone();
2014 let session_model_ref = session_effective_model_ref(&session);
2015 let requested_provider = session_model_ref
2016 .as_ref()
2017 .map(|model_ref| model_ref.provider.as_str())
2018 .unwrap_or(config.provider_name.as_str());
2019 let resolved_provider_name = match resolve_provider_routing_key(
2020 &config_snapshot,
2021 requested_provider,
2022 &self.provider_registry,
2023 ) {
2024 Ok(provider) => provider,
2025 Err(error) => {
2026 tracing::error!(
2027 session_id = %session_id,
2028 provider = requested_provider,
2029 %error,
2030 "child-completion resume provider is unavailable; refusing to fall back"
2031 );
2032 execution_reservation.abandon().await;
2033 return;
2034 }
2035 };
2036 let provider_override = if let Some(mut model_ref) = session_model_ref {
2037 model_ref.provider = resolved_provider_name.clone();
2038 persist_model_ref(&mut session, &model_ref);
2039 match self.provider_router.route(&model_ref) {
2040 Ok(provider) => Some(provider),
2041 Err(error) => {
2042 tracing::error!(
2043 session_id = %session_id,
2044 provider = %model_ref.provider,
2045 model = %model_ref.model,
2046 %error,
2047 "child-completion resume provider routing failed closed"
2048 );
2049 execution_reservation.abandon().await;
2050 return;
2051 }
2052 }
2053 } else {
2054 match self.provider_registry.get(&resolved_provider_name) {
2055 Some(provider) => Some(provider),
2056 None => {
2057 tracing::error!(
2058 session_id = %session_id,
2059 provider = %resolved_provider_name,
2060 "child-completion resume provider disappeared after resolution; refusing to fall back"
2061 );
2062 execution_reservation.abandon().await;
2063 return;
2064 }
2065 }
2066 };
2067 let resolved_fast_provider = resolve_fast_model(
2068 &config_snapshot,
2069 &resolved_provider_name,
2070 &self.provider_registry,
2071 )
2072 .map(|model| model.provider);
2073 let reasoning_effort = session.reasoning_effort;
2074 let reasoning_effort_source = session
2075 .metadata
2076 .get("reasoning_effort_source")
2077 .cloned()
2078 .unwrap_or_default();
2079 let gold_config = resolve_gold_config(
2080 &config_snapshot,
2081 session
2082 .metadata
2083 .get(GOLD_CONFIG_METADATA_KEY)
2084 .map(String::as_str),
2085 )
2086 .or(config.gold_config.clone());
2087
2088 let (mpsc_tx, _forwarder, history_commit_barrier) =
2089 crate::execution::create_event_forwarder_with_root_actor(
2090 session_id.clone(),
2091 execution_reservation.run_id().to_string(),
2092 event_sender,
2093 self.agent_runners.clone(),
2094 self.account_feed_inbox.clone(),
2095 root_publication,
2096 );
2097
2098 let config_handle = self.config.clone();
2099 let cached_config = Arc::new(StdRwLock::new(config_snapshot.clone()));
2100 let provider_registry = self.provider_registry.clone();
2101 let provider_name_for_aux = resolved_provider_name.clone();
2102 let auxiliary_model_resolver = std::sync::Arc::new(move || {
2103 let config_snapshot = read_config_snapshot(&config_handle, cached_config.as_ref());
2104 let areas = resolve_global_area_models(
2106 &config_snapshot,
2107 &provider_name_for_aux,
2108 &provider_registry,
2109 );
2110 crate::AuxiliaryModelConfig {
2111 fast_model_name: areas.fast.as_ref().map(|m| m.model_name.clone()),
2112 fast_model_provider: areas.fast.map(|m| m.provider),
2113 background_model_name: areas.background.as_ref().map(|m| m.model_name.clone()),
2114 planning_model_name: None,
2115 search_model_name: None,
2116 summarization_model_name: areas
2117 .summarization
2118 .as_ref()
2119 .map(|m| m.model_name.clone()),
2120 background_model_provider: areas.background.map(|m| m.provider),
2121 summarization_model_provider: areas.summarization.map(|m| m.provider),
2122 summarization_model_error: areas.summarization_error,
2123 }
2124 });
2125 let model_roster = crate::ModelRoster {
2126 model: Some(model),
2127 provider_name: Some(resolved_provider_name),
2128 provider_type: config.provider_type.clone(),
2129 fast: crate::RoleModel::from_parts(config.fast_model, resolved_fast_provider),
2130 background: crate::RoleModel::from_parts(
2131 config.background_model,
2132 config.background_model_provider,
2133 ),
2134 summarization: crate::RoleModel::from_parts(
2135 config.summarization_model,
2136 config.summarization_model_provider,
2137 ),
2138 };
2139
2140 let guardian_config = read_guardian_config(&session);
2145 let guardian_spawner = self.guardian_spawner.read().await.clone();
2146
2147 consume_pending_clarification_resume(&mut session);
2148 spawn_session_execution(SessionExecutionArgs {
2149 agent,
2150 session_id,
2151 session,
2152 execution_reservation,
2153 tools_override: Some(root_tools),
2154 provider_override,
2155 model_roster,
2156 reasoning_effort,
2157 reasoning_effort_source,
2158 auxiliary_model_resolver: Some(auxiliary_model_resolver),
2159 disabled_filter_resolver: None,
2162 disabled_tools: Some(config.disabled_tools),
2163 disabled_skill_ids: Some(config.disabled_skill_ids),
2164 selected_skill_ids: None,
2165 selected_skill_mode: None,
2166 mpsc_tx,
2167 history_commit_barrier,
2168 image_fallback: config.image_fallback,
2169 gold_config,
2170 guardian_config,
2171 guardian_spawner,
2172 bash_resume_hook: {
2173 let hook: Arc<dyn BashResumeHook> = Arc::new(self.clone());
2174 Some(hook)
2175 },
2176 bash_completion_sink: {
2177 let sink: Arc<dyn BashCompletionSink> = Arc::new(self.clone());
2180 Some(sink)
2181 },
2182 app_data_dir: Some(self.app_data_dir.clone()),
2183 run_budget: None,
2186 runners: self.agent_runners.clone(),
2187 sessions_cache: self.sessions.clone(),
2188 on_complete: None,
2189 child_completion_handler: Some(Arc::new(self.clone())),
2193 });
2194 }
2195}
2196
2197#[async_trait]
2202impl SessionActivationSpawner for ChildCompletionCoordinator {
2203 async fn reserve_activation(
2204 &self,
2205 target_session_id: &str,
2206 _inbox_generation: u64,
2207 ) -> Result<SessionActivationReserveOutcome, bamboo_domain::SessionActivationError> {
2208 let Some(inbox) = self.agent.session_inbox() else {
2209 return Err(bamboo_domain::SessionActivationError::Internal(
2210 "agent runtime has no SessionInbox".to_string(),
2211 ));
2212 };
2213 let backlog = inbox
2214 .inspect(target_session_id)
2215 .await
2216 .map_err(|error| bamboo_domain::SessionActivationError::Internal(error.to_string()))?;
2217 if !backlog.activation_pending() {
2218 return Ok(SessionActivationReserveOutcome::NoWork);
2219 }
2220 let prepared = prepare_session_inbox_activation(
2223 &self.persistence,
2224 target_session_id,
2225 backlog.interrupt_pending(),
2226 )
2227 .await
2228 .map_err(|error| {
2229 bamboo_domain::SessionActivationError::Internal(format!(
2230 "persist resumable SessionInbox target: {error}"
2231 ))
2232 })?;
2233 let Some((session, ready)) = prepared else {
2234 return Ok(SessionActivationReserveOutcome::NotFound);
2235 };
2236 if !ready {
2237 tracing::info!(
2238 session_id = target_session_id,
2239 "SessionInbox backlog is activation-eligible but a specific durable wait remains armed"
2240 );
2241 return Ok(SessionActivationReserveOutcome::NoWork);
2242 }
2243
2244 enum LaunchPlan {
2245 Root(Box<ResumeConfigSnapshot>),
2246 Child {
2247 scheduler: Arc<SpawnScheduler>,
2248 parent_session_id: String,
2249 model: String,
2250 disabled_tools: Option<Vec<String>>,
2251 },
2252 }
2253
2254 let launch_plan = match session.kind {
2257 SessionKind::Root => {
2258 if self.root_tools.read().await.is_none() {
2259 return Err(bamboo_domain::SessionActivationError::Internal(
2260 "root tool surface is not initialized".to_string(),
2261 ));
2262 }
2263 let config_snapshot = self.config.read().await.clone();
2264 LaunchPlan::Root(Box::new(
2265 self.build_resume_config(&session, &config_snapshot),
2266 ))
2267 }
2268 SessionKind::Child => {
2269 let scheduler = self.spawn_scheduler.read().await.upgrade().ok_or_else(|| {
2270 bamboo_domain::SessionActivationError::Internal(
2271 "child spawn scheduler is not initialized".to_string(),
2272 )
2273 })?;
2274 let parent_session_id = session
2275 .parent_session_id
2276 .as_deref()
2277 .map(str::trim)
2278 .filter(|id| !id.is_empty())
2279 .map(ToOwned::to_owned)
2280 .ok_or_else(|| {
2281 bamboo_domain::SessionActivationError::Internal(format!(
2282 "child SessionInbox target {target_session_id} has no parent owner"
2283 ))
2284 })?;
2285 let parent = self
2286 .storage
2287 .load_session(&parent_session_id)
2288 .await
2289 .map_err(|error| {
2290 bamboo_domain::SessionActivationError::Internal(format!(
2291 "load parent owner {parent_session_id}: {error}"
2292 ))
2293 })?
2294 .ok_or_else(|| {
2295 bamboo_domain::SessionActivationError::Internal(format!(
2296 "child SessionInbox parent owner {parent_session_id} disappeared"
2297 ))
2298 })?;
2299 let child_root = if session.root_session_id.trim().is_empty() {
2300 parent_session_id.as_str()
2301 } else {
2302 session.root_session_id.as_str()
2303 };
2304 let parent_root = if parent.root_session_id.trim().is_empty() {
2305 parent.id.as_str()
2306 } else {
2307 parent.root_session_id.as_str()
2308 };
2309 if child_root != parent_root {
2310 return Err(bamboo_domain::SessionActivationError::Internal(format!(
2311 "child SessionInbox target {target_session_id} does not share its parent owner's root"
2312 )));
2313 }
2314 let model = if session.model.trim().is_empty() {
2315 parent.model.clone()
2316 } else {
2317 session.model.clone()
2318 };
2319 if model.trim().is_empty() {
2320 return Err(bamboo_domain::SessionActivationError::Internal(format!(
2321 "child SessionInbox target {target_session_id} has no executable model"
2322 )));
2323 }
2324 let disabled_tools = match session.metadata.get("disabled_tools") {
2325 None => None,
2326 Some(raw) => {
2327 let tools = serde_json::from_str::<std::collections::BTreeSet<String>>(raw)
2328 .map_err(|error| {
2329 bamboo_domain::SessionActivationError::Internal(format!(
2330 "child SessionInbox target {target_session_id} has malformed disabled_tools: {error}"
2331 ))
2332 })?;
2333 (!tools.is_empty()).then(|| tools.into_iter().collect())
2334 }
2335 };
2336 LaunchPlan::Child {
2337 scheduler,
2338 parent_session_id,
2339 model,
2340 disabled_tools,
2341 }
2342 }
2343 };
2344 let event_sender =
2345 ResumeExecutionPort::get_or_create_event_sender(self, target_session_id).await;
2346
2347 let reservation = match reserve_runner_core(
2348 &self.agent_runners,
2349 &self.session_event_senders,
2350 target_session_id,
2351 &event_sender,
2352 )
2353 .await
2354 {
2355 ReserveOutcome::Reserved(reservation) => reservation,
2356 ReserveOutcome::AlreadyRunning(run_id) => {
2357 return Ok(SessionActivationReserveOutcome::AlreadyRunning { run_id });
2358 }
2359 };
2360 let run_id = reservation.run_id.clone();
2361 let launch = match launch_plan {
2362 LaunchPlan::Root(config) => {
2363 let mut execution_reservation =
2364 SessionExecutionReservation::from_activation_placeholder(
2365 target_session_id,
2366 reservation,
2367 self.agent
2368 .activation_router()
2369 .expect("SessionInbox activation requires an activation router")
2370 .clone(),
2371 self.agent_runners.clone(),
2372 );
2373 if self
2377 .agent
2378 .persistence()
2379 .root_actor_execution_required(&session)
2380 && self.root_account_sink.is_none()
2381 {
2382 execution_reservation
2383 .rollback_unpublished_activation()
2384 .await;
2385 return Err(bamboo_domain::SessionActivationError::Internal(
2386 "Root activation has no actual account sink".into(),
2387 ));
2388 }
2389 if let Err(error) = execution_reservation
2390 .bind_root_actor(&self.agent, &session)
2391 .await
2392 {
2393 execution_reservation
2394 .rollback_unpublished_activation()
2395 .await;
2396 return Err(bamboo_domain::SessionActivationError::Internal(format!(
2397 "Root Actor reservation rejected: {error}"
2398 )));
2399 }
2400 let reservation_cell = Arc::new(StdMutex::new(Some(execution_reservation)));
2404 let launch_reservation = reservation_cell.clone();
2405 let rollback_reservation = reservation_cell;
2406 let coordinator = self.clone();
2407 let launch_sessions = self.sessions.clone();
2408 let launch_session_id = session.id.clone();
2409 let launch_session = session.clone();
2410 let request_session_id = target_session_id.to_string();
2411 SessionActivationLaunch::new_with_async_rollback(
2412 run_id,
2413 move || {
2414 let mut execution_reservation = launch_reservation
2415 .lock()
2416 .unwrap_or_else(std::sync::PoisonError::into_inner)
2417 .take()
2418 .expect("activation reservation launches or rolls back exactly once");
2419 execution_reservation.mark_activation_published();
2420 if execution_reservation.root_actor_writer().is_none() {
2424 launch_sessions.insert(
2425 launch_session_id,
2426 Arc::new(crate::SessionSnapshot::new(launch_session)),
2427 );
2428 }
2429 let request = ResumeSpawnRequest {
2430 session_id: request_session_id,
2431 session,
2432 execution_reservation,
2433 event_sender,
2434 config: *config,
2435 };
2436 tokio::spawn(async move {
2437 ResumeExecutionPort::spawn_resume_execution(&coordinator, request)
2438 .await;
2439 });
2440 },
2441 move || async move {
2442 let reservation = rollback_reservation
2443 .lock()
2444 .unwrap_or_else(std::sync::PoisonError::into_inner)
2445 .take();
2446 if let Some(reservation) = reservation {
2447 reservation.rollback_unpublished_activation().await;
2448 }
2449 },
2450 )
2451 }
2452 LaunchPlan::Child {
2453 scheduler,
2454 parent_session_id,
2455 model,
2456 disabled_tools,
2457 } => {
2458 let execution_reservation =
2459 SessionExecutionReservation::from_activation_placeholder(
2460 target_session_id,
2461 reservation,
2462 self.agent
2463 .activation_router()
2464 .expect("SessionInbox activation requires an activation router")
2465 .clone(),
2466 self.agent_runners.clone(),
2467 );
2468 let reservation_cell = Arc::new(StdMutex::new(Some(execution_reservation)));
2469 let launch_reservation = reservation_cell.clone();
2470 let rollback_reservation = reservation_cell;
2471 let job = SpawnJob {
2472 parent_session_id,
2473 child_session_id: target_session_id.to_string(),
2474 model,
2475 disabled_tools,
2476 };
2477 let launch_sessions = self.sessions.clone();
2478 let launch_session_id = session.id.clone();
2479 let launch_session = session;
2480 SessionActivationLaunch::new_with_async_rollback(
2481 run_id,
2482 move || {
2483 let mut execution_reservation = launch_reservation
2484 .lock()
2485 .unwrap_or_else(std::sync::PoisonError::into_inner)
2486 .take()
2487 .expect("activation reservation launches or rolls back exactly once");
2488 execution_reservation.mark_activation_published();
2489 launch_sessions.insert(
2492 launch_session_id,
2493 Arc::new(crate::SessionSnapshot::new(launch_session)),
2494 );
2495 drop(scheduler.launch_reserved(job, execution_reservation));
2499 },
2500 move || async move {
2501 let reservation = rollback_reservation
2502 .lock()
2503 .unwrap_or_else(std::sync::PoisonError::into_inner)
2504 .take();
2505 if let Some(reservation) = reservation {
2506 reservation.rollback_unpublished_activation().await;
2507 }
2508 },
2509 )
2510 }
2511 };
2512 Ok(SessionActivationReserveOutcome::Reserved(launch))
2513 }
2514}
2515
2516fn bash_completion_resume_message(bash_ids: &[String], timed_out: bool) -> Message {
2526 let body = if timed_out {
2527 format!(
2528 "Runtime notification: the background-Bash wait ceiling was reached while one or more \
2529 shell(s) ({}) may still be running. The session is being resumed so it is not \
2530 stranded; verify their actual status with BashOutput before assuming completion.",
2531 bash_ids.join(", ")
2532 )
2533 } else {
2534 format!(
2535 "Runtime notification: all background Bash shell(s) ({}) have completed. \
2536 Review their output with BashOutput and resume the task from where you left off.",
2537 bash_ids.join(", ")
2538 )
2539 };
2540 let mut message = Message::user(body);
2541 message.metadata = Some(serde_json::json!({
2542 RUNTIME_RESUME_MESSAGE_HIDDEN_KEY: true,
2543 RUNTIME_RESUME_MESSAGE_KIND_KEY: BASH_COMPLETION_RESUME_KIND,
2544 }));
2545 message.never_compress = false;
2546 message
2547}
2548
2549fn bash_resume_should_retry(outcome: &ResumeOutcome, persisted_waiting_for_bash: bool) -> bool {
2562 match outcome {
2563 ResumeOutcome::Started { .. } | ResumeOutcome::NotFound => false,
2564 ResumeOutcome::Completed | ResumeOutcome::AlreadyRunning { .. } => {
2565 persisted_waiting_for_bash
2566 }
2567 }
2568}
2569
2570fn bash_completion_should_resume(
2578 loop_suspended_on_bash: bool,
2579 all_waited_shells_done: bool,
2580) -> bool {
2581 loop_suspended_on_bash && all_waited_shells_done
2582}
2583
2584#[derive(Debug, Clone, Copy, PartialEq, Eq)]
2585enum BashCompletionDeliveryPlan {
2586 DurableOnly,
2590 Activate,
2592 ClearWaitThenActivate,
2594}
2595
2596fn bash_completion_delivery_plan(
2597 loop_suspended_on_bash: bool,
2598 all_waited_shells_done: bool,
2599) -> BashCompletionDeliveryPlan {
2600 if bash_completion_should_resume(loop_suspended_on_bash, all_waited_shells_done) {
2601 BashCompletionDeliveryPlan::ClearWaitThenActivate
2602 } else if loop_suspended_on_bash {
2603 BashCompletionDeliveryPlan::DurableOnly
2604 } else {
2605 BashCompletionDeliveryPlan::Activate
2606 }
2607}
2608
2609fn background_bash_post_tool_payload(info: &BashCompletionInfo) -> HookPayload {
2610 let success = info.status == "completed" && info.exit_code == Some(0);
2611 let response = serde_json::json!({
2612 "bash_id": info.bash_id,
2613 "command": info.command,
2614 "exit_code": info.exit_code,
2615 "status": info.status,
2616 "output_tail": info.output_tail,
2617 });
2618 HookPayload::ToolResult {
2619 tool_name: "Bash".to_string(),
2620 tool_call_id: info.bash_id.clone(),
2621 outcome: HookToolOutcome {
2622 success,
2623 result: success.then(|| response.to_string()),
2624 error: (!success).then(|| response.to_string()),
2625 needs_human: false,
2626 duration_ms: 0,
2627 },
2628 }
2629}
2630
2631fn append_background_bash_hook_feedback(info: &mut BashCompletionInfo, feedback: Vec<String>) {
2632 let feedback = feedback
2633 .into_iter()
2634 .map(|text| text.trim().to_string())
2635 .filter(|text| !text.is_empty())
2636 .collect::<Vec<_>>();
2637 if feedback.is_empty() {
2638 return;
2639 }
2640 if !info.output_tail.is_empty() {
2641 info.output_tail.push_str("\n\n");
2642 }
2643 info.output_tail.push_str("<post_tool_use_feedback>\n");
2644 info.output_tail.push_str(&feedback.join("\n"));
2645 info.output_tail.push_str("\n</post_tool_use_feedback>");
2646}
2647
2648async fn run_background_bash_post_tool_hooks(
2649 config: &bamboo_config::LifecycleHooksConfig,
2650 fallback_cwd: Option<std::path::PathBuf>,
2651 session: &Session,
2652 info: &mut BashCompletionInfo,
2653) -> bool {
2654 let runner = crate::HookRunner::new().with_lifecycle_config(config, fallback_cwd);
2655 if !runner.has_hooks_for(AgentHookPoint::AfterToolExecution) {
2656 return false;
2657 }
2658
2659 let mut runtime_state = session
2660 .agent_runtime_state
2661 .clone()
2662 .unwrap_or_else(|| AgentRuntimeState::new(&session.id));
2663 let outcome = runner
2664 .run_observer_hooks(
2665 AgentHookPoint::AfterToolExecution,
2666 &background_bash_post_tool_payload(info),
2667 session,
2668 &mut runtime_state,
2669 None,
2670 )
2671 .await;
2672 append_background_bash_hook_feedback(info, outcome.injected_contexts);
2673 true
2674}
2675
2676fn apply_bash_resume_transition(session: &mut Session, resume_message: &Message) -> bool {
2684 let mut runtime_state = read_runtime_state(session);
2685 if runtime_state.waiting_for_bash.is_none() {
2686 return false;
2687 }
2688 runtime_state.waiting_for_bash = None;
2689 runtime_state.status = AgentStatusState::Idle;
2690 runtime_state.suspension = None;
2691 write_runtime_state(session, &runtime_state);
2692 session.metadata.remove("runtime.suspend_reason");
2693 session.add_message(resume_message.clone());
2694 true
2695}
2696
2697impl ChildCompletionCoordinator {
2699 async fn bash_self_resume(&self, session_id: String, bash_ids: Vec<String>) {
2715 let mut delay = Duration::from_secs(1);
2716 let max_delay = Duration::from_secs(30);
2717 let max_poll = Duration::from_secs(6 * 3600 + 600);
2721 let deadline = tokio::time::Instant::now() + max_poll;
2722
2723 loop {
2724 tokio::time::sleep(delay).await;
2725
2726 let Some(session) = self.load_session(&session_id).await else {
2727 tracing::info!(%session_id, "bash self-resume backstop: session gone; nothing to do");
2728 return;
2729 };
2730 if read_runtime_state(&session).waiting_for_bash.is_none() {
2731 return;
2734 }
2735
2736 let still_running =
2737 bamboo_tools::tools::bash_runtime::running_shells_for_session(&session_id);
2738 let timed_out = tokio::time::Instant::now() >= deadline;
2739 if still_running.is_empty() || timed_out {
2740 let guard = session_resume_lock(&session_id);
2745 let _held = guard.lock().await;
2746 tracing::warn!(
2747 %session_id,
2748 shell_count = bash_ids.len(),
2749 timed_out,
2750 "bash self-resume backstop engaged (push lost or wait ceiling reached)"
2751 );
2752 self.perform_bash_resume(
2753 &session_id,
2754 bash_completion_resume_message(&bash_ids, timed_out),
2755 )
2756 .await;
2757 return;
2758 }
2759
2760 delay = (delay * 2).min(max_delay);
2761 }
2762 }
2763
2764 async fn perform_bash_resume(&self, session_id: &str, resume_message: Message) {
2785 let retry_backoff = Duration::from_millis(200);
2786 const MAX_RESUME_ATTEMPTS: u8 = 5;
2787 for attempt in 0..MAX_RESUME_ATTEMPTS {
2788 if attempt > 0 {
2789 tokio::time::sleep(retry_backoff).await;
2790 }
2791
2792 let Some(mut session) = self.load_session(session_id).await else {
2793 tracing::warn!(%session_id, "bash resume: session not found; nothing to resume");
2794 return;
2795 };
2796
2797 if !apply_bash_resume_transition(&mut session, &resume_message) {
2798 tracing::info!(
2802 %session_id, attempt,
2803 "bash resume: persisted bash wait already cleared; nothing to resume"
2804 );
2805 return;
2806 }
2807 session.updated_at = Utc::now();
2808 self.save_and_cache(&mut session).await;
2809 tracing::info!(
2810 %session_id, attempt,
2811 "bash resume: cleared bash wait and appended resume message"
2812 );
2813
2814 let outcome = self.resume_parent(session_id.to_string()).await;
2815 match outcome {
2816 ResumeOutcome::Started { .. } => {
2817 tracing::info!(%session_id, attempt, "bash resume: resume fired");
2818 return;
2819 }
2820 ResumeOutcome::NotFound => {
2821 tracing::warn!(%session_id, "bash resume: session vanished during resume");
2822 return;
2823 }
2824 _ => {
2825 let clobbered = match self.load_session(session_id).await {
2831 Some(reloaded) => read_runtime_state(&reloaded).waiting_for_bash.is_some(),
2832 None => {
2833 tracing::warn!(%session_id, "bash resume: session vanished after resume");
2834 return;
2835 }
2836 };
2837 if bash_resume_should_retry(&outcome, clobbered) {
2838 tracing::warn!(
2839 %session_id, attempt,
2840 outcome = outcome.as_str(),
2841 "bash resume: persisted wait still set after resume (finalize-clobber); retrying"
2842 );
2843 continue;
2844 }
2845 tracing::info!(
2846 %session_id, attempt,
2847 outcome = outcome.as_str(),
2848 "bash resume: wait cleared and resume handled; stopping"
2849 );
2850 return;
2851 }
2852 }
2853 }
2854
2855 tracing::warn!(
2856 %session_id,
2857 attempts = MAX_RESUME_ATTEMPTS,
2858 "bash resume: exhausted clobber-retry budget without confirming resume; giving up"
2859 );
2860 }
2861}
2862
2863impl BashResumeHook for ChildCompletionCoordinator {
2864 fn arrange_bash_self_resume(&self, session_id: String, bash_ids: Vec<String>) {
2865 let coordinator = Arc::new(self.clone());
2866 tokio::spawn(async move {
2867 coordinator.bash_self_resume(session_id, bash_ids).await;
2868 });
2869 }
2870}
2871
2872fn bash_completion_injection_body(info: &BashCompletionInfo) -> String {
2877 let exit = match info.exit_code {
2878 Some(code) => code.to_string(),
2879 None => "none (signal/killed)".to_string(),
2880 };
2881 let mut body = format!(
2882 "Runtime notification: background shell `{}` (`{}`) finished — status {}, exit code {}.",
2883 info.bash_id, info.command, info.status, exit
2884 );
2885 if info.output_tail.trim().is_empty() {
2886 body.push_str(" It produced no captured output.");
2887 } else {
2888 body.push_str("\n\nOutput tail:\n");
2889 body.push_str(&info.output_tail);
2890 }
2891 body.push_str(&format!(
2892 "\n\nUse BashOutput with bash_id=\"{}\" for the full output, then continue the task.",
2893 info.bash_id
2894 ));
2895 body
2896}
2897
2898fn bash_completion_envelope(info: &BashCompletionInfo) -> bamboo_domain::SessionMessageEnvelope {
2899 let provider_message = bash_resume_message_from_info(info);
2900 let data = serde_json::json!({
2901 "session_id": info.session_id,
2902 "bash_id": info.bash_id,
2903 "command": info.command,
2904 "exit_code": info.exit_code,
2905 "status": info.status,
2906 "output_tail": info.output_tail,
2907 });
2908 let identity = data.clone();
2913 bamboo_domain::SessionMessageEnvelope {
2914 id: bamboo_domain::SessionMessageId::stable("background_bash_completion", &identity),
2915 source: bamboo_domain::SessionMessageSource::Runtime {
2916 subsystem: "background_bash".to_string(),
2917 },
2918 target_session_id: info.session_id.clone(),
2919 kind: bamboo_domain::SessionMessageKind::RuntimeInstruction,
2920 body: bamboo_domain::SessionMessageBody::RuntimeInstruction(
2921 bamboo_domain::SessionRuntimeInstruction {
2922 instruction: "background_bash_completed".to_string(),
2923 content: Some(bamboo_domain::SessionMessageContent::text(
2924 bash_completion_injection_body(info),
2925 )),
2926 data: Some(data),
2927 provider_message: Some(session_provider_message(&provider_message)),
2928 },
2929 ),
2930 created_at: Utc::now(),
2931 thread_id: None,
2932 in_reply_to: None,
2933 attempt: None,
2934 correlation_id: Some(info.bash_id.clone()),
2935 }
2936}
2937
2938async fn enqueue_bash_completion_injection(
2944 persistence: &LockedSessionStore,
2945 info: &BashCompletionInfo,
2946) -> std::io::Result<Option<Session>> {
2947 let body = bash_completion_injection_body(info);
2948 let queued = serde_json::json!({
2949 "content": body,
2950 "created_at": Utc::now(),
2951 });
2952 persistence
2953 .update_runtime_config(&info.session_id, move |session| {
2954 let mut pending = session.pending_injected_messages().unwrap_or_default();
2955 pending.push(queued);
2956 session.set_pending_injected_messages(pending);
2957 })
2958 .await
2959}
2960
2961fn bash_resume_message_from_info(info: &BashCompletionInfo) -> Message {
2969 let mut message = Message::user(bash_completion_injection_body(info));
2970 message.metadata = Some(serde_json::json!({
2971 RUNTIME_RESUME_MESSAGE_HIDDEN_KEY: true,
2972 RUNTIME_RESUME_MESSAGE_KIND_KEY: BASH_COMPLETION_RESUME_KIND,
2973 }));
2974 message.never_compress = false;
2975 message
2976}
2977
2978impl ChildCompletionCoordinator {
2979 async fn deliver_bash_completion(&self, mut info: BashCompletionInfo) {
2991 if let Some(hook_session) = self.load_session(&info.session_id).await {
2997 let config_snapshot = self.config.read().await.clone();
2998 let _ = run_background_bash_post_tool_hooks(
2999 &config_snapshot.lifecycle_hooks,
3000 Some(self.app_data_dir.clone()),
3001 &hook_session,
3002 &mut info,
3003 )
3004 .await;
3005 }
3006
3007 let guard = session_resume_lock(&info.session_id);
3008 let _held = guard.lock().await;
3009
3010 let Some(session) = self.load_session(&info.session_id).await else {
3011 tracing::warn!(
3012 session_id = %info.session_id,
3013 bash_id = %info.bash_id,
3014 "background bash completion: owning session not found; nothing to notify"
3015 );
3016 return;
3017 };
3018
3019 let waiting = read_runtime_state(&session).waiting_for_bash.is_some();
3020 let all_shells_done =
3026 bamboo_tools::tools::bash_runtime::running_shells_for_session(&info.session_id)
3027 .is_empty();
3028 let delivery_plan = bash_completion_delivery_plan(waiting, all_shells_done);
3029
3030 let messenger = self.agent.session_messenger().cloned();
3036 let admission = match messenger.as_ref() {
3037 Some(messenger) => match messenger.admit(bash_completion_envelope(&info)).await {
3038 Ok(admission) => Some(admission),
3039 Err(error) => {
3040 tracing::warn!(
3041 session_id = %info.session_id,
3042 bash_id = %info.bash_id,
3043 %error,
3044 "background bash completion: durable SessionInbox admission failed; leaving wait armed"
3045 );
3046 return;
3047 }
3048 },
3049 None => {
3050 tracing::warn!(
3051 session_id = %info.session_id,
3052 bash_id = %info.bash_id,
3053 "SessionMessenger unavailable; using compatibility injection before clearing wait"
3054 );
3055 match enqueue_bash_completion_injection(&self.persistence, &info).await {
3056 Ok(Some(_)) => None,
3057 Ok(None) => {
3058 tracing::warn!(
3059 session_id = %info.session_id,
3060 "background bash compatibility target disappeared"
3061 );
3062 return;
3063 }
3064 Err(error) => {
3065 tracing::warn!(
3066 session_id = %info.session_id,
3067 %error,
3068 "background bash compatibility admission failed; leaving wait armed"
3069 );
3070 return;
3071 }
3072 }
3073 }
3074 };
3075
3076 if delivery_plan == BashCompletionDeliveryPlan::DurableOnly {
3077 tracing::info!(
3078 session_id = %info.session_id,
3079 bash_id = %info.bash_id,
3080 "background bash completion is durable; sibling shells still run, so wait remains armed"
3081 );
3082 return;
3083 }
3084
3085 if let (Some(messenger), Some(admission)) = (messenger.as_ref(), admission.as_ref()) {
3086 if let Err(error) = messenger.prepare_activation(admission).await {
3087 tracing::warn!(
3088 session_id = %info.session_id,
3089 bash_id = %info.bash_id,
3090 %error,
3091 "background bash completion activation watermark failed; leaving wait armed"
3092 );
3093 return;
3094 }
3095 }
3096
3097 if delivery_plan == BashCompletionDeliveryPlan::ClearWaitThenActivate {
3098 tracing::info!(
3099 session_id = %info.session_id,
3100 bash_id = %info.bash_id,
3101 status = %info.status,
3102 "background bash completion: push-resuming suspended loop (event-driven)"
3103 );
3104 let mut resumable = session.clone();
3105 let mut runtime_state = read_runtime_state(&resumable);
3106 runtime_state.waiting_for_bash = None;
3107 runtime_state.status = AgentStatusState::Idle;
3108 runtime_state.suspension = None;
3109 write_runtime_state(&mut resumable, &runtime_state);
3110 resumable.metadata.remove("runtime.suspend_reason");
3111 resumable.updated_at = Utc::now();
3112 if let Err(error) = self.persistence.merge_save_runtime(&mut resumable).await {
3113 tracing::warn!(
3114 session_id = %info.session_id,
3115 %error,
3116 "background bash completion is durable but wait-state clear failed; leaving activation to the wait backstop"
3117 );
3118 return;
3119 }
3120 self.sessions.insert(
3121 resumable.id.clone(),
3122 Arc::new(crate::SessionSnapshot::new(resumable)),
3123 );
3124 }
3125
3126 let (Some(messenger), Some(admission)) = (messenger, admission) else {
3130 if delivery_plan == BashCompletionDeliveryPlan::ClearWaitThenActivate {
3131 let _ = self.resume_parent(info.session_id.clone()).await;
3132 }
3133 return;
3134 };
3135 match messenger.activate_prepared(&admission).await {
3136 Ok(receipt) => tracing::info!(
3137 session_id = %info.session_id,
3138 bash_id = %info.bash_id,
3139 status = %info.status,
3140 waiting,
3141 generation = receipt.delivery.generation,
3142 activation = ?receipt.activation,
3143 "background bash completion delivered through SessionMessenger"
3144 ),
3145 Err(error) => tracing::warn!(
3146 session_id = %info.session_id,
3147 bash_id = %info.bash_id,
3148 %error,
3149 "background bash completion: SessionMessenger delivery failed"
3150 ),
3151 }
3152 }
3153}
3154
3155impl BashCompletionSink for ChildCompletionCoordinator {
3156 fn on_bash_completed(&self, info: BashCompletionInfo) {
3157 let coordinator = Arc::new(self.clone());
3161 tokio::spawn(async move {
3162 coordinator.deliver_bash_completion(info).await;
3163 });
3164 }
3165}
3166
3167const CHILD_WAIT_SWEEP_INTERVAL_SECS: u64 = 30;
3173const CHILD_WAIT_REGISTRATION_GRACE_SECS: i64 = 60;
3177const DEAD_CHILD_GRACE_SECS: i64 = 120;
3180const STALE_RUNNER_SLACK_SECS: i64 = 600;
3186
3187fn is_dead_child_candidate_status(status: Option<&str>) -> bool {
3192 match status {
3193 Some(status) => !is_terminal_child_status(status) && status != "suspended",
3196 None => true,
3197 }
3198}
3199
3200fn completion_child_is_owned(reported_parent: &str, child_parent_linkage: Option<&str>) -> bool {
3208 child_parent_linkage == Some(reported_parent)
3209}
3210
3211fn child_wait_watchdog_resume_message(body: String) -> Message {
3212 let mut message = Message::user(body);
3213 message.metadata = Some(serde_json::json!({
3214 RUNTIME_RESUME_MESSAGE_HIDDEN_KEY: true,
3215 RUNTIME_RESUME_MESSAGE_KIND_KEY: "child_wait_watchdog_resume",
3216 }));
3217 message.never_compress = false;
3218 message
3219}
3220
3221fn empty_child_wait_message() -> Message {
3222 let inspection = serde_json::json!({"intent": "inspect", "message": "tree"});
3223 child_wait_watchdog_resume_message(format!(
3224 "Runtime notification: this session was suspended waiting for child sessions, but the \
3225 wait tracked no children (internal inconsistency). The session has been resumed; use \
3226 SubAgent with {inspection} to inspect child state and continue the task.",
3227 ))
3228}
3229
3230fn child_wait_lease_expired_message(child_ids: &[String]) -> Message {
3231 let inspection = serde_json::json!({"intent": "inspect", "message": "tree"});
3232 child_wait_watchdog_resume_message(format!(
3233 "Runtime notification: the wait lease for child session(s) [{}] expired before they all \
3234 reported completion. They were NOT cancelled and may still be running or already \
3235 finished — verify their actual status by calling SubAgent with {inspection} before assuming \
3236 anything, then continue the task.",
3237 child_ids.join(", ")
3238 ))
3239}
3240
3241impl ChildCompletionCoordinator {
3257 pub fn spawn_child_wait_watchdog(self: &Arc<Self>) {
3260 let coordinator = Arc::clone(self);
3261 tokio::spawn(async move {
3262 use futures::FutureExt;
3263 if std::panic::AssertUnwindSafe(coordinator.reconcile_orphans_at_boot())
3264 .catch_unwind()
3265 .await
3266 .is_err()
3267 {
3268 tracing::error!("child-wait watchdog: boot reconciliation panicked");
3269 }
3270 let mut ticker = tokio::time::interval(std::time::Duration::from_secs(
3271 CHILD_WAIT_SWEEP_INTERVAL_SECS,
3272 ));
3273 ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
3274 ticker.tick().await;
3276 loop {
3277 ticker.tick().await;
3278 if std::panic::AssertUnwindSafe(coordinator.sweep_child_waits())
3279 .catch_unwind()
3280 .await
3281 .is_err()
3282 {
3283 tracing::error!("child-wait watchdog: sweep panicked; continuing");
3284 }
3285 }
3286 });
3287 }
3288
3289 async fn reconcile_orphans_at_boot(&self) {
3294 let cutoff = Utc::now();
3295
3296 let running = self
3302 .storage
3303 .list_sessions_by_run_status("running")
3304 .await
3305 .unwrap_or_default();
3306 for (child_id, parent_id) in running {
3307 let Some(parent_id) = parent_id else { continue };
3308 if self.runner_is_running(&child_id).await {
3309 continue;
3310 }
3311 let Some(control_plane) = self.load_control_plane(&child_id).await else {
3312 continue;
3313 };
3314 if control_plane.updated_at >= cutoff {
3317 continue;
3318 }
3319 tracing::warn!(
3320 child_session_id = %child_id,
3321 parent_session_id = %parent_id,
3322 "boot reconciliation: child was running when the process died; \
3323 marking it error and waking the parent"
3324 );
3325 self.synthesize_child_completion(
3326 &parent_id,
3327 &child_id,
3328 "error",
3329 Some(
3330 "orphaned by server restart: the process died while this child session \
3331 was running"
3332 .to_string(),
3333 ),
3334 )
3335 .await;
3336 }
3337
3338 let suspended = self
3342 .storage
3343 .list_sessions_by_run_status("suspended")
3344 .await
3345 .unwrap_or_default();
3346 for (session_id, _) in suspended {
3347 let Some(control_plane) = self.load_control_plane(&session_id).await else {
3348 continue;
3349 };
3350 if control_plane
3351 .metadata
3352 .get("runtime.suspend_reason")
3353 .map(String::as_str)
3354 != Some("waiting_for_bash")
3355 {
3356 continue;
3357 }
3358 if let Some(wait) = read_runtime_state(&control_plane).waiting_for_bash {
3359 tracing::warn!(
3360 %session_id,
3361 "boot reconciliation: re-arming bash self-resume backstop lost in restart"
3362 );
3363 let coordinator = self.clone();
3364 tokio::spawn(async move {
3365 coordinator
3366 .bash_self_resume(session_id, wait.bash_ids)
3367 .await;
3368 });
3369 }
3370 }
3371 }
3372
3373 async fn runner_is_running(&self, session_id: &str) -> bool {
3374 let runners = self.agent_runners.read().await;
3375 runners
3376 .get(session_id)
3377 .is_some_and(|runner| matches!(runner.status, AgentStatus::Running))
3378 }
3379
3380 async fn load_control_plane(&self, session_id: &str) -> Option<Session> {
3381 match self.storage.load_runtime_control_plane(session_id).await {
3382 Ok(session) => session,
3383 Err(error) => {
3384 tracing::warn!(
3385 %session_id,
3386 %error,
3387 "child-wait watchdog: failed to load session control plane"
3388 );
3389 None
3390 }
3391 }
3392 }
3393
3394 async fn sweep_child_waits(&self) {
3398 let suspended = match self.storage.list_sessions_by_run_status("suspended").await {
3399 Ok(entries) => entries,
3400 Err(error) => {
3401 tracing::warn!(%error, "child-wait watchdog: failed to list suspended sessions");
3402 return;
3403 }
3404 };
3405 for (session_id, _) in suspended {
3406 self.sweep_one_suspended_session(&session_id).await;
3407 }
3408 }
3409
3410 async fn sweep_one_suspended_session(&self, session_id: &str) {
3411 if self.runner_is_running(session_id).await {
3414 return;
3415 }
3416 let Some(session) = self.load_control_plane(session_id).await else {
3417 return;
3418 };
3419 let runtime_state = read_runtime_state(&session);
3420 let suspend_reason = session
3421 .metadata
3422 .get("runtime.suspend_reason")
3423 .map(String::as_str)
3424 .unwrap_or_default()
3425 .to_string();
3426 match (
3427 suspend_reason.as_str(),
3428 runtime_state.waiting_for_children.clone(),
3429 ) {
3430 ("waiting_for_bash", _)
3432 | ("awaiting_clarification", _)
3433 | ("awaiting_parent_approval", _) => {}
3434 (_, Some(wait)) => self.sweep_child_wait(session_id, wait).await,
3435 ("waiting_for_children", None) | ("", None) => {
3438 self.rescue_stranded_resume(session_id).await;
3439 }
3440 _ => {}
3441 }
3442 }
3443
3444 async fn sweep_child_wait(&self, parent_session_id: &str, wait: WaitingForChildrenState) {
3449 let now = Utc::now();
3450
3451 if wait.child_session_ids.is_empty() {
3452 tracing::warn!(
3453 %parent_session_id,
3454 "child-wait watchdog: wait armed over an empty child set; force-resuming"
3455 );
3456 self.force_resume_child_wait(parent_session_id, empty_child_wait_message())
3457 .await;
3458 return;
3459 }
3460
3461 if wait.timeout_at.is_some_and(|deadline| now >= deadline) {
3465 tracing::warn!(
3466 %parent_session_id,
3467 "child-wait watchdog: wait lease expired; force-resuming parent"
3468 );
3469 self.force_resume_child_wait(
3470 parent_session_id,
3471 child_wait_lease_expired_message(&wait.child_session_ids),
3472 )
3473 .await;
3474 return;
3475 }
3476
3477 if now.signed_duration_since(wait.registered_at).num_seconds()
3479 < CHILD_WAIT_REGISTRATION_GRACE_SECS
3480 {
3481 return;
3482 }
3483
3484 let statuses: HashMap<String, Option<String>> = self
3485 .storage
3486 .list_child_run_statuses(parent_session_id)
3487 .await
3488 .unwrap_or_default()
3489 .into_iter()
3490 .collect();
3491
3492 struct DeadChild {
3493 child_id: String,
3494 status: String,
3495 reason: String,
3496 owned: bool,
3500 }
3501
3502 let mut terminal: Vec<(String, String)> = Vec::new();
3503 let mut dead: Vec<DeadChild> = Vec::new();
3504 for child_id in &wait.child_session_ids {
3505 let status = statuses.get(child_id).and_then(|status| status.as_deref());
3506 if let Some(status) = status {
3507 if is_terminal_child_status(status) {
3508 terminal.push((child_id.clone(), status.to_string()));
3509 continue;
3510 }
3511 }
3512 if !is_dead_child_candidate_status(status) {
3513 continue;
3514 }
3515
3516 let control_plane = self.load_control_plane(child_id).await;
3523 let owned = control_plane
3524 .as_ref()
3525 .is_some_and(|cp| cp.parent_session_id.as_deref() == Some(parent_session_id));
3526 if !owned {
3527 dead.push(DeadChild {
3528 child_id: child_id.clone(),
3529 status: "error".to_string(),
3530 reason: if control_plane.is_some() {
3531 "waited-on session id is not a child of this session; clearing it \
3532 from the wait without touching that session"
3533 .to_string()
3534 } else {
3535 "waited-on child session does not exist".to_string()
3536 },
3537 owned: false,
3538 });
3539 continue;
3540 }
3541
3542 let runner = { self.agent_runners.read().await.get(child_id).cloned() };
3543 match runner {
3544 Some(runner) if matches!(runner.status, AgentStatus::Running) => {
3545 let last_activity = runner.last_activity_at().unwrap_or(runner.started_at);
3551 let idle_secs = now.signed_duration_since(last_activity).num_seconds();
3552 let total_secs = now.signed_duration_since(runner.started_at).num_seconds();
3553 let policy = match &control_plane {
3554 Some(child) => {
3555 crate::runtime::execution::spawn::watchdog_policy_for_session(child)
3556 }
3557 None => Default::default(),
3558 };
3559 let idle_limit = policy.max_idle_secs.saturating_add(STALE_RUNNER_SLACK_SECS);
3560 let total_limit = policy
3561 .max_total_secs
3562 .saturating_add(STALE_RUNNER_SLACK_SECS);
3563 if idle_secs >= idle_limit || total_secs >= total_limit {
3564 runner.cancel_token.cancel();
3565 dead.push(DeadChild {
3566 child_id: child_id.clone(),
3567 status: "timeout".to_string(),
3568 reason: format!(
3569 "child runner stalled: no events for {idle_secs}s \
3570 (limit {idle_limit}s including watchdog slack); \
3571 force-finalized by the child-wait watchdog"
3572 ),
3573 owned: true,
3574 });
3575 }
3576 }
3577 _ => {
3578 let quiet_secs = control_plane
3582 .as_ref()
3583 .map(|child| now.signed_duration_since(child.updated_at).num_seconds())
3584 .unwrap_or(i64::MAX);
3585 if quiet_secs >= DEAD_CHILD_GRACE_SECS {
3586 dead.push(DeadChild {
3587 child_id: child_id.clone(),
3588 status: "error".to_string(),
3589 reason: format!(
3590 "child runner lost (crashed task, dropped spawn job, or \
3591 process restart): index status {status:?} with no live \
3592 runner driving it"
3593 ),
3594 owned: true,
3595 });
3596 }
3597 }
3598 }
3599 }
3600
3601 if !dead.is_empty() {
3602 for entry in dead {
3603 tracing::warn!(
3604 %parent_session_id,
3605 child_session_id = %entry.child_id,
3606 status = %entry.status,
3607 reason = %entry.reason,
3608 owned = entry.owned,
3609 "child-wait watchdog: synthesizing terminal completion for dead child"
3610 );
3611 if entry.owned {
3612 self.synthesize_child_completion(
3613 parent_session_id,
3614 &entry.child_id,
3615 &entry.status,
3616 Some(entry.reason),
3617 )
3618 .await;
3619 } else {
3620 self.publish_synthetic_completion(
3622 parent_session_id,
3623 &entry.child_id,
3624 &entry.status,
3625 Some(entry.reason),
3626 )
3627 .await;
3628 }
3629 }
3630 return;
3632 }
3633
3634 terminal.sort_by_key(|(_, status)| !is_error_like(status));
3639 for (child_id, status) in terminal {
3640 tracing::debug!(
3641 %parent_session_id,
3642 child_session_id = %child_id,
3643 "child-wait watchdog: replaying terminal child outcome"
3644 );
3645 let control_plane = self.load_control_plane(&child_id).await;
3646 let mut completion = ChildCompletion {
3647 parent_session_id: parent_session_id.into(),
3648 child_session_id: child_id.clone(),
3649 status,
3650 error: control_plane.as_ref().and_then(Session::last_run_error),
3651 completed_at: Utc::now(),
3652 source: None,
3653 };
3654 if control_plane
3657 .as_ref()
3658 .is_some_and(|child| child.parent_session_id.as_deref() == Some(parent_session_id))
3659 {
3660 let Ok(Some(child)) = self.storage.load_session(&child_id).await else {
3661 continue;
3662 };
3663 completion.source = ChildCompletionSource::from_committed_session(&child)
3664 .filter(|source| source.matches_completion(&completion));
3665 if ChildCompletionSource::has_source_record(&child) && completion.source.is_none() {
3666 tracing::warn!(%child_id,
3667 "unavailable committed replay source; leaving wait armed");
3668 continue;
3669 }
3670 completion.completed_at = child.updated_at;
3671 }
3672 self.publish_completion(completion).await;
3673 }
3674 }
3675
3676 async fn synthesize_child_completion(
3682 &self,
3683 parent_session_id: &str,
3684 child_session_id: &str,
3685 status: &str,
3686 error: Option<String>,
3687 ) {
3688 match self.storage.load_session(child_session_id).await {
3689 Ok(Some(mut child)) => {
3690 if child.parent_session_id.as_deref() != Some(parent_session_id) {
3695 tracing::warn!(
3696 %parent_session_id,
3697 child_session_id = %child.id,
3698 "child-wait watchdog: refusing to synthesize status onto a session \
3699 that is not a child of this parent"
3700 );
3701 self.publish_synthetic_completion(
3702 parent_session_id,
3703 child_session_id,
3704 status,
3705 error,
3706 )
3707 .await;
3708 return;
3709 }
3710 child.set_last_run_status(status);
3711 match &error {
3712 Some(message) => child.set_last_run_error(message.clone()),
3713 None => child.clear_last_run_error(),
3714 }
3715 child.updated_at = Utc::now();
3716 if let Err(save_error) = self.persistence.merge_save_runtime(&mut child).await {
3717 tracing::warn!(
3718 child_session_id = %child.id,
3719 %save_error,
3720 "child-wait watchdog: failed to persist synthesized terminal status"
3721 );
3722 }
3723 self.sessions.insert(
3724 child.id.clone(),
3725 Arc::new(crate::SessionSnapshot::new(child)),
3726 );
3727 }
3728 Ok(None) => {}
3729 Err(load_error) => {
3730 tracing::warn!(
3731 %child_session_id,
3732 %load_error,
3733 "child-wait watchdog: failed to load child for synthesized terminal status"
3734 );
3735 }
3736 }
3737 finalize_runner(
3738 &self.agent_runners,
3739 child_session_id,
3740 &Err(bamboo_agent_core::AgentError::LLM(
3741 error
3742 .clone()
3743 .unwrap_or_else(|| format!("synthesized {status}")),
3744 )),
3745 )
3746 .await;
3747 self.publish_synthetic_completion(parent_session_id, child_session_id, status, error)
3748 .await;
3749 }
3750
3751 async fn publish_synthetic_completion(
3752 &self,
3753 parent_session_id: &str,
3754 child_session_id: &str,
3755 status: &str,
3756 error: Option<String>,
3757 ) {
3758 self.publish_completion(ChildCompletion {
3759 parent_session_id: parent_session_id.into(),
3760 child_session_id: child_session_id.into(),
3761 status: status.into(),
3762 error,
3763 completed_at: Utc::now(),
3764 source: None,
3765 })
3766 .await;
3767 }
3768
3769 async fn publish_completion(&self, completion: ChildCompletion) {
3770 let publisher =
3771 crate::runtime::execution::session_events::ReplayableSessionEventPublisher::new(
3772 self.agent_runners.clone(),
3773 self.session_event_senders.clone(),
3774 self.account_feed_inbox.clone(),
3775 );
3776 let handler: Arc<dyn ChildCompletionHandler> = Arc::new(self.clone());
3777 crate::runtime::execution::spawn::publish_child_completion(
3778 &publisher,
3779 Some(handler),
3780 completion,
3781 )
3782 .await;
3783 }
3784
3785 async fn rescue_stranded_resume(&self, session_id: &str) {
3791 let Some(session) = self.load_session(session_id).await else {
3792 return;
3793 };
3794 let pending_runtime_resume = session.messages.last().is_some_and(|message| {
3795 matches!(message.role, Role::User)
3796 && message
3797 .metadata
3798 .as_ref()
3799 .is_some_and(|meta| meta.get(RUNTIME_RESUME_MESSAGE_KIND_KEY).is_some())
3800 });
3801 if !pending_runtime_resume {
3802 return;
3803 }
3804 tracing::warn!(
3805 %session_id,
3806 "child-wait watchdog: stranded resume detected (wait cleared, resume never \
3807 spawned); resuming"
3808 );
3809 self.resume_parent(session_id.to_string()).await;
3810 }
3811
3812 async fn force_resume_child_wait(&self, session_id: &str, resume_message: Message) {
3818 const MAX_ATTEMPTS: u8 = 5;
3819 let lock = session_resume_lock(session_id);
3820 for attempt in 0..MAX_ATTEMPTS {
3821 if attempt > 0 {
3822 tokio::time::sleep(Duration::from_millis(200)).await;
3823 }
3824 {
3825 let _held = lock.lock().await;
3826 let Some(mut session) = self.load_session(session_id).await else {
3827 return;
3828 };
3829 let mut runtime_state = read_runtime_state(&session);
3830 if runtime_state.waiting_for_children.is_none() {
3831 return;
3833 }
3834 runtime_state.waiting_for_children = None;
3835 runtime_state.status = AgentStatusState::Idle;
3836 runtime_state.suspension = None;
3837 write_runtime_state(&mut session, &runtime_state);
3838 session.metadata.remove("runtime.suspend_reason");
3839 session.add_message(resume_message.clone());
3840 session.updated_at = Utc::now();
3841 self.save_and_cache(&mut session).await;
3842 }
3843 let outcome = self.resume_parent(session_id.to_string()).await;
3844 match outcome {
3845 ResumeOutcome::Started { .. } | ResumeOutcome::NotFound => return,
3846 ResumeOutcome::Completed | ResumeOutcome::AlreadyRunning { .. } => {
3847 let clobbered = self
3851 .load_session(session_id)
3852 .await
3853 .map(|session| read_runtime_state(&session).waiting_for_children.is_some())
3854 .unwrap_or(false);
3855 if !clobbered {
3856 return;
3857 }
3858 }
3859 }
3860 }
3861 tracing::error!(
3862 %session_id,
3863 "child-wait watchdog: force-resume exhausted its clobber-retry budget"
3864 );
3865 }
3866}
3867
3868#[cfg(test)]
3869mod tests {
3870 use super::*;
3871 include!("child_completion_background_tests.rs");
3872 use bamboo_domain::session::runtime_state::SuspensionState;
3873
3874 #[test]
3875 fn skill_request_child_provider_constructor_keeps_data_absent_and_rejects_injection() {
3876 let provider = Message::user("rendered child outcome");
3877 let presentation = session_provider_message(&provider);
3878 assert_eq!(presentation.content.skill_request, None);
3879 assert_eq!(presentation.content.text, provider.content);
3880 let mut envelope = SessionMessageEnvelope::user_input("parent", "unused");
3881 envelope.source = SessionMessageSource::Runtime {
3882 subsystem: "child_completion_coordinator".into(),
3883 };
3884 envelope.kind = SessionMessageKind::ChildOutcome;
3885 envelope.body = SessionMessageBody::ChildOutcome(SessionChildOutcome {
3886 child_session_id: "child".into(),
3887 status: "completed".into(),
3888 result: Some("typed result".into()),
3889 error: None,
3890 provider_message: Some(presentation),
3891 });
3892 envelope.validate().unwrap();
3893 assert_eq!(
3894 envelope.to_provider_message().unwrap().content,
3895 "rendered child outcome"
3896 );
3897 if let SessionMessageBody::ChildOutcome(outcome) = &mut envelope.body {
3898 outcome
3899 .provider_message
3900 .as_mut()
3901 .unwrap()
3902 .content
3903 .skill_request = Some(bamboo_domain::SessionSkillRequest {
3904 mode: None,
3905 selections: vec![bamboo_domain::SessionSkillSelection {
3906 id: "review".into(),
3907 source: "builtin".into(),
3908 revision: 7,
3909 args: serde_json::Value::Null,
3910 }],
3911 });
3912 }
3913 assert_eq!(
3914 envelope.validate(),
3915 Err(bamboo_domain::SessionMessageValidationError::KindSourceBodyMismatch)
3916 );
3917 }
3918
3919 #[tokio::test]
3920 async fn cancelled_resume_waiters_do_not_retain_historical_parent_ids() {
3921 for index in 0..512 {
3922 let id = format!("resume-lock-reclaim-{index}");
3923 let owner = session_resume_lock(&id);
3924 let held = owner.lock().await;
3925 let waiter = session_resume_lock(&id);
3926 let mut waiting = Box::pin(waiter.lock());
3927 assert!(futures::poll!(waiting.as_mut()).is_pending());
3928 drop(held);
3929 drop(owner);
3930 drop(waiting);
3931 drop(waiter);
3932 assert!(!parent_locks().lock().recover_poison().contains_key(&id));
3933 }
3934 }
3935
3936 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
3937 async fn resume_lock_reclamation_preserves_exclusive_parent_wake_ownership() {
3938 use std::sync::atomic::{AtomicUsize, Ordering};
3939 let active = Arc::new(AtomicUsize::new(0));
3940 let mut tasks = Vec::new();
3941 for _ in 0..32 {
3942 let active = active.clone();
3943 tasks.push(tokio::spawn(async move {
3944 for _ in 0..16 {
3945 let lease = session_resume_lock("resume-lock-exclusive");
3946 let _guard = lease.lock().await;
3947 assert_eq!(active.fetch_add(1, Ordering::SeqCst), 0);
3948 tokio::task::yield_now().await;
3949 assert_eq!(active.fetch_sub(1, Ordering::SeqCst), 1);
3950 }
3951 }));
3952 }
3953 for task in tasks {
3954 task.await.unwrap();
3955 }
3956 assert!(!parent_locks()
3957 .lock()
3958 .recover_poison()
3959 .contains_key("resume-lock-exclusive"));
3960 }
3961 use bamboo_agent_core::Message;
3962 use bamboo_domain::SessionInboxPort;
3963 use futures::stream;
3964 use std::sync::atomic::{AtomicUsize, Ordering};
3965
3966 struct FailingRuntimeSaveStorage {
3967 inner: Arc<bamboo_storage::SessionStoreV2>,
3968 }
3969
3970 #[async_trait]
3971 impl Storage for FailingRuntimeSaveStorage {
3972 async fn save_session(&self, session: &Session) -> std::io::Result<()> {
3973 self.inner.save_session(session).await
3974 }
3975
3976 async fn load_session(&self, id: &str) -> std::io::Result<Option<Session>> {
3977 self.inner.load_session(id).await
3978 }
3979
3980 async fn delete_session(&self, id: &str) -> std::io::Result<bool> {
3981 self.inner.delete_session(id).await
3982 }
3983
3984 async fn load_runtime_control_plane(&self, id: &str) -> std::io::Result<Option<Session>> {
3985 self.inner.load_runtime_control_plane(id).await
3986 }
3987
3988 async fn save_runtime_state(&self, _session: &Session) -> std::io::Result<()> {
3989 Err(std::io::Error::other("injected sidecar save failure"))
3990 }
3991 }
3992
3993 struct EmptyTools;
3994
3995 #[async_trait]
3996 impl bamboo_agent_core::tools::ToolExecutor for EmptyTools {
3997 async fn execute(
3998 &self,
3999 _call: &bamboo_agent_core::tools::ToolCall,
4000 ) -> Result<bamboo_agent_core::tools::ToolResult, bamboo_agent_core::tools::ToolError>
4001 {
4002 Err(bamboo_agent_core::tools::ToolError::NotFound(
4003 "no tools".to_string(),
4004 ))
4005 }
4006
4007 fn list_tools(&self) -> Vec<bamboo_agent_core::tools::ToolSchema> {
4008 Vec::new()
4009 }
4010 }
4011
4012 struct CompletedTestProvider;
4013
4014 #[async_trait]
4015 impl bamboo_llm::LLMProvider for CompletedTestProvider {
4016 async fn chat_stream(
4017 &self,
4018 _messages: &[Message],
4019 _tools: &[bamboo_agent_core::tools::ToolSchema],
4020 _max_output_tokens: Option<u32>,
4021 _model: &str,
4022 ) -> Result<bamboo_llm::LLMStream, bamboo_llm::LLMError> {
4023 let chunks: Vec<bamboo_llm::provider::Result<bamboo_llm::LLMChunk>> = vec![
4024 Ok(bamboo_llm::LLMChunk::Token("done".to_string())),
4025 Ok(bamboo_llm::LLMChunk::Done),
4026 ];
4027 Ok(Box::pin(stream::iter(chunks)))
4028 }
4029 }
4030
4031 struct CountingActivationSpawner {
4032 reservations: Arc<AtomicUsize>,
4033 launches: Arc<AtomicUsize>,
4034 }
4035
4036 #[async_trait]
4037 impl SessionActivationSpawner for CountingActivationSpawner {
4038 async fn reserve_activation(
4039 &self,
4040 target_session_id: &str,
4041 inbox_generation: u64,
4042 ) -> Result<SessionActivationReserveOutcome, bamboo_domain::SessionActivationError>
4043 {
4044 self.reservations.fetch_add(1, Ordering::SeqCst);
4045 let launches = self.launches.clone();
4046 Ok(SessionActivationReserveOutcome::Reserved(
4047 SessionActivationLaunch::new(
4048 format!("{target_session_id}-{inbox_generation}"),
4049 move || {
4050 launches.fetch_add(1, Ordering::SeqCst);
4051 },
4052 ),
4053 ))
4054 }
4055 }
4056
4057 async fn save_sourced_test_child(store: &bamboo_storage::SessionStoreV2, child: &Session) {
4058 let mut saved = child.clone();
4059 ChildCompletionSource::prepare(
4060 &mut saved,
4061 &format!("test-run-{}", child.id),
4062 &Default::default(),
4063 );
4064 store.save_session(&saved).await.unwrap();
4065 }
4066
4067 async fn test_child_source(
4068 store: &bamboo_storage::SessionStoreV2,
4069 id: &str,
4070 ) -> Option<ChildCompletionSource> {
4071 ChildCompletionSource::from_committed_session(
4072 &store.load_session(id).await.unwrap().unwrap(),
4073 )
4074 }
4075
4076 fn typed_test_report() -> serde_json::Value {
4077 serde_json::json!({"version":1,"outcome":"blocked","summary":"Quoted \"claim\" \\ 用户🪷\n",
4078 "reported_evidence":[{"description":"reported only","reference":"/never/read/reported-ref","sha256":null}],
4079 "reported_verification":[{"check":"fixture","reported_status":"passed","details":"a model claim"}],
4080 "proposals":["Try the next step"],"blockers":["Need approval"],"open_decisions":["Which route?"]})
4081 }
4082
4083 fn typed_test_child(parent: &Session, id: &str, report: &str) -> Session {
4084 let mut child = Session::new_child(id, &parent.id, "model", "Child");
4085 let packet = bamboo_domain::ChildContextPacket {
4086 version: 1,
4087 objective: "bounded task".into(),
4088 constraints: vec![],
4089 acceptance: vec!["report only".into()],
4090 non_goals: vec![],
4091 necessary_user_instructions: vec![],
4092 recorded_decisions: vec![],
4093 source_user_message_ids: vec![parent.messages[0].id.clone()],
4094 background_message_ids: vec![],
4095 };
4096 let resolved = packet.resolve(parent, "bounded task").unwrap();
4097 let binding = bamboo_domain::ChildContextBinding::new(
4098 parent,
4099 id,
4100 resolved.required_brief.clone(),
4101 resolved,
4102 )
4103 .unwrap();
4104 binding.install(&mut child).unwrap();
4105 child.add_message(binding.assignment_message());
4106 child.add_message(Message::assistant(report, None));
4107 child.set_last_run_status("completed");
4108 ChildCompletionSource::prepare(&mut child, "sealed-report-run", &Default::default());
4109 child
4110 }
4111
4112 fn typed_test_completion(child: &Session) -> ChildCompletion {
4113 ChildCompletion {
4114 parent_session_id: child.parent_session_id.clone().unwrap(),
4115 child_session_id: child.id.clone(),
4116 status: child.last_run_status().unwrap(),
4117 error: child.last_run_error(),
4118 completed_at: child.updated_at,
4119 source: ChildCompletionSource::from_committed_session(child),
4120 }
4121 }
4122
4123 fn unusable_typed_test_child(
4124 parent: &Session,
4125 id: &str,
4126 case: &str,
4127 ) -> (Session, &'static str) {
4128 let mut child = typed_test_child(parent, id, &typed_test_report().to_string());
4129 let mut prior = Default::default();
4130 let reason = match case {
4131 "prose" => {
4132 child.messages.last_mut().unwrap().content = "UNUSABLE_RAW_PROSE".repeat(64);
4133 "report_malformed"
4134 }
4135 "malformed" => {
4136 child.messages.last_mut().unwrap().content =
4137 "{\"version\":1,UNUSABLE_RAW_JSON".into();
4138 "report_malformed"
4139 }
4140 "oversized" => {
4141 child
4142 .messages
4143 .last_mut()
4144 .unwrap()
4145 .content
4146 .push_str(&" ".repeat(8193));
4147 "result_budget_exceeded"
4148 }
4149 "multipart" => {
4150 child.messages.last_mut().unwrap().content_parts = Some(
4151 serde_json::from_value(
4152 serde_json::json!([{"type":"text","text":"UNUSABLE_RAW_PART"}]),
4153 )
4154 .unwrap(),
4155 );
4156 "report_not_current_final"
4157 }
4158 "commentary" => {
4159 child.messages.last_mut().unwrap().phase =
4160 Some(bamboo_domain::MessagePhase::Commentary);
4161 "report_not_current_final"
4162 }
4163 "blank" => {
4164 child.messages.last_mut().unwrap().content = " ".into();
4165 "report_not_current_final"
4166 }
4167 "old_final" => {
4168 prior = child.messages.iter().map(|m| m.id.clone()).collect();
4169 "report_not_current_final"
4170 }
4171 "error" => {
4172 prior = child.messages.iter().map(|m| m.id.clone()).collect();
4173 child.set_last_run_status("error");
4174 child.set_last_run_error("UNUSABLE_RAW_ERROR\"\\\n".repeat(4096));
4175 "report_not_current_final"
4176 }
4177 "absent" => {
4178 child.messages.pop();
4179 "report_absent"
4180 }
4181 _ => panic!("unknown unusable report case"),
4182 };
4183 ChildCompletionSource::prepare(&mut child, "unusable-report-run", &prior);
4184 (child, reason)
4185 }
4186
4187 #[test]
4188 fn sealed_unusable_reports_keep_current_source_boundary_and_escape_budget() {
4189 let mut parent = Session::new("unavailable-parent", "model");
4190 parent.add_message(Message::user("required source"));
4191 for case in [
4192 "prose",
4193 "malformed",
4194 "oversized",
4195 "multipart",
4196 "commentary",
4197 "blank",
4198 "old_final",
4199 "error",
4200 "absent",
4201 ] {
4202 let (child, reason) = unusable_typed_test_child(&parent, "unavailable-child", case);
4203 let completion = typed_test_completion(&child);
4204 assert!(completion.source.is_some(), "valid seal: {case}");
4205 let mut projection = sealed_child_report(&completion, &child, &parent).unwrap();
4206 assert_eq!(
4207 projection,
4208 unavailable_child_result("typed_result", reason),
4209 "{case}"
4210 );
4211 let mut message = typed_child_resume_message(&completion, 0, &projection);
4212 let raw = completion.source.as_ref().unwrap().result(&child, &parent);
4213 let mut envelope =
4214 child_completion_envelope(&completion, Utc::now(), raw.clone(), &message);
4215 assert!(bound_typed_child_delivery(
4216 &mut envelope,
4217 &mut message,
4218 &mut projection,
4219 &completion,
4220 0
4221 ));
4222 assert!(fits_child_result_budget(&envelope));
4223 assert!(fits_child_result_budget(&message));
4224 assert!(fits_child_result_budget(
4225 &envelope.to_provider_message().unwrap()
4226 ));
4227 assert!(message.content.contains("SubAgent"));
4228 assert_eq!(
4229 message.metadata.as_ref().unwrap()["child_final_response_included"],
4230 false
4231 );
4232 assert!(projection.get("child_report").is_none());
4233 assert!(projection.get("host_observation").is_none());
4234 let SessionMessageBody::ChildOutcome(outcome) = envelope.body else {
4235 panic!("child outcome")
4236 };
4237 assert!(outcome.result.is_none());
4238 assert!(outcome.error.is_none());
4239 if let Some(raw) = raw {
4240 assert!(!message.content.contains(&raw), "no raw report: {case}");
4241 }
4242 let mut stale = child.clone();
4245 stale
4246 .messages
4247 .last_mut()
4248 .unwrap()
4249 .content
4250 .push_str("changed");
4251 assert!(sealed_child_report(&completion, &stale, &parent).is_none());
4252 }
4253 }
4254
4255 #[test]
4256 fn unavailable_delivery_counts_escaped_wrappers_at_exact_limit() {
4257 let mut parent = Session::new("unavailable-parent", "model");
4258 parent.add_message(Message::user("required source"));
4259 let (child, _) = unusable_typed_test_child(&parent, "unavailable-child", "oversized");
4260 let completion = typed_test_completion(&child);
4261 let mut projection = sealed_child_report(&completion, &child, &parent).unwrap();
4262 let mut message = typed_child_resume_message(&completion, 0, &projection);
4263 let mut base = child_completion_envelope(&completion, Utc::now(), None, &message);
4264 assert!(bound_typed_child_delivery(
4265 &mut base,
4266 &mut message,
4267 &mut projection,
4268 &completion,
4269 0
4270 ));
4271 let bytes = serde_json::to_vec(&base).unwrap().len().max(
4272 serde_json::to_vec(&base.to_provider_message().unwrap())
4273 .unwrap()
4274 .len(),
4275 );
4276 for extra in [8192 - bytes, 8193 - bytes] {
4277 let mut envelope = base.clone();
4278 let padding = format!("{}{}", "\"".repeat(extra / 2), "x".repeat(extra % 2));
4280 envelope.correlation_id.as_mut().unwrap().push_str(&padding);
4281 let mut projection = projection.clone();
4282 let mut message = message.clone();
4283 assert_eq!(
4284 bound_typed_child_delivery(
4285 &mut envelope,
4286 &mut message,
4287 &mut projection,
4288 &completion,
4289 0
4290 ),
4291 bytes + extra == 8192
4292 );
4293 assert_eq!(
4294 serde_json::to_vec(&envelope).unwrap().len().max(
4295 serde_json::to_vec(&envelope.to_provider_message().unwrap())
4296 .unwrap()
4297 .len()
4298 ),
4299 bytes + extra
4300 );
4301 assert!(fits_child_result_budget(&message));
4302 }
4303 }
4304
4305 #[test]
4306 fn sealed_typed_projection_keeps_host_source_distinct_from_all_reported_claims() {
4307 let mut parent = Session::new("typed-parent", "model");
4308 parent.add_message(Message::user("required user constraint"));
4309 let report = typed_test_report();
4310 let child = typed_test_child(&parent, "typed-child", &report.to_string());
4311 let completion = typed_test_completion(&child);
4312 let projection = sealed_child_report(&completion, &child, &parent).unwrap();
4313 assert_eq!(projection["child_report"], report);
4314 assert_eq!(
4315 projection["host_observation"]["terminal_source"],
4316 serde_json::to_value(&completion.source).unwrap()
4317 );
4318 assert_eq!(
4319 projection["host_observation"]["kind"],
4320 "committed_terminal_source"
4321 );
4322 assert_eq!(
4323 projection["host_observation"]["last_run_status"],
4324 "completed"
4325 );
4326 assert_eq!(projection["child_report"]["outcome"], "blocked");
4327 assert_eq!(
4328 projection["child_report"]["reported_verification"][0]["reported_status"],
4329 "passed"
4330 );
4331 assert!(projection["host_observation"].get("verified").is_none());
4332 for excluded in ["guardian", "resident", "legacy", "missing_source"] {
4333 let mut excluded_child = child.clone();
4334 match excluded {
4335 "guardian" => {
4336 excluded_child
4337 .metadata
4338 .insert("subagent_type".into(), "guardian".into());
4339 }
4340 "resident" => {
4341 excluded_child
4342 .metadata
4343 .insert("lifecycle".into(), "resident".into());
4344 }
4345 "legacy" => {
4346 excluded_child =
4347 Session::new_child("typed-child", &parent.id, "model", "Child");
4348 excluded_child.add_message(Message::assistant(report.to_string(), None));
4349 excluded_child.set_last_run_status("completed");
4350 }
4351 _ => {}
4352 }
4353 ChildCompletionSource::prepare(
4354 &mut excluded_child,
4355 "excluded-run",
4356 &Default::default(),
4357 );
4358 let mut excluded_completion = typed_test_completion(&excluded_child);
4359 if excluded == "missing_source" {
4360 excluded_completion.source = None;
4361 }
4362 assert!(
4363 sealed_child_report(&excluded_completion, &excluded_child, &parent).is_none(),
4364 "{excluded}"
4365 );
4366 }
4367 }
4368
4369 #[tokio::test]
4370 async fn typed_delivery_counts_complete_escaped_envelope_and_final_correlation() {
4371 let mut parent = Session::new("typed-parent", "model");
4372 parent.add_message(Message::user("required source"));
4373 let child = typed_test_child(&parent, "typed-child", &typed_test_report().to_string());
4374 let completion = typed_test_completion(&child);
4375 let original = sealed_child_report(&completion, &child, &parent).unwrap();
4376 let presentation = typed_child_resume_message(&completion, 0, &original);
4377 let mut base = child_completion_envelope(
4378 &completion,
4379 Utc::now(),
4380 completion.source.as_ref().unwrap().result(&child, &parent),
4381 &presentation,
4382 );
4383 let mut projection = original.clone();
4384 let mut message = presentation.clone();
4385 assert!(bound_typed_child_delivery(
4386 &mut base,
4387 &mut message,
4388 &mut projection,
4389 &completion,
4390 0
4391 ));
4392 assert_eq!(projection, original);
4393 let temp = tempfile::tempdir().unwrap();
4394 let store = Arc::new(
4395 bamboo_storage::SessionStoreV2::new(temp.path().to_path_buf())
4396 .await
4397 .unwrap(),
4398 );
4399 store.save_session(&parent).await.unwrap();
4400 let inbox = bamboo_storage::FileSessionInbox::new(store, Default::default());
4401 let first = inbox.deliver(&base).await.unwrap();
4402 let stable_id = base.id.clone();
4403 let base_bytes = serde_json::to_vec(&base.to_provider_message().unwrap())
4404 .unwrap()
4405 .len();
4406 assert!(base_bytes < 8192);
4407 let correlation = base.correlation_id.as_ref().unwrap().clone();
4408 for (extra, available) in [(8192 - base_bytes, true), (8193 - base_bytes, false)] {
4409 let mut envelope = base.clone();
4410 envelope.correlation_id = Some(format!("{correlation}{}", "x".repeat(extra)));
4411 assert_eq!(
4412 serde_json::to_vec(&envelope.to_provider_message().unwrap())
4413 .unwrap()
4414 .len(),
4415 base_bytes + extra
4416 );
4417 let mut message = presentation.clone();
4418 let mut projection = original.clone();
4419 assert!(bound_typed_child_delivery(
4420 &mut envelope,
4421 &mut message,
4422 &mut projection,
4423 &completion,
4424 0
4425 ));
4426 assert_eq!(projection["available"], available);
4427 assert!(fits_child_result_budget(&envelope));
4428 assert!(fits_child_result_budget(&message));
4429 assert!(fits_child_result_budget(
4430 &envelope.to_provider_message().unwrap()
4431 ));
4432 assert_eq!(
4433 envelope.idempotency_semantics(),
4434 base.idempotency_semantics()
4435 );
4436 assert_eq!(envelope.id, stable_id);
4437 assert_eq!(
4438 inbox.deliver(&envelope).await.unwrap(),
4439 first,
4440 "presentation overflow cannot change delivery semantics"
4441 );
4442 if !available {
4443 assert_eq!(projection["reason"], "result_budget_exceeded");
4444 assert!(projection.get("child_report").is_none());
4445 assert!(message.content.contains("SubAgent"));
4446 assert!(!message.content.contains("Bounded tail"));
4447 }
4448 }
4449 let mut large = typed_test_report();
4450 large["summary"] = serde_json::json!("\"\\\n".repeat(500));
4451 large["proposals"] = serde_json::json!(["\"\\\n".repeat(250)]);
4452 let child = typed_test_child(&parent, "escaped-child", &large.to_string());
4453 assert!(decode_report(&large.to_string()).is_ok());
4454 let completion = typed_test_completion(&child);
4455 let mut projection = sealed_child_report(&completion, &child, &parent).unwrap();
4456 let mut message = typed_child_resume_message(&completion, 0, &projection);
4457 let mut envelope =
4458 child_completion_envelope(&completion, Utc::now(), Some(large.to_string()), &message);
4459 assert!(serde_json::to_vec(&envelope).unwrap().len() > 8192);
4460 assert!(bound_typed_child_delivery(
4461 &mut envelope,
4462 &mut message,
4463 &mut projection,
4464 &completion,
4465 0
4466 ));
4467 assert_eq!(projection["available"], false);
4468 assert!(fits_child_result_budget(&envelope));
4469 assert!(fits_child_result_budget(&message));
4470 assert!(fits_child_result_budget(
4471 &envelope.to_provider_message().unwrap()
4472 ));
4473 }
4474
4475 #[tokio::test]
4476 async fn typed_source_mutations_reject_before_admission_and_wait_clear() {
4477 for mutation in ["birth", "assignment", "input", "result", "parent_source"] {
4478 let (_temp, store, inbox, coordinator, _, launches) = completion_inbox_fixture().await;
4479 save_waiting_test_parent(
4480 &store,
4481 "typed-parent",
4482 vec!["typed-child".into()],
4483 ChildWaitPolicy::All,
4484 )
4485 .await;
4486 let mut parent = store.load_session("typed-parent").await.unwrap().unwrap();
4487 parent.add_message(Message::user("required source"));
4488 store.save_session(&parent).await.unwrap();
4489 let mut child =
4490 typed_test_child(&parent, "typed-child", &typed_test_report().to_string());
4491 store.save_session(&child).await.unwrap();
4492 let mut completion = typed_test_completion(&child);
4493 match mutation {
4494 "birth" => {
4495 completion.source.as_mut().unwrap().child_created_at +=
4496 chrono::Duration::seconds(1)
4497 }
4498 "assignment" => {
4499 child.messages[0].content.push_str("changed assignment");
4500 }
4501 "input" => child.add_message(Message::user("new user input")),
4502 "result" => child
4503 .messages
4504 .last_mut()
4505 .unwrap()
4506 .content
4507 .push_str("changed"),
4508 _ => {
4509 parent.messages[0].content.push_str("changed");
4510 store.save_session(&parent).await.unwrap();
4511 }
4512 }
4513 store.save_session(&child).await.unwrap();
4514 coordinator.on_child_completed(completion).await;
4515 assert_eq!(
4516 inbox.inspect(&parent.id).await.unwrap().pending,
4517 0,
4518 "{mutation}"
4519 );
4520 assert_eq!(launches.load(Ordering::SeqCst), 0, "{mutation}");
4521 assert!(
4522 read_runtime_state(&store.load_session(&parent.id).await.unwrap().unwrap())
4523 .waiting_for_children
4524 .is_some(),
4525 "{mutation}"
4526 );
4527 }
4528 }
4529
4530 #[tokio::test]
4531 async fn typed_cold_watchdog_and_terminal_siblings_admit_each_source_once() {
4532 for case in [
4533 "valid",
4534 "prose",
4535 "malformed",
4536 "oversized",
4537 "multipart",
4538 "commentary",
4539 "blank",
4540 "old_final",
4541 "error",
4542 "absent",
4543 ] {
4544 for cold_watchdog in [false, true] {
4545 let (_temp, store, inbox, coordinator, reservations, launches) =
4546 completion_inbox_fixture().await;
4547 let ids = vec!["typed-a".into(), "typed-b".into()];
4548 save_waiting_test_parent(&store, "typed-parent", ids.clone(), ChildWaitPolicy::All)
4549 .await;
4550 let mut parent = store.load_session("typed-parent").await.unwrap().unwrap();
4551 parent.add_message(Message::user("required source"));
4552 store.save_session(&parent).await.unwrap();
4553 let mut completions = Vec::new();
4554 for id in &ids {
4555 let child = if case == "valid" {
4556 typed_test_child(&parent, id, &typed_test_report().to_string())
4557 } else {
4558 unusable_typed_test_child(&parent, id, case).0
4559 };
4560 store.save_session(&child).await.unwrap();
4561 completions.push(typed_test_completion(&child));
4562 }
4563 if cold_watchdog {
4564 let mut wait = read_runtime_state(&parent).waiting_for_children.unwrap();
4566 wait.registered_at = Utc::now() - chrono::Duration::seconds(61);
4567 coordinator.sweep_child_wait(&parent.id, wait.clone()).await;
4568 coordinator.sweep_child_wait(&parent.id, wait).await;
4569 } else {
4570 coordinator.on_child_completed(completions[0].clone()).await;
4572 }
4573 for completion in completions {
4574 coordinator.on_child_completed(completion).await;
4575 }
4576 let claims = inbox.claim(&parent.id, 8).await.unwrap();
4577 assert_eq!(claims.len(), 2);
4578 assert_eq!(reservations.load(Ordering::SeqCst), 1);
4579 assert_eq!(launches.load(Ordering::SeqCst), 1);
4580 for claim in claims {
4581 assert!(fits_child_result_budget(&claim.envelope));
4582 assert!(fits_child_result_budget(
4583 &claim.envelope.to_provider_message().unwrap()
4584 ));
4585 let actual = claim.envelope.to_provider_message().unwrap();
4586 assert!(!actual.content.contains("UNUSABLE_RAW"));
4587 let SessionMessageBody::ChildOutcome(outcome) = claim.envelope.body else {
4588 panic!("child outcome");
4589 };
4590 let projection: serde_json::Value = serde_json::from_str(
4591 outcome
4592 .provider_message
4593 .as_ref()
4594 .unwrap()
4595 .content
4596 .text
4597 .split_once("Child typed result:\n")
4598 .unwrap()
4599 .1
4600 .lines()
4601 .next()
4602 .unwrap(),
4603 )
4604 .unwrap();
4605 if case != "valid" {
4606 let expected =
4607 unusable_typed_test_child(&parent, &outcome.child_session_id, case).1;
4608 assert_eq!(
4609 projection,
4610 unavailable_child_result("typed_result", expected)
4611 );
4612 assert!(outcome.result.is_none());
4613 assert!(outcome.error.is_none());
4614 assert!(fits_child_result_budget(&outcome.provider_message));
4615 continue;
4616 }
4617 assert_eq!(projection["child_report"], typed_test_report());
4618 assert_eq!(
4619 projection["host_observation"]["terminal_source"]["child_session_id"],
4620 outcome.child_session_id
4621 );
4622 assert_eq!(
4623 projection["host_observation"]["last_run_status"],
4624 "completed"
4625 );
4626 assert!(fits_child_result_budget(&outcome.provider_message));
4627 }
4628 }
4629 }
4630 }
4631
4632 async fn completion_inbox_fixture() -> (
4633 tempfile::TempDir,
4634 Arc<bamboo_storage::SessionStoreV2>,
4635 Arc<dyn SessionInboxPort>,
4636 Arc<ChildCompletionCoordinator>,
4637 Arc<AtomicUsize>,
4638 Arc<AtomicUsize>,
4639 ) {
4640 completion_inbox_fixture_with_limits(bamboo_domain::SessionInboxLimits::default()).await
4641 }
4642
4643 async fn completion_inbox_fixture_with_limits(
4644 limits: bamboo_domain::SessionInboxLimits,
4645 ) -> (
4646 tempfile::TempDir,
4647 Arc<bamboo_storage::SessionStoreV2>,
4648 Arc<dyn SessionInboxPort>,
4649 Arc<ChildCompletionCoordinator>,
4650 Arc<AtomicUsize>,
4651 Arc<AtomicUsize>,
4652 ) {
4653 let temp = tempfile::tempdir().unwrap();
4654 let store = Arc::new(
4655 bamboo_storage::SessionStoreV2::new(temp.path().to_path_buf())
4656 .await
4657 .unwrap(),
4658 );
4659 let storage: Arc<dyn Storage> = store.clone();
4660 let locked = Arc::new(LockedSessionStore::new(storage.clone()));
4661 let inbox: Arc<dyn SessionInboxPort> =
4662 Arc::new(bamboo_storage::FileSessionInbox::new(store.clone(), limits));
4663 let router = crate::SessionActivationRouter::new();
4664 let messenger = Arc::new(crate::SessionMessenger::new(
4665 storage.clone(),
4666 inbox.clone(),
4667 router.clone(),
4668 ));
4669 let reservations = Arc::new(AtomicUsize::new(0));
4670 let launches = Arc::new(AtomicUsize::new(0));
4671 router
4672 .set_spawner(Arc::new(CountingActivationSpawner {
4673 reservations: reservations.clone(),
4674 launches: launches.clone(),
4675 }))
4676 .await;
4677 let provider: Arc<dyn bamboo_llm::LLMProvider> = Arc::new(CompletedTestProvider);
4678 let config = Arc::new(RwLock::new(Config::default()));
4679 let metrics = bamboo_metrics::MetricsCollector::spawn(
4680 Arc::new(bamboo_metrics::SqliteMetricsStorage::new(
4681 temp.path().join("metrics.db"),
4682 )),
4683 7,
4684 );
4685 let tools: Arc<dyn ToolExecutor> = Arc::new(EmptyTools);
4686 let agent = Arc::new(
4687 Agent::builder()
4688 .storage(storage.clone())
4689 .persistence(locked.clone())
4690 .session_inbox(inbox.clone())
4691 .activation_router(router)
4692 .session_messenger(messenger)
4693 .attachment_reader(store.clone())
4694 .skill_manager(Arc::new(bamboo_skills::SkillManager::new()))
4695 .metrics_collector(metrics)
4696 .config(config.clone())
4697 .provider(provider.clone())
4698 .default_tools(tools)
4699 .build()
4700 .unwrap(),
4701 );
4702 let mut providers = HashMap::new();
4703 providers.insert("test".to_string(), provider);
4704 let registry = Arc::new(ProviderRegistry::new(providers, "test".to_string()));
4705 let provider_router = Arc::new(ProviderModelRouter::new(registry.clone()));
4706 let coordinator = Arc::new(ChildCompletionCoordinator::new(
4707 storage,
4708 locked,
4709 Arc::default(),
4710 Arc::new(RwLock::new(HashMap::new())),
4711 Arc::new(RwLock::new(HashMap::new())),
4712 agent,
4713 config,
4714 registry,
4715 provider_router,
4716 temp.path().to_path_buf(),
4717 None,
4718 ));
4719 (temp, store, inbox, coordinator, reservations, launches)
4720 }
4721
4722 async fn save_waiting_test_parent(
4723 store: &bamboo_storage::SessionStoreV2,
4724 parent_id: &str,
4725 ids: Vec<String>,
4726 policy: ChildWaitPolicy,
4727 ) {
4728 let mut parent = Session::new(parent_id, "model");
4729 let mut runtime = AgentRuntimeState::new("waiting-run");
4730 runtime.status = AgentStatusState::Suspended;
4731 runtime.waiting_for_children = Some(WaitingForChildrenState::for_children(
4732 ids,
4733 policy,
4734 Utc::now(),
4735 ));
4736 runtime.suspension = Some(SuspensionState {
4737 reason: "waiting_for_children".into(),
4738 suspended_at: Utc::now(),
4739 resumable: true,
4740 hook_point: Some("ChildCompletion".into()),
4741 });
4742 write_runtime_state(&mut parent, &runtime);
4743 parent.metadata.insert(
4744 "runtime.suspend_reason".into(),
4745 "waiting_for_children".into(),
4746 );
4747 store.save_session(&parent).await.unwrap();
4748 }
4749
4750 async fn save_completion_test_child(
4751 store: &bamboo_storage::SessionStoreV2,
4752 parent_id: &str,
4753 child_id: &str,
4754 status: &str,
4755 ) -> ChildCompletion {
4756 let mut child = store
4757 .load_session(child_id)
4758 .await
4759 .unwrap()
4760 .unwrap_or_else(|| Session::new_child(child_id, parent_id, "model", "Child"));
4761 child.add_message(Message::assistant(format!("result-{child_id}"), None));
4762 child.set_last_run_status(status);
4763 save_sourced_test_child(store, &child).await;
4764 ChildCompletion {
4765 parent_session_id: parent_id.into(),
4766 child_session_id: child_id.into(),
4767 status: status.into(),
4768 error: None,
4769 completed_at: child.updated_at,
4770 source: test_child_source(store, child_id).await,
4771 }
4772 }
4773
4774 #[tokio::test]
4775 async fn unrelated_invalid_pending_body_does_not_block_incremental_child_activation() {
4776 for queue in ["new", "cur"] {
4777 let (_temp, store, inbox, coordinator, reservations, launches) =
4778 completion_inbox_fixture().await;
4779 let parent_id = "invalid-pending-parent";
4780 save_waiting_test_parent(
4781 &store,
4782 parent_id,
4783 vec!["invalid-pending-fast".into(), "invalid-pending-slow".into()],
4784 ChildWaitPolicy::All,
4785 )
4786 .await;
4787 let before = read_runtime_state(&store.load_session(parent_id).await.unwrap().unwrap());
4788 let fast =
4789 save_completion_test_child(&store, parent_id, "invalid-pending-fast", "completed")
4790 .await;
4791 save_completion_test_child(&store, parent_id, "invalid-pending-slow", "running").await;
4792
4793 let mut unrelated = SessionMessageEnvelope::user_input(parent_id, "unrelated bash");
4794 unrelated.source = SessionMessageSource::Runtime {
4795 subsystem: "bash".into(),
4796 };
4797 unrelated.kind = SessionMessageKind::RuntimeInstruction;
4798 unrelated.body =
4799 SessionMessageBody::RuntimeInstruction(bamboo_domain::SessionRuntimeInstruction {
4800 instruction: "bash_completion".into(),
4801 content: Some(bamboo_domain::SessionMessageContent::text("unrelated bash")),
4802 data: None,
4803 provider_message: None,
4804 });
4805 inbox.deliver(&unrelated).await.unwrap();
4806 let dir = store
4807 .bamboo_home_dir()
4808 .join(store.resolve_rel_path(parent_id).await.unwrap())
4809 .join("inbox");
4810 let staged = tokio::fs::read_dir(dir.join("new"))
4811 .await
4812 .unwrap()
4813 .next_entry()
4814 .await
4815 .unwrap()
4816 .unwrap()
4817 .path();
4818 let mut transport: serde_json::Value =
4819 serde_json::from_slice(&tokio::fs::read(&staged).await.unwrap()).unwrap();
4820 transport["body"] = serde_json::json!({"not": "an envelope"});
4821 let invalid_bytes = serde_json::to_vec(&transport).unwrap();
4822 tokio::fs::write(&staged, &invalid_bytes).await.unwrap();
4823 let invalid_path = if queue == "cur" {
4824 let destination = dir.join("cur").join(staged.file_name().unwrap());
4825 tokio::fs::create_dir_all(dir.join("cur")).await.unwrap();
4826 tokio::fs::rename(&staged, &destination).await.unwrap();
4827 destination
4828 } else {
4829 staged
4830 };
4831
4832 coordinator.on_child_completed(fast.clone()).await;
4833 let outcomes = inbox
4834 .pending_child_outcomes(parent_id, "invalid-pending-fast")
4835 .await
4836 .unwrap();
4837 assert_eq!(outcomes.len(), 1);
4838 assert!(fits_child_result_budget(&outcomes[0]));
4839 let after = read_runtime_state(&store.load_session(parent_id).await.unwrap().unwrap());
4840 assert_eq!(after, before, "All wait and suspension remain armed");
4841 assert_eq!(reservations.load(Ordering::SeqCst), 1);
4842 assert_eq!(launches.load(Ordering::SeqCst), 1);
4843 let backlog = inbox.inspect(parent_id).await.unwrap();
4844 assert!(backlog.interrupt_pending());
4845 assert_eq!(backlog.coordinator_generation, 0);
4846 let ready = inbox
4847 .inspect_wake_readiness(parent_id, Utc::now())
4848 .await
4849 .unwrap()
4850 .ready
4851 .unwrap();
4852 assert_eq!(ready.id, outcomes[0].id);
4853 assert_eq!(
4854 ready.activation_policy,
4855 SessionActivationPolicy::InterruptSpecificWait
4856 );
4857 assert_eq!(tokio::fs::read(&invalid_path).await.unwrap(), invalid_bytes);
4858 assert!(!inbox.was_admitted(parent_id, &unrelated.id).await.unwrap());
4859
4860 let claims = inbox.claim(parent_id, 8).await.unwrap();
4861 assert_eq!(claims.len(), 1);
4862 assert_eq!(claims[0].envelope.id, outcomes[0].id);
4863 inbox.ack(parent_id, &claims[0]).await.unwrap();
4864 coordinator.on_child_completed(fast).await;
4865 assert_eq!(launches.load(Ordering::SeqCst), 1);
4866 assert!(!inbox.was_admitted(parent_id, &unrelated.id).await.unwrap());
4867 }
4868 }
4869
4870 #[tokio::test]
4871 async fn partial_all_outcome_preserves_wait_and_active_parent_control_state() {
4872 let (_temp, store, inbox, coordinator, _, launches) = completion_inbox_fixture().await;
4873 let parent_id = "incremental-parent";
4874 save_waiting_test_parent(
4875 &store,
4876 parent_id,
4877 vec!["ready-child".into(), "slow-child".into()],
4878 ChildWaitPolicy::All,
4879 )
4880 .await;
4881 let mut parent = store.load_session(parent_id).await.unwrap().unwrap();
4882 let mut runtime = read_runtime_state(&parent);
4883 runtime.status = AgentStatusState::Running;
4884 runtime.suspension = None;
4885 parent.metadata.remove("runtime.suspend_reason");
4886 write_runtime_state(&mut parent, &runtime);
4887 store.save_session(&parent).await.unwrap();
4888 let ready = save_completion_test_child(&store, parent_id, "ready-child", "completed").await;
4889 save_completion_test_child(&store, parent_id, "slow-child", "running").await;
4890
4891 coordinator.on_child_completed(ready.clone()).await;
4892 let after = store.load_session(parent_id).await.unwrap().unwrap();
4893 assert_eq!(read_runtime_state(&after), runtime);
4894 assert_eq!(after.updated_at, parent.updated_at);
4895 assert_eq!(launches.load(Ordering::SeqCst), 1);
4896 let backlog = inbox.inspect(parent_id).await.unwrap();
4897 assert!(backlog.interrupt_pending());
4898 assert_eq!(backlog.coordinator_generation, 0, "no blanket prefix grant");
4899 let claims = inbox.claim(parent_id, 8).await.unwrap();
4900 assert_eq!(claims.len(), 1);
4901 inbox.ack(parent_id, &claims[0]).await.unwrap();
4902
4903 coordinator.on_child_completed(ready).await;
4904 assert_eq!(inbox.inspect(parent_id).await.unwrap().pending, 0);
4905 let slow = save_completion_test_child(&store, parent_id, "slow-child", "completed").await;
4906 coordinator.on_child_completed(slow).await;
4907 assert!(
4908 read_runtime_state(&store.load_session(parent_id).await.unwrap().unwrap())
4909 .waiting_for_children
4910 .is_none()
4911 );
4912 let claims = inbox.claim(parent_id, 8).await.unwrap();
4913 assert_eq!(
4914 claims.len(),
4915 1,
4916 "the acknowledged first result is not repeated at wait release"
4917 );
4918 let SessionMessageBody::ChildOutcome(outcome) = &claims[0].envelope.body else {
4919 panic!("child outcome");
4920 };
4921 assert_eq!(outcome.child_session_id, "slow-child");
4922 }
4923
4924 #[tokio::test]
4925 async fn incremental_completions_during_owner_exit_share_one_successor() {
4926 let (_temp, store, inbox, coordinator, reservations, launches) =
4927 completion_inbox_fixture().await;
4928 let parent_id = "handoff-parent";
4929 save_waiting_test_parent(
4930 &store,
4931 parent_id,
4932 vec![
4933 "handoff-a".into(),
4934 "handoff-b".into(),
4935 "handoff-slow".into(),
4936 ],
4937 ChildWaitPolicy::All,
4938 )
4939 .await;
4940 let a = save_completion_test_child(&store, parent_id, "handoff-a", "completed").await;
4941 let b = save_completion_test_child(&store, parent_id, "handoff-b", "completed").await;
4942 save_completion_test_child(&store, parent_id, "handoff-slow", "running").await;
4943 let router = coordinator.agent.activation_router().unwrap();
4944 router.set_inbox(inbox.clone());
4945 let mut owner = router
4946 .register_run(parent_id, "old-parent-run")
4947 .await
4948 .unwrap();
4949
4950 coordinator.on_child_completed(a.clone()).await;
4951 assert!(inbox
4952 .claim_for_turn(parent_id, 8, Some("old-parent-run"))
4953 .await
4954 .unwrap()
4955 .is_empty());
4956 owner.begin_finalization().await;
4957 coordinator.on_child_completed(b.clone()).await;
4958 assert_eq!(reservations.load(Ordering::SeqCst), 0);
4959 assert_eq!(launches.load(Ordering::SeqCst), 0);
4960 owner.finish(0).await.unwrap();
4961 assert_eq!(reservations.load(Ordering::SeqCst), 1);
4962 assert_eq!(launches.load(Ordering::SeqCst), 1);
4963 let successor = router.current_run_id(parent_id).await.unwrap();
4964 let claims = inbox
4965 .claim_for_turn(parent_id, 8, Some(&successor))
4966 .await
4967 .unwrap();
4968 assert_eq!(claims.len(), 2);
4969 for claim in &claims {
4970 inbox.ack(parent_id, claim).await.unwrap();
4971 }
4972 tokio::join!(
4973 coordinator.on_child_completed(a),
4974 coordinator.on_child_completed(b)
4975 );
4976 assert_eq!(inbox.inspect(parent_id).await.unwrap().pending, 0);
4977 assert_eq!(reservations.load(Ordering::SeqCst), 1);
4978 assert!(
4979 read_runtime_state(&store.load_session(parent_id).await.unwrap().unwrap())
4980 .waiting_for_children
4981 .is_some()
4982 );
4983 }
4984
4985 #[tokio::test]
4986 async fn cold_watchdog_replays_each_ready_child_before_all_wait_is_satisfied() {
4987 let (_temp, store, inbox, coordinator, _, launches) = completion_inbox_fixture().await;
4988 let parent_id = "cold-incremental-parent";
4989 save_waiting_test_parent(
4990 &store,
4991 parent_id,
4992 vec!["cold-a".into(), "cold-b".into(), "cold-slow".into()],
4993 ChildWaitPolicy::All,
4994 )
4995 .await;
4996 let mut parent = store.load_session(parent_id).await.unwrap().unwrap();
4997 let mut runtime = read_runtime_state(&parent);
4998 runtime.waiting_for_children.as_mut().unwrap().registered_at -=
4999 chrono::Duration::seconds(61);
5000 write_runtime_state(&mut parent, &runtime);
5001 store.save_session(&parent).await.unwrap();
5002 save_completion_test_child(&store, parent_id, "cold-a", "completed").await;
5003 save_completion_test_child(&store, parent_id, "cold-b", "completed").await;
5004 save_completion_test_child(&store, parent_id, "cold-slow", "running").await;
5005
5006 let wait = runtime.waiting_for_children.unwrap();
5007 coordinator.sweep_child_wait(parent_id, wait.clone()).await;
5008 let reopened = bamboo_storage::FileSessionInbox::new(store.clone(), Default::default());
5009 let backlog = reopened.inspect(parent_id).await.unwrap();
5010 assert_eq!(backlog.pending, 2);
5011 assert!(backlog.interrupt_pending());
5012 assert_eq!(backlog.coordinator_generation, 0);
5013 assert_eq!(launches.load(Ordering::SeqCst), 1);
5014 let claims = reopened.claim(parent_id, 8).await.unwrap();
5015 assert_eq!(claims.len(), 2);
5016 for claim in &claims {
5017 reopened.ack(parent_id, claim).await.unwrap();
5018 }
5019 coordinator.sweep_child_wait(parent_id, wait).await;
5020 assert_eq!(inbox.inspect(parent_id).await.unwrap().pending, 0);
5021 assert!(
5022 read_runtime_state(&store.load_session(parent_id).await.unwrap().unwrap())
5023 .waiting_for_children
5024 .is_some()
5025 );
5026 }
5027
5028 #[tokio::test]
5029 async fn any_wait_delivers_a_late_direct_child_without_repeating_the_first_outcome() {
5030 let (_temp, store, inbox, coordinator, _, _) = completion_inbox_fixture().await;
5031 let parent_id = "any-incremental-parent";
5032 save_waiting_test_parent(
5033 &store,
5034 parent_id,
5035 vec!["any-first".into(), "any-late".into()],
5036 ChildWaitPolicy::Any,
5037 )
5038 .await;
5039 let first = save_completion_test_child(&store, parent_id, "any-first", "completed").await;
5040 save_completion_test_child(&store, parent_id, "any-late", "running").await;
5041 coordinator.on_child_completed(first.clone()).await;
5042 assert!(
5043 read_runtime_state(&store.load_session(parent_id).await.unwrap().unwrap())
5044 .waiting_for_children
5045 .is_none()
5046 );
5047 let first_claims = inbox.claim(parent_id, 8).await.unwrap();
5048 assert_eq!(first_claims.len(), 1);
5049 inbox.ack(parent_id, &first_claims[0]).await.unwrap();
5050
5051 let late = save_completion_test_child(&store, parent_id, "any-late", "completed").await;
5052 coordinator.on_child_completed(late.clone()).await;
5053 coordinator.on_child_completed(first).await;
5054 coordinator.on_child_completed(late).await;
5055 let claims = inbox.claim(parent_id, 8).await.unwrap();
5056 assert_eq!(claims.len(), 1);
5057 let SessionMessageBody::ChildOutcome(outcome) = &claims[0].envelope.body else {
5058 panic!("child outcome");
5059 };
5060 assert_eq!(outcome.child_session_id, "any-late");
5061 assert!(
5062 read_runtime_state(&store.load_session(parent_id).await.unwrap().unwrap())
5063 .waiting_for_children
5064 .is_none()
5065 );
5066 }
5067
5068 #[tokio::test]
5069 async fn terminal_batch_upgrades_lower_pending_outcome_below_an_acknowledged_sibling() {
5070 let (_temp, store, inbox, coordinator, _, _) = completion_inbox_fixture().await;
5071 let parent_id = "terminal-batch-parent";
5072 save_waiting_test_parent(
5073 &store,
5074 parent_id,
5075 vec!["batch-low".into(), "batch-high".into()],
5076 ChildWaitPolicy::All,
5077 )
5078 .await;
5079 let mut parent = store.load_session(parent_id).await.unwrap().unwrap();
5080 let wait = read_runtime_state(&parent).waiting_for_children.unwrap();
5081 let low = save_completion_test_child(&store, parent_id, "batch-low", "completed").await;
5082 let high = save_completion_test_child(&store, parent_id, "batch-high", "completed").await;
5083 let mut receipts = Vec::new();
5084 for completion in [&low, &high] {
5085 let result = format!("result-{}", completion.child_session_id);
5086 let presentation = runtime_resume_message(completion, 0, Some(&result));
5087 let envelope = child_completion_envelope(
5088 completion,
5089 wait.registered_at,
5090 Some(result),
5091 &presentation,
5092 );
5093 receipts.push(inbox.deliver(&envelope).await.unwrap());
5094 }
5095 inbox
5096 .mark_message_activation_eligible(
5097 parent_id,
5098 &receipts[1],
5099 SessionActivationPolicy::InterruptSpecificWait,
5100 )
5101 .await
5102 .unwrap();
5103 let claims = inbox.claim(parent_id, 8).await.unwrap();
5104 assert_eq!(claims.len(), 1);
5105 assert_eq!(claims[0].generation, receipts[1].generation);
5106 parent.add_message(claims[0].envelope.to_provider_message().unwrap());
5107 store.save_session(&parent).await.unwrap();
5108 inbox.ack(parent_id, &claims[0]).await.unwrap();
5109
5110 coordinator.on_child_completed(low).await;
5111 let mut rearmed = store.load_session(parent_id).await.unwrap().unwrap();
5112 let mut runtime = read_runtime_state(&rearmed);
5113 assert!(runtime.waiting_for_children.is_none());
5114 let new_wait = WaitingForChildrenState::for_children(
5115 vec!["next-slow-child".into()],
5116 ChildWaitPolicy::All,
5117 Utc::now(),
5118 );
5119 runtime.waiting_for_children = Some(new_wait.clone());
5120 runtime.status = AgentStatusState::Suspended;
5121 write_runtime_state(&mut rearmed, &runtime);
5122 store.save_session(&rearmed).await.unwrap();
5123 let backlog = inbox.inspect(parent_id).await.unwrap();
5124 assert_eq!(backlog.pending, 1);
5125 assert!(
5126 backlog.interrupt_pending(),
5127 "the low generation has its own grant"
5128 );
5129 let (prepared, ready) = prepare_session_inbox_activation(
5130 &coordinator.persistence,
5131 parent_id,
5132 backlog.interrupt_pending(),
5133 )
5134 .await
5135 .unwrap()
5136 .unwrap();
5137 assert!(ready);
5138 assert_eq!(
5139 read_runtime_state(&prepared).waiting_for_children,
5140 Some(new_wait)
5141 );
5142 let claims = inbox.claim(parent_id, 8).await.unwrap();
5143 assert_eq!(claims.len(), 1);
5144 assert_eq!(claims[0].generation, receipts[0].generation);
5145 }
5146
5147 #[tokio::test]
5148 async fn large_legal_typed_source_falls_back_within_the_delivery_budget() {
5149 let (_temp, store, inbox, coordinator, _, launches) = completion_inbox_fixture().await;
5150 let parent_id = "large-source-parent";
5151 let child_id = "large-source-child";
5152 save_waiting_test_parent(
5153 &store,
5154 parent_id,
5155 vec![child_id.into()],
5156 ChildWaitPolicy::All,
5157 )
5158 .await;
5159 let mut parent = store.load_session(parent_id).await.unwrap().unwrap();
5160 parent.add_message(Message::user("required original source"));
5161 store.save_session(&parent).await.unwrap();
5162 let mut child = typed_test_child(&parent, child_id, &typed_test_report().to_string());
5163 let source = ChildCompletionSource::from_committed_session(&child).unwrap();
5164 let run_length =
5165 3900 - serde_json::to_vec(&source).unwrap().len() + source.activation_run_id.len();
5166 ChildCompletionSource::prepare(&mut child, &"r".repeat(run_length), &Default::default());
5167 let completion = typed_test_completion(&child);
5168 let source_bytes = serde_json::to_vec(completion.source.as_ref().unwrap())
5169 .unwrap()
5170 .len();
5171 assert!((3800..=4096).contains(&source_bytes));
5172 store.save_session(&child).await.unwrap();
5173
5174 coordinator.on_child_completed(completion).await;
5175 let claims = inbox.claim(parent_id, 8).await.unwrap();
5176 assert_eq!(claims.len(), 1);
5177 assert_eq!(launches.load(Ordering::SeqCst), 1);
5178 assert!(fits_child_result_budget(&claims[0].envelope));
5179 let message = claims[0].envelope.to_provider_message().unwrap();
5180 assert!(fits_child_result_budget(&message));
5181 assert!(message.content.contains("result_budget_exceeded"));
5182 }
5183
5184 #[tokio::test]
5185 async fn upgrade_promotes_one_existing_legacy_partial_delivery_without_a_second_id() {
5186 let (_temp, store, inbox, coordinator, _, launches) = completion_inbox_fixture().await;
5187 let parent_id = "legacy-partial-parent";
5188 save_waiting_test_parent(
5189 &store,
5190 parent_id,
5191 vec!["legacy-ready".into(), "legacy-slow".into()],
5192 ChildWaitPolicy::All,
5193 )
5194 .await;
5195 let completion =
5196 save_completion_test_child(&store, parent_id, "legacy-ready", "completed").await;
5197 save_completion_test_child(&store, parent_id, "legacy-slow", "running").await;
5198 let parent = store.load_session(parent_id).await.unwrap().unwrap();
5199 let wait = read_runtime_state(&parent).waiting_for_children.unwrap();
5200 let result = "result-legacy-ready";
5201 let presentation = runtime_resume_message(&completion, 1, Some(result));
5202 let mut legacy = child_completion_envelope(
5203 &completion,
5204 wait.registered_at,
5205 Some(result.into()),
5206 &presentation,
5207 );
5208 legacy.id = child_completion_id(&completion, wait.registered_at, Some(result), true);
5209 let receipt = inbox.deliver(&legacy).await.unwrap();
5210 assert!(!inbox.inspect(parent_id).await.unwrap().activation_pending());
5211
5212 coordinator.on_child_completed(completion.clone()).await;
5213 coordinator.on_child_completed(completion).await;
5214 let claims = inbox.claim(parent_id, 8).await.unwrap();
5215 assert_eq!(claims.len(), 1);
5216 assert_eq!(claims[0].envelope.id, legacy.id);
5217 assert_eq!(claims[0].generation, receipt.generation);
5218 assert_eq!(launches.load(Ordering::SeqCst), 1);
5219 assert_eq!(
5220 read_runtime_state(&store.load_session(parent_id).await.unwrap().unwrap())
5221 .waiting_for_children
5222 .as_ref(),
5223 Some(&wait)
5224 );
5225 }
5226
5227 #[tokio::test]
5228 async fn upgrade_keeps_admitted_legacy_raw_guardian_and_typed_sources_deduplicated_without_a_wait(
5229 ) {
5230 for presentation_kind in ["raw", "guardian", "typed", "typed_unavailable_digest"] {
5231 let (_temp, store, inbox, coordinator, _, launches) = completion_inbox_fixture().await;
5232 let parent_id = "legacy-admitted-parent";
5233 let child_id = "legacy-admitted-child";
5234 let mut parent = Session::new(parent_id, "model");
5235 parent.add_message(Message::user("required original source"));
5236 store.save_session(&parent).await.unwrap();
5237 let completion = if presentation_kind.starts_with("typed") {
5238 let child = typed_test_child(&parent, child_id, &typed_test_report().to_string());
5239 store.save_session(&child).await.unwrap();
5240 typed_test_completion(&child)
5241 } else {
5242 save_completion_test_child(&store, parent_id, child_id, "completed").await
5243 };
5244 let child = store.load_session(child_id).await.unwrap().unwrap();
5245 let result = completion.source.as_ref().unwrap().result(&child, &parent);
5246 let mut projection = sealed_child_report(&completion, &child, &parent);
5247 if presentation_kind == "typed_unavailable_digest" {
5248 projection = Some(unavailable_child_result(
5249 "typed_result",
5250 "result_budget_exceeded",
5251 ));
5252 }
5253 let mut presentation = match presentation_kind {
5254 "guardian" => guardian_resume_message(&completion, &GuardianVerdict::approved()),
5255 "typed" | "typed_unavailable_digest" => {
5256 typed_child_resume_message(&completion, 0, projection.as_ref().unwrap())
5257 }
5258 _ => runtime_resume_message(&completion, 0, result.as_deref()),
5259 };
5260 if presentation_kind == "typed" {
5261 presentation
5262 .metadata
5263 .as_mut()
5264 .unwrap()
5265 .as_object_mut()
5266 .unwrap()
5267 .remove("child_completion_source_sha256");
5268 }
5269 let old_wait = Utc::now() - chrono::Duration::minutes(2);
5270 let mut legacy =
5271 child_completion_envelope(&completion, old_wait, result.clone(), &presentation);
5272 legacy.id = child_completion_id(&completion, old_wait, result.as_deref(), true);
5273 if let Some(projection) = projection.as_mut() {
5274 assert!(bound_typed_child_delivery(
5275 &mut legacy,
5276 &mut presentation,
5277 projection,
5278 &completion,
5279 0
5280 ));
5281 }
5282 let receipt = inbox.deliver(&legacy).await.unwrap();
5283 inbox
5284 .mark_message_activation_eligible(
5285 parent_id,
5286 &receipt,
5287 SessionActivationPolicy::InterruptSpecificWait,
5288 )
5289 .await
5290 .unwrap();
5291 let claims = inbox.claim(parent_id, 1).await.unwrap();
5292 parent.add_message(claims[0].envelope.to_provider_message().unwrap());
5293 store.save_session(&parent).await.unwrap();
5294 inbox.ack(parent_id, &claims[0]).await.unwrap();
5295
5296 coordinator.on_child_completed(completion).await;
5297 assert_eq!(
5298 inbox.inspect(parent_id).await.unwrap().generation,
5299 receipt.generation,
5300 "{presentation_kind}"
5301 );
5302 assert_eq!(
5303 inbox.inspect(parent_id).await.unwrap().pending,
5304 0,
5305 "{presentation_kind}"
5306 );
5307 assert_eq!(launches.load(Ordering::SeqCst), 0, "{presentation_kind}");
5308 }
5309 }
5310
5311 #[tokio::test]
5312 async fn upgrade_reuses_a_pending_legacy_source_after_its_wait_was_cleared() {
5313 for (arm, acked) in [
5314 ("none", false),
5315 ("partial", false),
5316 ("terminal", false),
5317 ("terminal", true),
5318 ] {
5319 let (_temp, store, inbox, coordinator, _, launches) = completion_inbox_fixture().await;
5320 let parent_id = "legacy-pending-parent";
5321 let child_id = "legacy-pending-child";
5322 let mut parent = Session::new(parent_id, "model");
5323 store.save_session(&parent).await.unwrap();
5324 let completion =
5325 save_completion_test_child(&store, parent_id, child_id, "completed").await;
5326 let result = "result-legacy-pending-child";
5327 let presentation = runtime_resume_message(&completion, 0, Some(result));
5328 let old_wait = Utc::now() - chrono::Duration::minutes(2);
5329 let mut legacy = child_completion_envelope(
5330 &completion,
5331 old_wait,
5332 Some(result.into()),
5333 &presentation,
5334 );
5335 legacy.id = child_completion_id(&completion, old_wait, Some(result), true);
5336 let receipt = inbox.deliver(&legacy).await.unwrap();
5337 inbox
5338 .mark_message_activation_eligible(
5339 parent_id,
5340 &receipt,
5341 SessionActivationPolicy::RespectSpecificWait,
5342 )
5343 .await
5344 .unwrap();
5345 if acked {
5346 let claims = inbox.claim(parent_id, 1).await.unwrap();
5347 parent.add_message(claims[0].envelope.to_provider_message().unwrap());
5348 store.save_session(&parent).await.unwrap();
5349 inbox.ack(parent_id, &claims[0]).await.unwrap();
5350 }
5351 let new_wait = if arm != "none" {
5352 let mut ids = vec![child_id.into()];
5353 if arm == "partial" {
5354 ids.push("legacy-pending-slow".into());
5355 save_completion_test_child(&store, parent_id, "legacy-pending-slow", "running")
5356 .await;
5357 }
5358 let wait =
5359 WaitingForChildrenState::for_children(ids, ChildWaitPolicy::All, Utc::now());
5360 let mut runtime = read_runtime_state(&parent);
5361 runtime.waiting_for_children = Some(wait.clone());
5362 runtime.status = AgentStatusState::Suspended;
5363 write_runtime_state(&mut parent, &runtime);
5364 store.save_session(&parent).await.unwrap();
5365 Some(wait)
5366 } else {
5367 None
5368 };
5369
5370 coordinator.on_child_completed(completion.clone()).await;
5371 coordinator.on_child_completed(completion).await;
5372 let backlog = inbox.inspect(parent_id).await.unwrap();
5373 assert_eq!(
5374 backlog.generation, receipt.generation,
5375 "{arm}, acked={acked}"
5376 );
5377 assert_eq!(backlog.pending, usize::from(!acked), "{arm}, acked={acked}");
5378 assert_eq!(backlog.interrupt_pending(), !acked, "{arm}, acked={acked}");
5379 assert_eq!(
5383 launches.load(Ordering::SeqCst),
5384 usize::from(!acked || arm == "terminal"),
5385 "{arm}, acked={acked}"
5386 );
5387 let wait = read_runtime_state(&store.load_session(parent_id).await.unwrap().unwrap())
5388 .waiting_for_children;
5389 assert_eq!(wait, if arm == "partial" { new_wait } else { None });
5390 let claims = inbox.claim(parent_id, 8).await.unwrap();
5391 assert_eq!(claims.len(), usize::from(!acked));
5392 if !acked {
5393 assert_eq!(claims[0].envelope.id, legacy.id);
5394 assert_eq!(claims[0].generation, receipt.generation);
5395 }
5396 }
5397 }
5398
5399 #[tokio::test]
5400 async fn admitted_legacy_guardian_source_still_commits_a_pending_verdict() {
5401 let (_temp, store, inbox, coordinator, _, launches) = completion_inbox_fixture().await;
5402 let parent_id = "legacy-pending-guardian-parent";
5403 let child_id = "legacy-pending-guardian-child";
5404 let mut parent = Session::new(parent_id, "model");
5405 let mut guardian = crate::runtime::guardian_state::ensure_guardian_state(&parent);
5406 guardian.record_spawn(child_id);
5407 write_guardian_state(&mut parent, guardian);
5408 store.save_session(&parent).await.unwrap();
5409 let verdict = GuardianVerdict::approved();
5410 let mut child = Session::new_child(child_id, parent_id, "model", "Guardian");
5411 child.set_subagent_type("guardian");
5412 child.add_message(Message::assistant(
5413 serde_json::to_string(&verdict).unwrap(),
5414 None,
5415 ));
5416 child.set_last_run_status("completed");
5417 save_sourced_test_child(&store, &child).await;
5418 let completion = ChildCompletion {
5419 parent_session_id: parent_id.into(),
5420 child_session_id: child_id.into(),
5421 status: "completed".into(),
5422 error: None,
5423 completed_at: child.updated_at,
5424 source: test_child_source(&store, child_id).await,
5425 };
5426 let result = serde_json::to_string(&verdict).unwrap();
5427 let old_wait = Utc::now() - chrono::Duration::minutes(2);
5428 let presentation = guardian_resume_message(&completion, &verdict);
5429 let mut legacy =
5430 child_completion_envelope(&completion, old_wait, Some(result.clone()), &presentation);
5431 legacy.id = child_completion_id(&completion, old_wait, Some(&result), true);
5432 let receipt = inbox.deliver(&legacy).await.unwrap();
5433 inbox
5434 .mark_message_activation_eligible(
5435 parent_id,
5436 &receipt,
5437 SessionActivationPolicy::InterruptSpecificWait,
5438 )
5439 .await
5440 .unwrap();
5441 let claims = inbox.claim(parent_id, 1).await.unwrap();
5442 parent.add_message(claims[0].envelope.to_provider_message().unwrap());
5443 store.save_session(&parent).await.unwrap();
5444 inbox.ack(parent_id, &claims[0]).await.unwrap();
5445
5446 coordinator.on_child_completed(completion.clone()).await;
5447 let saved = store.load_session(parent_id).await.unwrap().unwrap();
5448 let guardian = read_guardian_state(&saved).unwrap();
5449 assert!(guardian.phase.is_reviewed());
5450 assert!(guardian.last_approved());
5451 assert_eq!(guardian.review_count, 1);
5452 let saved_guardian = saved.metadata["guardian.state"].clone();
5453 coordinator.on_child_completed(completion).await;
5454 assert_eq!(
5455 store
5456 .load_session(parent_id)
5457 .await
5458 .unwrap()
5459 .unwrap()
5460 .metadata["guardian.state"],
5461 saved_guardian
5462 );
5463 let backlog = inbox.inspect(parent_id).await.unwrap();
5464 assert_eq!(backlog.generation, receipt.generation);
5465 assert_eq!(backlog.pending, 0);
5466 assert_eq!(launches.load(Ordering::SeqCst), 0);
5467 }
5468
5469 #[tokio::test]
5470 async fn terminal_index_ahead_of_callbacks_preserves_every_waited_child_outcome() {
5471 for partial_first in [false, true] {
5472 let (_temp, store, inbox, coordinator, reservations, launches) =
5473 completion_inbox_fixture().await;
5474 let parent_id = "parallel-parent";
5475 let ids: Vec<_> = (0..4)
5476 .map(|slot| format!("parallel-child-{slot}"))
5477 .collect();
5478 save_waiting_test_parent(&store, parent_id, ids.clone(), ChildWaitPolicy::All).await;
5479 for (slot, id) in ids.iter().enumerate() {
5480 let mut child = Session::new_child(id, parent_id, "model", "Child");
5481 child.add_message(Message::assistant(format!("result-{id}"), None));
5482 child.set_last_run_status(if partial_first && slot > 0 {
5483 "running"
5484 } else {
5485 "completed"
5486 });
5487 save_sourced_test_child(&store, &child).await;
5488 }
5489 let mut sources = HashMap::new();
5490 for id in &ids {
5491 sources.insert(id.clone(), test_child_source(&store, id).await);
5492 }
5493 let completion = |id: &str| ChildCompletion {
5494 parent_session_id: parent_id.into(),
5495 child_session_id: id.into(),
5496 status: "completed".into(),
5497 error: None,
5498 completed_at: Utc::now(),
5499 source: sources.get(id).cloned().flatten(),
5500 };
5501 if partial_first {
5502 ChildCompletionHandler::on_child_completed(
5503 coordinator.as_ref(),
5504 completion(&ids[0]),
5505 )
5506 .await;
5507 assert!(
5508 read_runtime_state(&store.load_session(parent_id).await.unwrap().unwrap())
5509 .waiting_for_children
5510 .is_some()
5511 );
5512 assert_eq!(inbox.inspect(parent_id).await.unwrap().pending, 1);
5513 assert_eq!(launches.load(Ordering::SeqCst), 1);
5514 for id in &ids[1..] {
5515 let mut child = store.load_session(id).await.unwrap().unwrap();
5516 child.set_last_run_status("completed");
5517 save_sourced_test_child(&store, &child).await;
5518 }
5519 }
5520 for id in &ids {
5523 sources.insert(id.clone(), test_child_source(&store, id).await);
5524 }
5525 let completion = |id: &str| ChildCompletion {
5526 parent_session_id: parent_id.into(),
5527 child_session_id: id.into(),
5528 status: "completed".into(),
5529 error: None,
5530 completed_at: Utc::now(),
5531 source: sources.get(id).cloned().flatten(),
5532 };
5533 ChildCompletionHandler::on_child_completed(coordinator.as_ref(), completion(&ids[1]))
5534 .await;
5535 assert!(
5536 read_runtime_state(&store.load_session(parent_id).await.unwrap().unwrap())
5537 .waiting_for_children
5538 .is_none()
5539 );
5540 assert_eq!(inbox.inspect(parent_id).await.unwrap().pending, 4);
5541 assert_eq!(reservations.load(Ordering::SeqCst), 1);
5542 assert_eq!(launches.load(Ordering::SeqCst), 1);
5543 for id in &ids {
5544 ChildCompletionHandler::on_child_completed(coordinator.as_ref(), completion(id))
5545 .await;
5546 }
5547 let claims = inbox.claim(parent_id, 8).await.unwrap();
5548 assert_eq!(claims.len(), 4, "late callbacks cannot duplicate outcomes");
5549 assert_eq!(reservations.load(Ordering::SeqCst), 1);
5550 for id in &ids {
5551 let matching: Vec<_> = claims
5552 .iter()
5553 .filter_map(|claim| match &claim.envelope.body {
5554 SessionMessageBody::ChildOutcome(outcome)
5555 if &outcome.child_session_id == id =>
5556 {
5557 Some(outcome)
5558 }
5559 _ => None,
5560 })
5561 .collect();
5562 assert_eq!(matching.len(), 1);
5563 assert_eq!(
5564 matching[0].result.as_deref(),
5565 Some(format!("result-{id}").as_str())
5566 );
5567 }
5568 }
5569 }
5570
5571 #[tokio::test]
5572 async fn saved_a_callback_barrier_never_attributes_completed_b_to_a() {
5573 for guardian in [false, true] {
5574 let (_temp, store, inbox, coordinator, _, launches) = completion_inbox_fixture().await;
5575 let parent_id = "source-barrier-parent";
5576 let child_id = "source-barrier-child";
5577 save_waiting_test_parent(
5578 &store,
5579 parent_id,
5580 vec![child_id.into()],
5581 ChildWaitPolicy::All,
5582 )
5583 .await;
5584 if guardian {
5585 let mut parent = store.load_session(parent_id).await.unwrap().unwrap();
5586 let mut state = crate::runtime::guardian_state::ensure_guardian_state(&parent);
5587 state.record_spawn(child_id);
5588 write_guardian_state(&mut parent, state);
5589 store.save_session(&parent).await.unwrap();
5590 }
5591 let mut child = Session::new_child(child_id, parent_id, "model", "Child");
5592 if guardian {
5593 child
5594 .metadata
5595 .insert("subagent_type".into(), "guardian".into());
5596 }
5597 child.add_message(Message::user("assignment A"));
5598 store.save_session(&child).await.unwrap();
5599 let prior = child.messages.iter().map(|m| m.id.clone()).collect();
5600 child.add_message(Message::assistant("answer A", None));
5601 child.set_last_run_status("completed");
5602 ChildCompletionSource::prepare(&mut child, "actual-run-A", &prior);
5603 coordinator
5604 .agent
5605 .persistence()
5606 .save_runtime_session(&mut child)
5607 .await
5608 .unwrap();
5609 let completion_a = ChildCompletion {
5610 parent_session_id: parent_id.into(),
5611 child_session_id: child_id.into(),
5612 status: "completed".into(),
5613 error: None,
5614 completed_at: Utc::now(),
5615 source: ChildCompletionSource::after_final_save(&child, true),
5616 };
5617 assert_eq!(
5618 completion_a.source.as_ref().unwrap().activation_run_id,
5619 "actual-run-A"
5620 );
5621 let (release, barrier) = tokio::sync::oneshot::channel();
5624 let target = coordinator.clone();
5625 let callback = tokio::spawn(async move {
5626 barrier.await.unwrap();
5627 target.on_child_completed(completion_a).await;
5628 });
5629 let prior = child.messages.iter().map(|m| m.id.clone()).collect();
5630 child.add_message(Message::user("assignment B"));
5631 let answer_b = if guardian {
5632 r#"{"approve":true,"summary":"B approved","findings":[]}"#
5633 } else {
5634 "answer B"
5635 };
5636 child.add_message(Message::assistant(answer_b, None));
5637 child.set_last_run_status("completed");
5638 ChildCompletionSource::prepare(&mut child, "actual-run-B", &prior);
5639 coordinator
5640 .agent
5641 .persistence()
5642 .save_runtime_session(&mut child)
5643 .await
5644 .unwrap();
5645 release.send(()).unwrap();
5646 callback.await.unwrap();
5647 assert_eq!(inbox.inspect(parent_id).await.unwrap().pending, 0);
5648 let parent = store.load_session(parent_id).await.unwrap().unwrap();
5649 assert!(read_runtime_state(&parent).waiting_for_children.is_some());
5650 assert_eq!(launches.load(Ordering::SeqCst), 0);
5651 if guardian {
5652 assert!(read_guardian_state(&parent).unwrap().phase.is_pending());
5653 }
5654
5655 let completion_b = ChildCompletion {
5656 parent_session_id: parent_id.into(),
5657 child_session_id: child_id.into(),
5658 status: "completed".into(),
5659 error: None,
5660 completed_at: Utc::now(),
5661 source: ChildCompletionSource::after_final_save(&child, true),
5662 };
5663 coordinator.on_child_completed(completion_b.clone()).await;
5664 coordinator.on_child_completed(completion_b).await;
5665 let claims = inbox.claim(parent_id, 8).await.unwrap();
5666 assert_eq!(
5667 claims.len(),
5668 1,
5669 "B is admitted once with B's own provenance"
5670 );
5671 let SessionMessageBody::ChildOutcome(outcome) = &claims[0].envelope.body else {
5672 panic!("expected ChildOutcome");
5673 };
5674 assert_eq!(outcome.result.as_deref(), Some(answer_b));
5675 assert_eq!(launches.load(Ordering::SeqCst), 1);
5676 }
5677 }
5678
5679 #[tokio::test]
5680 async fn missing_source_cannot_import_latest_guardian_approval() {
5681 let (_temp, store, inbox, coordinator, _, _) = completion_inbox_fixture().await;
5682 let parent_id = "unavailable-guardian-parent";
5683 let child_id = "unavailable-guardian-child";
5684 save_waiting_test_parent(
5685 &store,
5686 parent_id,
5687 vec![child_id.into()],
5688 ChildWaitPolicy::All,
5689 )
5690 .await;
5691 let mut parent = store.load_session(parent_id).await.unwrap().unwrap();
5692 let mut state = crate::runtime::guardian_state::ensure_guardian_state(&parent);
5693 state.record_spawn(child_id);
5694 write_guardian_state(&mut parent, state);
5695 store.save_session(&parent).await.unwrap();
5696 let mut child = Session::new_child(child_id, parent_id, "model", "Child");
5697 child
5698 .metadata
5699 .insert("subagent_type".into(), "guardian".into());
5700 child.add_message(Message::assistant(
5701 r#"{"approve":true,"summary":"unbound latest","findings":[]}"#,
5702 None,
5703 ));
5704 child.set_last_run_status("completed");
5705 store.save_session(&child).await.unwrap();
5706 coordinator
5707 .on_child_completed(ChildCompletion {
5708 parent_session_id: parent_id.into(),
5709 child_session_id: child_id.into(),
5710 status: "completed".into(),
5711 error: None,
5712 completed_at: Utc::now(),
5713 source: None,
5714 })
5715 .await;
5716 let parent = store.load_session(parent_id).await.unwrap().unwrap();
5717 let state = read_guardian_state(&parent).unwrap();
5718 assert!(state.phase.is_reviewed());
5719 assert!(
5720 !state.last_approved(),
5721 "source-less latest approval is never authoritative"
5722 );
5723 let claims = inbox.claim(parent_id, 8).await.unwrap();
5724 let SessionMessageBody::ChildOutcome(outcome) = &claims[0].envelope.body else {
5725 panic!("expected ChildOutcome");
5726 };
5727 assert!(outcome.result.is_none());
5728 }
5729
5730 #[tokio::test]
5731 async fn lost_callback_replay_uses_the_committed_seal() {
5732 for valid in [true, false] {
5733 let (_temp, store, inbox, coordinator, _, _) = completion_inbox_fixture().await;
5734 save_waiting_test_parent(
5735 &store,
5736 "replay-parent",
5737 vec!["replay-child".into()],
5738 ChildWaitPolicy::All,
5739 )
5740 .await;
5741 let mut child = Session::new_child("replay-child", "replay-parent", "model", "Child");
5742 child.add_message(Message::assistant("sealed replay result", None));
5743 child.set_last_run_status("completed");
5744 save_sourced_test_child(&store, &child).await;
5745 if !valid {
5746 let mut child = store.load_session("replay-child").await.unwrap().unwrap();
5747 child.messages[0].content = "unsealed successor result".into();
5748 store.save_session(&child).await.unwrap();
5749 }
5750 let parent = store.load_session("replay-parent").await.unwrap().unwrap();
5751 let mut wait = read_runtime_state(&parent).waiting_for_children.unwrap();
5752 wait.registered_at = Utc::now() - chrono::Duration::seconds(61);
5753 coordinator.sweep_child_wait("replay-parent", wait).await;
5754 let claims = inbox.claim("replay-parent", 1).await.unwrap();
5755 if !valid {
5756 assert!(claims.is_empty());
5757 let parent = store.load_session("replay-parent").await.unwrap().unwrap();
5758 assert!(read_runtime_state(&parent).waiting_for_children.is_some());
5759 continue;
5760 }
5761 let SessionMessageBody::ChildOutcome(outcome) = &claims[0].envelope.body else {
5762 panic!("expected ChildOutcome");
5763 };
5764 assert_eq!(outcome.result.as_deref(), Some("sealed replay result"));
5765 }
5766 }
5767
5768 #[tokio::test]
5769 async fn partial_admission_then_source_mutation_ignores_stale_retry() {
5770 for parent_changed in [false, true] {
5771 let (_temp, store, inbox, coordinator, _, launches) = completion_inbox_fixture().await;
5772 let parent_id = "partial-source-parent";
5773 let child_id = "partial-source-child";
5774 save_waiting_test_parent(
5775 &store,
5776 parent_id,
5777 vec![child_id.into(), "still-running-child".into()],
5778 ChildWaitPolicy::All,
5779 )
5780 .await;
5781 let mut child = Session::new_child(child_id, parent_id, "model", "Child");
5782 if parent_changed {
5783 let mut parent = store.load_session(parent_id).await.unwrap().unwrap();
5784 let mut source = Message::user("required parent constraint");
5785 source.id = "parent-source".into();
5786 parent.add_message(source);
5787 store.save_session(&parent).await.unwrap();
5788 let packet = bamboo_domain::ChildContextPacket {
5789 version: 1,
5790 objective: "task".into(),
5791 constraints: vec![],
5792 acceptance: vec!["source remains attributable".into()],
5793 non_goals: vec![],
5794 necessary_user_instructions: vec![],
5795 recorded_decisions: vec![],
5796 source_user_message_ids: vec!["parent-source".into()],
5797 background_message_ids: vec![],
5798 };
5799 let resolved = packet.resolve(&parent, "task").unwrap();
5800 let binding = bamboo_domain::ChildContextBinding::new(
5801 &parent,
5802 child_id,
5803 resolved.required_brief.clone(),
5804 resolved,
5805 )
5806 .unwrap();
5807 binding.install(&mut child).unwrap();
5808 child.add_message(binding.assignment_message());
5809 }
5810 child.add_message(Message::assistant("original source", None));
5811 child.set_last_run_status("completed");
5812 save_sourced_test_child(&store, &child).await;
5813 let completion = ChildCompletion {
5814 parent_session_id: parent_id.into(),
5815 child_session_id: child_id.into(),
5816 status: "completed".into(),
5817 error: None,
5818 completed_at: Utc::now(),
5819 source: test_child_source(&store, child_id).await,
5820 };
5821 coordinator.on_child_completed(completion.clone()).await;
5822 assert_eq!(inbox.inspect(parent_id).await.unwrap().pending, 1);
5823 if parent_changed {
5824 let mut parent = store.load_session(parent_id).await.unwrap().unwrap();
5825 parent.messages[0].content = "modified parent source".into();
5826 store.save_session(&parent).await.unwrap();
5827 } else {
5828 child = store.load_session(child_id).await.unwrap().unwrap();
5829 child.messages[0].content = "modified after admission".into();
5830 store.save_session(&child).await.unwrap();
5831 }
5832 coordinator.on_child_completed(completion).await;
5833 assert_eq!(inbox.inspect(parent_id).await.unwrap().pending, 1);
5834 assert_eq!(launches.load(Ordering::SeqCst), 1);
5835 assert!(
5836 read_runtime_state(&store.load_session(parent_id).await.unwrap().unwrap())
5837 .waiting_for_children
5838 .is_some()
5839 );
5840 let mut sibling =
5841 Session::new_child("still-running-child", parent_id, "model", "Child");
5842 sibling.add_message(Message::assistant("sibling result", None));
5843 sibling.set_last_run_status("completed");
5844 save_sourced_test_child(&store, &sibling).await;
5845 coordinator
5846 .on_child_completed(ChildCompletion {
5847 parent_session_id: parent_id.into(),
5848 child_session_id: sibling.id.clone(),
5849 status: "completed".into(),
5850 error: None,
5851 completed_at: Utc::now(),
5852 source: test_child_source(&store, &sibling.id).await,
5853 })
5854 .await;
5855 assert_eq!(
5856 inbox.inspect(parent_id).await.unwrap().pending,
5857 2,
5858 "sibling completion cannot restage mutated A as a neutral outcome"
5859 );
5860 assert_eq!(launches.load(Ordering::SeqCst), 1);
5861 assert!(
5862 read_runtime_state(&store.load_session(parent_id).await.unwrap().unwrap())
5863 .waiting_for_children
5864 .is_some()
5865 );
5866 assert_eq!(
5867 inbox.inspect(parent_id).await.unwrap().claimed,
5868 0,
5869 "stale replay does not create another claim or reasoning owner"
5870 );
5871 }
5872 }
5873
5874 #[tokio::test]
5875 async fn short_circuit_wait_stages_terminal_siblings_without_waiting_or_foreign_content() {
5876 for policy in [ChildWaitPolicy::Any, ChildWaitPolicy::FirstError] {
5877 let (_temp, store, inbox, coordinator, _, launches) = completion_inbox_fixture().await;
5878 let parent_id = "short-circuit-parent";
5879 let ids = vec![
5880 "failed-child".into(),
5881 "done-child".into(),
5882 "running-child".into(),
5883 "foreign-child".into(),
5884 ];
5885 save_waiting_test_parent(&store, parent_id, ids, policy).await;
5886 for (id, owner, status, result) in [
5887 ("failed-child", parent_id, "error", "failed-owned-result"),
5888 ("done-child", parent_id, "completed", "done-owned-result"),
5889 ("running-child", parent_id, "running", "not-terminal"),
5890 (
5891 "foreign-child",
5892 "another-parent",
5893 "completed",
5894 "foreign-secret",
5895 ),
5896 ] {
5897 let mut child = Session::new_child(id, owner, "model", "Child");
5898 child.add_message(Message::assistant(result, None));
5899 child.set_last_run_status(status);
5900 save_sourced_test_child(&store, &child).await;
5901 }
5902 ChildCompletionHandler::on_child_completed(
5903 coordinator.as_ref(),
5904 ChildCompletion {
5905 parent_session_id: parent_id.into(),
5906 child_session_id: "failed-child".into(),
5907 status: "error".into(),
5908 error: None,
5909 completed_at: Utc::now(),
5910 source: test_child_source(&store, "failed-child").await,
5911 },
5912 )
5913 .await;
5914 assert!(
5915 read_runtime_state(&store.load_session(parent_id).await.unwrap().unwrap())
5916 .waiting_for_children
5917 .is_none()
5918 );
5919 let claims = inbox.claim(parent_id, 8).await.unwrap();
5920 assert_eq!(claims.len(), 2);
5921 let content = serde_json::to_string(
5922 &claims
5923 .iter()
5924 .map(|claim| &claim.envelope)
5925 .collect::<Vec<_>>(),
5926 )
5927 .unwrap();
5928 assert!(content.contains("done-owned-result"));
5929 assert!(!content.contains("foreign-secret"));
5930 assert!(!content.contains("not-terminal"));
5931 assert_eq!(
5932 store
5933 .load_session("running-child")
5934 .await
5935 .unwrap()
5936 .unwrap()
5937 .last_run_status()
5938 .as_deref(),
5939 Some("running")
5940 );
5941 assert_eq!(launches.load(Ordering::SeqCst), 1);
5942 }
5943 }
5944
5945 #[tokio::test]
5946 async fn sibling_admission_failure_keeps_wait_armed_without_activation() {
5947 let limits = bamboo_domain::SessionInboxLimits {
5948 max_backlog: 1,
5949 ..Default::default()
5950 };
5951 let (_temp, store, inbox, coordinator, reservations, launches) =
5952 completion_inbox_fixture_with_limits(limits).await;
5953 let parent_id = "partial-stage-parent";
5954 let ids = vec!["first-child".into(), "second-child".into()];
5955 save_waiting_test_parent(&store, parent_id, ids, ChildWaitPolicy::All).await;
5956 for id in ["first-child", "second-child"] {
5957 let mut child = Session::new_child(id, parent_id, "model", "Child");
5958 child.add_message(Message::assistant(format!("result-{id}"), None));
5959 child.set_last_run_status("completed");
5960 save_sourced_test_child(&store, &child).await;
5961 }
5962 ChildCompletionHandler::on_child_completed(
5963 coordinator.as_ref(),
5964 ChildCompletion {
5965 parent_session_id: parent_id.into(),
5966 child_session_id: "first-child".into(),
5967 status: "completed".into(),
5968 error: None,
5969 completed_at: Utc::now(),
5970 source: test_child_source(&store, "first-child").await,
5971 },
5972 )
5973 .await;
5974 let durable = store.load_session(parent_id).await.unwrap().unwrap();
5975 assert!(read_runtime_state(&durable).waiting_for_children.is_some());
5976 assert_eq!(
5977 read_runtime_state(&durable).status,
5978 AgentStatusState::Suspended
5979 );
5980 assert_eq!(inbox.inspect(parent_id).await.unwrap().pending, 1);
5981 let backlog = inbox.inspect(parent_id).await.unwrap();
5982 assert!(backlog.activation_pending());
5983 assert!(
5984 !backlog.interrupt_pending(),
5985 "failed wait transition is still respected"
5986 );
5987 assert_eq!(reservations.load(Ordering::SeqCst), 0);
5988 assert_eq!(launches.load(Ordering::SeqCst), 0);
5989 }
5990
5991 #[tokio::test]
5992 async fn supervisor_resume_rejects_old_incarnation_without_replacing_cache() {
5993 let (_temp, store, _inbox, coordinator, _reservations, _launches) =
5994 completion_inbox_fixture().await;
5995 let original = store
5996 .get_or_create_default_supervisor("model")
5997 .await
5998 .unwrap();
5999 let mut stale = store
6000 .load_session(&original.session_id)
6001 .await
6002 .unwrap()
6003 .unwrap();
6004 store.delete_session(&original.session_id).await.unwrap();
6005 let recreated = store
6006 .get_or_create_default_supervisor("replacement")
6007 .await
6008 .unwrap();
6009 assert_ne!(original.incarnation_id, recreated.incarnation_id);
6010 let mut current = store
6011 .load_session(&recreated.session_id)
6012 .await
6013 .unwrap()
6014 .unwrap();
6015 coordinator.save_and_cache(&mut current).await;
6016 let cached = coordinator
6017 .sessions
6018 .get(¤t.id)
6019 .unwrap()
6020 .value()
6021 .clone();
6022 coordinator.save_and_cache(&mut stale).await;
6023 assert!(Arc::ptr_eq(
6024 &cached,
6025 coordinator.sessions.get(¤t.id).unwrap().value()
6026 ));
6027 assert_eq!(
6028 store
6029 .load_session(¤t.id)
6030 .await
6031 .unwrap()
6032 .unwrap()
6033 .authority_identity,
6034 current.authority_identity
6035 );
6036 }
6037
6038 #[test]
6041 fn dead_child_candidate_status_matrix() {
6042 assert!(is_dead_child_candidate_status(None));
6044 assert!(is_dead_child_candidate_status(Some("running")));
6047 assert!(is_dead_child_candidate_status(Some("pending")));
6048 assert!(!is_dead_child_candidate_status(Some("suspended")));
6050 for status in ["completed", "error", "timeout", "cancelled", "skipped"] {
6052 assert!(!is_dead_child_candidate_status(Some(status)), "{status}");
6053 }
6054 }
6055
6056 #[test]
6057 fn completion_child_ownership_gates_content_fold() {
6058 assert!(completion_child_is_owned("parent-1", Some("parent-1")));
6060 assert!(!completion_child_is_owned("parent-1", Some("parent-2")));
6063 assert!(!completion_child_is_owned("parent-1", None));
6065 }
6066
6067 #[test]
6068 fn watchdog_resume_messages_are_hidden_runtime_messages() {
6069 for message in [
6070 empty_child_wait_message(),
6071 child_wait_lease_expired_message(&["c-1".to_string(), "c-2".to_string()]),
6072 ] {
6073 assert!(matches!(message.role, Role::User));
6074 let meta = message.metadata.expect("hidden runtime metadata");
6075 assert_eq!(meta[RUNTIME_RESUME_MESSAGE_HIDDEN_KEY], true);
6076 assert_eq!(
6077 meta[RUNTIME_RESUME_MESSAGE_KIND_KEY],
6078 "child_wait_watchdog_resume"
6079 );
6080 }
6081 let lease = child_wait_lease_expired_message(&["c-1".to_string()]);
6082 assert!(lease.content.contains("NOT cancelled"));
6084 assert!(lease.content.contains("c-1"));
6085 }
6086
6087 #[test]
6088 fn runtime_child_guidance_uses_compact_inspection_calls() {
6089 let completion = make_completion("completed");
6090 let child_inspection = serde_json::json!({
6091 "intent": "inspect", "target": "child-1", "message": "messages",
6092 })
6093 .to_string();
6094 let tree_inspection =
6095 serde_json::json!({"intent": "inspect", "message": "tree"}).to_string();
6096 for (message, expected_call) in [
6097 (
6098 runtime_resume_message(&completion, 0, Some("evidence")),
6099 &child_inspection,
6100 ),
6101 (
6102 guardian_resume_message(&completion, &GuardianVerdict::approved()),
6103 &child_inspection,
6104 ),
6105 (empty_child_wait_message(), &tree_inspection),
6106 (
6107 child_wait_lease_expired_message(&["child-1".to_string()]),
6108 &tree_inspection,
6109 ),
6110 ] {
6111 assert!(message.content.contains(expected_call));
6112 for legacy in [
6113 "SubAgent.get",
6114 "SubAgent.list",
6115 "SubAgent.wait",
6116 "child_session_id=",
6117 ] {
6118 assert!(!message.content.contains(legacy), "stale hint: {legacy}");
6119 }
6120 }
6121 }
6122
6123 #[test]
6126 fn non_terminal_statuses_never_satisfy_wait_policies() {
6127 assert!(!is_terminal_child_status("suspended"));
6130 assert!(!is_terminal_child_status("running"));
6131 assert!(!is_terminal_child_status("pending"));
6132 }
6133
6134 fn make_completion(status: &str) -> ChildCompletion {
6135 ChildCompletion {
6136 parent_session_id: "parent-1".to_string(),
6137 child_session_id: "child-1".to_string(),
6138 status: status.to_string(),
6139 error: None,
6140 completed_at: Utc::now(),
6141 source: None,
6142 }
6143 }
6144
6145 #[test]
6146 fn oversized_child_outcome_is_bounded_and_keeps_full_content_identity() {
6147 let mut completion = make_completion("completed");
6148 let wait_registered_at = Utc::now();
6149 let huge = format!("prefix-A-{}", "x".repeat(300 * 1024));
6150 let presentation = runtime_resume_message(&completion, 0, Some(&huge));
6151 let first = child_completion_envelope(
6152 &completion,
6153 wait_registered_at,
6154 Some(huge.clone()),
6155 &presentation,
6156 );
6157 assert!(
6158 serde_json::to_vec(&first).unwrap().len()
6159 < bamboo_domain::SessionInboxLimits::default().max_payload_bytes
6160 );
6161 let SessionMessageBody::ChildOutcome(outcome) = &first.body else {
6162 panic!("typed child outcome");
6163 };
6164 let stored = outcome.result.as_deref().unwrap();
6165 assert!(stored.contains("sha256="));
6166 assert!(stored.contains("\"intent\":\"inspect\""));
6167 assert!(stored.contains("\"target\":\"child-1\""));
6168 assert!(stored.contains("\"message\":\"result\""));
6169 assert!(!stored.contains("SubAgent.get"));
6170 assert!(stored.len() < CHILD_COMPLETION_INLINE_FIELD_BYTES);
6171
6172 completion.completed_at += chrono::Duration::seconds(1);
6175 let exact_retry = child_completion_envelope(
6176 &completion,
6177 wait_registered_at,
6178 Some(huge.clone()),
6179 &runtime_resume_message(&completion, 9, Some(&huge)),
6180 );
6181 assert_eq!(exact_retry.id, first.id);
6182 let changed = format!("prefix-B-{}", "x".repeat(300 * 1024));
6183 let corrected = child_completion_envelope(
6184 &completion,
6185 wait_registered_at,
6186 Some(changed.clone()),
6187 &runtime_resume_message(&completion, 0, Some(&changed)),
6188 );
6189 assert_ne!(corrected.id, first.id);
6190 }
6191
6192 #[test]
6193 fn oversized_child_error_points_to_bounded_error_view() {
6194 let mut completion = make_completion("error");
6195 completion.error = Some("失败🙂".repeat(30_000));
6196 let presentation = runtime_resume_message(&completion, 0, None);
6197 let envelope = child_completion_envelope(&completion, Utc::now(), None, &presentation);
6198 assert!(
6199 serde_json::to_vec(&envelope).unwrap().len()
6200 < bamboo_domain::SessionInboxLimits::default().max_payload_bytes
6201 );
6202 let SessionMessageBody::ChildOutcome(outcome) = &envelope.body else {
6203 panic!("typed child outcome");
6204 };
6205 let stored = outcome.error.as_deref().expect("bounded error");
6206 assert!(stored.contains(r#""message":"diagnostics""#));
6207 assert!(stored.contains("authenticated inspector"));
6208 assert!(!stored.contains("bounded slices"));
6209 assert!(!stored.contains("cursors"));
6210 assert!(stored.len() < CHILD_COMPLETION_INLINE_FIELD_BYTES);
6211 let metadata_error = outcome
6212 .provider_message
6213 .as_ref()
6214 .and_then(|provider| provider.metadata.get("child_error"))
6215 .and_then(serde_json::Value::as_str)
6216 .expect("bounded metadata error");
6217 assert_eq!(metadata_error, stored);
6218 assert!(!metadata_error.contains(&"失败🙂".repeat(30_000)));
6219 }
6220
6221 #[test]
6222 fn escaped_and_mixed_child_fields_fit_the_serialized_inbox_budget() {
6223 let mut completion = make_completion("error");
6224 completion.error = Some("\u{0001}".repeat(CHILD_COMPLETION_INLINE_FIELD_BYTES));
6225 let result = "\n".repeat(CHILD_COMPLETION_INLINE_FIELD_BYTES);
6226 let wait_registered_at = Utc::now();
6227 let presentation = runtime_resume_message(&completion, 0, Some(&result));
6228 let envelope = child_completion_envelope(
6229 &completion,
6230 wait_registered_at,
6231 Some(result.clone()),
6232 &presentation,
6233 );
6234 assert!(
6235 serde_json::to_vec(&envelope).unwrap().len()
6236 < bamboo_domain::SessionInboxLimits::default().max_payload_bytes
6237 );
6238 let SessionMessageBody::ChildOutcome(outcome) = &envelope.body else {
6239 panic!("typed child outcome");
6240 };
6241 let error = outcome.error.as_deref().unwrap();
6242 let stored_result = outcome.result.as_deref().unwrap();
6243 assert!(error.contains(r#""message":"diagnostics""#));
6244 assert!(stored_result.contains(r#""message":"result""#));
6245 assert!(!error.contains("Bounded tail"));
6246 assert!(!stored_result.contains("Bounded tail"));
6247 assert_eq!(
6248 outcome
6249 .provider_message
6250 .as_ref()
6251 .and_then(|provider| provider.metadata.get("child_error"))
6252 .and_then(serde_json::Value::as_str),
6253 Some(error)
6254 );
6255 let retry = child_completion_envelope(
6256 &completion,
6257 wait_registered_at,
6258 Some(result),
6259 &runtime_resume_message(&completion, 8, None),
6260 );
6261 assert_eq!(envelope.id, retry.id);
6262 }
6263
6264 #[test]
6265 fn compact_guardian_outcome_keeps_rejected_verdict_and_result_retrieval() {
6266 let completion = make_completion("completed");
6267 let result = format!(
6268 "{}{{\"approve\":false,\"findings\":[\"needs fix\"]}}",
6269 "\u{0001}".repeat(CHILD_COMPLETION_INLINE_FIELD_BYTES - 80)
6270 );
6271 assert!(result.len() <= CHILD_COMPLETION_INLINE_FIELD_BYTES);
6272 let verdict = parse_guardian_verdict(&result).expect("trailing guardian verdict");
6273 let presentation = guardian_resume_message(&completion, &verdict);
6274 assert!(
6275 serde_json::to_vec(&presentation).unwrap().len() <= CHILD_COMPLETION_INLINE_FIELD_BYTES,
6276 "the original guardian presentation must fit the inline cap"
6277 );
6278 let wait_registered_at = Utc::now();
6279 let expanded = build_child_completion_envelope(
6280 &completion,
6281 wait_registered_at,
6282 Some(&result),
6283 &presentation,
6284 false,
6285 );
6286 assert!(
6287 serde_json::to_vec(&expanded).unwrap().len()
6288 > bamboo_domain::SessionInboxLimits::default().max_payload_bytes,
6289 "JSON escaping must trigger the compact fallback"
6290 );
6291
6292 let envelope =
6293 child_completion_envelope(&completion, wait_registered_at, Some(result), &presentation);
6294 assert!(
6295 serde_json::to_vec(&envelope).unwrap().len()
6296 <= bamboo_domain::SessionInboxLimits::default().max_payload_bytes
6297 );
6298 let SessionMessageBody::ChildOutcome(outcome) = &envelope.body else {
6299 panic!("typed child outcome");
6300 };
6301 let provider = outcome.provider_message.as_ref().unwrap();
6302 assert!(provider.content.text.contains("Guardian review REJECTED"));
6303 assert!(provider.content.text.contains(r#""message":"result""#));
6304 assert_eq!(
6305 provider.metadata.get("guardian_approved"),
6306 Some(&serde_json::json!(false))
6307 );
6308 assert!(envelope
6309 .to_provider_message()
6310 .unwrap()
6311 .content
6312 .contains("Guardian review REJECTED"));
6313 }
6314
6315 #[tokio::test]
6316 async fn oversized_multibyte_child_error_is_durable_and_replayable() {
6317 let home = tempfile::tempdir().unwrap();
6318 let store = Arc::new(
6319 bamboo_storage::SessionStoreV2::new(home.path().to_path_buf())
6320 .await
6321 .unwrap(),
6322 );
6323 let parent_id = "large-error-parent";
6324 store
6325 .save_session(&Session::new(parent_id, "model"))
6326 .await
6327 .unwrap();
6328 let inbox = bamboo_storage::FileSessionInbox::new(
6329 store.clone(),
6330 bamboo_domain::SessionInboxLimits::default(),
6331 );
6332 let mut completion = make_completion("error");
6333 completion.parent_session_id = parent_id.to_string();
6334 completion.error = Some("失败🙂".repeat(30_000));
6335 let presentation = runtime_resume_message(&completion, 0, None);
6336 let envelope = child_completion_envelope(&completion, Utc::now(), None, &presentation);
6337 inbox.deliver(&envelope).await.unwrap();
6338 completion.child_session_id = "escaped-error-child".to_string();
6339 completion.error = Some("\u{0001}".repeat(CHILD_COMPLETION_INLINE_FIELD_BYTES));
6340 let result = "\n".repeat(CHILD_COMPLETION_INLINE_FIELD_BYTES);
6341 let mixed = child_completion_envelope(
6342 &completion,
6343 Utc::now(),
6344 Some(result.clone()),
6345 &runtime_resume_message(&completion, 0, Some(&result)),
6346 );
6347 let receipt = inbox.deliver(&mixed).await.unwrap();
6348 inbox
6349 .mark_activation_eligible(
6350 parent_id,
6351 receipt.generation,
6352 bamboo_domain::SessionActivationPolicy::RespectSpecificWait,
6353 )
6354 .await
6355 .unwrap();
6356 drop(inbox);
6357 drop(store);
6358
6359 let reopened_store = Arc::new(
6360 bamboo_storage::SessionStoreV2::new(home.path().to_path_buf())
6361 .await
6362 .unwrap(),
6363 );
6364 let reopened = bamboo_storage::FileSessionInbox::new(
6365 reopened_store,
6366 bamboo_domain::SessionInboxLimits::default(),
6367 );
6368 let claims = reopened.claim(parent_id, 2).await.unwrap();
6369 assert_eq!(claims.len(), 2);
6370 assert_eq!(claims[0].envelope.id, envelope.id);
6371 assert_eq!(claims[1].envelope.id, mixed.id);
6372 for claim in claims {
6373 assert!(
6374 serde_json::to_vec(&claim.envelope).unwrap().len()
6375 < bamboo_domain::SessionInboxLimits::default().max_payload_bytes
6376 );
6377 }
6378 }
6379
6380 #[tokio::test]
6381 async fn failed_child_wait_sidecar_save_keeps_cache_wait_and_transcript() {
6382 let (_temp, store, inbox, mut coordinator, reservations, launches) =
6383 completion_inbox_fixture().await;
6384 let parent_id = "failed-sidecar-parent";
6385 let child_id = "failed-sidecar-child";
6386 let now = Utc::now();
6387 let mut parent = Session::new(parent_id, "model");
6388 parent.add_message(Message::user("durable root turn"));
6389 let mut runtime = AgentRuntimeState::new("waiting-run");
6390 runtime.status = AgentStatusState::Suspended;
6391 runtime.waiting_for_children = Some(WaitingForChildrenState::for_children(
6392 vec![child_id.to_string()],
6393 ChildWaitPolicy::All,
6394 now,
6395 ));
6396 write_runtime_state(&mut parent, &runtime);
6397 parent.metadata.insert(
6398 "runtime.suspend_reason".to_string(),
6399 "waiting_for_children".to_string(),
6400 );
6401 store.save_session(&parent).await.unwrap();
6402 let mut cached = parent.clone();
6403 cached.add_message(Message::assistant("newer cached root turn", None));
6404 coordinator.sessions.insert(
6405 parent_id.to_string(),
6406 Arc::new(crate::SessionSnapshot::new(cached)),
6407 );
6408
6409 let mut child = Session::new_child(child_id, parent_id, "model", "Child");
6410 child.set_last_run_status("completed");
6411 save_sourced_test_child(&store, &child).await;
6412 Arc::get_mut(&mut coordinator).unwrap().persistence = Arc::new(LockedSessionStore::new(
6413 Arc::new(FailingRuntimeSaveStorage {
6414 inner: store.clone(),
6415 }),
6416 ));
6417
6418 ChildCompletionHandler::on_child_completed(
6419 coordinator.as_ref(),
6420 ChildCompletion {
6421 parent_session_id: parent_id.to_string(),
6422 child_session_id: child_id.to_string(),
6423 status: "completed".to_string(),
6424 error: None,
6425 completed_at: Utc::now(),
6426 source: None,
6427 },
6428 )
6429 .await;
6430
6431 let durable = store.load_session(parent_id).await.unwrap().unwrap();
6432 assert!(read_runtime_state(&durable).waiting_for_children.is_some());
6433 assert_eq!(durable.messages.len(), 1);
6434 assert_eq!(durable.messages[0].content, "durable root turn");
6435 let cached = coordinator.sessions.get(parent_id).unwrap();
6436 let cached = cached.value().read();
6437 assert!(read_runtime_state(&cached).waiting_for_children.is_some());
6438 assert!(cached
6439 .messages
6440 .iter()
6441 .any(|message| message.content == "newer cached root turn"));
6442 let backlog = inbox.inspect(parent_id).await.unwrap();
6443 assert_eq!(backlog.pending + backlog.claimed, 1);
6444 assert_eq!(reservations.load(Ordering::SeqCst), 0);
6445 assert_eq!(launches.load(Ordering::SeqCst), 0);
6446 }
6447
6448 #[tokio::test]
6449 async fn oversized_child_completion_clears_wait_and_activates_exactly_once() {
6450 let (_temp, store, inbox, coordinator, reservations, launches) =
6451 completion_inbox_fixture().await;
6452 let parent_id = "oversized-parent";
6453 let child_id = "oversized-child";
6454 let now = Utc::now();
6455 let mut parent = Session::new(parent_id, "model");
6456 let mut parent_runtime = AgentRuntimeState::new("waiting-run");
6457 parent_runtime.status = AgentStatusState::Suspended;
6458 parent_runtime.waiting_for_children = Some(WaitingForChildrenState::for_children(
6459 vec![child_id.to_string()],
6460 ChildWaitPolicy::All,
6461 now,
6462 ));
6463 parent_runtime.suspension = Some(SuspensionState {
6464 reason: "waiting_for_children".to_string(),
6465 suspended_at: now,
6466 resumable: true,
6467 hook_point: Some("ChildCompletion".to_string()),
6468 });
6469 write_runtime_state(&mut parent, &parent_runtime);
6470 parent.metadata.insert(
6471 "runtime.suspend_reason".to_string(),
6472 "waiting_for_children".to_string(),
6473 );
6474 store.save_session(&parent).await.unwrap();
6475 let mut cached_parent = parent.clone();
6479 cached_parent.add_message(Message::assistant("concurrent root turn", None));
6480 coordinator.sessions.insert(
6481 parent_id.to_string(),
6482 Arc::new(crate::SessionSnapshot::new(cached_parent)),
6483 );
6484
6485 let mut child = Session::new_child(child_id, parent_id, "model", "Child");
6486 child.add_message(Message::assistant("z".repeat(300 * 1024), None));
6487 child.set_last_run_status("completed");
6488 save_sourced_test_child(&store, &child).await;
6489 let completion = ChildCompletion {
6490 parent_session_id: parent_id.to_string(),
6491 child_session_id: child_id.to_string(),
6492 status: "completed".to_string(),
6493 error: None,
6494 completed_at: Utc::now(),
6495 source: test_child_source(&store, child_id).await,
6496 };
6497
6498 ChildCompletionHandler::on_child_completed(coordinator.as_ref(), completion.clone()).await;
6499 let durable_parent = store.load_session(parent_id).await.unwrap().unwrap();
6500 let durable_runtime = read_runtime_state(&durable_parent);
6501 assert!(durable_runtime.waiting_for_children.is_none());
6502 assert_eq!(durable_runtime.status, AgentStatusState::Idle);
6503 assert!(!durable_parent
6504 .metadata
6505 .contains_key("runtime.suspend_reason"));
6506 let cached_parent = coordinator.sessions.get(parent_id).unwrap();
6507 let cached_parent = cached_parent.value().read();
6508 assert!(cached_parent
6509 .messages
6510 .iter()
6511 .any(|message| message.content == "concurrent root turn"));
6512 assert!(read_runtime_state(&cached_parent)
6513 .waiting_for_children
6514 .is_none());
6515 let backlog = inbox.inspect(parent_id).await.unwrap();
6516 assert_eq!(backlog.pending + backlog.claimed, 1);
6517 assert!(backlog.activation_pending());
6518 assert_eq!(reservations.load(Ordering::SeqCst), 1);
6519 assert_eq!(launches.load(Ordering::SeqCst), 1);
6520
6521 let claim = inbox.claim(parent_id, 1).await.unwrap().remove(0);
6522 assert!(
6523 serde_json::to_vec(&claim.envelope).unwrap().len()
6524 < bamboo_domain::SessionInboxLimits::default().max_payload_bytes
6525 );
6526 ChildCompletionHandler::on_child_completed(coordinator.as_ref(), completion).await;
6529 assert_eq!(reservations.load(Ordering::SeqCst), 1);
6530 assert_eq!(launches.load(Ordering::SeqCst), 1);
6531 assert_eq!(inbox.inspect(parent_id).await.unwrap().claimed, 1);
6532 }
6533
6534 #[tokio::test]
6535 async fn completion_before_parent_wait_does_not_checkpoint_stale_root() {
6536 let (_temp, store, inbox, coordinator, reservations, launches) =
6537 completion_inbox_fixture().await;
6538 let parent_id = "parent-without-wait";
6539 let child_id = "already-finished-child";
6540 let mut parent = Session::new(parent_id, "model");
6541 parent.add_message(Message::assistant("root is still reasoning", None));
6542 store.save_session(&parent).await.unwrap();
6543 let before = store.load_session(parent_id).await.unwrap().unwrap();
6544
6545 let mut child = Session::new_child(child_id, parent_id, "model", "Child");
6546 child.set_last_run_status("completed");
6547 save_sourced_test_child(&store, &child).await;
6548 ChildCompletionHandler::on_child_completed(
6549 coordinator.as_ref(),
6550 ChildCompletion {
6551 parent_session_id: parent_id.to_string(),
6552 child_session_id: child_id.to_string(),
6553 status: "completed".to_string(),
6554 error: None,
6555 completed_at: Utc::now(),
6556 source: None,
6557 },
6558 )
6559 .await;
6560
6561 let after = store.load_session(parent_id).await.unwrap().unwrap();
6562 assert_eq!(after.updated_at, before.updated_at);
6563 assert_eq!(
6564 serde_json::to_value(&after.messages).unwrap(),
6565 serde_json::to_value(&before.messages).unwrap()
6566 );
6567 assert!(read_runtime_state(&after).waiting_for_children.is_none());
6568 assert_eq!(inbox.inspect(parent_id).await.unwrap().pending, 0);
6569 assert_eq!(reservations.load(Ordering::SeqCst), 0);
6570 assert_eq!(launches.load(Ordering::SeqCst), 0);
6571 }
6572
6573 #[tokio::test]
6574 async fn latest_locked_activation_mutation_preserves_wait_armed_after_stale_load() {
6575 let temp = tempfile::tempdir().unwrap();
6576 let store = Arc::new(
6577 bamboo_storage::SessionStoreV2::new(temp.path().to_path_buf())
6578 .await
6579 .unwrap(),
6580 );
6581 let storage: Arc<dyn Storage> = store.clone();
6582 let locked = LockedSessionStore::new(storage);
6583 let mut session = Session::new("activation-stale-wait", "model");
6584 session.agent_runtime_state = Some(AgentRuntimeState::new("old-run"));
6585 store.save_session(&session).await.unwrap();
6586
6587 let stale = store.load_session(&session.id).await.unwrap().unwrap();
6589 assert!(read_runtime_state(&stale).waiting_for_children.is_none());
6590
6591 locked
6592 .update_runtime_config(&session.id, |latest| {
6593 let mut state = read_runtime_state(latest);
6594 state.status = AgentStatusState::Suspended;
6595 state.waiting_for_children = Some(WaitingForChildrenState::for_children(
6596 vec!["child-new".to_string()],
6597 ChildWaitPolicy::All,
6598 Utc::now(),
6599 ));
6600 state.suspension = Some(SuspensionState {
6601 reason: "waiting_for_children".to_string(),
6602 suspended_at: Utc::now(),
6603 resumable: true,
6604 hook_point: None,
6605 });
6606 write_runtime_state(latest, &state);
6607 latest.metadata.insert(
6608 "runtime.suspend_reason".to_string(),
6609 "waiting_for_children".to_string(),
6610 );
6611 })
6612 .await
6613 .unwrap();
6614
6615 let (prepared, ready) = prepare_session_inbox_activation(&locked, &session.id, false)
6616 .await
6617 .unwrap()
6618 .unwrap();
6619 assert!(!ready, "latest durable specific wait must block activation");
6620 let state = read_runtime_state(&prepared);
6621 assert!(state.waiting_for_children.is_some());
6622 assert_eq!(state.status, AgentStatusState::Suspended);
6623 assert_eq!(
6624 prepared
6625 .metadata
6626 .get("runtime.suspend_reason")
6627 .map(String::as_str),
6628 Some("waiting_for_children")
6629 );
6630 }
6631
6632 #[tokio::test]
6633 async fn partial_child_and_bash_backlogs_remain_inert_after_inbox_reopen() {
6634 let temp = tempfile::tempdir().unwrap();
6635 let store = Arc::new(
6636 bamboo_storage::SessionStoreV2::new(temp.path().to_path_buf())
6637 .await
6638 .unwrap(),
6639 );
6640 store
6641 .save_session(&Session::new("restart-child-parent", "model"))
6642 .await
6643 .unwrap();
6644 store
6645 .save_session(&Session::new("restart-bash-parent", "model"))
6646 .await
6647 .unwrap();
6648 let inbox = bamboo_storage::FileSessionInbox::new(
6649 store.clone(),
6650 bamboo_domain::SessionInboxLimits::default(),
6651 );
6652
6653 let completion = ChildCompletion {
6654 parent_session_id: "restart-child-parent".to_string(),
6655 child_session_id: "child-a".to_string(),
6656 status: "completed".to_string(),
6657 error: None,
6658 completed_at: Utc::now(),
6659 source: None,
6660 };
6661 let child_resume = runtime_resume_message(&completion, 1, Some("first child"));
6662 inbox
6663 .deliver(&child_completion_envelope(
6664 &completion,
6665 Utc::now(),
6666 Some("first child".to_string()),
6667 &child_resume,
6668 ))
6669 .await
6670 .unwrap();
6671 let bash = BashCompletionInfo {
6672 session_id: "restart-bash-parent".to_string(),
6673 bash_id: "bash-a".to_string(),
6674 command: "true".to_string(),
6675 exit_code: Some(0),
6676 status: "completed".to_string(),
6677 output_tail: String::new(),
6678 };
6679 inbox
6680 .deliver(&bash_completion_envelope(&bash))
6681 .await
6682 .unwrap();
6683
6684 let reopened = bamboo_storage::FileSessionInbox::new(
6685 store,
6686 bamboo_domain::SessionInboxLimits::default(),
6687 );
6688 for session_id in ["restart-child-parent", "restart-bash-parent"] {
6689 let backlog = reopened.inspect(session_id).await.unwrap();
6690 assert_eq!(backlog.pending, 1);
6691 assert_eq!(backlog.activation_generation, 0);
6692 assert!(
6693 !backlog.activation_pending(),
6694 "startup must not run a partial wait backlog for {session_id}"
6695 );
6696 }
6697 }
6698
6699 struct StubChildIndex {
6702 children: Vec<(String, Option<String>)>,
6703 }
6704
6705 #[async_trait]
6706 impl Storage for StubChildIndex {
6707 async fn save_session(&self, _session: &Session) -> std::io::Result<()> {
6708 Ok(())
6709 }
6710 async fn load_session(&self, _id: &str) -> std::io::Result<Option<Session>> {
6711 Ok(None)
6712 }
6713 async fn delete_session(&self, _id: &str) -> std::io::Result<bool> {
6714 Ok(false)
6715 }
6716 async fn list_child_run_statuses(
6717 &self,
6718 _parent_session_id: &str,
6719 ) -> std::io::Result<Vec<(String, Option<String>)>> {
6720 Ok(self.children.clone())
6721 }
6722 }
6723
6724 #[tokio::test]
6725 async fn derive_completed_only_includes_terminal_children() {
6726 let storage: Arc<dyn Storage> = Arc::new(StubChildIndex {
6727 children: vec![
6728 ("a".into(), Some("completed".into())),
6729 ("b".into(), Some("running".into())),
6730 ("c".into(), Some("error".into())),
6731 ("d".into(), None),
6732 ],
6733 });
6734 let completed = derive_completed_child_ids(&storage, "parent-1", "b").await;
6735 assert_eq!(
6737 completed,
6738 vec!["a".to_string(), "b".to_string(), "c".to_string()]
6739 );
6740 }
6741
6742 #[tokio::test]
6743 async fn derive_completed_folds_in_just_completed_when_index_lags() {
6744 let storage: Arc<dyn Storage> = Arc::new(StubChildIndex {
6746 children: vec![("only".into(), Some("running".into()))],
6747 });
6748 let completed = derive_completed_child_ids(&storage, "parent-1", "only").await;
6749 assert_eq!(completed, vec!["only".to_string()]);
6750 }
6751
6752 #[test]
6753 fn wait_policy_all_uses_derived_completed_set() {
6754 let waited = vec!["a".to_string(), "b".to_string()];
6755 assert!(!wait_policy_satisfied(
6756 ChildWaitPolicy::All,
6757 &waited,
6758 &["a".to_string()],
6759 "a",
6760 "completed"
6761 ));
6762 assert!(wait_policy_satisfied(
6763 ChildWaitPolicy::All,
6764 &waited,
6765 &["a".to_string(), "b".to_string()],
6766 "b",
6767 "completed"
6768 ));
6769 }
6770
6771 #[test]
6772 fn wait_policy_first_error_requires_tracked_membership() {
6773 let waited = vec!["a".to_string(), "b".to_string()];
6774 assert!(wait_policy_satisfied(
6776 ChildWaitPolicy::FirstError,
6777 &waited,
6778 &["a".to_string()],
6779 "a",
6780 "error"
6781 ));
6782 assert!(!wait_policy_satisfied(
6785 ChildWaitPolicy::FirstError,
6786 &waited,
6787 &["a".to_string()],
6788 "stray-child",
6789 "timeout"
6790 ));
6791 assert!(wait_policy_satisfied(
6793 ChildWaitPolicy::FirstError,
6794 &waited,
6795 &["a".to_string(), "b".to_string()],
6796 "stray-child",
6797 "completed"
6798 ));
6799 }
6800
6801 #[test]
6802 fn child_final_assistant_text_returns_last_assistant() {
6803 let mut session = Session::new("child-1", "gpt-4");
6804 session.messages.push(Message::user("hi"));
6805 session
6806 .messages
6807 .push(Message::assistant("first answer", None));
6808 session.messages.push(Message::user("again"));
6809 session
6810 .messages
6811 .push(Message::assistant("final answer", None));
6812
6813 assert_eq!(
6814 child_final_assistant_text(&session).as_deref(),
6815 Some("final answer")
6816 );
6817 }
6818
6819 #[test]
6820 fn child_final_assistant_text_returns_none_when_blank() {
6821 let mut session = Session::new("child-1", "gpt-4");
6822 session.messages.push(Message::assistant(" ", None));
6823 assert!(child_final_assistant_text(&session).is_none());
6824 }
6825
6826 #[test]
6827 fn child_final_assistant_text_returns_none_when_no_assistant() {
6828 let mut session = Session::new("child-1", "gpt-4");
6829 session.messages.push(Message::user("hi"));
6830 assert!(child_final_assistant_text(&session).is_none());
6831 }
6832
6833 #[test]
6834 fn runtime_resume_message_folds_full_response_without_truncation() {
6835 let completion = make_completion("completed");
6838 let long: String = "a".repeat(10_000);
6839 let message = runtime_resume_message(&completion, 0, Some(&long));
6840 assert!(message.content.contains(&long));
6841 assert!(!message.content.contains("truncated"));
6842 }
6843
6844 #[test]
6845 fn runtime_resume_message_includes_child_response_when_provided() {
6846 let completion = make_completion("completed");
6847 let message = runtime_resume_message(&completion, 0, Some("the answer is 42"));
6848
6849 assert!(matches!(message.role, Role::User));
6850 assert!(!message.never_compress);
6853 assert!(message.content.contains("Child final response:"));
6854 assert!(message.content.contains("the answer is 42"));
6855
6856 let metadata = message.metadata.expect("metadata present");
6857 assert_eq!(
6858 metadata.get("hidden_from_ui").and_then(|v| v.as_bool()),
6859 Some(true)
6860 );
6861 assert_eq!(
6862 metadata.get("runtime_kind").and_then(|v| v.as_str()),
6863 Some("child_completion_resume")
6864 );
6865 assert_eq!(
6866 metadata
6867 .get("child_final_response_included")
6868 .and_then(|v| v.as_bool()),
6869 Some(true)
6870 );
6871 }
6872
6873 #[test]
6874 fn runtime_resume_message_falls_back_to_error_when_no_response() {
6875 let mut completion = make_completion("error");
6876 completion.error = Some("boom".to_string());
6877
6878 let message = runtime_resume_message(&completion, 1, None);
6879 assert!(message.content.contains("Child error:"));
6880 assert!(message.content.contains("boom"));
6881 let metadata = message.metadata.expect("metadata present");
6882 assert_eq!(
6883 metadata
6884 .get("child_final_response_included")
6885 .and_then(|v| v.as_bool()),
6886 Some(false)
6887 );
6888 }
6889
6890 #[test]
6891 fn runtime_resume_message_minimal_when_no_response_and_no_error() {
6892 let completion = make_completion("completed");
6893 let message = runtime_resume_message(&completion, 2, None);
6894 assert!(!message.content.contains("Child final response:"));
6895 assert!(!message.content.contains("Child error:"));
6896 assert!(message.content.contains("Resume the parent task"));
6897 }
6898
6899 #[test]
6900 fn read_config_snapshot_refreshes_cached_snapshot_from_live_config() {
6901 let runtime = tokio::runtime::Runtime::new().expect("runtime");
6902
6903 runtime.block_on(async {
6904 let config = Arc::new(RwLock::new(Config::default()));
6905 config.write().await.provider = "copilot".to_string();
6906 let cached_config = StdRwLock::new(Config::default());
6907
6908 let snapshot = read_config_snapshot(&config, &cached_config);
6909
6910 assert_eq!(snapshot.provider, "copilot");
6911 assert_eq!(
6912 cached_config.read().expect("cached snapshot lock").provider,
6913 "copilot"
6914 );
6915 });
6916 }
6917
6918 #[test]
6919 fn read_config_snapshot_uses_cached_snapshot_when_live_lock_is_busy() {
6920 let runtime = tokio::runtime::Runtime::new().expect("runtime");
6921
6922 runtime.block_on(async {
6923 let mut cached_snapshot = Config::default();
6924 cached_snapshot.provider = "cached-provider".to_string();
6925
6926 let config = Arc::new(RwLock::new(Config::default()));
6927 let cached_config = StdRwLock::new(cached_snapshot);
6928 let _write_guard = config.write().await;
6929
6930 let snapshot = read_config_snapshot(&config, &cached_config);
6931
6932 assert_eq!(snapshot.provider, "cached-provider");
6933 });
6934 }
6935
6936 #[test]
6939 fn bash_completion_resume_message_normal_announces_completion() {
6940 let ids = vec!["bg-1".to_string(), "bg-2".to_string()];
6941 let message = bash_completion_resume_message(&ids, false);
6942 assert!(
6944 message.content.contains("have completed"),
6945 "normal resume message must announce completion: {}",
6946 message.content
6947 );
6948 let metadata = message.metadata.expect("metadata present");
6950 assert_eq!(
6951 metadata
6952 .get(RUNTIME_RESUME_MESSAGE_HIDDEN_KEY)
6953 .and_then(|v| v.as_bool()),
6954 Some(true),
6955 "resume message must be hidden from the UI"
6956 );
6957 assert_eq!(
6958 metadata
6959 .get(RUNTIME_RESUME_MESSAGE_KIND_KEY)
6960 .and_then(|v| v.as_str()),
6961 Some(BASH_COMPLETION_RESUME_KIND),
6962 "resume message must carry the bash-completion kind discriminant"
6963 );
6964 }
6965
6966 #[test]
6967 fn bash_completion_resume_message_deadline_does_not_claim_completion() {
6968 let ids = vec!["bg-long".to_string()];
6972 let message = bash_completion_resume_message(&ids, true);
6973 assert!(
6974 !message.content.contains("have completed"),
6975 "deadline resume message must NOT claim the shells completed: {}",
6976 message.content
6977 );
6978 assert!(
6979 message.content.contains("may still be running"),
6980 "deadline resume message must warn shells may still be running: {}",
6981 message.content
6982 );
6983 assert!(
6984 message.content.contains("BashOutput"),
6985 "deadline resume message must direct verification via BashOutput: {}",
6986 message.content
6987 );
6988 let metadata = message.metadata.expect("metadata present");
6990 assert_eq!(
6991 metadata
6992 .get(RUNTIME_RESUME_MESSAGE_KIND_KEY)
6993 .and_then(|v| v.as_str()),
6994 Some(BASH_COMPLETION_RESUME_KIND)
6995 );
6996 }
6997
6998 #[test]
6999 fn bash_resume_should_retry_matrix() {
7000 assert!(!bash_resume_should_retry(
7006 &ResumeOutcome::Started { run_id: "r".into() },
7007 true
7008 ));
7009 assert!(!bash_resume_should_retry(
7010 &ResumeOutcome::Started { run_id: "r".into() },
7011 false
7012 ));
7013
7014 assert!(!bash_resume_should_retry(&ResumeOutcome::NotFound, true));
7016 assert!(!bash_resume_should_retry(&ResumeOutcome::NotFound, false));
7017
7018 assert!(bash_resume_should_retry(&ResumeOutcome::Completed, true));
7020 assert!(!bash_resume_should_retry(&ResumeOutcome::Completed, false));
7023
7024 assert!(bash_resume_should_retry(
7027 &ResumeOutcome::AlreadyRunning { run_id: "r".into() },
7028 true
7029 ));
7030 assert!(!bash_resume_should_retry(
7032 &ResumeOutcome::AlreadyRunning { run_id: "r".into() },
7033 false
7034 ));
7035 }
7036
7037 #[test]
7040 fn injection_body_includes_status_exit_command_and_tail() {
7041 let info = BashCompletionInfo {
7042 session_id: "s".into(),
7043 bash_id: "abc123".into(),
7044 command: "make build".into(),
7045 exit_code: Some(0),
7046 status: "completed".into(),
7047 output_tail: "BUILD OK".into(),
7048 };
7049 let body = bash_completion_injection_body(&info);
7050 assert!(body.contains("abc123"), "body: {body}");
7051 assert!(body.contains("make build"), "body: {body}");
7052 assert!(body.contains("completed"), "body: {body}");
7053 assert!(body.contains("exit code 0"), "body: {body}");
7054 assert!(body.contains("BUILD OK"), "body: {body}");
7055 assert!(body.contains("BashOutput"), "body: {body}");
7057 assert!(body.contains("bash_id=\"abc123\""), "body: {body}");
7058 }
7059
7060 #[test]
7061 fn injection_body_handles_no_output_and_signal_kill() {
7062 let info = BashCompletionInfo {
7063 session_id: "s".into(),
7064 bash_id: "xyz".into(),
7065 command: "sleep 99".into(),
7066 exit_code: None,
7067 status: "killed".into(),
7068 output_tail: String::new(),
7069 };
7070 let body = bash_completion_injection_body(&info);
7071 assert!(body.contains("killed"), "body: {body}");
7072 assert!(body.contains("none (signal/killed)"), "body: {body}");
7073 assert!(body.contains("no captured output"), "body: {body}");
7074 assert!(!body.contains("Output tail:"), "body: {body}");
7076 }
7077
7078 #[test]
7079 fn background_completion_builds_post_tool_use_payload_and_feedback() {
7080 let mut info = BashCompletionInfo {
7081 session_id: "s".into(),
7082 bash_id: "bg-7".into(),
7083 command: "cargo test".into(),
7084 exit_code: Some(0),
7085 status: "completed".into(),
7086 output_tail: "test result: ok".into(),
7087 };
7088
7089 let payload = background_bash_post_tool_payload(&info);
7090 match payload {
7091 HookPayload::ToolResult {
7092 tool_name,
7093 tool_call_id,
7094 outcome,
7095 } => {
7096 assert_eq!(tool_name, "Bash");
7097 assert_eq!(tool_call_id, "bg-7");
7098 assert!(outcome.success);
7099 let response: serde_json::Value =
7100 serde_json::from_str(outcome.result.as_deref().unwrap()).unwrap();
7101 assert_eq!(response["command"], "cargo test");
7102 assert_eq!(response["exit_code"], 0);
7103 assert_eq!(response["status"], "completed");
7104 assert_eq!(response["output_tail"], "test result: ok");
7105 }
7106 other => panic!("expected PostToolUse payload, got {other:?}"),
7107 }
7108
7109 append_background_bash_hook_feedback(
7110 &mut info,
7111 vec!["Run the formatter before continuing".to_string()],
7112 );
7113 assert!(info.output_tail.contains("<post_tool_use_feedback>"));
7114 assert!(info
7115 .output_tail
7116 .contains("Run the formatter before continuing"));
7117 }
7118
7119 #[tokio::test]
7120 async fn bash_completion_payload_identity_matches_file_inbox_idempotency() {
7121 let baseline = BashCompletionInfo {
7122 session_id: "session".into(),
7123 bash_id: "bg-7".into(),
7124 command: "cargo test".into(),
7125 exit_code: Some(0),
7126 status: "completed".into(),
7127 output_tail: "first tail".into(),
7128 };
7129 let mut retried = baseline.clone();
7130 retried.output_tail = "first tail\nlater bytes\n<hook feedback>".into();
7131 retried.status = "completed-after-hook".into();
7132 let baseline_envelope = bash_completion_envelope(&baseline);
7133 let exact_retry = bash_completion_envelope(&baseline);
7134 let corrected_envelope = bash_completion_envelope(&retried);
7135 assert_eq!(baseline_envelope.id, exact_retry.id);
7136 assert_ne!(
7137 baseline_envelope.id, corrected_envelope.id,
7138 "changed payload semantics must receive a distinct id"
7139 );
7140
7141 let mut other_shell = baseline.clone();
7142 other_shell.bash_id = "bg-8".into();
7143 assert_ne!(
7144 bash_completion_envelope(&baseline).id,
7145 bash_completion_envelope(&other_shell).id
7146 );
7147
7148 let temp = tempfile::tempdir().unwrap();
7149 let store = Arc::new(
7150 bamboo_storage::SessionStoreV2::new(temp.path().to_path_buf())
7151 .await
7152 .unwrap(),
7153 );
7154 store
7155 .save_session(&Session::new("session", "model"))
7156 .await
7157 .unwrap();
7158 let inbox = bamboo_storage::FileSessionInbox::new(
7159 store,
7160 bamboo_domain::SessionInboxLimits::default(),
7161 );
7162 let first = inbox.deliver(&baseline_envelope).await.unwrap();
7163 let duplicate = inbox.deliver(&exact_retry).await.unwrap();
7164 let corrected = inbox.deliver(&corrected_envelope).await.unwrap();
7165 assert_eq!(duplicate, first, "exact payload retry is idempotent");
7166 assert_ne!(corrected.id, first.id);
7167 assert_eq!(corrected.generation, first.generation + 1);
7168 assert_eq!(inbox.inspect("session").await.unwrap().pending, 2);
7169 }
7170
7171 #[cfg(unix)]
7172 #[tokio::test]
7173 async fn background_completion_fires_configured_post_tool_use_command() {
7174 use bamboo_config::{
7175 LifecycleHookGroup, LifecycleHookHandler, LifecycleHooksConfig,
7176 DEFAULT_LIFECYCLE_HOOK_TIMEOUT_MS,
7177 };
7178
7179 let dir = tempfile::tempdir().unwrap();
7180 let output = dir.path().join("background-post-tool.json");
7181 let command = format!(
7182 "cat > '{}'; printf '%s' '{{\"additional_context\":\"inspect the completed build log\"}}'",
7183 output.display()
7184 );
7185 let config = LifecycleHooksConfig {
7186 enabled: true,
7187 post_tool_use: vec![LifecycleHookGroup {
7188 enabled: true,
7189 matcher: Some("^Bash$".to_string()),
7190 hooks: vec![LifecycleHookHandler::command(
7191 command,
7192 DEFAULT_LIFECYCLE_HOOK_TIMEOUT_MS,
7193 )],
7194 }],
7195 ..Default::default()
7196 };
7197 let mut session = Session::new("session-bg-hook", "test-model");
7198 session.workspace = Some(dir.path().to_string_lossy().into_owned());
7199 let mut info = BashCompletionInfo {
7200 session_id: session.id.clone(),
7201 bash_id: "bg-9".into(),
7202 command: "cargo test".into(),
7203 exit_code: Some(0),
7204 status: "completed".into(),
7205 output_tail: "test result: ok".into(),
7206 };
7207
7208 assert!(run_background_bash_post_tool_hooks(&config, None, &session, &mut info).await);
7209 let envelope: serde_json::Value =
7210 serde_json::from_str(&std::fs::read_to_string(output).unwrap()).unwrap();
7211 assert_eq!(envelope["hook_event_name"], "PostToolUse");
7212 assert_eq!(envelope["tool_name"], "Bash");
7213 assert_eq!(envelope["payload"]["tool_call_id"], "bg-9");
7214 let response = envelope["tool_response"]["result"]
7215 .as_str()
7216 .map(serde_json::from_str::<serde_json::Value>)
7217 .transpose()
7218 .unwrap()
7219 .unwrap();
7220 assert_eq!(response["command"], "cargo test");
7221 assert_eq!(response["status"], "completed");
7222 assert!(info.output_tail.contains("inspect the completed build log"));
7223 }
7224
7225 async fn temp_store() -> (tempfile::TempDir, Arc<dyn Storage>, LockedSessionStore) {
7226 let temp = tempfile::tempdir().unwrap();
7227 let storage: Arc<dyn Storage> = Arc::new(
7228 bamboo_storage::v2::SessionStoreV2::new(temp.path().to_path_buf())
7229 .await
7230 .expect("storage init"),
7231 );
7232 let persistence = LockedSessionStore::new(storage.clone());
7233 (temp, storage, persistence)
7234 }
7235
7236 #[tokio::test]
7237 async fn enqueue_writes_pending_injection_and_preserves_messages() {
7238 let (_temp, storage, persistence) = temp_store().await;
7239
7240 let mut session = Session::new("sess-enq", "test-model");
7241 session.add_message(Message::user("do the build"));
7242 storage.save_session(&session).await.unwrap();
7243
7244 let info = BashCompletionInfo {
7245 session_id: "sess-enq".into(),
7246 bash_id: "sh-1".into(),
7247 command: "make".into(),
7248 exit_code: Some(0),
7249 status: "completed".into(),
7250 output_tail: "done".into(),
7251 };
7252 let saved = enqueue_bash_completion_injection(&persistence, &info)
7253 .await
7254 .expect("enqueue io ok")
7255 .expect("session exists");
7256
7257 let pending = saved
7258 .pending_injected_messages()
7259 .expect("pending injection present");
7260 assert_eq!(pending.len(), 1);
7261 let content = pending[0].get("content").and_then(|v| v.as_str()).unwrap();
7262 assert!(content.contains("sh-1"), "content: {content}");
7263 assert!(content.contains("make"), "content: {content}");
7264 assert!(content.contains("done"), "content: {content}");
7265 assert_eq!(saved.messages.len(), 1);
7267 }
7268
7269 #[tokio::test]
7270 async fn enqueue_returns_none_for_missing_session() {
7271 let (_temp, _storage, persistence) = temp_store().await;
7272 let info = BashCompletionInfo {
7273 session_id: "does-not-exist".into(),
7274 bash_id: "x".into(),
7275 command: "true".into(),
7276 exit_code: Some(0),
7277 status: "completed".into(),
7278 output_tail: String::new(),
7279 };
7280 let result = enqueue_bash_completion_injection(&persistence, &info)
7281 .await
7282 .expect("io ok");
7283 assert!(result.is_none(), "no session → nothing enqueued");
7284 }
7285
7286 #[test]
7294 fn apply_bash_resume_transition_clears_wait_and_appends_message() {
7295 use bamboo_domain::session::runtime_state::WaitingForBashState;
7296
7297 let mut session = Session::new("sess-resume", "test-model");
7298 session.add_message(Message::user("kick off the build"));
7299 let mut rt = read_runtime_state(&session);
7300 rt.status = AgentStatusState::Running;
7301 rt.waiting_for_bash = Some(WaitingForBashState::for_bash(
7302 vec!["sh-1".into()],
7303 Utc::now(),
7304 ));
7305 write_runtime_state(&mut session, &rt);
7306 session.metadata.insert(
7307 "runtime.suspend_reason".to_string(),
7308 "waiting_for_bash".to_string(),
7309 );
7310
7311 let resume = bash_completion_resume_message(&["sh-1".to_string()], false);
7312 let did = apply_bash_resume_transition(&mut session, &resume);
7313
7314 assert!(did, "a suspended session must transition");
7315 let after = read_runtime_state(&session);
7316 assert!(
7317 after.waiting_for_bash.is_none(),
7318 "bash wait must be cleared"
7319 );
7320 assert_eq!(after.status, AgentStatusState::Idle, "runtime must be Idle");
7321 assert!(
7322 !session.metadata.contains_key("runtime.suspend_reason"),
7323 "suspend-reason marker must be removed"
7324 );
7325 assert_eq!(session.messages.len(), 2, "resume message must be appended");
7326 assert!(matches!(
7327 session.messages.last().map(|m| &m.role),
7328 Some(Role::User)
7329 ));
7330 }
7331
7332 #[test]
7336 fn apply_bash_resume_transition_noops_when_not_waiting() {
7337 let mut session = Session::new("sess-live", "test-model");
7338 session.add_message(Message::user("hi"));
7339
7340 let resume = bash_completion_resume_message(&["sh-1".to_string()], false);
7341 let did = apply_bash_resume_transition(&mut session, &resume);
7342
7343 assert!(!did, "a non-waiting session must not transition");
7344 assert_eq!(session.messages.len(), 1, "no resume message appended");
7345 }
7346
7347 #[test]
7351 fn bash_completion_should_resume_only_when_suspended_and_all_done() {
7352 assert!(bash_completion_should_resume(true, true));
7353 assert!(!bash_completion_should_resume(true, false)); assert!(!bash_completion_should_resume(false, true)); assert!(!bash_completion_should_resume(false, false));
7356 }
7357
7358 #[test]
7359 fn two_shell_delivery_stages_backlog_before_one_final_activation() {
7360 let mut waiting = true;
7361 let mut durable_backlog = 0;
7362 let mut reservations = 0;
7363
7364 durable_backlog += 1;
7366 let first = bash_completion_delivery_plan(waiting, false);
7367 assert_eq!(first, BashCompletionDeliveryPlan::DurableOnly);
7368 assert!(waiting);
7369 assert_eq!(durable_backlog, 1);
7370 assert_eq!(reservations, 0);
7371
7372 durable_backlog += 1;
7375 let last = bash_completion_delivery_plan(waiting, true);
7376 assert_eq!(last, BashCompletionDeliveryPlan::ClearWaitThenActivate);
7377 waiting = false;
7378 reservations += 1;
7379 assert!(!waiting);
7380 assert_eq!(durable_backlog, 2);
7381 assert_eq!(reservations, 1);
7382 }
7383
7384 #[test]
7388 fn bash_resume_message_from_info_carries_bashid_tail_and_kind() {
7389 let info = BashCompletionInfo {
7390 session_id: "s".into(),
7391 bash_id: "sh-42".into(),
7392 command: "cargo test".into(),
7393 exit_code: Some(0),
7394 status: "completed".into(),
7395 output_tail: "test result: ok".into(),
7396 };
7397 let msg = bash_resume_message_from_info(&info);
7398
7399 assert!(matches!(msg.role, Role::User));
7400 assert!(msg.content.contains("sh-42"), "content: {}", msg.content);
7401 assert!(
7402 msg.content.contains("cargo test"),
7403 "content: {}",
7404 msg.content
7405 );
7406 assert!(
7407 msg.content.contains("test result: ok"),
7408 "content: {}",
7409 msg.content
7410 );
7411 assert!(
7412 msg.content.contains("BashOutput"),
7413 "content: {}",
7414 msg.content
7415 );
7416 let meta = serde_json::to_string(&msg.metadata).unwrap();
7417 assert!(
7418 meta.contains(BASH_COMPLETION_RESUME_KIND),
7419 "resume message must be tagged as a bash-completion resume: {meta}"
7420 );
7421 }
7422}