use std::collections::HashMap;
use std::sync::{Arc, OnceLock, RwLock as StdRwLock};
use std::time::Duration;
use bamboo_domain::poison::PoisonRecover;
use crate::execution::{
create_event_forwarder, finalize_runner, spawn_session_execution, try_reserve_runner,
AgentRunner, AgentStatus, ChildCompletion, ChildCompletionHandler, RunnerReservation,
SessionExecutionArgs,
};
use crate::runtime::config::{BashResumeHook, GuardianSpawner, BASH_COMPLETION_RESUME_KIND};
use crate::runtime::guardian_state::{
parse_guardian_verdict, read_guardian_config, read_guardian_state, write_guardian_state,
GuardianVerdict,
};
use crate::Agent;
use async_trait::async_trait;
use bamboo_agent_core::storage::Storage;
use bamboo_agent_core::tools::ToolExecutor;
use bamboo_agent_core::{
AgentEvent, BashCompletionInfo, BashCompletionSink, Message, Role, Session,
};
use bamboo_domain::session::runtime_state::{
AgentRuntimeState, AgentStatusState, ChildWaitPolicy, SuspensionState, WaitingForChildrenState,
};
use bamboo_llm::{Config, ProviderModelRouter, ProviderRegistry};
use bamboo_storage::LockedSessionStore;
use chrono::Utc;
use tokio::sync::{broadcast, RwLock};
use crate::model_areas::resolve_global_area_models;
use crate::model_config_helper::{
resolve_fast_model, resolve_gold_config, GOLD_CONFIG_METADATA_KEY,
};
use crate::session_app::provider_model::session_effective_model_ref;
use crate::session_app::resume::{
resume_session_execution, ResumeExecutionPort, ResumeSpawnRequest,
};
use crate::session_app::types::{ResumeConfigSnapshot, ResumeOutcome};
const AGENT_RUNTIME_STATE_METADATA_KEY: &str = "agent.runtime.state";
const RUNTIME_RESUME_MESSAGE_HIDDEN_KEY: &str = "hidden_from_ui";
const RUNTIME_RESUME_MESSAGE_KIND_KEY: &str = "runtime_kind";
fn read_runtime_state(session: &Session) -> AgentRuntimeState {
session
.agent_runtime_state
.clone()
.or_else(|| {
session
.metadata
.get(AGENT_RUNTIME_STATE_METADATA_KEY)
.and_then(|raw| serde_json::from_str::<AgentRuntimeState>(raw).ok())
})
.unwrap_or_else(|| AgentRuntimeState::new(format!("{}-child-wait", session.id)))
}
fn write_runtime_state(session: &mut Session, runtime_state: &AgentRuntimeState) {
session.agent_runtime_state = Some(runtime_state.clone());
if let Ok(serialized) = serde_json::to_string(runtime_state) {
session
.metadata
.insert(AGENT_RUNTIME_STATE_METADATA_KEY.to_string(), serialized);
}
}
fn is_error_like(status: &str) -> bool {
matches!(status, "error" | "timeout" | "cancelled")
}
fn is_terminal_child_status(status: &str) -> bool {
matches!(
status,
"completed" | "error" | "timeout" | "cancelled" | "skipped"
)
}
async fn derive_completed_child_ids(
storage: &Arc<dyn Storage>,
parent_session_id: &str,
just_completed_child_id: &str,
) -> Vec<String> {
let mut completed: Vec<String> = storage
.list_child_run_statuses(parent_session_id)
.await
.unwrap_or_default()
.into_iter()
.filter(|(_, status)| status.as_deref().is_some_and(is_terminal_child_status))
.map(|(id, _)| id)
.collect();
if !completed.iter().any(|id| id == just_completed_child_id) {
completed.push(just_completed_child_id.to_string());
}
completed.sort();
completed.dedup();
completed
}
fn read_config_snapshot(config: &Arc<RwLock<Config>>, cached_config: &StdRwLock<Config>) -> Config {
if let Ok(config_guard) = config.try_read() {
let snapshot = config_guard.clone();
if let Ok(mut cached_guard) = cached_config.try_write() {
*cached_guard = snapshot.clone();
}
snapshot
} else {
cached_config
.try_read()
.map(|guard| guard.clone())
.unwrap_or_default()
}
}
fn parent_locks() -> &'static std::sync::Mutex<HashMap<String, Arc<tokio::sync::Mutex<()>>>> {
static LOCKS: OnceLock<std::sync::Mutex<HashMap<String, Arc<tokio::sync::Mutex<()>>>>> =
OnceLock::new();
LOCKS.get_or_init(|| std::sync::Mutex::new(HashMap::new()))
}
fn session_resume_lock(session_id: &str) -> Arc<tokio::sync::Mutex<()>> {
let mut map = parent_locks().lock().recover_poison();
map.entry(session_id.to_string())
.or_insert_with(|| Arc::new(tokio::sync::Mutex::new(())))
.clone()
}
fn wait_policy_satisfied(
policy: ChildWaitPolicy,
wait_child_ids: &[String],
completed_child_ids: &[String],
latest_child_id: &str,
latest_status: &str,
) -> bool {
if wait_child_ids.is_empty() {
return false;
}
match policy {
ChildWaitPolicy::All => wait_child_ids
.iter()
.all(|id| completed_child_ids.iter().any(|completed| completed == id)),
ChildWaitPolicy::Any => completed_child_ids
.iter()
.any(|id| wait_child_ids.iter().any(|wait_id| wait_id == id)),
ChildWaitPolicy::FirstError => {
(is_error_like(latest_status) && wait_child_ids.iter().any(|id| id == latest_child_id))
|| wait_child_ids
.iter()
.all(|id| completed_child_ids.iter().any(|completed| completed == id))
}
}
}
fn child_final_assistant_text(child: &Session) -> Option<String> {
child
.messages
.iter()
.rev()
.find(|message| matches!(message.role, Role::Assistant))
.map(|message| message.content.clone())
.filter(|content| !content.trim().is_empty())
}
fn runtime_resume_message(
completion: &ChildCompletion,
remaining_children: usize,
child_final_response: Option<&str>,
) -> Message {
let mut body = format!(
"Runtime notification: child session `{}` finished with status `{}`. Remaining child sessions: {}.",
completion.child_session_id, completion.status, remaining_children
);
let final_response = child_final_response.map(str::to_string);
if let Some(response) = final_response.as_deref() {
body.push_str("\n\nChild final response:\n");
body.push_str(response);
} else if let Some(error) = completion.error.as_deref() {
if !error.is_empty() {
body.push_str("\n\nChild error:\n");
body.push_str(error);
}
}
body.push_str(
"\n\nResume the parent task using this child result and continue from the previous plan. \
If you need the full child transcript, call SubAgent.get(child_session_id).",
);
let mut message = Message::user(body);
message.metadata = Some(serde_json::json!({
RUNTIME_RESUME_MESSAGE_HIDDEN_KEY: true,
RUNTIME_RESUME_MESSAGE_KIND_KEY: "child_completion_resume",
"child_session_id": completion.child_session_id,
"child_status": completion.status,
"child_error": completion.error,
"completed_at": completion.completed_at,
"child_final_response_included": final_response.is_some(),
}));
message.never_compress = false;
message
}
fn guardian_resume_message(completion: &ChildCompletion, verdict: &GuardianVerdict) -> Message {
let mut body = if verdict.approve {
String::from(
"Guardian review APPROVED: an independent reviewer verified the work and found no blocking issues. You may finalize the task.",
)
} else {
String::from(
"Guardian review REJECTED: an independent reviewer found issues. Address every finding below before completing — do NOT declare the task complete until they are resolved.",
)
};
if let Some(summary) = verdict.summary.as_deref().filter(|s| !s.trim().is_empty()) {
body.push_str("\n\nReviewer summary: ");
body.push_str(summary);
}
if !verdict.findings.is_empty() {
body.push_str("\n\nFindings:");
for (idx, finding) in verdict.findings.iter().enumerate() {
body.push_str(&format!("\n{}. {}", idx + 1, finding));
}
}
body.push_str(
"\n\nIf you need the full guardian transcript, call SubAgent.get(child_session_id).",
);
let mut message = Message::user(body);
message.metadata = Some(serde_json::json!({
RUNTIME_RESUME_MESSAGE_HIDDEN_KEY: true,
RUNTIME_RESUME_MESSAGE_KIND_KEY: "guardian_review_resume",
"child_session_id": completion.child_session_id,
"child_status": completion.status,
"guardian_approved": verdict.approve,
"completed_at": completion.completed_at,
}));
message.never_compress = false;
message
}
#[derive(Clone)]
pub struct ChildCompletionCoordinator {
storage: Arc<dyn Storage>,
persistence: Arc<bamboo_storage::LockedSessionStore>,
sessions: crate::SessionCache,
agent_runners: Arc<RwLock<HashMap<String, AgentRunner>>>,
session_event_senders: Arc<RwLock<HashMap<String, broadcast::Sender<AgentEvent>>>>,
agent: Arc<Agent>,
config: Arc<RwLock<Config>>,
provider_registry: Arc<ProviderRegistry>,
provider_router: Arc<ProviderModelRouter>,
app_data_dir: std::path::PathBuf,
account_feed_inbox: Option<crate::execution::AccountFeedInbox>,
root_tools: Arc<RwLock<Option<Arc<dyn ToolExecutor>>>>,
guardian_spawner: Arc<RwLock<Option<Arc<dyn GuardianSpawner>>>>,
}
impl ChildCompletionCoordinator {
#[allow(clippy::too_many_arguments)]
pub fn new(
storage: Arc<dyn Storage>,
persistence: Arc<LockedSessionStore>,
sessions: crate::SessionCache,
agent_runners: Arc<RwLock<HashMap<String, AgentRunner>>>,
session_event_senders: Arc<RwLock<HashMap<String, broadcast::Sender<AgentEvent>>>>,
agent: Arc<Agent>,
config: Arc<RwLock<Config>>,
provider_registry: Arc<ProviderRegistry>,
provider_router: Arc<ProviderModelRouter>,
app_data_dir: std::path::PathBuf,
account_feed_inbox: Option<crate::execution::AccountFeedInbox>,
) -> Self {
Self {
storage,
persistence,
sessions,
agent_runners,
session_event_senders,
agent,
config,
provider_registry,
provider_router,
app_data_dir,
account_feed_inbox,
root_tools: Arc::new(RwLock::new(None)),
guardian_spawner: Arc::new(RwLock::new(None)),
}
}
pub async fn set_root_tools(&self, tools: Arc<dyn ToolExecutor>) {
*self.root_tools.write().await = Some(tools);
}
pub async fn set_guardian_spawner(&self, spawner: Arc<dyn GuardianSpawner>) {
*self.guardian_spawner.write().await = Some(spawner);
}
fn build_resume_config(
&self,
session: &Session,
config_snapshot: &Config,
) -> ResumeConfigSnapshot {
crate::session_app::resolution::resolve_resume_config_snapshot(
config_snapshot,
&self.provider_registry,
session,
None,
)
}
async fn resume_parent(&self, parent_session_id: String) -> ResumeOutcome {
for attempt in 0..=5u8 {
if attempt > 0 {
tokio::time::sleep(Duration::from_millis(250 * attempt as u64)).await;
}
let Some(session) = self.load_session(&parent_session_id).await else {
tracing::warn!(%parent_session_id, "cannot resume parent after child completion: session not found");
return ResumeOutcome::NotFound;
};
let config_snapshot = self.config.read().await.clone();
let resume_config = self.build_resume_config(&session, &config_snapshot);
let outcome = resume_session_execution(self, &parent_session_id, resume_config).await;
tracing::info!(
%parent_session_id,
attempt,
outcome = outcome.as_str(),
"child completion requested parent resume"
);
if !matches!(outcome, ResumeOutcome::AlreadyRunning { .. }) {
return outcome;
}
}
tracing::error!(
%parent_session_id,
"parent resume gave up after AlreadyRunning retry budget; \
relying on the child-wait watchdog backstop"
);
ResumeOutcome::AlreadyRunning {
run_id: String::new(),
}
}
async fn save_and_cache(&self, session: &mut Session) {
if let Err(error) = self.persistence.merge_save_runtime(session).await {
tracing::warn!(session_id = %session.id, %error, "failed to persist session");
}
self.sessions.insert(
session.id.clone(),
Arc::new(parking_lot::RwLock::new(session.clone())),
);
}
}
#[async_trait]
impl ChildCompletionHandler for ChildCompletionCoordinator {
async fn on_child_completed(&self, completion: ChildCompletion) {
if !is_terminal_child_status(&completion.status) {
tracing::info!(
parent_session_id = %completion.parent_session_id,
child_session_id = %completion.child_session_id,
status = %completion.status,
"non-terminal child status; leaving the parent wait armed"
);
return;
}
let per_parent = session_resume_lock(&completion.parent_session_id);
let _per_parent_guard = per_parent.lock().await;
let Some(mut parent) = self.load_session(&completion.parent_session_id).await else {
tracing::warn!(
parent_session_id = %completion.parent_session_id,
child_session_id = %completion.child_session_id,
"child completion received for missing parent"
);
return;
};
let mut runtime_state = read_runtime_state(&parent);
let completed_child_ids = derive_completed_child_ids(
&self.storage,
&completion.parent_session_id,
&completion.child_session_id,
)
.await;
let mut should_resume = false;
let mut remaining_children = 0usize;
if let Some(wait) = runtime_state.waiting_for_children.clone() {
remaining_children = wait
.child_session_ids
.iter()
.filter(|id| !completed_child_ids.iter().any(|completed| completed == *id))
.count();
should_resume = wait_policy_satisfied(
wait.wait_for,
&wait.child_session_ids,
&completed_child_ids,
&completion.child_session_id,
&completion.status,
);
if should_resume {
runtime_state.waiting_for_children = None;
runtime_state.status = AgentStatusState::Idle;
runtime_state.suspension = None;
}
}
if should_resume {
parent.metadata.remove("runtime.suspend_reason");
let reported_child_owned = match self
.storage
.load_runtime_control_plane(&completion.child_session_id)
.await
{
Ok(Some(control_plane)) => completion_child_is_owned(
&completion.parent_session_id,
control_plane.parent_session_id.as_deref(),
),
_ => false,
};
let loaded_child = if reported_child_owned {
match self
.storage
.load_session(&completion.child_session_id)
.await
{
Ok(child) => child,
Err(error) => {
tracing::warn!(
child_session_id = %completion.child_session_id,
%error,
"failed to load child session for runtime resume message"
);
None
}
}
} else {
tracing::warn!(
parent_session_id = %completion.parent_session_id,
child_session_id = %completion.child_session_id,
"completion child is not a child of this parent; resuming with a neutral \
message and NOT folding its content"
);
None
};
let reviewed_round = runtime_state.round.current_round;
let guardian_resume = loaded_child.as_ref().and_then(|child| {
if child.subagent_type().as_deref() != Some("guardian") {
return None;
}
let mut guardian_state = read_guardian_state(&parent)?;
if guardian_state.guardian_child_id.as_deref()
!= Some(completion.child_session_id.as_str())
{
tracing::warn!(
parent_session_id = %completion.parent_session_id,
child_session_id = %completion.child_session_id,
expected = ?guardian_state.guardian_child_id,
"guardian completion does not match recorded guardian_child_id; using generic resume"
);
return None;
}
let verdict = child_final_assistant_text(child)
.and_then(|text| match parse_guardian_verdict(&text) {
Ok(verdict) => Some(verdict),
Err(error) => {
tracing::warn!(
child_session_id = %completion.child_session_id,
%error,
"guardian verdict unparseable; recording a synthetic reject"
);
None
}
})
.unwrap_or_else(|| {
GuardianVerdict::rejected(vec![
"The guardian reviewer did not return a usable verdict (it errored or \
emitted unparseable output); the work has NOT been independently \
verified."
.to_string(),
])
});
let approved = verdict.approve;
let message = guardian_resume_message(&completion, &verdict);
guardian_state.record_verdict(verdict, reviewed_round);
write_guardian_state(&mut parent, guardian_state);
tracing::info!(
parent_session_id = %completion.parent_session_id,
child_session_id = %completion.child_session_id,
approved,
"guardian verdict recorded; resuming parent"
);
Some(message)
});
let resume_message = guardian_resume.unwrap_or_else(|| {
runtime_resume_message(
&completion,
remaining_children,
loaded_child
.as_ref()
.and_then(child_final_assistant_text)
.as_deref(),
)
});
parent.add_message(resume_message);
} else if runtime_state.waiting_for_children.is_some() {
runtime_state.status = AgentStatusState::Suspended;
runtime_state.suspension = Some(SuspensionState {
reason: "waiting_for_children".to_string(),
suspended_at: Utc::now(),
resumable: true,
hook_point: Some("ChildCompletion".to_string()),
});
}
parent.updated_at = Utc::now();
write_runtime_state(&mut parent, &runtime_state);
self.save_and_cache(&mut parent).await;
let resume_parent_id = parent.id.clone();
drop(_per_parent_guard);
if should_resume {
self.resume_parent(resume_parent_id).await;
}
}
}
#[async_trait]
impl ResumeExecutionPort for ChildCompletionCoordinator {
async fn load_session(&self, session_id: &str) -> Option<Session> {
match self.storage.load_session(session_id).await {
Ok(Some(session)) => Some(session),
Ok(None) => self
.sessions
.get(session_id)
.map(|e| e.value().clone())
.map(|arc| arc.read().clone()),
Err(error) => {
tracing::warn!(%session_id, %error, "failed to load session from storage");
self.sessions
.get(session_id)
.map(|e| e.value().clone())
.map(|arc| arc.read().clone())
}
}
}
async fn save_and_cache_session(&self, session: &mut Session) {
self.save_and_cache(session).await;
}
async fn try_reserve_runner(
&self,
session_id: &str,
event_sender: &broadcast::Sender<AgentEvent>,
) -> Option<RunnerReservation> {
try_reserve_runner(
&self.agent_runners,
&self.session_event_senders,
session_id,
event_sender,
)
.await
}
async fn get_existing_runner_run_id(&self, session_id: &str) -> Option<String> {
let runners = self.agent_runners.read().await;
runners.get(session_id).map(|r| r.run_id.clone())
}
async fn get_or_create_event_sender(&self, session_id: &str) -> broadcast::Sender<AgentEvent> {
crate::execution::session_events::get_or_create_event_sender(
&self.session_event_senders,
session_id,
)
.await
}
async fn spawn_resume_execution(&self, request: ResumeSpawnRequest) {
let ResumeSpawnRequest {
session_id,
session,
cancel_token,
run_id: _,
event_sender,
config,
} = request;
let Some(root_tools) = self.root_tools.read().await.clone() else {
tracing::error!(%session_id, "cannot resume parent after child completion: root tool surface is not initialized");
return;
};
let model = session.model.clone();
let resolved_provider_name = session_effective_model_ref(&session)
.map(|model_ref| model_ref.provider)
.unwrap_or(config.provider_name);
let provider_override = session_effective_model_ref(&session)
.and_then(|model_ref| match self.provider_router.route(&model_ref) {
Ok(provider) => Some(provider),
Err(error) => {
tracing::warn!(
session_id = %session_id,
provider = %model_ref.provider,
model = %model_ref.model,
error = %error,
"failed to resolve provider override for child-completion parent resume; falling back to runtime provider"
);
None
}
});
let config_snapshot = self.config.read().await.clone();
let resolved_fast_provider = resolve_fast_model(
&config_snapshot,
&resolved_provider_name,
&self.provider_registry,
)
.map(|model| model.provider);
let reasoning_effort = session.reasoning_effort;
let reasoning_effort_source = session
.metadata
.get("reasoning_effort_source")
.cloned()
.unwrap_or_default();
let gold_config = resolve_gold_config(
&config_snapshot,
session
.metadata
.get(GOLD_CONFIG_METADATA_KEY)
.map(String::as_str),
)
.or(config.gold_config.clone());
let (mpsc_tx, _forwarder) = create_event_forwarder(
session_id.clone(),
event_sender,
self.agent_runners.clone(),
self.account_feed_inbox.clone(),
);
let config_handle = self.config.clone();
let cached_config = Arc::new(StdRwLock::new(config_snapshot.clone()));
let provider_registry = self.provider_registry.clone();
let provider_name_for_aux = resolved_provider_name.clone();
let auxiliary_model_resolver = std::sync::Arc::new(move || {
let config_snapshot = read_config_snapshot(&config_handle, cached_config.as_ref());
let areas = resolve_global_area_models(
&config_snapshot,
&provider_name_for_aux,
&provider_registry,
);
crate::AuxiliaryModelConfig {
fast_model_name: areas.fast.as_ref().map(|m| m.model_name.clone()),
fast_model_provider: areas.fast.map(|m| m.provider),
background_model_name: areas.background.as_ref().map(|m| m.model_name.clone()),
planning_model_name: None,
search_model_name: None,
summarization_model_name: areas
.summarization
.as_ref()
.map(|m| m.model_name.clone()),
background_model_provider: areas.background.map(|m| m.provider),
summarization_model_provider: areas.summarization.map(|m| m.provider),
}
});
let model_roster = crate::ModelRoster {
model: Some(model),
provider_name: Some(resolved_provider_name),
provider_type: config.provider_type.clone(),
fast: crate::RoleModel::from_parts(config.fast_model, resolved_fast_provider),
background: crate::RoleModel::from_parts(
config.background_model,
config.background_model_provider,
),
summarization: crate::RoleModel::from_parts(
config.summarization_model,
config.summarization_model_provider,
),
};
let guardian_config = read_guardian_config(&session);
let guardian_spawner = self.guardian_spawner.read().await.clone();
spawn_session_execution(SessionExecutionArgs {
agent: self.agent.clone(),
session_id,
session,
tools_override: Some(root_tools),
provider_override,
model_roster,
reasoning_effort,
reasoning_effort_source,
auxiliary_model_resolver: Some(auxiliary_model_resolver),
disabled_filter_resolver: None,
disabled_tools: Some(config.disabled_tools),
disabled_skill_ids: Some(config.disabled_skill_ids),
selected_skill_ids: None,
selected_skill_mode: None,
cancel_token,
mpsc_tx,
image_fallback: config.image_fallback,
gold_config,
guardian_config,
guardian_spawner,
bash_resume_hook: {
let hook: Arc<dyn BashResumeHook> = Arc::new(self.clone());
Some(hook)
},
bash_completion_sink: {
let sink: Arc<dyn BashCompletionSink> = Arc::new(self.clone());
Some(sink)
},
app_data_dir: Some(self.app_data_dir.clone()),
run_budget: None,
runners: self.agent_runners.clone(),
sessions_cache: self.sessions.clone(),
on_complete: None,
child_completion_handler: Some(Arc::new(self.clone())),
});
}
}
fn bash_completion_resume_message(bash_ids: &[String], timed_out: bool) -> Message {
let body = if timed_out {
format!(
"Runtime notification: the background-Bash wait ceiling was reached while one or more \
shell(s) ({}) may still be running. The session is being resumed so it is not \
stranded; verify their actual status with BashOutput before assuming completion.",
bash_ids.join(", ")
)
} else {
format!(
"Runtime notification: all background Bash shell(s) ({}) have completed. \
Review their output with BashOutput and resume the task from where you left off.",
bash_ids.join(", ")
)
};
let mut message = Message::user(body);
message.metadata = Some(serde_json::json!({
RUNTIME_RESUME_MESSAGE_HIDDEN_KEY: true,
RUNTIME_RESUME_MESSAGE_KIND_KEY: BASH_COMPLETION_RESUME_KIND,
}));
message.never_compress = false;
message
}
fn bash_resume_should_retry(outcome: &ResumeOutcome, persisted_waiting_for_bash: bool) -> bool {
match outcome {
ResumeOutcome::Started { .. } | ResumeOutcome::NotFound => false,
ResumeOutcome::Completed | ResumeOutcome::AlreadyRunning { .. } => {
persisted_waiting_for_bash
}
}
}
fn bash_completion_should_resume(
loop_suspended_on_bash: bool,
all_waited_shells_done: bool,
) -> bool {
loop_suspended_on_bash && all_waited_shells_done
}
fn apply_bash_resume_transition(session: &mut Session, resume_message: &Message) -> bool {
let mut runtime_state = read_runtime_state(session);
if runtime_state.waiting_for_bash.is_none() {
return false;
}
runtime_state.waiting_for_bash = None;
runtime_state.status = AgentStatusState::Idle;
runtime_state.suspension = None;
write_runtime_state(session, &runtime_state);
session.metadata.remove("runtime.suspend_reason");
session.add_message(resume_message.clone());
true
}
impl ChildCompletionCoordinator {
async fn bash_self_resume(&self, session_id: String, bash_ids: Vec<String>) {
let mut delay = Duration::from_secs(1);
let max_delay = Duration::from_secs(30);
let max_poll = Duration::from_secs(6 * 3600 + 600);
let deadline = tokio::time::Instant::now() + max_poll;
loop {
tokio::time::sleep(delay).await;
let Some(session) = self.load_session(&session_id).await else {
tracing::info!(%session_id, "bash self-resume backstop: session gone; nothing to do");
return;
};
if read_runtime_state(&session).waiting_for_bash.is_none() {
return;
}
let still_running =
bamboo_tools::tools::bash_runtime::running_shells_for_session(&session_id);
let timed_out = tokio::time::Instant::now() >= deadline;
if still_running.is_empty() || timed_out {
let guard = session_resume_lock(&session_id);
let _held = guard.lock().await;
tracing::warn!(
%session_id,
shell_count = bash_ids.len(),
timed_out,
"bash self-resume backstop engaged (push lost or wait ceiling reached)"
);
self.perform_bash_resume(
&session_id,
bash_completion_resume_message(&bash_ids, timed_out),
)
.await;
return;
}
delay = (delay * 2).min(max_delay);
}
}
async fn perform_bash_resume(&self, session_id: &str, resume_message: Message) {
let retry_backoff = Duration::from_millis(200);
const MAX_RESUME_ATTEMPTS: u8 = 5;
for attempt in 0..MAX_RESUME_ATTEMPTS {
if attempt > 0 {
tokio::time::sleep(retry_backoff).await;
}
let Some(mut session) = self.load_session(session_id).await else {
tracing::warn!(%session_id, "bash resume: session not found; nothing to resume");
return;
};
if !apply_bash_resume_transition(&mut session, &resume_message) {
tracing::info!(
%session_id, attempt,
"bash resume: persisted bash wait already cleared; nothing to resume"
);
return;
}
session.updated_at = Utc::now();
self.save_and_cache(&mut session).await;
tracing::info!(
%session_id, attempt,
"bash resume: cleared bash wait and appended resume message"
);
let outcome = self.resume_parent(session_id.to_string()).await;
match outcome {
ResumeOutcome::Started { .. } => {
tracing::info!(%session_id, attempt, "bash resume: resume fired");
return;
}
ResumeOutcome::NotFound => {
tracing::warn!(%session_id, "bash resume: session vanished during resume");
return;
}
_ => {
let clobbered = match self.load_session(session_id).await {
Some(reloaded) => read_runtime_state(&reloaded).waiting_for_bash.is_some(),
None => {
tracing::warn!(%session_id, "bash resume: session vanished after resume");
return;
}
};
if bash_resume_should_retry(&outcome, clobbered) {
tracing::warn!(
%session_id, attempt,
outcome = outcome.as_str(),
"bash resume: persisted wait still set after resume (finalize-clobber); retrying"
);
continue;
}
tracing::info!(
%session_id, attempt,
outcome = outcome.as_str(),
"bash resume: wait cleared and resume handled; stopping"
);
return;
}
}
}
tracing::warn!(
%session_id,
attempts = MAX_RESUME_ATTEMPTS,
"bash resume: exhausted clobber-retry budget without confirming resume; giving up"
);
}
}
impl BashResumeHook for ChildCompletionCoordinator {
fn arrange_bash_self_resume(&self, session_id: String, bash_ids: Vec<String>) {
let coordinator = Arc::new(self.clone());
tokio::spawn(async move {
coordinator.bash_self_resume(session_id, bash_ids).await;
});
}
}
fn bash_completion_injection_body(info: &BashCompletionInfo) -> String {
let exit = match info.exit_code {
Some(code) => code.to_string(),
None => "none (signal/killed)".to_string(),
};
let mut body = format!(
"Runtime notification: background shell `{}` (`{}`) finished — status {}, exit code {}.",
info.bash_id, info.command, info.status, exit
);
if info.output_tail.trim().is_empty() {
body.push_str(" It produced no captured output.");
} else {
body.push_str("\n\nOutput tail:\n");
body.push_str(&info.output_tail);
}
body.push_str(&format!(
"\n\nUse BashOutput with bash_id=\"{}\" for the full output, then continue the task.",
info.bash_id
));
body
}
async fn enqueue_bash_completion_injection(
persistence: &LockedSessionStore,
info: &BashCompletionInfo,
) -> std::io::Result<Option<Session>> {
let body = bash_completion_injection_body(info);
let queued = serde_json::json!({
"content": body,
"created_at": Utc::now(),
});
persistence
.update_runtime_config(&info.session_id, move |session| {
let mut pending = session.pending_injected_messages().unwrap_or_default();
pending.push(queued);
session.set_pending_injected_messages(pending);
})
.await
}
fn bash_resume_message_from_info(info: &BashCompletionInfo) -> Message {
let mut message = Message::user(bash_completion_injection_body(info));
message.metadata = Some(serde_json::json!({
RUNTIME_RESUME_MESSAGE_HIDDEN_KEY: true,
RUNTIME_RESUME_MESSAGE_KIND_KEY: BASH_COMPLETION_RESUME_KIND,
}));
message.never_compress = false;
message
}
impl ChildCompletionCoordinator {
async fn deliver_bash_completion(&self, info: BashCompletionInfo) {
let guard = session_resume_lock(&info.session_id);
let _held = guard.lock().await;
let Some(session) = self.load_session(&info.session_id).await else {
tracing::warn!(
session_id = %info.session_id,
bash_id = %info.bash_id,
"background bash completion: owning session not found; nothing to notify"
);
return;
};
let waiting = read_runtime_state(&session).waiting_for_bash.is_some();
let all_shells_done =
bamboo_tools::tools::bash_runtime::running_shells_for_session(&info.session_id)
.is_empty();
if bash_completion_should_resume(waiting, all_shells_done) {
tracing::info!(
session_id = %info.session_id,
bash_id = %info.bash_id,
status = %info.status,
"background bash completion: push-resuming suspended loop (event-driven)"
);
self.perform_bash_resume(&info.session_id, bash_resume_message_from_info(&info))
.await;
return;
}
match enqueue_bash_completion_injection(&self.persistence, &info).await {
Ok(Some(_)) => tracing::info!(
session_id = %info.session_id,
bash_id = %info.bash_id,
status = %info.status,
waiting,
"background bash completion queued for injection at the next round boundary"
),
Ok(None) => tracing::warn!(
session_id = %info.session_id,
bash_id = %info.bash_id,
"background bash completion: owning session not found; nothing to notify"
),
Err(error) => tracing::warn!(
session_id = %info.session_id,
bash_id = %info.bash_id,
%error,
"background bash completion: failed to queue injection"
),
}
}
}
impl BashCompletionSink for ChildCompletionCoordinator {
fn on_bash_completed(&self, info: BashCompletionInfo) {
let coordinator = Arc::new(self.clone());
tokio::spawn(async move {
coordinator.deliver_bash_completion(info).await;
});
}
}
const CHILD_WAIT_SWEEP_INTERVAL_SECS: u64 = 30;
const CHILD_WAIT_REGISTRATION_GRACE_SECS: i64 = 60;
const DEAD_CHILD_GRACE_SECS: i64 = 120;
const STALE_RUNNER_SLACK_SECS: i64 = 600;
fn is_dead_child_candidate_status(status: Option<&str>) -> bool {
match status {
Some(status) => !is_terminal_child_status(status) && status != "suspended",
None => true,
}
}
fn completion_child_is_owned(reported_parent: &str, child_parent_linkage: Option<&str>) -> bool {
child_parent_linkage == Some(reported_parent)
}
fn select_replay_child(terminal: &[(String, String)]) -> Option<&(String, String)> {
terminal
.iter()
.find(|(_, status)| is_error_like(status))
.or_else(|| terminal.last())
}
fn child_wait_watchdog_resume_message(body: String) -> Message {
let mut message = Message::user(body);
message.metadata = Some(serde_json::json!({
RUNTIME_RESUME_MESSAGE_HIDDEN_KEY: true,
RUNTIME_RESUME_MESSAGE_KIND_KEY: "child_wait_watchdog_resume",
}));
message.never_compress = false;
message
}
fn empty_child_wait_message() -> Message {
child_wait_watchdog_resume_message(
"Runtime notification: this session was suspended waiting for child sessions, but the \
wait tracked no children (internal inconsistency). The session has been resumed; use \
SubAgent.list to inspect child state and continue the task."
.to_string(),
)
}
fn child_wait_lease_expired_message(child_ids: &[String]) -> Message {
child_wait_watchdog_resume_message(format!(
"Runtime notification: the wait lease for child session(s) [{}] expired before they all \
reported completion. They were NOT cancelled and may still be running or already \
finished — verify their actual status with SubAgent.list / SubAgent.get before assuming \
anything, then continue the task.",
child_ids.join(", ")
))
}
impl ChildCompletionCoordinator {
pub fn spawn_child_wait_watchdog(self: &Arc<Self>) {
let coordinator = Arc::clone(self);
tokio::spawn(async move {
use futures::FutureExt;
if std::panic::AssertUnwindSafe(coordinator.reconcile_orphans_at_boot())
.catch_unwind()
.await
.is_err()
{
tracing::error!("child-wait watchdog: boot reconciliation panicked");
}
let mut ticker = tokio::time::interval(std::time::Duration::from_secs(
CHILD_WAIT_SWEEP_INTERVAL_SECS,
));
ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
ticker.tick().await;
loop {
ticker.tick().await;
if std::panic::AssertUnwindSafe(coordinator.sweep_child_waits())
.catch_unwind()
.await
.is_err()
{
tracing::error!("child-wait watchdog: sweep panicked; continuing");
}
}
});
}
async fn reconcile_orphans_at_boot(&self) {
let cutoff = Utc::now();
let running = self
.storage
.list_sessions_by_run_status("running")
.await
.unwrap_or_default();
for (child_id, parent_id) in running {
let Some(parent_id) = parent_id else { continue };
if self.runner_is_running(&child_id).await {
continue;
}
let Some(control_plane) = self.load_control_plane(&child_id).await else {
continue;
};
if control_plane.updated_at >= cutoff {
continue;
}
tracing::warn!(
child_session_id = %child_id,
parent_session_id = %parent_id,
"boot reconciliation: child was running when the process died; \
marking it error and waking the parent"
);
self.synthesize_child_completion(
&parent_id,
&child_id,
"error",
Some(
"orphaned by server restart: the process died while this child session \
was running"
.to_string(),
),
)
.await;
}
let suspended = self
.storage
.list_sessions_by_run_status("suspended")
.await
.unwrap_or_default();
for (session_id, _) in suspended {
let Some(control_plane) = self.load_control_plane(&session_id).await else {
continue;
};
if control_plane
.metadata
.get("runtime.suspend_reason")
.map(String::as_str)
!= Some("waiting_for_bash")
{
continue;
}
if let Some(wait) = read_runtime_state(&control_plane).waiting_for_bash {
tracing::warn!(
%session_id,
"boot reconciliation: re-arming bash self-resume backstop lost in restart"
);
let coordinator = self.clone();
tokio::spawn(async move {
coordinator
.bash_self_resume(session_id, wait.bash_ids)
.await;
});
}
}
}
async fn runner_is_running(&self, session_id: &str) -> bool {
let runners = self.agent_runners.read().await;
runners
.get(session_id)
.is_some_and(|runner| matches!(runner.status, AgentStatus::Running))
}
async fn load_control_plane(&self, session_id: &str) -> Option<Session> {
match self.storage.load_runtime_control_plane(session_id).await {
Ok(session) => session,
Err(error) => {
tracing::warn!(
%session_id,
%error,
"child-wait watchdog: failed to load session control plane"
);
None
}
}
}
async fn sweep_child_waits(&self) {
let suspended = match self.storage.list_sessions_by_run_status("suspended").await {
Ok(entries) => entries,
Err(error) => {
tracing::warn!(%error, "child-wait watchdog: failed to list suspended sessions");
return;
}
};
for (session_id, _) in suspended {
self.sweep_one_suspended_session(&session_id).await;
}
}
async fn sweep_one_suspended_session(&self, session_id: &str) {
if self.runner_is_running(session_id).await {
return;
}
let Some(session) = self.load_control_plane(session_id).await else {
return;
};
let runtime_state = read_runtime_state(&session);
let suspend_reason = session
.metadata
.get("runtime.suspend_reason")
.map(String::as_str)
.unwrap_or_default()
.to_string();
match (
suspend_reason.as_str(),
runtime_state.waiting_for_children.clone(),
) {
("waiting_for_bash", _)
| ("awaiting_clarification", _)
| ("awaiting_parent_approval", _) => {}
(_, Some(wait)) => self.sweep_child_wait(session_id, wait).await,
("waiting_for_children", None) | ("", None) => {
self.rescue_stranded_resume(session_id).await;
}
_ => {}
}
}
async fn sweep_child_wait(&self, parent_session_id: &str, wait: WaitingForChildrenState) {
let now = Utc::now();
if wait.child_session_ids.is_empty() {
tracing::warn!(
%parent_session_id,
"child-wait watchdog: wait armed over an empty child set; force-resuming"
);
self.force_resume_child_wait(parent_session_id, empty_child_wait_message())
.await;
return;
}
if wait.timeout_at.is_some_and(|deadline| now >= deadline) {
tracing::warn!(
%parent_session_id,
"child-wait watchdog: wait lease expired; force-resuming parent"
);
self.force_resume_child_wait(
parent_session_id,
child_wait_lease_expired_message(&wait.child_session_ids),
)
.await;
return;
}
if now.signed_duration_since(wait.registered_at).num_seconds()
< CHILD_WAIT_REGISTRATION_GRACE_SECS
{
return;
}
let statuses: HashMap<String, Option<String>> = self
.storage
.list_child_run_statuses(parent_session_id)
.await
.unwrap_or_default()
.into_iter()
.collect();
struct DeadChild {
child_id: String,
status: String,
reason: String,
owned: bool,
}
let mut terminal: Vec<(String, String)> = Vec::new();
let mut dead: Vec<DeadChild> = Vec::new();
for child_id in &wait.child_session_ids {
let status = statuses.get(child_id).and_then(|status| status.as_deref());
if let Some(status) = status {
if is_terminal_child_status(status) {
terminal.push((child_id.clone(), status.to_string()));
continue;
}
}
if !is_dead_child_candidate_status(status) {
continue;
}
let control_plane = self.load_control_plane(child_id).await;
let owned = control_plane
.as_ref()
.is_some_and(|cp| cp.parent_session_id.as_deref() == Some(parent_session_id));
if !owned {
dead.push(DeadChild {
child_id: child_id.clone(),
status: "error".to_string(),
reason: if control_plane.is_some() {
"waited-on session id is not a child of this session; clearing it \
from the wait without touching that session"
.to_string()
} else {
"waited-on child session does not exist".to_string()
},
owned: false,
});
continue;
}
let runner = { self.agent_runners.read().await.get(child_id).cloned() };
match runner {
Some(runner) if matches!(runner.status, AgentStatus::Running) => {
let last_activity = runner.last_event_at.unwrap_or(runner.started_at);
let idle_secs = now.signed_duration_since(last_activity).num_seconds();
let total_secs = now.signed_duration_since(runner.started_at).num_seconds();
let policy = match &control_plane {
Some(child) => {
crate::runtime::execution::spawn::watchdog_policy_for_session(child)
}
None => Default::default(),
};
let idle_limit = policy.max_idle_secs.saturating_add(STALE_RUNNER_SLACK_SECS);
let total_limit = policy
.max_total_secs
.saturating_add(STALE_RUNNER_SLACK_SECS);
if idle_secs >= idle_limit || total_secs >= total_limit {
runner.cancel_token.cancel();
dead.push(DeadChild {
child_id: child_id.clone(),
status: "timeout".to_string(),
reason: format!(
"child runner stalled: no events for {idle_secs}s \
(limit {idle_limit}s including watchdog slack); \
force-finalized by the child-wait watchdog"
),
owned: true,
});
}
}
_ => {
let quiet_secs = control_plane
.as_ref()
.map(|child| now.signed_duration_since(child.updated_at).num_seconds())
.unwrap_or(i64::MAX);
if quiet_secs >= DEAD_CHILD_GRACE_SECS {
dead.push(DeadChild {
child_id: child_id.clone(),
status: "error".to_string(),
reason: format!(
"child runner lost (crashed task, dropped spawn job, or \
process restart): index status {status:?} with no live \
runner driving it"
),
owned: true,
});
}
}
}
}
if !dead.is_empty() {
for entry in dead {
tracing::warn!(
%parent_session_id,
child_session_id = %entry.child_id,
status = %entry.status,
reason = %entry.reason,
owned = entry.owned,
"child-wait watchdog: synthesizing terminal completion for dead child"
);
if entry.owned {
self.synthesize_child_completion(
parent_session_id,
&entry.child_id,
&entry.status,
Some(entry.reason),
)
.await;
} else {
self.publish_synthetic_completion(
parent_session_id,
&entry.child_id,
&entry.status,
Some(entry.reason),
)
.await;
}
}
return;
}
let terminal_ids: Vec<String> = terminal.iter().map(|(id, _)| id.clone()).collect();
if let Some((child_id, status)) = select_replay_child(&terminal) {
if wait_policy_satisfied(
wait.wait_for,
&wait.child_session_ids,
&terminal_ids,
child_id,
status,
) {
tracing::warn!(
%parent_session_id,
child_session_id = %child_id,
"child-wait watchdog: wait already satisfied but parent still suspended \
(lost wake); replaying the completion"
);
let error = self
.load_control_plane(child_id)
.await
.and_then(|child| child.last_run_error());
self.publish_synthetic_completion(parent_session_id, child_id, status, error)
.await;
}
}
}
async fn synthesize_child_completion(
&self,
parent_session_id: &str,
child_session_id: &str,
status: &str,
error: Option<String>,
) {
match self.storage.load_session(child_session_id).await {
Ok(Some(mut child)) => {
if child.parent_session_id.as_deref() != Some(parent_session_id) {
tracing::warn!(
%parent_session_id,
child_session_id = %child.id,
"child-wait watchdog: refusing to synthesize status onto a session \
that is not a child of this parent"
);
self.publish_synthetic_completion(
parent_session_id,
child_session_id,
status,
error,
)
.await;
return;
}
child.set_last_run_status(status);
match &error {
Some(message) => child.set_last_run_error(message.clone()),
None => child.clear_last_run_error(),
}
child.updated_at = Utc::now();
if let Err(save_error) = self.persistence.merge_save_runtime(&mut child).await {
tracing::warn!(
child_session_id = %child.id,
%save_error,
"child-wait watchdog: failed to persist synthesized terminal status"
);
}
self.sessions
.insert(child.id.clone(), Arc::new(parking_lot::RwLock::new(child)));
}
Ok(None) => {}
Err(load_error) => {
tracing::warn!(
%child_session_id,
%load_error,
"child-wait watchdog: failed to load child for synthesized terminal status"
);
}
}
finalize_runner(
&self.agent_runners,
child_session_id,
&Err(bamboo_agent_core::AgentError::LLM(
error
.clone()
.unwrap_or_else(|| format!("synthesized {status}")),
)),
)
.await;
self.publish_synthetic_completion(parent_session_id, child_session_id, status, error)
.await;
}
async fn publish_synthetic_completion(
&self,
parent_session_id: &str,
child_session_id: &str,
status: &str,
error: Option<String>,
) {
let parent_tx = crate::execution::session_events::get_or_create_event_sender(
&self.session_event_senders,
parent_session_id,
)
.await;
let handler: Arc<dyn ChildCompletionHandler> = Arc::new(self.clone());
crate::runtime::execution::spawn::publish_child_completion_parts(
&parent_tx,
Some(handler),
parent_session_id.to_string(),
child_session_id.to_string(),
status.to_string(),
error,
)
.await;
}
async fn rescue_stranded_resume(&self, session_id: &str) {
let Some(session) = self.load_session(session_id).await else {
return;
};
let pending_runtime_resume = session.messages.last().is_some_and(|message| {
matches!(message.role, Role::User)
&& message
.metadata
.as_ref()
.is_some_and(|meta| meta.get(RUNTIME_RESUME_MESSAGE_KIND_KEY).is_some())
});
if !pending_runtime_resume {
return;
}
tracing::warn!(
%session_id,
"child-wait watchdog: stranded resume detected (wait cleared, resume never \
spawned); resuming"
);
self.resume_parent(session_id.to_string()).await;
}
async fn force_resume_child_wait(&self, session_id: &str, resume_message: Message) {
const MAX_ATTEMPTS: u8 = 5;
let lock = session_resume_lock(session_id);
for attempt in 0..MAX_ATTEMPTS {
if attempt > 0 {
tokio::time::sleep(Duration::from_millis(200)).await;
}
{
let _held = lock.lock().await;
let Some(mut session) = self.load_session(session_id).await else {
return;
};
let mut runtime_state = read_runtime_state(&session);
if runtime_state.waiting_for_children.is_none() {
return;
}
runtime_state.waiting_for_children = None;
runtime_state.status = AgentStatusState::Idle;
runtime_state.suspension = None;
write_runtime_state(&mut session, &runtime_state);
session.metadata.remove("runtime.suspend_reason");
session.add_message(resume_message.clone());
session.updated_at = Utc::now();
self.save_and_cache(&mut session).await;
}
let outcome = self.resume_parent(session_id.to_string()).await;
match outcome {
ResumeOutcome::Started { .. } | ResumeOutcome::NotFound => return,
ResumeOutcome::Completed | ResumeOutcome::AlreadyRunning { .. } => {
let clobbered = self
.load_session(session_id)
.await
.map(|session| read_runtime_state(&session).waiting_for_children.is_some())
.unwrap_or(false);
if !clobbered {
return;
}
}
}
}
tracing::error!(
%session_id,
"child-wait watchdog: force-resume exhausted its clobber-retry budget"
);
}
}
#[cfg(test)]
mod tests {
use super::*;
use bamboo_agent_core::Message;
#[test]
fn dead_child_candidate_status_matrix() {
assert!(is_dead_child_candidate_status(None));
assert!(is_dead_child_candidate_status(Some("running")));
assert!(is_dead_child_candidate_status(Some("pending")));
assert!(!is_dead_child_candidate_status(Some("suspended")));
for status in ["completed", "error", "timeout", "cancelled", "skipped"] {
assert!(!is_dead_child_candidate_status(Some(status)), "{status}");
}
}
#[test]
fn completion_child_ownership_gates_content_fold() {
assert!(completion_child_is_owned("parent-1", Some("parent-1")));
assert!(!completion_child_is_owned("parent-1", Some("parent-2")));
assert!(!completion_child_is_owned("parent-1", None));
}
#[test]
fn replay_child_prefers_error_like_for_first_error_policy() {
let terminal = vec![
("c-ok".to_string(), "completed".to_string()),
("c-err".to_string(), "timeout".to_string()),
("c-late".to_string(), "completed".to_string()),
];
let (id, status) = select_replay_child(&terminal).expect("non-empty");
assert_eq!(id, "c-err");
assert_eq!(status, "timeout");
let all_ok = vec![
("c-1".to_string(), "completed".to_string()),
("c-2".to_string(), "completed".to_string()),
];
let (id, _) = select_replay_child(&all_ok).expect("non-empty");
assert_eq!(id, "c-2");
assert!(select_replay_child(&[]).is_none());
}
#[test]
fn watchdog_resume_messages_are_hidden_runtime_messages() {
for message in [
empty_child_wait_message(),
child_wait_lease_expired_message(&["c-1".to_string(), "c-2".to_string()]),
] {
assert!(matches!(message.role, Role::User));
let meta = message.metadata.expect("hidden runtime metadata");
assert_eq!(meta[RUNTIME_RESUME_MESSAGE_HIDDEN_KEY], true);
assert_eq!(
meta[RUNTIME_RESUME_MESSAGE_KIND_KEY],
"child_wait_watchdog_resume"
);
}
let lease = child_wait_lease_expired_message(&["c-1".to_string()]);
assert!(lease.content.contains("NOT cancelled"));
assert!(lease.content.contains("c-1"));
}
#[test]
fn non_terminal_statuses_never_satisfy_wait_policies() {
assert!(!is_terminal_child_status("suspended"));
assert!(!is_terminal_child_status("running"));
assert!(!is_terminal_child_status("pending"));
}
fn make_completion(status: &str) -> ChildCompletion {
ChildCompletion {
parent_session_id: "parent-1".to_string(),
child_session_id: "child-1".to_string(),
status: status.to_string(),
error: None,
completed_at: Utc::now(),
}
}
struct StubChildIndex {
children: Vec<(String, Option<String>)>,
}
#[async_trait]
impl Storage for StubChildIndex {
async fn save_session(&self, _session: &Session) -> std::io::Result<()> {
Ok(())
}
async fn load_session(&self, _id: &str) -> std::io::Result<Option<Session>> {
Ok(None)
}
async fn delete_session(&self, _id: &str) -> std::io::Result<bool> {
Ok(false)
}
async fn list_child_run_statuses(
&self,
_parent_session_id: &str,
) -> std::io::Result<Vec<(String, Option<String>)>> {
Ok(self.children.clone())
}
}
#[tokio::test]
async fn derive_completed_only_includes_terminal_children() {
let storage: Arc<dyn Storage> = Arc::new(StubChildIndex {
children: vec![
("a".into(), Some("completed".into())),
("b".into(), Some("running".into())),
("c".into(), Some("error".into())),
("d".into(), None),
],
});
let completed = derive_completed_child_ids(&storage, "parent-1", "b").await;
assert_eq!(
completed,
vec!["a".to_string(), "b".to_string(), "c".to_string()]
);
}
#[tokio::test]
async fn derive_completed_folds_in_just_completed_when_index_lags() {
let storage: Arc<dyn Storage> = Arc::new(StubChildIndex {
children: vec![("only".into(), Some("running".into()))],
});
let completed = derive_completed_child_ids(&storage, "parent-1", "only").await;
assert_eq!(completed, vec!["only".to_string()]);
}
#[test]
fn wait_policy_all_uses_derived_completed_set() {
let waited = vec!["a".to_string(), "b".to_string()];
assert!(!wait_policy_satisfied(
ChildWaitPolicy::All,
&waited,
&["a".to_string()],
"a",
"completed"
));
assert!(wait_policy_satisfied(
ChildWaitPolicy::All,
&waited,
&["a".to_string(), "b".to_string()],
"b",
"completed"
));
}
#[test]
fn wait_policy_first_error_requires_tracked_membership() {
let waited = vec!["a".to_string(), "b".to_string()];
assert!(wait_policy_satisfied(
ChildWaitPolicy::FirstError,
&waited,
&["a".to_string()],
"a",
"error"
));
assert!(!wait_policy_satisfied(
ChildWaitPolicy::FirstError,
&waited,
&["a".to_string()],
"stray-child",
"timeout"
));
assert!(wait_policy_satisfied(
ChildWaitPolicy::FirstError,
&waited,
&["a".to_string(), "b".to_string()],
"stray-child",
"completed"
));
}
#[test]
fn child_final_assistant_text_returns_last_assistant() {
let mut session = Session::new("child-1", "gpt-4");
session.messages.push(Message::user("hi"));
session
.messages
.push(Message::assistant("first answer", None));
session.messages.push(Message::user("again"));
session
.messages
.push(Message::assistant("final answer", None));
assert_eq!(
child_final_assistant_text(&session).as_deref(),
Some("final answer")
);
}
#[test]
fn child_final_assistant_text_returns_none_when_blank() {
let mut session = Session::new("child-1", "gpt-4");
session.messages.push(Message::assistant(" ", None));
assert!(child_final_assistant_text(&session).is_none());
}
#[test]
fn child_final_assistant_text_returns_none_when_no_assistant() {
let mut session = Session::new("child-1", "gpt-4");
session.messages.push(Message::user("hi"));
assert!(child_final_assistant_text(&session).is_none());
}
#[test]
fn runtime_resume_message_folds_full_response_without_truncation() {
let completion = make_completion("completed");
let long: String = "a".repeat(10_000);
let message = runtime_resume_message(&completion, 0, Some(&long));
assert!(message.content.contains(&long));
assert!(!message.content.contains("truncated"));
}
#[test]
fn runtime_resume_message_includes_child_response_when_provided() {
let completion = make_completion("completed");
let message = runtime_resume_message(&completion, 0, Some("the answer is 42"));
assert!(matches!(message.role, Role::User));
assert!(!message.never_compress);
assert!(message.content.contains("Child final response:"));
assert!(message.content.contains("the answer is 42"));
let metadata = message.metadata.expect("metadata present");
assert_eq!(
metadata.get("hidden_from_ui").and_then(|v| v.as_bool()),
Some(true)
);
assert_eq!(
metadata.get("runtime_kind").and_then(|v| v.as_str()),
Some("child_completion_resume")
);
assert_eq!(
metadata
.get("child_final_response_included")
.and_then(|v| v.as_bool()),
Some(true)
);
}
#[test]
fn runtime_resume_message_falls_back_to_error_when_no_response() {
let mut completion = make_completion("error");
completion.error = Some("boom".to_string());
let message = runtime_resume_message(&completion, 1, None);
assert!(message.content.contains("Child error:"));
assert!(message.content.contains("boom"));
let metadata = message.metadata.expect("metadata present");
assert_eq!(
metadata
.get("child_final_response_included")
.and_then(|v| v.as_bool()),
Some(false)
);
}
#[test]
fn runtime_resume_message_minimal_when_no_response_and_no_error() {
let completion = make_completion("completed");
let message = runtime_resume_message(&completion, 2, None);
assert!(!message.content.contains("Child final response:"));
assert!(!message.content.contains("Child error:"));
assert!(message.content.contains("Resume the parent task"));
}
#[test]
fn read_config_snapshot_refreshes_cached_snapshot_from_live_config() {
let runtime = tokio::runtime::Runtime::new().expect("runtime");
runtime.block_on(async {
let config = Arc::new(RwLock::new(Config::default()));
config.write().await.provider = "copilot".to_string();
let cached_config = StdRwLock::new(Config::default());
let snapshot = read_config_snapshot(&config, &cached_config);
assert_eq!(snapshot.provider, "copilot");
assert_eq!(
cached_config.read().expect("cached snapshot lock").provider,
"copilot"
);
});
}
#[test]
fn read_config_snapshot_uses_cached_snapshot_when_live_lock_is_busy() {
let runtime = tokio::runtime::Runtime::new().expect("runtime");
runtime.block_on(async {
let cached_snapshot = Config {
provider: "cached-provider".to_string(),
..Default::default()
};
let config = Arc::new(RwLock::new(Config::default()));
let cached_config = StdRwLock::new(cached_snapshot);
let _write_guard = config.write().await;
let snapshot = read_config_snapshot(&config, &cached_config);
assert_eq!(snapshot.provider, "cached-provider");
});
}
#[test]
fn bash_completion_resume_message_normal_announces_completion() {
let ids = vec!["bg-1".to_string(), "bg-2".to_string()];
let message = bash_completion_resume_message(&ids, false);
assert!(
message.content.contains("have completed"),
"normal resume message must announce completion: {}",
message.content
);
let metadata = message.metadata.expect("metadata present");
assert_eq!(
metadata
.get(RUNTIME_RESUME_MESSAGE_HIDDEN_KEY)
.and_then(|v| v.as_bool()),
Some(true),
"resume message must be hidden from the UI"
);
assert_eq!(
metadata
.get(RUNTIME_RESUME_MESSAGE_KIND_KEY)
.and_then(|v| v.as_str()),
Some(BASH_COMPLETION_RESUME_KIND),
"resume message must carry the bash-completion kind discriminant"
);
}
#[test]
fn bash_completion_resume_message_deadline_does_not_claim_completion() {
let ids = vec!["bg-long".to_string()];
let message = bash_completion_resume_message(&ids, true);
assert!(
!message.content.contains("have completed"),
"deadline resume message must NOT claim the shells completed: {}",
message.content
);
assert!(
message.content.contains("may still be running"),
"deadline resume message must warn shells may still be running: {}",
message.content
);
assert!(
message.content.contains("BashOutput"),
"deadline resume message must direct verification via BashOutput: {}",
message.content
);
let metadata = message.metadata.expect("metadata present");
assert_eq!(
metadata
.get(RUNTIME_RESUME_MESSAGE_KIND_KEY)
.and_then(|v| v.as_str()),
Some(BASH_COMPLETION_RESUME_KIND)
);
}
#[test]
fn bash_resume_should_retry_matrix() {
assert!(!bash_resume_should_retry(
&ResumeOutcome::Started { run_id: "r".into() },
true
));
assert!(!bash_resume_should_retry(
&ResumeOutcome::Started { run_id: "r".into() },
false
));
assert!(!bash_resume_should_retry(&ResumeOutcome::NotFound, true));
assert!(!bash_resume_should_retry(&ResumeOutcome::NotFound, false));
assert!(bash_resume_should_retry(&ResumeOutcome::Completed, true));
assert!(!bash_resume_should_retry(&ResumeOutcome::Completed, false));
assert!(bash_resume_should_retry(
&ResumeOutcome::AlreadyRunning { run_id: "r".into() },
true
));
assert!(!bash_resume_should_retry(
&ResumeOutcome::AlreadyRunning { run_id: "r".into() },
false
));
}
#[test]
fn injection_body_includes_status_exit_command_and_tail() {
let info = BashCompletionInfo {
session_id: "s".into(),
bash_id: "abc123".into(),
command: "make build".into(),
exit_code: Some(0),
status: "completed".into(),
output_tail: "BUILD OK".into(),
};
let body = bash_completion_injection_body(&info);
assert!(body.contains("abc123"), "body: {body}");
assert!(body.contains("make build"), "body: {body}");
assert!(body.contains("completed"), "body: {body}");
assert!(body.contains("exit code 0"), "body: {body}");
assert!(body.contains("BUILD OK"), "body: {body}");
assert!(body.contains("BashOutput"), "body: {body}");
assert!(body.contains("bash_id=\"abc123\""), "body: {body}");
}
#[test]
fn injection_body_handles_no_output_and_signal_kill() {
let info = BashCompletionInfo {
session_id: "s".into(),
bash_id: "xyz".into(),
command: "sleep 99".into(),
exit_code: None,
status: "killed".into(),
output_tail: String::new(),
};
let body = bash_completion_injection_body(&info);
assert!(body.contains("killed"), "body: {body}");
assert!(body.contains("none (signal/killed)"), "body: {body}");
assert!(body.contains("no captured output"), "body: {body}");
assert!(!body.contains("Output tail:"), "body: {body}");
}
async fn temp_store() -> (tempfile::TempDir, Arc<dyn Storage>, LockedSessionStore) {
let temp = tempfile::tempdir().unwrap();
let storage: Arc<dyn Storage> = Arc::new(
bamboo_storage::v2::SessionStoreV2::new(temp.path().to_path_buf())
.await
.expect("storage init"),
);
let persistence = LockedSessionStore::new(storage.clone());
(temp, storage, persistence)
}
#[tokio::test]
async fn enqueue_writes_pending_injection_and_preserves_messages() {
let (_temp, storage, persistence) = temp_store().await;
let mut session = Session::new("sess-enq", "test-model");
session.add_message(Message::user("do the build"));
storage.save_session(&session).await.unwrap();
let info = BashCompletionInfo {
session_id: "sess-enq".into(),
bash_id: "sh-1".into(),
command: "make".into(),
exit_code: Some(0),
status: "completed".into(),
output_tail: "done".into(),
};
let saved = enqueue_bash_completion_injection(&persistence, &info)
.await
.expect("enqueue io ok")
.expect("session exists");
let pending = saved
.pending_injected_messages()
.expect("pending injection present");
assert_eq!(pending.len(), 1);
let content = pending[0].get("content").and_then(|v| v.as_str()).unwrap();
assert!(content.contains("sh-1"), "content: {content}");
assert!(content.contains("make"), "content: {content}");
assert!(content.contains("done"), "content: {content}");
assert_eq!(saved.messages.len(), 1);
}
#[tokio::test]
async fn enqueue_returns_none_for_missing_session() {
let (_temp, _storage, persistence) = temp_store().await;
let info = BashCompletionInfo {
session_id: "does-not-exist".into(),
bash_id: "x".into(),
command: "true".into(),
exit_code: Some(0),
status: "completed".into(),
output_tail: String::new(),
};
let result = enqueue_bash_completion_injection(&persistence, &info)
.await
.expect("io ok");
assert!(result.is_none(), "no session → nothing enqueued");
}
#[test]
fn apply_bash_resume_transition_clears_wait_and_appends_message() {
use bamboo_domain::session::runtime_state::WaitingForBashState;
let mut session = Session::new("sess-resume", "test-model");
session.add_message(Message::user("kick off the build"));
let mut rt = read_runtime_state(&session);
rt.status = AgentStatusState::Running;
rt.waiting_for_bash = Some(WaitingForBashState::for_bash(
vec!["sh-1".into()],
Utc::now(),
));
write_runtime_state(&mut session, &rt);
session.metadata.insert(
"runtime.suspend_reason".to_string(),
"waiting_for_bash".to_string(),
);
let resume = bash_completion_resume_message(&["sh-1".to_string()], false);
let did = apply_bash_resume_transition(&mut session, &resume);
assert!(did, "a suspended session must transition");
let after = read_runtime_state(&session);
assert!(
after.waiting_for_bash.is_none(),
"bash wait must be cleared"
);
assert_eq!(after.status, AgentStatusState::Idle, "runtime must be Idle");
assert!(
!session.metadata.contains_key("runtime.suspend_reason"),
"suspend-reason marker must be removed"
);
assert_eq!(session.messages.len(), 2, "resume message must be appended");
assert!(matches!(
session.messages.last().map(|m| &m.role),
Some(Role::User)
));
}
#[test]
fn apply_bash_resume_transition_noops_when_not_waiting() {
let mut session = Session::new("sess-live", "test-model");
session.add_message(Message::user("hi"));
let resume = bash_completion_resume_message(&["sh-1".to_string()], false);
let did = apply_bash_resume_transition(&mut session, &resume);
assert!(!did, "a non-waiting session must not transition");
assert_eq!(session.messages.len(), 1, "no resume message appended");
}
#[test]
fn bash_completion_should_resume_only_when_suspended_and_all_done() {
assert!(bash_completion_should_resume(true, true));
assert!(!bash_completion_should_resume(true, false)); assert!(!bash_completion_should_resume(false, true)); assert!(!bash_completion_should_resume(false, false));
}
#[test]
fn bash_resume_message_from_info_carries_bashid_tail_and_kind() {
let info = BashCompletionInfo {
session_id: "s".into(),
bash_id: "sh-42".into(),
command: "cargo test".into(),
exit_code: Some(0),
status: "completed".into(),
output_tail: "test result: ok".into(),
};
let msg = bash_resume_message_from_info(&info);
assert!(matches!(msg.role, Role::User));
assert!(msg.content.contains("sh-42"), "content: {}", msg.content);
assert!(
msg.content.contains("cargo test"),
"content: {}",
msg.content
);
assert!(
msg.content.contains("test result: ok"),
"content: {}",
msg.content
);
assert!(
msg.content.contains("BashOutput"),
"content: {}",
msg.content
);
let meta = serde_json::to_string(&msg.metadata).unwrap();
assert!(
meta.contains(BASH_COMPLETION_RESUME_KIND),
"resume message must be tagged as a bash-completion resume: {meta}"
);
}
}