Skip to main content

bamboo_engine/session_app/
child_completion_coordinator.rs

1//! Child-session completion coordinator.
2//!
3//! Receives terminal child runner notifications from `bamboo-engine`, updates
4//! durable parent wait state, and resumes the parent when the configured wait
5//! policy is satisfied.
6
7use std::collections::HashMap;
8use std::sync::{Arc, Mutex as StdMutex, OnceLock, RwLock as StdRwLock, Weak};
9use std::time::Duration;
10
11use bamboo_domain::poison::PoisonRecover;
12use bamboo_domain::{
13    AgentHookPoint, HookPayload, HookToolOutcome, SessionChildOutcome, SessionMessageBody,
14    SessionMessageContent, SessionMessageEnvelope, SessionMessageId, SessionMessageKind,
15    SessionMessageSource, SessionProviderMessage,
16};
17
18use crate::execution::{
19    create_event_forwarder, finalize_runner, reserve_runner_core, reserve_session_execution,
20    spawn_session_execution, AgentRunner, AgentStatus, ChildCompletion, ChildCompletionHandler,
21    ReserveOutcome, SessionExecutionArgs, SessionExecutionReservation,
22    SessionExecutionReserveOutcome, SpawnJob, SpawnScheduler,
23};
24use crate::runtime::config::{BashResumeHook, GuardianSpawner, BASH_COMPLETION_RESUME_KIND};
25use crate::runtime::guardian_state::{
26    parse_guardian_verdict, read_guardian_config, read_guardian_state, write_guardian_state,
27    GuardianVerdict,
28};
29use crate::Agent;
30use async_trait::async_trait;
31use bamboo_agent_core::storage::Storage;
32use bamboo_agent_core::tools::ToolExecutor;
33use bamboo_agent_core::{
34    AgentEvent, BashCompletionInfo, BashCompletionSink, Message, Role, Session, SessionKind,
35};
36use bamboo_domain::session::runtime_state::{
37    AgentRuntimeState, AgentStatusState, ChildWaitPolicy, SuspensionState, WaitingForChildrenState,
38};
39use bamboo_llm::{Config, ProviderModelRouter, ProviderRegistry};
40use bamboo_storage::LockedSessionStore;
41use chrono::Utc;
42use sha2::{Digest, Sha256};
43use tokio::sync::{broadcast, RwLock};
44
45use crate::model_areas::resolve_global_area_models;
46use crate::model_config_helper::{
47    resolve_fast_model, resolve_gold_config, resolve_provider_routing_key, GOLD_CONFIG_METADATA_KEY,
48};
49use crate::session_activation::{
50    SessionActivationLaunch, SessionActivationReserveOutcome, SessionActivationSpawner,
51};
52use crate::session_app::execute::consume_pending_clarification_resume;
53use crate::session_app::provider_model::{persist_model_ref, session_effective_model_ref};
54use crate::session_app::resume::{
55    resume_session_execution, ResumeExecutionPort, ResumeSpawnRequest,
56};
57use crate::session_app::types::{ResumeConfigSnapshot, ResumeOutcome};
58
59const AGENT_RUNTIME_STATE_METADATA_KEY: &str = "agent.runtime.state";
60const RUNTIME_RESUME_MESSAGE_HIDDEN_KEY: &str = "hidden_from_ui";
61const RUNTIME_RESUME_MESSAGE_KIND_KEY: &str = "runtime_kind";
62const CHILD_COMPLETION_INLINE_FIELD_BYTES: usize = 48 * 1024;
63const CHILD_COMPLETION_OVERSIZE_TAIL_BYTES: usize = 8 * 1024;
64
65fn read_runtime_state(session: &Session) -> AgentRuntimeState {
66    session
67        .agent_runtime_state
68        .clone()
69        .or_else(|| {
70            session
71                .metadata
72                .get(AGENT_RUNTIME_STATE_METADATA_KEY)
73                .and_then(|raw| serde_json::from_str::<AgentRuntimeState>(raw).ok())
74        })
75        .unwrap_or_else(|| AgentRuntimeState::new(format!("{}-child-wait", session.id)))
76}
77
78fn write_runtime_state(session: &mut Session, runtime_state: &AgentRuntimeState) {
79    session.agent_runtime_state = Some(runtime_state.clone());
80    if let Ok(serialized) = serde_json::to_string(runtime_state) {
81        session
82            .metadata
83            .insert(AGENT_RUNTIME_STATE_METADATA_KEY.to_string(), serialized);
84    }
85}
86
87/// Re-read and prepare an activation target under the same per-session
88/// persistence lock that commits the generic suspension clear.
89///
90/// The boolean is false when a specific child/Bash wait is present in the
91/// latest durable snapshot; callers must leave the activation unreserved.
92async fn prepare_session_inbox_activation(
93    persistence: &LockedSessionStore,
94    session_id: &str,
95    interrupt_specific_wait: bool,
96) -> std::io::Result<Option<(Session, bool)>> {
97    let ready = Arc::new(std::sync::atomic::AtomicBool::new(false));
98    let ready_for_mutation = ready.clone();
99    let saved = persistence
100        .update_runtime_config(session_id, move |latest| {
101            let mut runtime_state = read_runtime_state(latest);
102            let specifically_waiting = runtime_state.waiting_for_children.is_some()
103                || runtime_state.waiting_for_bash.is_some();
104            if specifically_waiting && !interrupt_specific_wait {
105                return;
106            }
107            // Explicit steering interrupts only this reasoning gate. The
108            // durable child/Bash wait remains owned so later terminal events
109            // still have exactly one coordinator and can authorize their
110            // staged outcomes. End-of-run bookkeeping re-suspends if that wait
111            // is still present.
112            runtime_state.status = AgentStatusState::Idle;
113            runtime_state.suspension = None;
114            write_runtime_state(latest, &runtime_state);
115            latest.metadata.remove("runtime.suspend_reason");
116            latest.updated_at = Utc::now();
117            ready_for_mutation.store(true, std::sync::atomic::Ordering::Release);
118        })
119        .await?;
120    Ok(saved.map(|session| (session, ready.load(std::sync::atomic::Ordering::Acquire))))
121}
122
123fn is_error_like(status: &str) -> bool {
124    matches!(status, "error" | "timeout" | "cancelled")
125}
126
127/// Terminal child run statuses, as mirrored into the session index.
128fn is_terminal_child_status(status: &str) -> bool {
129    matches!(
130        status,
131        "completed" | "error" | "timeout" | "cancelled" | "skipped"
132    )
133}
134
135fn child_completion_envelope(
136    completion: &ChildCompletion,
137    wait_registered_at: chrono::DateTime<Utc>,
138    result: Option<String>,
139    provider_message: &Message,
140) -> SessionMessageEnvelope {
141    fn bounded_terminal_field(
142        label: &str,
143        child_session_id: &str,
144        value: Option<String>,
145    ) -> (Option<String>, serde_json::Value, bool) {
146        let Some(value) = value else {
147            return (None, serde_json::Value::Null, false);
148        };
149        if value.len() <= CHILD_COMPLETION_INLINE_FIELD_BYTES {
150            return (Some(value.clone()), serde_json::Value::String(value), false);
151        }
152        let digest = hex::encode(Sha256::digest(value.as_bytes()));
153        let mut tail_start = value
154            .len()
155            .saturating_sub(CHILD_COMPLETION_OVERSIZE_TAIL_BYTES);
156        while tail_start < value.len() && !value.is_char_boundary(tail_start) {
157            tail_start += 1;
158        }
159        let tail = &value[tail_start..];
160        let summary = format!(
161            "Child {label} exceeded the durable inline limit ({} UTF-8 bytes, sha256={digest}). \
162             Retrieve the full child transcript with SubAgent.get(child_session_id=\"{child_session_id}\").\
163             \n\nBounded tail:\n{tail}",
164            value.len()
165        );
166        (
167            Some(summary),
168            serde_json::json!({
169                "oversized": true,
170                "utf8_bytes": value.len(),
171                "sha256": digest,
172            }),
173            true,
174        )
175    }
176
177    let (stored_result, result_identity, _result_oversized) =
178        bounded_terminal_field("result", &completion.child_session_id, result);
179    let (stored_error, error_identity, _error_oversized) = bounded_terminal_field(
180        "error",
181        &completion.child_session_id,
182        completion.error.clone(),
183    );
184    let mut bounded_provider_message = provider_message.clone();
185    let provider_oversized = serde_json::to_vec(provider_message)
186        .map(|bytes| bytes.len() > CHILD_COMPLETION_INLINE_FIELD_BYTES)
187        .unwrap_or(true);
188    if provider_oversized {
189        let mut content = format!(
190            "Runtime notification: child session `{}` finished with status `{}`.",
191            completion.child_session_id, completion.status
192        );
193        if let Some(result) = stored_result.as_deref() {
194            content.push_str("\n\n");
195            content.push_str(result);
196        }
197        if let Some(error) = stored_error.as_deref() {
198            content.push_str("\n\n");
199            content.push_str(error);
200        }
201        bounded_provider_message.content = content;
202        bounded_provider_message.content_parts = None;
203    }
204    let body = SessionMessageBody::ChildOutcome(SessionChildOutcome {
205        child_session_id: completion.child_session_id.clone(),
206        status: completion.status.clone(),
207        result: stored_result,
208        error: stored_error,
209        provider_message: Some(session_provider_message(&bounded_provider_message)),
210    });
211    let semantic = serde_json::json!({
212        "parent_session_id": completion.parent_session_id,
213        "child_session_id": completion.child_session_id,
214        "status": completion.status,
215        "error": error_identity,
216        "result": result_identity,
217        "wait_registered_at": wait_registered_at,
218    });
219    SessionMessageEnvelope {
220        id: SessionMessageId::stable("session_child_completion", &semantic),
221        source: SessionMessageSource::Runtime {
222            subsystem: "child_completion_coordinator".to_string(),
223        },
224        target_session_id: completion.parent_session_id.clone(),
225        kind: SessionMessageKind::ChildOutcome,
226        body,
227        created_at: completion.completed_at,
228        thread_id: None,
229        in_reply_to: None,
230        attempt: None,
231        correlation_id: Some(format!("child_completion:{}", completion.child_session_id)),
232    }
233}
234
235fn session_provider_message(message: &Message) -> SessionProviderMessage {
236    SessionProviderMessage {
237        content: SessionMessageContent {
238            text: message.content.clone(),
239            parts: message.content_parts.clone().unwrap_or_default(),
240        },
241        metadata: message
242            .metadata
243            .as_ref()
244            .and_then(serde_json::Value::as_object)
245            .cloned()
246            .unwrap_or_default(),
247        never_compress: message.never_compress,
248    }
249}
250
251/// Reconstruct the set of completed child session ids for a parent from the
252/// session index (the single source of truth), folding in the child whose
253/// completion event is being processed so a momentarily-lagging index can never
254/// stall the parent's resume.
255async fn derive_completed_child_ids(
256    storage: &Arc<dyn Storage>,
257    parent_session_id: &str,
258    just_completed_child_id: &str,
259) -> Vec<String> {
260    let mut completed: Vec<String> = storage
261        .list_child_run_statuses(parent_session_id)
262        .await
263        .unwrap_or_default()
264        .into_iter()
265        .filter(|(_, status)| status.as_deref().is_some_and(is_terminal_child_status))
266        .map(|(id, _)| id)
267        .collect();
268    if !completed.iter().any(|id| id == just_completed_child_id) {
269        completed.push(just_completed_child_id.to_string());
270    }
271    completed.sort();
272    completed.dedup();
273    completed
274}
275
276fn read_config_snapshot(config: &Arc<RwLock<Config>>, cached_config: &StdRwLock<Config>) -> Config {
277    if let Ok(config_guard) = config.try_read() {
278        let snapshot = config_guard.clone();
279
280        if let Ok(mut cached_guard) = cached_config.try_write() {
281            *cached_guard = snapshot.clone();
282        }
283
284        snapshot
285    } else {
286        cached_config
287            .try_read()
288            .map(|guard| guard.clone())
289            .unwrap_or_default()
290    }
291}
292
293/// Per-parent async locks that serialize concurrent `on_child_completed`
294/// invocations for the same parent session.
295///
296/// Race eliminated: when `wait_for=Any` and two child sessions complete
297/// simultaneously, both invocations load the parent with
298/// `waiting_for_children=Some` before either persists the cleared state, so
299/// both pass `wait_policy_satisfied`, both clear `waiting_for_children`, add a
300/// duplicate resume message, and call `resume_parent` — a double resume.
301/// Holding this per-parent `tokio::sync::Mutex` across the load-check-save
302/// critical section makes the second caller observe the already-cleared state.
303///
304/// The inner `std::sync::Mutex` guards only the brief HashMap lookup/insert
305/// (no await inside); the per-parent `tokio::sync::Mutex` is the one held
306/// across the async critical section. Entries accumulate but are small
307/// (`Arc<tokio::sync::Mutex<()>>` ≈ 24 bytes) and bounded by the number of
308/// distinct parent sessions.
309fn parent_locks() -> &'static std::sync::Mutex<HashMap<String, Arc<tokio::sync::Mutex<()>>>> {
310    static LOCKS: OnceLock<std::sync::Mutex<HashMap<String, Arc<tokio::sync::Mutex<()>>>>> =
311        OnceLock::new();
312    LOCKS.get_or_init(|| std::sync::Mutex::new(HashMap::new()))
313}
314
315/// Fetch (or create) the per-session async lock from [`parent_locks`]. Held
316/// across the load-check-clear-resume critical section so the three resume
317/// sources for one session — child completion, the loop-facing bash **push**
318/// ([`BashCompletionSink::on_bash_completed`]), and the bash **backstop** poll
319/// ([`ChildCompletionCoordinator::bash_self_resume`]) — can never double-resume.
320/// The inner sync `Mutex` guards only the brief map lookup (no await inside).
321fn session_resume_lock(session_id: &str) -> Arc<tokio::sync::Mutex<()>> {
322    let mut map = parent_locks().lock().recover_poison();
323    map.entry(session_id.to_string())
324        .or_insert_with(|| Arc::new(tokio::sync::Mutex::new(())))
325        .clone()
326}
327
328fn wait_policy_satisfied(
329    policy: ChildWaitPolicy,
330    wait_child_ids: &[String],
331    completed_child_ids: &[String],
332    latest_child_id: &str,
333    latest_status: &str,
334) -> bool {
335    if wait_child_ids.is_empty() {
336        return false;
337    }
338
339    match policy {
340        ChildWaitPolicy::All => wait_child_ids
341            .iter()
342            .all(|id| completed_child_ids.iter().any(|completed| completed == id)),
343        ChildWaitPolicy::Any => completed_child_ids
344            .iter()
345            .any(|id| wait_child_ids.iter().any(|wait_id| wait_id == id)),
346        ChildWaitPolicy::FirstError => {
347            // The error short-circuit only counts a completion from a child
348            // this wait actually tracks (issue #546): a stray/duplicate
349            // completion from an untracked child — e.g. a frozen runner's
350            // task waking up after the watchdog already synthesized its
351            // timeout, in a later run's wait — must not resume the parent.
352            (is_error_like(latest_status) && wait_child_ids.iter().any(|id| id == latest_child_id))
353                || wait_child_ids
354                    .iter()
355                    .all(|id| completed_child_ids.iter().any(|completed| completed == id))
356        }
357    }
358}
359
360/// Extract the child session's last assistant content, if any. Returns `None`
361/// when the child produced no assistant message (e.g. errored before the first
362/// model response, or only emitted tool messages).
363fn child_final_assistant_text(child: &Session) -> Option<String> {
364    child
365        .messages
366        .iter()
367        .rev()
368        .find(|message| matches!(message.role, Role::Assistant))
369        .map(|message| message.content.clone())
370        .filter(|content| !content.trim().is_empty())
371}
372
373fn runtime_resume_message(
374    completion: &ChildCompletion,
375    remaining_children: usize,
376    child_final_response: Option<&str>,
377) -> Message {
378    let mut body = format!(
379        "Runtime notification: child session `{}` finished with status `{}`. Remaining child sessions: {}.",
380        completion.child_session_id, completion.status, remaining_children
381    );
382
383    // Fold the child's full final response back into the parent — no
384    // truncation. Sub-agents are first-class agents whose complete conclusion
385    // should be available to the parent without an extra `SubAgent.get` round
386    // trip. The message is left compressible (see `never_compress` below) so a
387    // long transcript can still be reclaimed under parent compaction.
388    let final_response = child_final_response.map(str::to_string);
389    if let Some(response) = final_response.as_deref() {
390        body.push_str("\n\nChild final response:\n");
391        body.push_str(response);
392    } else if let Some(error) = completion.error.as_deref() {
393        if !error.is_empty() {
394            body.push_str("\n\nChild error:\n");
395            body.push_str(error);
396        }
397    }
398
399    body.push_str(
400        "\n\nResume the parent task using this child result and continue from the previous plan. \
401         If you need the full child transcript, call SubAgent.get(child_session_id).",
402    );
403
404    let mut message = Message::user(body);
405    message.metadata = Some(serde_json::json!({
406        RUNTIME_RESUME_MESSAGE_HIDDEN_KEY: true,
407        RUNTIME_RESUME_MESSAGE_KIND_KEY: "child_completion_resume",
408        "child_session_id": completion.child_session_id,
409        "child_status": completion.status,
410        "child_error": completion.error,
411        "child_final_response_included": final_response.is_some(),
412    }));
413    // Allow parent-side compaction to reclaim this (now untruncated) message if
414    // the parent context grows — important once children nest and fold full
415    // results upward. The `SubAgent.get` hint preserves recoverability.
416    message.never_compress = false;
417    message
418}
419
420/// The hidden resume message for a completed **guardian** review: a directive,
421/// verdict-tailored note that carries the reviewer's findings straight into the
422/// parent (so it can act without a `SubAgent.get`), mirroring
423/// [`runtime_resume_message`]'s hidden/compressible shape.
424fn guardian_resume_message(completion: &ChildCompletion, verdict: &GuardianVerdict) -> Message {
425    let mut body = if verdict.approve {
426        String::from(
427            "Guardian review APPROVED: an independent reviewer verified the work and found no blocking issues. You may finalize the task.",
428        )
429    } else {
430        String::from(
431            "Guardian review REJECTED: an independent reviewer found issues. Address every finding below before completing — do NOT declare the task complete until they are resolved.",
432        )
433    };
434    if let Some(summary) = verdict.summary.as_deref().filter(|s| !s.trim().is_empty()) {
435        body.push_str("\n\nReviewer summary: ");
436        body.push_str(summary);
437    }
438    if !verdict.findings.is_empty() {
439        body.push_str("\n\nFindings:");
440        for (idx, finding) in verdict.findings.iter().enumerate() {
441            body.push_str(&format!("\n{}. {}", idx + 1, finding));
442        }
443    }
444    body.push_str(
445        "\n\nIf you need the full guardian transcript, call SubAgent.get(child_session_id).",
446    );
447
448    let mut message = Message::user(body);
449    message.metadata = Some(serde_json::json!({
450        RUNTIME_RESUME_MESSAGE_HIDDEN_KEY: true,
451        RUNTIME_RESUME_MESSAGE_KIND_KEY: "guardian_review_resume",
452        "child_session_id": completion.child_session_id,
453        "child_status": completion.status,
454        "guardian_approved": verdict.approve,
455    }));
456    message.never_compress = false;
457    message
458}
459
460#[derive(Clone)]
461pub struct ChildCompletionCoordinator {
462    storage: Arc<dyn Storage>,
463    persistence: Arc<bamboo_storage::LockedSessionStore>,
464    sessions: crate::SessionCache,
465    agent_runners: Arc<RwLock<HashMap<String, AgentRunner>>>,
466    session_event_senders: Arc<RwLock<HashMap<String, broadcast::Sender<AgentEvent>>>>,
467    agent: Arc<Agent>,
468    config: Arc<RwLock<Config>>,
469    provider_registry: Arc<ProviderRegistry>,
470    provider_router: Arc<ProviderModelRouter>,
471    app_data_dir: std::path::PathBuf,
472    account_feed_inbox: Option<crate::execution::AccountFeedInbox>,
473    root_tools: Arc<RwLock<Option<Arc<dyn ToolExecutor>>>>,
474    /// Late-bound guardian reviewer spawner, set post-construction by the server
475    /// (mirrors `root_tools`). Re-injected into resumed runs so a guardian's
476    /// reject→fix verdict can be re-reviewed across the suspend/resume boundary.
477    guardian_spawner: Arc<RwLock<Option<Arc<dyn GuardianSpawner>>>>,
478    /// Weak late binding avoids the scheduler -> completion handler ->
479    /// scheduler ownership cycle while still routing idle child activation
480    /// through the canonical placement-aware spawn core.
481    spawn_scheduler: Arc<RwLock<Weak<SpawnScheduler>>>,
482}
483
484impl ChildCompletionCoordinator {
485    #[allow(clippy::too_many_arguments)]
486    pub fn new(
487        storage: Arc<dyn Storage>,
488        persistence: Arc<LockedSessionStore>,
489        sessions: crate::SessionCache,
490        agent_runners: Arc<RwLock<HashMap<String, AgentRunner>>>,
491        session_event_senders: Arc<RwLock<HashMap<String, broadcast::Sender<AgentEvent>>>>,
492        agent: Arc<Agent>,
493        config: Arc<RwLock<Config>>,
494        provider_registry: Arc<ProviderRegistry>,
495        provider_router: Arc<ProviderModelRouter>,
496        app_data_dir: std::path::PathBuf,
497        account_feed_inbox: Option<crate::execution::AccountFeedInbox>,
498    ) -> Self {
499        Self {
500            storage,
501            persistence,
502            sessions,
503            agent_runners,
504            session_event_senders,
505            agent,
506            config,
507            provider_registry,
508            provider_router,
509            app_data_dir,
510            account_feed_inbox,
511            root_tools: Arc::new(RwLock::new(None)),
512            guardian_spawner: Arc::new(RwLock::new(None)),
513            spawn_scheduler: Arc::new(RwLock::new(Weak::new())),
514        }
515    }
516
517    pub async fn set_root_tools(&self, tools: Arc<dyn ToolExecutor>) {
518        *self.root_tools.write().await = Some(tools);
519    }
520
521    pub async fn set_spawn_scheduler(&self, scheduler: &Arc<SpawnScheduler>) {
522        *self.spawn_scheduler.write().await = Arc::downgrade(scheduler);
523    }
524
525    /// Wire the guardian reviewer spawner (server-provided), so resumed runs can
526    /// re-spawn a guardian to re-review a fix after a reject verdict.
527    pub async fn set_guardian_spawner(&self, spawner: Arc<dyn GuardianSpawner>) {
528        *self.guardian_spawner.write().await = Some(spawner);
529    }
530
531    fn build_resume_config(
532        &self,
533        session: &Session,
534        config_snapshot: &Config,
535    ) -> ResumeConfigSnapshot {
536        crate::session_app::resolution::resolve_resume_config_snapshot(
537            config_snapshot,
538            &self.provider_registry,
539            session,
540            None,
541        )
542    }
543
544    /// Drive a parent-resume and return the final [`ResumeOutcome`] so callers
545    /// can distinguish a successful spawn (`Started`) from a gate-blocked
546    /// attempt (`Completed`). The bash self-resume poll task uses this to
547    /// detect the finalize-clobber case — its appended resume message was
548    /// reverted by the suspending runner's final `merge_save_runtime`, so the
549    /// resume port's `has_pending_user_message` gate fails and nothing spawns —
550    /// and retry the clear→append→resume (see [`Self::bash_self_resume`]).
551    async fn resume_parent(&self, parent_session_id: String) -> ResumeOutcome {
552        for attempt in 0..=5u8 {
553            if attempt > 0 {
554                tokio::time::sleep(Duration::from_millis(250 * attempt as u64)).await;
555            }
556
557            let Some(session) = self.load_session(&parent_session_id).await else {
558                tracing::warn!(%parent_session_id, "cannot resume parent after child completion: session not found");
559                return ResumeOutcome::NotFound;
560            };
561            let config_snapshot = self.config.read().await.clone();
562            let resume_config = self.build_resume_config(&session, &config_snapshot);
563            let outcome = resume_session_execution(self, &parent_session_id, resume_config).await;
564            tracing::info!(
565                %parent_session_id,
566                attempt,
567                outcome = outcome.as_str(),
568                "child completion requested parent resume"
569            );
570
571            if !matches!(outcome, ResumeOutcome::AlreadyRunning { .. }) {
572                return outcome;
573            }
574        }
575        // Exhausted the AlreadyRunning retry budget; surface the final state.
576        // The wait state was already cleared and the resume message persisted,
577        // so nothing event-driven will retry — the child-wait watchdog is the
578        // backstop that picks this stranded parent up (it resumes suspended
579        // sessions that hold a pending runtime resume message but no runner).
580        tracing::error!(
581            %parent_session_id,
582            "parent resume gave up after AlreadyRunning retry budget; \
583             relying on the child-wait watchdog backstop"
584        );
585        ResumeOutcome::AlreadyRunning {
586            run_id: String::new(),
587        }
588    }
589
590    async fn save_and_cache(&self, session: &mut Session) {
591        if let Err(error) = self.persistence.merge_save_runtime(session).await {
592            tracing::warn!(session_id = %session.id, %error, "failed to persist session");
593        }
594        self.sessions.insert(
595            session.id.clone(),
596            Arc::new(parking_lot::RwLock::new(session.clone())),
597        );
598    }
599}
600
601#[async_trait]
602impl ChildCompletionHandler for ChildCompletionCoordinator {
603    async fn on_child_completed(&self, completion: ChildCompletion) {
604        // Terminality guard: a child that reports a NON-terminal status (e.g.
605        // "suspended" — awaiting parent approval, its own bash wait, or its own
606        // grandchildren) is not done. It must never satisfy the parent's wait:
607        // `derive_completed_child_ids` folds the just-reported child in
608        // unconditionally, so without this guard a suspending child would
609        // resume the parent with a premature "finished with status
610        // `suspended`" message. The child will publish a real terminal
611        // completion when it later resumes and finishes.
612        if !is_terminal_child_status(&completion.status) {
613            tracing::info!(
614                parent_session_id = %completion.parent_session_id,
615                child_session_id = %completion.child_session_id,
616                status = %completion.status,
617                "non-terminal child status; leaving the parent wait armed"
618            );
619            return;
620        }
621
622        // Acquire the per-session async lock to eliminate the concurrent
623        // double-resume race (see `parent_locks` for the full scenario). The
624        // inner std::sync::Mutex is released immediately so no sync lock is
625        // held across the await that follows.
626        let per_parent = session_resume_lock(&completion.parent_session_id);
627        let _per_parent_guard = per_parent.lock().await;
628
629        let Some(mut parent) = self.load_session(&completion.parent_session_id).await else {
630            tracing::warn!(
631                parent_session_id = %completion.parent_session_id,
632                child_session_id = %completion.child_session_id,
633                "child completion received for missing parent"
634            );
635            return;
636        };
637
638        // A parent may itself be a child (nested sub-agents): the rest of this
639        // handler is kind-agnostic — it operates on `completion.parent_session_id`,
640        // inspects that session's own `waiting_for_children` runtime state, and
641        // resumes it. (Previously this bailed unless the parent was Root, which
642        // silently dropped grandchild completions.)
643        let mut runtime_state = read_runtime_state(&parent);
644
645        // Single source of truth: reconstruct the completed-child set from the
646        // session index rather than from a denormalized copy on the parent file.
647        let completed_child_ids = derive_completed_child_ids(
648            &self.storage,
649            &completion.parent_session_id,
650            &completion.child_session_id,
651        )
652        .await;
653
654        let mut should_resume = false;
655        let mut remaining_children = 0usize;
656        let active_wait = runtime_state.waiting_for_children.clone();
657        if let Some(wait) = active_wait.as_ref() {
658            remaining_children = wait
659                .child_session_ids
660                .iter()
661                .filter(|id| !completed_child_ids.iter().any(|completed| completed == *id))
662                .count();
663            should_resume = wait_policy_satisfied(
664                wait.wait_for,
665                &wait.child_session_ids,
666                &completed_child_ids,
667                &completion.child_session_id,
668                &completion.status,
669            );
670        }
671
672        // READ-SIDE OWNERSHIP GUARD (issue #546): `SubAgent.wait` ids are
673        // model-provided and unvalidated, and the watchdog unstrands a wait
674        // over a FOREIGN/unknown id by publishing a synthetic completion
675        // here. We must resume the parent (so it is not stranded) but MUST
676        // NOT fold that foreign session's transcript into the parent — that
677        // would be a cross-session disclosure primitive. Decide ownership
678        // from the child's OWN parent linkage (control-plane only, no
679        // messages loaded), and only load its full content when it is truly
680        // this parent's child. An unowned id resumes with the neutral/error
681        // message (`runtime_resume_message` falls back to `completion.error`
682        // when no child content is supplied).
683        let reported_child_owned = match self
684            .storage
685            .load_runtime_control_plane(&completion.child_session_id)
686            .await
687        {
688            Ok(Some(control_plane)) => completion_child_is_owned(
689                &completion.parent_session_id,
690                control_plane.parent_session_id.as_deref(),
691            ),
692            _ => false,
693        };
694
695        // Load the completed child once, ONLY when owned. The guardian
696        // branch inspects its subagent_type + final verdict; the generic
697        // path folds its final assistant content into the hidden resume
698        // message (avoiding an extra `SubAgent.get` round trip after resume).
699        let loaded_child = if reported_child_owned {
700            match self
701                .storage
702                .load_session(&completion.child_session_id)
703                .await
704            {
705                Ok(child) => child,
706                Err(error) => {
707                    tracing::warn!(
708                        child_session_id = %completion.child_session_id,
709                        %error,
710                        "failed to load child session for runtime resume message"
711                    );
712                    None
713                }
714            }
715        } else {
716            tracing::warn!(
717                parent_session_id = %completion.parent_session_id,
718                child_session_id = %completion.child_session_id,
719                "completion child is not a child of this parent; resuming with a neutral \
720                 message and NOT folding its content"
721            );
722            None
723        };
724
725        let child_final_response = loaded_child.as_ref().and_then(child_final_assistant_text);
726        // Select the exact provider-facing resume message before durable
727        // admission. The typed body carries its content/parts and safe runtime
728        // metadata, so the canonical path is semantically identical to the
729        // rolling-upgrade transcript fallback.
730        let guardian_resume = if should_resume {
731            let reviewed_round = runtime_state.round.current_round;
732            loaded_child.as_ref().and_then(|child| {
733                if child.subagent_type().as_deref() != Some("guardian") {
734                    return None;
735                }
736                let mut guardian_state = read_guardian_state(&parent)?;
737                if guardian_state.guardian_child_id.as_deref()
738                    != Some(completion.child_session_id.as_str())
739                {
740                    // A *different* guardian is legitimately still in flight —
741                    // leave its Pending state intact and use the generic resume.
742                    tracing::warn!(
743                        parent_session_id = %completion.parent_session_id,
744                        child_session_id = %completion.child_session_id,
745                        expected = ?guardian_state.guardian_child_id,
746                        "guardian completion does not match recorded guardian_child_id; using generic resume"
747                    );
748                    return None;
749                }
750                // This IS the guardian we dispatched, so we MUST advance the
751                // phase out of `Pending` — otherwise the next terminal gate's
752                // `Pending => return None` would let the run complete unreviewed.
753                // A reviewer that errored or produced unparseable output is
754                // treated as a SYNTHETIC REJECT (never a silent pass), so the
755                // budgeted re-review loop governs the outcome: fail-closed, but
756                // still bounded by `max_reviews`.
757                let verdict = child_final_assistant_text(child)
758                    .and_then(|text| match parse_guardian_verdict(&text) {
759                        Ok(verdict) => Some(verdict),
760                        Err(error) => {
761                            tracing::warn!(
762                                child_session_id = %completion.child_session_id,
763                                %error,
764                                "guardian verdict unparseable; recording a synthetic reject"
765                            );
766                            None
767                        }
768                    })
769                    .unwrap_or_else(|| {
770                        GuardianVerdict::rejected(vec![
771                            "The guardian reviewer did not return a usable verdict (it errored or \
772                             emitted unparseable output); the work has NOT been independently \
773                             verified."
774                                .to_string(),
775                        ])
776                    });
777                let approved = verdict.approve;
778                let message = guardian_resume_message(&completion, &verdict);
779                guardian_state.record_verdict(verdict, reviewed_round);
780                write_guardian_state(&mut parent, guardian_state);
781                tracing::info!(
782                    parent_session_id = %completion.parent_session_id,
783                    child_session_id = %completion.child_session_id,
784                    approved,
785                    "guardian verdict recorded; resuming parent"
786                );
787                Some(message)
788            })
789        } else {
790            None
791        };
792        let resume_message = guardian_resume.unwrap_or_else(|| {
793            runtime_resume_message(
794                &completion,
795                remaining_children,
796                child_final_response.as_deref(),
797            )
798        });
799
800        // Stage the typed child outcome before clearing any durable wait. A
801        // crash after this admission leaves the parent suspended with an
802        // inspectable envelope; only a durably committed policy transition
803        // below is allowed to activate it.
804        let messenger = self.agent.session_messenger().cloned();
805        let child_admission =
806            if let (Some(wait), Some(messenger)) = (active_wait.as_ref(), messenger.as_ref()) {
807                let envelope = child_completion_envelope(
808                    &completion,
809                    wait.registered_at,
810                    child_final_response,
811                    &resume_message,
812                );
813                match messenger.admit(envelope).await {
814                    Ok(admission) => Some(admission),
815                    Err(error) => {
816                        tracing::warn!(
817                            parent_session_id = %completion.parent_session_id,
818                            child_session_id = %completion.child_session_id,
819                            %error,
820                            "child outcome SessionInbox admission failed; leaving parent wait armed"
821                        );
822                        return;
823                    }
824                }
825            } else {
826                None
827            };
828
829        if should_resume {
830            if let (Some(messenger), Some(admission)) =
831                (messenger.as_ref(), child_admission.as_ref())
832            {
833                if let Err(error) = messenger.prepare_activation(admission).await {
834                    tracing::warn!(
835                        parent_session_id = %completion.parent_session_id,
836                        child_session_id = %completion.child_session_id,
837                        %error,
838                        "child outcome activation watermark failed; leaving parent wait armed"
839                    );
840                    return;
841                }
842            }
843        }
844
845        if should_resume {
846            runtime_state.waiting_for_children = None;
847            runtime_state.status = AgentStatusState::Idle;
848            runtime_state.suspension = None;
849            parent.metadata.remove("runtime.suspend_reason");
850
851            if child_admission.is_none() {
852                // Rolling-upgrade fallback only. The canonical path keeps the
853                // child outcome solely in SessionInbox until the next safe
854                // reasoning boundary.
855                parent.add_message(resume_message);
856            }
857        } else if runtime_state.waiting_for_children.is_some() {
858            runtime_state.status = AgentStatusState::Suspended;
859            runtime_state.suspension = Some(SuspensionState {
860                reason: "waiting_for_children".to_string(),
861                suspended_at: Utc::now(),
862                resumable: true,
863                hook_point: Some("ChildCompletion".to_string()),
864            });
865        }
866
867        parent.updated_at = Utc::now();
868        write_runtime_state(&mut parent, &runtime_state);
869        if let Err(error) = self
870            .persistence
871            .checkpoint_runtime_session(&mut parent)
872            .await
873        {
874            tracing::warn!(
875                parent_session_id = %completion.parent_session_id,
876                child_session_id = %completion.child_session_id,
877                %error,
878                "child outcome is durable but parent wait transition failed; leaving activation deferred"
879            );
880            return;
881        }
882        self.sessions.insert(
883            parent.id.clone(),
884            Arc::new(parking_lot::RwLock::new(parent.clone())),
885        );
886
887        // Capture before releasing the per-parent lock so the borrow checker
888        // is satisfied; `resume_parent` has its own retry loop and should not
889        // hold the per-parent lock (it would block other completions for the
890        // same parent, and the state is already durably settled above).
891        let resume_parent_id = parent.id.clone();
892        drop(_per_parent_guard);
893
894        if should_resume {
895            if let (Some(messenger), Some(admission)) = (messenger, child_admission) {
896                if let Err(error) = messenger.activate_prepared(&admission).await {
897                    tracing::warn!(
898                        parent_session_id = %resume_parent_id,
899                        %error,
900                        "child outcome and wait transition are durable but activation failed"
901                    );
902                }
903            } else {
904                self.resume_parent(resume_parent_id).await;
905            }
906        }
907    }
908}
909
910#[async_trait]
911impl ResumeExecutionPort for ChildCompletionCoordinator {
912    async fn load_session(&self, session_id: &str) -> Option<Session> {
913        match self.storage.load_session(session_id).await {
914            Ok(Some(session)) => Some(session),
915            Ok(None) => self
916                .sessions
917                .get(session_id)
918                .map(|e| e.value().clone())
919                .map(|arc| arc.read().clone()),
920            Err(error) => {
921                tracing::warn!(%session_id, %error, "failed to load session from storage");
922                self.sessions
923                    .get(session_id)
924                    .map(|e| e.value().clone())
925                    .map(|arc| arc.read().clone())
926            }
927        }
928    }
929
930    async fn save_and_cache_session(&self, session: &mut Session) {
931        self.save_and_cache(session).await;
932    }
933
934    async fn reserve_session_execution(
935        &self,
936        session_id: &str,
937        event_sender: &broadcast::Sender<AgentEvent>,
938    ) -> SessionExecutionReserveOutcome {
939        reserve_session_execution(
940            &self.agent,
941            &self.agent_runners,
942            &self.session_event_senders,
943            session_id,
944            event_sender,
945        )
946        .await
947    }
948
949    async fn get_or_create_event_sender(&self, session_id: &str) -> broadcast::Sender<AgentEvent> {
950        crate::execution::session_events::get_or_create_event_sender(
951            &self.session_event_senders,
952            session_id,
953        )
954        .await
955    }
956
957    fn dispatch_resume_execution(
958        &self,
959        request: ResumeSpawnRequest,
960    ) -> Result<(), ResumeSpawnRequest> {
961        let owner = self.clone();
962        tokio::spawn(async move {
963            ResumeExecutionPort::spawn_resume_execution(&owner, request).await;
964        });
965        Ok(())
966    }
967
968    async fn spawn_resume_execution(&self, request: ResumeSpawnRequest) {
969        let ResumeSpawnRequest {
970            session_id,
971            mut session,
972            mut execution_reservation,
973            event_sender,
974            config,
975        } = request;
976        if let Err(error) = execution_reservation.ensure_registered().await {
977            tracing::warn!(
978                %session_id,
979                run_id = %execution_reservation.run_id(),
980                %error,
981                "cannot resume after child completion without exact router ownership"
982            );
983            return;
984        }
985
986        let Some(root_tools) = self.root_tools.read().await.clone() else {
987            tracing::error!(%session_id, "cannot resume parent after child completion: root tool surface is not initialized");
988            return;
989        };
990
991        let config_snapshot = self.config.read().await.clone();
992        let model = session.model.clone();
993        let session_model_ref = session_effective_model_ref(&session);
994        let requested_provider = session_model_ref
995            .as_ref()
996            .map(|model_ref| model_ref.provider.as_str())
997            .unwrap_or(config.provider_name.as_str());
998        let resolved_provider_name = match resolve_provider_routing_key(
999            &config_snapshot,
1000            requested_provider,
1001            &self.provider_registry,
1002        ) {
1003            Ok(provider) => provider,
1004            Err(error) => {
1005                tracing::error!(
1006                    session_id = %session_id,
1007                    provider = requested_provider,
1008                    %error,
1009                    "child-completion resume provider is unavailable; refusing to fall back"
1010                );
1011                execution_reservation.abandon().await;
1012                return;
1013            }
1014        };
1015        let provider_override = if let Some(mut model_ref) = session_model_ref {
1016            model_ref.provider = resolved_provider_name.clone();
1017            persist_model_ref(&mut session, &model_ref);
1018            match self.provider_router.route(&model_ref) {
1019                Ok(provider) => Some(provider),
1020                Err(error) => {
1021                    tracing::error!(
1022                        session_id = %session_id,
1023                        provider = %model_ref.provider,
1024                        model = %model_ref.model,
1025                        %error,
1026                        "child-completion resume provider routing failed closed"
1027                    );
1028                    execution_reservation.abandon().await;
1029                    return;
1030                }
1031            }
1032        } else {
1033            match self.provider_registry.get(&resolved_provider_name) {
1034                Some(provider) => Some(provider),
1035                None => {
1036                    tracing::error!(
1037                        session_id = %session_id,
1038                        provider = %resolved_provider_name,
1039                        "child-completion resume provider disappeared after resolution; refusing to fall back"
1040                    );
1041                    execution_reservation.abandon().await;
1042                    return;
1043                }
1044            }
1045        };
1046        let resolved_fast_provider = resolve_fast_model(
1047            &config_snapshot,
1048            &resolved_provider_name,
1049            &self.provider_registry,
1050        )
1051        .map(|model| model.provider);
1052        let reasoning_effort = session.reasoning_effort;
1053        let reasoning_effort_source = session
1054            .metadata
1055            .get("reasoning_effort_source")
1056            .cloned()
1057            .unwrap_or_default();
1058        let gold_config = resolve_gold_config(
1059            &config_snapshot,
1060            session
1061                .metadata
1062                .get(GOLD_CONFIG_METADATA_KEY)
1063                .map(String::as_str),
1064        )
1065        .or(config.gold_config.clone());
1066
1067        let (mpsc_tx, _forwarder) = create_event_forwarder(
1068            session_id.clone(),
1069            execution_reservation.run_id().to_string(),
1070            event_sender,
1071            self.agent_runners.clone(),
1072            self.account_feed_inbox.clone(),
1073        );
1074
1075        let config_handle = self.config.clone();
1076        let cached_config = Arc::new(StdRwLock::new(config_snapshot.clone()));
1077        let provider_registry = self.provider_registry.clone();
1078        let provider_name_for_aux = resolved_provider_name.clone();
1079        let auxiliary_model_resolver = std::sync::Arc::new(move || {
1080            let config_snapshot = read_config_snapshot(&config_handle, cached_config.as_ref());
1081            // Auxiliary models are global (config-derived), never session-bound.
1082            let areas = resolve_global_area_models(
1083                &config_snapshot,
1084                &provider_name_for_aux,
1085                &provider_registry,
1086            );
1087            crate::AuxiliaryModelConfig {
1088                fast_model_name: areas.fast.as_ref().map(|m| m.model_name.clone()),
1089                fast_model_provider: areas.fast.map(|m| m.provider),
1090                background_model_name: areas.background.as_ref().map(|m| m.model_name.clone()),
1091                planning_model_name: None,
1092                search_model_name: None,
1093                summarization_model_name: areas
1094                    .summarization
1095                    .as_ref()
1096                    .map(|m| m.model_name.clone()),
1097                background_model_provider: areas.background.map(|m| m.provider),
1098                summarization_model_provider: areas.summarization.map(|m| m.provider),
1099            }
1100        });
1101        let model_roster = crate::ModelRoster {
1102            model: Some(model),
1103            provider_name: Some(resolved_provider_name),
1104            provider_type: config.provider_type.clone(),
1105            fast: crate::RoleModel::from_parts(config.fast_model, resolved_fast_provider),
1106            background: crate::RoleModel::from_parts(
1107                config.background_model,
1108                config.background_model_provider,
1109            ),
1110            summarization: crate::RoleModel::from_parts(
1111                config.summarization_model,
1112                config.summarization_model_provider,
1113            ),
1114        };
1115
1116        // Re-inject guardian state on resume so a reject→fix verdict can be
1117        // re-reviewed: config from the session (persisted at first spawn),
1118        // spawner from the coordinator-held handle. Absent guardian config this
1119        // stays `None`, and the approve→complete path is unchanged.
1120        let guardian_config = read_guardian_config(&session);
1121        let guardian_spawner = self.guardian_spawner.read().await.clone();
1122
1123        consume_pending_clarification_resume(&mut session);
1124        spawn_session_execution(SessionExecutionArgs {
1125            agent: self.agent.clone(),
1126            session_id,
1127            session,
1128            execution_reservation,
1129            tools_override: Some(root_tools),
1130            provider_override,
1131            model_roster,
1132            reasoning_effort,
1133            reasoning_effort_source,
1134            auxiliary_model_resolver: Some(auxiliary_model_resolver),
1135            // Resumed child runs keep the spawn-time disabled snapshot (#136 lives
1136            // on the long-running main agent path; children are short-lived).
1137            disabled_filter_resolver: None,
1138            disabled_tools: Some(config.disabled_tools),
1139            disabled_skill_ids: Some(config.disabled_skill_ids),
1140            selected_skill_ids: None,
1141            selected_skill_mode: None,
1142            mpsc_tx,
1143            image_fallback: config.image_fallback,
1144            gold_config,
1145            guardian_config,
1146            guardian_spawner,
1147            bash_resume_hook: {
1148                let hook: Arc<dyn BashResumeHook> = Arc::new(self.clone());
1149                Some(hook)
1150            },
1151            bash_completion_sink: {
1152                // Resumed runs keep the push wired too, so a background shell
1153                // launched after resume still notifies the loop.
1154                let sink: Arc<dyn BashCompletionSink> = Arc::new(self.clone());
1155                Some(sink)
1156            },
1157            app_data_dir: Some(self.app_data_dir.clone()),
1158            // Resume does not carry a fresh per-request override; the
1159            // config-level default (issue #221) still applies.
1160            run_budget: None,
1161            runners: self.agent_runners.clone(),
1162            sessions_cache: self.sessions.clone(),
1163            on_complete: None,
1164            // A resumed session that is itself a CHILD (nested sub-agents)
1165            // must publish its terminal completion so ITS parent is woken
1166            // in turn (issue #546).
1167            child_completion_handler: Some(Arc::new(self.clone())),
1168        });
1169    }
1170}
1171
1172/// Real SessionInbox activation adapter. It reserves through the exact same
1173/// runner registry as every existing resume path, but deliberately bypasses
1174/// `has_pending_user_message`: the typed envelope is still in the durable inbox
1175/// and will be admitted by the loop's first safe turn boundary.
1176#[async_trait]
1177impl SessionActivationSpawner for ChildCompletionCoordinator {
1178    async fn reserve_activation(
1179        &self,
1180        target_session_id: &str,
1181        _inbox_generation: u64,
1182    ) -> Result<SessionActivationReserveOutcome, bamboo_domain::SessionActivationError> {
1183        let Some(inbox) = self.agent.session_inbox() else {
1184            return Err(bamboo_domain::SessionActivationError::Internal(
1185                "agent runtime has no SessionInbox".to_string(),
1186            ));
1187        };
1188        let backlog = inbox
1189            .inspect(target_session_id)
1190            .await
1191            .map_err(|error| bamboo_domain::SessionActivationError::Internal(error.to_string()))?;
1192        if !backlog.activation_pending() {
1193            return Ok(SessionActivationReserveOutcome::NoWork);
1194        }
1195        // Load and mutate under the persistence lock. A coordinator that armed
1196        // a child/Bash wait immediately before this activation therefore wins.
1197        let prepared = prepare_session_inbox_activation(
1198            &self.persistence,
1199            target_session_id,
1200            backlog.interrupt_pending(),
1201        )
1202        .await
1203        .map_err(|error| {
1204            bamboo_domain::SessionActivationError::Internal(format!(
1205                "persist resumable SessionInbox target: {error}"
1206            ))
1207        })?;
1208        let Some((session, ready)) = prepared else {
1209            return Ok(SessionActivationReserveOutcome::NotFound);
1210        };
1211        if !ready {
1212            tracing::info!(
1213                session_id = target_session_id,
1214                "SessionInbox backlog is activation-eligible but a specific durable wait remains armed"
1215            );
1216            return Ok(SessionActivationReserveOutcome::NoWork);
1217        }
1218
1219        enum LaunchPlan {
1220            Root(Box<ResumeConfigSnapshot>),
1221            Child {
1222                scheduler: Arc<SpawnScheduler>,
1223                parent_session_id: String,
1224                model: String,
1225                disabled_tools: Option<Vec<String>>,
1226            },
1227        }
1228
1229        // Derive every execution/security input from the latest locked session
1230        // returned above, never from a stale pre-lock snapshot.
1231        let launch_plan = match session.kind {
1232            SessionKind::Root => {
1233                if self.root_tools.read().await.is_none() {
1234                    return Err(bamboo_domain::SessionActivationError::Internal(
1235                        "root tool surface is not initialized".to_string(),
1236                    ));
1237                }
1238                let config_snapshot = self.config.read().await.clone();
1239                LaunchPlan::Root(Box::new(
1240                    self.build_resume_config(&session, &config_snapshot),
1241                ))
1242            }
1243            SessionKind::Child => {
1244                let scheduler = self.spawn_scheduler.read().await.upgrade().ok_or_else(|| {
1245                    bamboo_domain::SessionActivationError::Internal(
1246                        "child spawn scheduler is not initialized".to_string(),
1247                    )
1248                })?;
1249                let parent_session_id = session
1250                    .parent_session_id
1251                    .as_deref()
1252                    .map(str::trim)
1253                    .filter(|id| !id.is_empty())
1254                    .map(ToOwned::to_owned)
1255                    .ok_or_else(|| {
1256                        bamboo_domain::SessionActivationError::Internal(format!(
1257                            "child SessionInbox target {target_session_id} has no parent owner"
1258                        ))
1259                    })?;
1260                let parent = self
1261                    .storage
1262                    .load_session(&parent_session_id)
1263                    .await
1264                    .map_err(|error| {
1265                        bamboo_domain::SessionActivationError::Internal(format!(
1266                            "load parent owner {parent_session_id}: {error}"
1267                        ))
1268                    })?
1269                    .ok_or_else(|| {
1270                        bamboo_domain::SessionActivationError::Internal(format!(
1271                            "child SessionInbox parent owner {parent_session_id} disappeared"
1272                        ))
1273                    })?;
1274                let child_root = if session.root_session_id.trim().is_empty() {
1275                    parent_session_id.as_str()
1276                } else {
1277                    session.root_session_id.as_str()
1278                };
1279                let parent_root = if parent.root_session_id.trim().is_empty() {
1280                    parent.id.as_str()
1281                } else {
1282                    parent.root_session_id.as_str()
1283                };
1284                if child_root != parent_root {
1285                    return Err(bamboo_domain::SessionActivationError::Internal(format!(
1286                        "child SessionInbox target {target_session_id} does not share its parent owner's root"
1287                    )));
1288                }
1289                let model = if session.model.trim().is_empty() {
1290                    parent.model.clone()
1291                } else {
1292                    session.model.clone()
1293                };
1294                if model.trim().is_empty() {
1295                    return Err(bamboo_domain::SessionActivationError::Internal(format!(
1296                        "child SessionInbox target {target_session_id} has no executable model"
1297                    )));
1298                }
1299                let disabled_tools = match session.metadata.get("disabled_tools") {
1300                    None => None,
1301                    Some(raw) => {
1302                        let tools = serde_json::from_str::<std::collections::BTreeSet<String>>(raw)
1303                            .map_err(|error| {
1304                                bamboo_domain::SessionActivationError::Internal(format!(
1305                                    "child SessionInbox target {target_session_id} has malformed disabled_tools: {error}"
1306                                ))
1307                            })?;
1308                        (!tools.is_empty()).then(|| tools.into_iter().collect())
1309                    }
1310                };
1311                LaunchPlan::Child {
1312                    scheduler,
1313                    parent_session_id,
1314                    model,
1315                    disabled_tools,
1316                }
1317            }
1318        };
1319        let event_sender =
1320            ResumeExecutionPort::get_or_create_event_sender(self, target_session_id).await;
1321
1322        let reservation = match reserve_runner_core(
1323            &self.agent_runners,
1324            &self.session_event_senders,
1325            target_session_id,
1326            &event_sender,
1327        )
1328        .await
1329        {
1330            ReserveOutcome::Reserved(reservation) => reservation,
1331            ReserveOutcome::AlreadyRunning(run_id) => {
1332                return Ok(SessionActivationReserveOutcome::AlreadyRunning { run_id });
1333            }
1334        };
1335        let run_id = reservation.run_id.clone();
1336        let launch = match launch_plan {
1337            LaunchPlan::Root(config) => {
1338                let execution_reservation =
1339                    SessionExecutionReservation::from_activation_placeholder(
1340                        target_session_id,
1341                        reservation,
1342                        self.agent
1343                            .activation_router()
1344                            .expect("SessionInbox activation requires an activation router")
1345                            .clone(),
1346                        self.agent_runners.clone(),
1347                    );
1348                // Launch and rollback share one exact RAII reservation. Dropping
1349                // an unlaunched SessionActivationLaunch cannot race a raw slot
1350                // removal against the reservation's router-placeholder cleanup.
1351                let reservation_cell = Arc::new(StdMutex::new(Some(execution_reservation)));
1352                let launch_reservation = reservation_cell.clone();
1353                let rollback_reservation = reservation_cell;
1354                let coordinator = self.clone();
1355                let launch_sessions = self.sessions.clone();
1356                let launch_session_id = session.id.clone();
1357                let launch_session = session.clone();
1358                let request_session_id = target_session_id.to_string();
1359                SessionActivationLaunch::new_with_async_rollback(
1360                    run_id,
1361                    move || {
1362                        let mut execution_reservation = launch_reservation
1363                            .lock()
1364                            .unwrap_or_else(std::sync::PoisonError::into_inner)
1365                            .take()
1366                            .expect("activation reservation launches or rolls back exactly once");
1367                        execution_reservation.mark_activation_published();
1368                        // The router publishes the exact owner before invoking
1369                        // this closure, so only now may the prepared snapshot
1370                        // replace the shared cache entry.
1371                        launch_sessions.insert(
1372                            launch_session_id,
1373                            Arc::new(parking_lot::RwLock::new(launch_session)),
1374                        );
1375                        let request = ResumeSpawnRequest {
1376                            session_id: request_session_id,
1377                            session,
1378                            execution_reservation,
1379                            event_sender,
1380                            config: *config,
1381                        };
1382                        tokio::spawn(async move {
1383                            ResumeExecutionPort::spawn_resume_execution(&coordinator, request)
1384                                .await;
1385                        });
1386                    },
1387                    move || async move {
1388                        let reservation = rollback_reservation
1389                            .lock()
1390                            .unwrap_or_else(std::sync::PoisonError::into_inner)
1391                            .take();
1392                        if let Some(reservation) = reservation {
1393                            reservation.rollback_unpublished_activation().await;
1394                        }
1395                    },
1396                )
1397            }
1398            LaunchPlan::Child {
1399                scheduler,
1400                parent_session_id,
1401                model,
1402                disabled_tools,
1403            } => {
1404                let execution_reservation =
1405                    SessionExecutionReservation::from_activation_placeholder(
1406                        target_session_id,
1407                        reservation,
1408                        self.agent
1409                            .activation_router()
1410                            .expect("SessionInbox activation requires an activation router")
1411                            .clone(),
1412                        self.agent_runners.clone(),
1413                    );
1414                let reservation_cell = Arc::new(StdMutex::new(Some(execution_reservation)));
1415                let launch_reservation = reservation_cell.clone();
1416                let rollback_reservation = reservation_cell;
1417                let job = SpawnJob {
1418                    parent_session_id,
1419                    child_session_id: target_session_id.to_string(),
1420                    model,
1421                    disabled_tools,
1422                };
1423                let launch_sessions = self.sessions.clone();
1424                let launch_session_id = session.id.clone();
1425                let launch_session = session;
1426                SessionActivationLaunch::new_with_async_rollback(
1427                    run_id,
1428                    move || {
1429                        let mut execution_reservation = launch_reservation
1430                            .lock()
1431                            .unwrap_or_else(std::sync::PoisonError::into_inner)
1432                            .take()
1433                            .expect("activation reservation launches or rolls back exactly once");
1434                        execution_reservation.mark_activation_published();
1435                        // As with root activation, publish the prepared cache
1436                        // snapshot only after router ownership commits.
1437                        launch_sessions.insert(
1438                            launch_session_id,
1439                            Arc::new(parking_lot::RwLock::new(launch_session)),
1440                        );
1441                        // Dropping a JoinHandle detaches the task. The captured
1442                        // combined reservation remains RAII-protected if the
1443                        // task is later aborted or unwinds during setup.
1444                        drop(scheduler.launch_reserved(job, execution_reservation));
1445                    },
1446                    move || async move {
1447                        let reservation = rollback_reservation
1448                            .lock()
1449                            .unwrap_or_else(std::sync::PoisonError::into_inner)
1450                            .take();
1451                        if let Some(reservation) = reservation {
1452                            reservation.rollback_unpublished_activation().await;
1453                        }
1454                    },
1455                )
1456            }
1457        };
1458        Ok(SessionActivationReserveOutcome::Reserved(launch))
1459    }
1460}
1461
1462/// Hidden resume message for a bash-completion self-resume (issue #84 Phase 2b).
1463/// Mirrors [`runtime_resume_message`]'s hidden/compressible shape so the resume
1464/// port's `has_pending_user_message` gate is satisfied.
1465///
1466/// `timed_out` selects the wording: the normal path (all shells finished)
1467/// announces completion; the deadline path (the 6h+10m wait ceiling was hit with
1468/// shells STILL running) must NOT claim the shells completed — it says they may
1469/// still be running so the model verifies with BashOutput instead of assuming
1470/// success on a false premise.
1471fn bash_completion_resume_message(bash_ids: &[String], timed_out: bool) -> Message {
1472    let body = if timed_out {
1473        format!(
1474            "Runtime notification: the background-Bash wait ceiling was reached while one or more \
1475             shell(s) ({}) may still be running. The session is being resumed so it is not \
1476             stranded; verify their actual status with BashOutput before assuming completion.",
1477            bash_ids.join(", ")
1478        )
1479    } else {
1480        format!(
1481            "Runtime notification: all background Bash shell(s) ({}) have completed. \
1482             Review their output with BashOutput and resume the task from where you left off.",
1483            bash_ids.join(", ")
1484        )
1485    };
1486    let mut message = Message::user(body);
1487    message.metadata = Some(serde_json::json!({
1488        RUNTIME_RESUME_MESSAGE_HIDDEN_KEY: true,
1489        RUNTIME_RESUME_MESSAGE_KIND_KEY: BASH_COMPLETION_RESUME_KIND,
1490    }));
1491    message.never_compress = false;
1492    message
1493}
1494
1495/// Decide whether the bash self-resume should retry its clear→append→resume
1496/// sequence after a resume attempt returned `outcome`, given that the persisted
1497/// bash wait is (`true`) / is not (`false`) still set on reload.
1498///
1499/// Retry **only** when the resume did NOT spawn (`Completed` — no pending user
1500/// message, i.e. our resume message was dropped — or `AlreadyRunning`) AND the
1501/// persisted bash wait is still set: the signature of the finalize-clobber, where
1502/// the suspending runner's one-shot final `merge_save_runtime` lands after our
1503/// save and reverts `waiting_for_bash=Some` while dropping our resume message, so
1504/// `has_pending_user_message` fails and nothing spawns. `Started` (resume fired)
1505/// and `NotFound` (session gone) never retry. Pure helper so the clobber
1506/// detection is unit-testable in isolation from async I/O.
1507fn bash_resume_should_retry(outcome: &ResumeOutcome, persisted_waiting_for_bash: bool) -> bool {
1508    match outcome {
1509        ResumeOutcome::Started { .. } | ResumeOutcome::NotFound => false,
1510        ResumeOutcome::Completed | ResumeOutcome::AlreadyRunning { .. } => {
1511            persisted_waiting_for_bash
1512        }
1513    }
1514}
1515
1516/// Whether a background-shell completion push should **resume** the owning loop
1517/// (vs merely enqueue an injection). Resume only when the loop is actually
1518/// suspended on a bash wait AND every shell it was waiting on has now finished —
1519/// resuming while other waited shells are still running would drop them back into
1520/// a foreground turn prematurely. The last shell to finish (or the backstop)
1521/// drives the resume; earlier ones enqueue their notice. Pure so the invariant is
1522/// unit-testable in isolation.
1523fn bash_completion_should_resume(
1524    loop_suspended_on_bash: bool,
1525    all_waited_shells_done: bool,
1526) -> bool {
1527    loop_suspended_on_bash && all_waited_shells_done
1528}
1529
1530#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1531enum BashCompletionDeliveryPlan {
1532    /// Preserve the durable wait and do not reserve a successor. The last
1533    /// sibling completion (or the wait backstop) activates the accumulated
1534    /// inbox in order.
1535    DurableOnly,
1536    /// The loop is live/not waiting; notify its current owner after admission.
1537    Activate,
1538    /// The last waited shell finished: clear the wait durably, then activate.
1539    ClearWaitThenActivate,
1540}
1541
1542fn bash_completion_delivery_plan(
1543    loop_suspended_on_bash: bool,
1544    all_waited_shells_done: bool,
1545) -> BashCompletionDeliveryPlan {
1546    if bash_completion_should_resume(loop_suspended_on_bash, all_waited_shells_done) {
1547        BashCompletionDeliveryPlan::ClearWaitThenActivate
1548    } else if loop_suspended_on_bash {
1549        BashCompletionDeliveryPlan::DurableOnly
1550    } else {
1551        BashCompletionDeliveryPlan::Activate
1552    }
1553}
1554
1555fn background_bash_post_tool_payload(info: &BashCompletionInfo) -> HookPayload {
1556    let success = info.status == "completed" && info.exit_code == Some(0);
1557    let response = serde_json::json!({
1558        "bash_id": info.bash_id,
1559        "command": info.command,
1560        "exit_code": info.exit_code,
1561        "status": info.status,
1562        "output_tail": info.output_tail,
1563    });
1564    HookPayload::ToolResult {
1565        tool_name: "Bash".to_string(),
1566        tool_call_id: info.bash_id.clone(),
1567        outcome: HookToolOutcome {
1568            success,
1569            result: success.then(|| response.to_string()),
1570            error: (!success).then(|| response.to_string()),
1571            needs_human: false,
1572            duration_ms: 0,
1573        },
1574    }
1575}
1576
1577fn append_background_bash_hook_feedback(info: &mut BashCompletionInfo, feedback: Vec<String>) {
1578    let feedback = feedback
1579        .into_iter()
1580        .map(|text| text.trim().to_string())
1581        .filter(|text| !text.is_empty())
1582        .collect::<Vec<_>>();
1583    if feedback.is_empty() {
1584        return;
1585    }
1586    if !info.output_tail.is_empty() {
1587        info.output_tail.push_str("\n\n");
1588    }
1589    info.output_tail.push_str("<post_tool_use_feedback>\n");
1590    info.output_tail.push_str(&feedback.join("\n"));
1591    info.output_tail.push_str("\n</post_tool_use_feedback>");
1592}
1593
1594async fn run_background_bash_post_tool_hooks(
1595    config: &bamboo_config::LifecycleHooksConfig,
1596    fallback_cwd: Option<std::path::PathBuf>,
1597    session: &Session,
1598    info: &mut BashCompletionInfo,
1599) -> bool {
1600    let runner = crate::HookRunner::new().with_lifecycle_config(config, fallback_cwd);
1601    if !runner.has_hooks_for(AgentHookPoint::AfterToolExecution) {
1602        return false;
1603    }
1604
1605    let mut runtime_state = session
1606        .agent_runtime_state
1607        .clone()
1608        .unwrap_or_else(|| AgentRuntimeState::new(&session.id));
1609    let outcome = runner
1610        .run_observer_hooks(
1611            AgentHookPoint::AfterToolExecution,
1612            &background_bash_post_tool_payload(info),
1613            session,
1614            &mut runtime_state,
1615            None,
1616        )
1617        .await;
1618    append_background_bash_hook_feedback(info, outcome.injected_contexts);
1619    true
1620}
1621
1622/// Apply the bash-resume state transition to a loaded session **in place**: clear
1623/// the `waiting_for_bash` wait, mark the runtime Idle, drop the suspension +
1624/// `runtime.suspend_reason`, and append `resume_message`. Returns `false` (a
1625/// no-op) when the session was not actually waiting on bash — the double-resume
1626/// guard shared by the push and the backstop. Pure (no I/O) so both
1627/// [`ChildCompletionCoordinator::perform_bash_resume`] and unit tests exercise the
1628/// exact same transition.
1629fn apply_bash_resume_transition(session: &mut Session, resume_message: &Message) -> bool {
1630    let mut runtime_state = read_runtime_state(session);
1631    if runtime_state.waiting_for_bash.is_none() {
1632        return false;
1633    }
1634    runtime_state.waiting_for_bash = None;
1635    runtime_state.status = AgentStatusState::Idle;
1636    runtime_state.suspension = None;
1637    write_runtime_state(session, &runtime_state);
1638    session.metadata.remove("runtime.suspend_reason");
1639    session.add_message(resume_message.clone());
1640    true
1641}
1642
1643/// Bash self-resume support (issue #84 Phase 2b; push follow-up).
1644impl ChildCompletionCoordinator {
1645    /// **Backstop** for a session suspended on `waiting_for_bash`. The primary,
1646    /// event-driven wake is the loop-facing push
1647    /// ([`BashCompletionSink::on_bash_completed`] → [`Self::deliver_bash_completion`]):
1648    /// the shell's completion task fires it the instant the process exits, and it
1649    /// resumes the loop directly. This task exists ONLY to catch a **lost push** —
1650    /// the completion landing in the window before the suspend was persisted (so
1651    /// the push saw no `waiting_for_bash` and only queued an injection), or a
1652    /// configuration with no sink wired — and to honour the wait ceiling.
1653    ///
1654    /// So it is deliberately NOT a hot spin: a coarse backoff (1 s → 30 s) that
1655    /// **yields to the push**. In the happy path the push has already cleared
1656    /// `waiting_for_bash` before the first check fires, so this returns after one
1657    /// cheap load with no registry polling at all. It only performs a resume when
1658    /// the shell(s) have finished but the loop is somehow still suspended, or the
1659    /// 6 h wait ceiling is reached.
1660    async fn bash_self_resume(&self, session_id: String, bash_ids: Vec<String>) {
1661        let mut delay = Duration::from_secs(1);
1662        let max_delay = Duration::from_secs(30);
1663        // Hard ceiling: the wait lease (6 h) + the registry GC TTL (5 min) +
1664        // margin. After this the shells are gone from the registry regardless,
1665        // so force-resume to avoid stranding the session on a GC edge case.
1666        let max_poll = Duration::from_secs(6 * 3600 + 600);
1667        let deadline = tokio::time::Instant::now() + max_poll;
1668
1669        loop {
1670            tokio::time::sleep(delay).await;
1671
1672            let Some(session) = self.load_session(&session_id).await else {
1673                tracing::info!(%session_id, "bash self-resume backstop: session gone; nothing to do");
1674                return;
1675            };
1676            if read_runtime_state(&session).waiting_for_bash.is_none() {
1677                // The push (or another path) already resumed. This is the common
1678                // case — the backstop yields silently after a single load.
1679                return;
1680            }
1681
1682            let still_running =
1683                bamboo_tools::tools::bash_runtime::running_shells_for_session(&session_id);
1684            let timed_out = tokio::time::Instant::now() >= deadline;
1685            if still_running.is_empty() || timed_out {
1686                // The shell(s) finished but the loop is still suspended → the push
1687                // was lost (pre-persist window / no sink), or the ceiling hit.
1688                // Resume under the shared per-session lock so we never race the
1689                // push or a concurrent child-completion resume.
1690                let guard = session_resume_lock(&session_id);
1691                let _held = guard.lock().await;
1692                tracing::warn!(
1693                    %session_id,
1694                    shell_count = bash_ids.len(),
1695                    timed_out,
1696                    "bash self-resume backstop engaged (push lost or wait ceiling reached)"
1697                );
1698                self.perform_bash_resume(
1699                    &session_id,
1700                    bash_completion_resume_message(&bash_ids, timed_out),
1701                )
1702                .await;
1703                return;
1704            }
1705
1706            delay = (delay * 2).min(max_delay);
1707        }
1708    }
1709
1710    /// Clear a session's `waiting_for_bash` state, append `resume_message`, and
1711    /// drive the parent resume — the shared clear→append→resume used by BOTH the
1712    /// event-driven push ([`Self::deliver_bash_completion`]) and the backstop poll
1713    /// ([`Self::bash_self_resume`]).
1714    ///
1715    /// **The caller MUST hold the [`session_resume_lock`] for `session_id`** so the
1716    /// load-check-clear-resume critical section is serialized against every other
1717    /// resume source (no double resume). No-op when the persisted wait was already
1718    /// cleared (another source handled it first).
1719    ///
1720    /// The clear→append→resume is a **bounded retry loop** that closes the
1721    /// finalize-clobber strand. The suspending runner's `finalize_task_context`
1722    /// runs a full `save_runtime_session` (same `merge_save_runtime`, which
1723    /// overwrites the whole `messages` array) that can land AFTER our save,
1724    /// reverting `waiting_for_bash=Some` and dropping our resume message, so
1725    /// `has_pending_user_message` fails and `resume_parent` returns `Completed`
1726    /// without spawning. We detect that (persisted wait still set after a
1727    /// non-`Started` outcome) and re-clear/re-append/re-resume. It converges
1728    /// because the runner's finalize persist is one-shot: once landed, our retry's
1729    /// save is the last writer, the message sticks, and resume fires.
1730    async fn perform_bash_resume(&self, session_id: &str, resume_message: Message) {
1731        let retry_backoff = Duration::from_millis(200);
1732        const MAX_RESUME_ATTEMPTS: u8 = 5;
1733        for attempt in 0..MAX_RESUME_ATTEMPTS {
1734            if attempt > 0 {
1735                tokio::time::sleep(retry_backoff).await;
1736            }
1737
1738            let Some(mut session) = self.load_session(session_id).await else {
1739                tracing::warn!(%session_id, "bash resume: session not found; nothing to resume");
1740                return;
1741            };
1742
1743            if !apply_bash_resume_transition(&mut session, &resume_message) {
1744                // Double-resume guard: the wait was already cleared by another
1745                // source (the push, the backstop, or a user-driven resume). Do
1746                // not append a duplicate message or request a redundant resume.
1747                tracing::info!(
1748                    %session_id, attempt,
1749                    "bash resume: persisted bash wait already cleared; nothing to resume"
1750                );
1751                return;
1752            }
1753            session.updated_at = Utc::now();
1754            self.save_and_cache(&mut session).await;
1755            tracing::info!(
1756                %session_id, attempt,
1757                "bash resume: cleared bash wait and appended resume message"
1758            );
1759
1760            let outcome = self.resume_parent(session_id.to_string()).await;
1761            match outcome {
1762                ResumeOutcome::Started { .. } => {
1763                    tracing::info!(%session_id, attempt, "bash resume: resume fired");
1764                    return;
1765                }
1766                ResumeOutcome::NotFound => {
1767                    tracing::warn!(%session_id, "bash resume: session vanished during resume");
1768                    return;
1769                }
1770                _ => {
1771                    // Completed (no pending user message ⇒ our resume message was
1772                    // dropped by the runner's finalize persist) or AlreadyRunning.
1773                    // Decide via the persisted bash wait: still set ⇒
1774                    // finalize-clobber ⇒ retry; cleared ⇒ the session is being
1775                    // handled (by us or a concurrent resume) ⇒ stop.
1776                    let clobbered = match self.load_session(session_id).await {
1777                        Some(reloaded) => read_runtime_state(&reloaded).waiting_for_bash.is_some(),
1778                        None => {
1779                            tracing::warn!(%session_id, "bash resume: session vanished after resume");
1780                            return;
1781                        }
1782                    };
1783                    if bash_resume_should_retry(&outcome, clobbered) {
1784                        tracing::warn!(
1785                            %session_id, attempt,
1786                            outcome = outcome.as_str(),
1787                            "bash resume: persisted wait still set after resume (finalize-clobber); retrying"
1788                        );
1789                        continue;
1790                    }
1791                    tracing::info!(
1792                        %session_id, attempt,
1793                        outcome = outcome.as_str(),
1794                        "bash resume: wait cleared and resume handled; stopping"
1795                    );
1796                    return;
1797                }
1798            }
1799        }
1800
1801        tracing::warn!(
1802            %session_id,
1803            attempts = MAX_RESUME_ATTEMPTS,
1804            "bash resume: exhausted clobber-retry budget without confirming resume; giving up"
1805        );
1806    }
1807}
1808
1809impl BashResumeHook for ChildCompletionCoordinator {
1810    fn arrange_bash_self_resume(&self, session_id: String, bash_ids: Vec<String>) {
1811        let coordinator = Arc::new(self.clone());
1812        tokio::spawn(async move {
1813            coordinator.bash_self_resume(session_id, bash_ids).await;
1814        });
1815    }
1816}
1817
1818/// Build the injected user-message body for a completed background shell — a
1819/// concise notice plus a bounded output tail — so the model can act on the
1820/// result without a mandatory `BashOutput` round-trip (issue #84 Phase 2b
1821/// follow-up).
1822fn bash_completion_injection_body(info: &BashCompletionInfo) -> String {
1823    let exit = match info.exit_code {
1824        Some(code) => code.to_string(),
1825        None => "none (signal/killed)".to_string(),
1826    };
1827    let mut body = format!(
1828        "Runtime notification: background shell `{}` (`{}`) finished — status {}, exit code {}.",
1829        info.bash_id, info.command, info.status, exit
1830    );
1831    if info.output_tail.trim().is_empty() {
1832        body.push_str(" It produced no captured output.");
1833    } else {
1834        body.push_str("\n\nOutput tail:\n");
1835        body.push_str(&info.output_tail);
1836    }
1837    body.push_str(&format!(
1838        "\n\nUse BashOutput with bash_id=\"{}\" for the full output, then continue the task.",
1839        info.bash_id
1840    ));
1841    body
1842}
1843
1844fn bash_completion_envelope(info: &BashCompletionInfo) -> bamboo_domain::SessionMessageEnvelope {
1845    let provider_message = bash_resume_message_from_info(info);
1846    let data = serde_json::json!({
1847        "session_id": info.session_id,
1848        "bash_id": info.bash_id,
1849        "command": info.command,
1850        "exit_code": info.exit_code,
1851        "status": info.status,
1852        "output_tail": info.output_tail,
1853    });
1854    // The id covers the immutable completion snapshot. An exact delivery retry
1855    // (whose transport timestamp may differ) is idempotent; a changed status,
1856    // output tail, command, or exit code is a distinct correction rather than
1857    // silently reusing one id with different semantics.
1858    let identity = data.clone();
1859    bamboo_domain::SessionMessageEnvelope {
1860        id: bamboo_domain::SessionMessageId::stable("background_bash_completion", &identity),
1861        source: bamboo_domain::SessionMessageSource::Runtime {
1862            subsystem: "background_bash".to_string(),
1863        },
1864        target_session_id: info.session_id.clone(),
1865        kind: bamboo_domain::SessionMessageKind::RuntimeInstruction,
1866        body: bamboo_domain::SessionMessageBody::RuntimeInstruction(
1867            bamboo_domain::SessionRuntimeInstruction {
1868                instruction: "background_bash_completed".to_string(),
1869                content: Some(bamboo_domain::SessionMessageContent::text(
1870                    bash_completion_injection_body(info),
1871                )),
1872                data: Some(data),
1873                provider_message: Some(session_provider_message(&provider_message)),
1874            },
1875        ),
1876        created_at: Utc::now(),
1877        thread_id: None,
1878        in_reply_to: None,
1879        attempt: None,
1880        correlation_id: Some(info.bash_id.clone()),
1881    }
1882}
1883
1884/// Enqueue a completed shell's summary as a pending injected message on the
1885/// owning session for the rolling-upgrade polling backstop. New push delivery
1886/// uses [`SessionMessenger`]; this helper remains only so an older process that
1887/// has not wired the typed delivery plane can still resume persisted sessions.
1888/// Race-safe: `update_runtime_config` loads and saves under the per-session lock.
1889async fn enqueue_bash_completion_injection(
1890    persistence: &LockedSessionStore,
1891    info: &BashCompletionInfo,
1892) -> std::io::Result<Option<Session>> {
1893    let body = bash_completion_injection_body(info);
1894    let queued = serde_json::json!({
1895        "content": body,
1896        "created_at": Utc::now(),
1897    });
1898    persistence
1899        .update_runtime_config(&info.session_id, move |session| {
1900            let mut pending = session.pending_injected_messages().unwrap_or_default();
1901            pending.push(queued);
1902            session.set_pending_injected_messages(pending);
1903        })
1904        .await
1905}
1906
1907/// Build the hidden, compressible resume message for a completed background
1908/// shell — the same rich notice body used for a live-loop injection
1909/// ([`bash_completion_injection_body`]), but tagged as a resume message so it
1910/// satisfies the `has_pending_user_message` gate that lets a suspended session
1911/// spawn. This is what the **push** appends when it wakes a suspended loop, so
1912/// the model gets the shell's status + output tail in one shot without a
1913/// separate `BashOutput` round-trip.
1914fn bash_resume_message_from_info(info: &BashCompletionInfo) -> Message {
1915    let mut message = Message::user(bash_completion_injection_body(info));
1916    message.metadata = Some(serde_json::json!({
1917        RUNTIME_RESUME_MESSAGE_HIDDEN_KEY: true,
1918        RUNTIME_RESUME_MESSAGE_KIND_KEY: BASH_COMPLETION_RESUME_KIND,
1919    }));
1920    message.never_compress = false;
1921    message
1922}
1923
1924impl ChildCompletionCoordinator {
1925    /// Loop-facing delivery of a completed background shell. Two paths, chosen
1926    /// under the per-session resume lock so we never race the backstop poll or a
1927    /// concurrent child-completion resume:
1928    ///
1929    /// - **Suspended loop** (the model ended its turn with the shell running, so
1930    ///   `waiting_for_bash` is set) AND every waited shell has now finished →
1931    ///   **resume the loop directly**, event-driven, appending the rich completion
1932    ///   notice as the resume message. This is the push's whole point: no polling.
1933    /// - Every notice is then delivered through the same typed SessionMessenger
1934    ///   as peer/user steering. The durable inbox and activation router decide
1935    ///   whether to notify the current owner or reserve one successor.
1936    async fn deliver_bash_completion(&self, mut info: BashCompletionInfo) {
1937        // A background shell completes outside the originating tool call, so
1938        // fire its PostToolUse seam here before routing the completion into the
1939        // next round/resume message. Do not hold the resume lock while an
1940        // arbitrary command hook runs: its configured timeout must never block
1941        // the independent liveness backstop.
1942        if let Some(hook_session) = self.load_session(&info.session_id).await {
1943            let config_snapshot = self.config.read().await.clone();
1944            let _ = run_background_bash_post_tool_hooks(
1945                &config_snapshot.lifecycle_hooks,
1946                Some(self.app_data_dir.clone()),
1947                &hook_session,
1948                &mut info,
1949            )
1950            .await;
1951        }
1952
1953        let guard = session_resume_lock(&info.session_id);
1954        let _held = guard.lock().await;
1955
1956        let Some(session) = self.load_session(&info.session_id).await else {
1957            tracing::warn!(
1958                session_id = %info.session_id,
1959                bash_id = %info.bash_id,
1960                "background bash completion: owning session not found; nothing to notify"
1961            );
1962            return;
1963        };
1964
1965        let waiting = read_runtime_state(&session).waiting_for_bash.is_some();
1966        // The producer flips the shell's `running` flag false BEFORE firing this
1967        // push, so a now-empty per-session registry means every shell the loop was
1968        // waiting on has finished — safe to resume. If OTHER waited shells are
1969        // still running, fall through to the enqueue path and let the last one (or
1970        // the backstop) drive the resume.
1971        let all_shells_done =
1972            bamboo_tools::tools::bash_runtime::running_shells_for_session(&info.session_id)
1973                .is_empty();
1974        let delivery_plan = bash_completion_delivery_plan(waiting, all_shells_done);
1975
1976        // Stage 1: make the completion durable BEFORE clearing the wait. The
1977        // activation is intentionally deferred until the wait-state mutation
1978        // below is durably committed; otherwise a successor can observe the old
1979        // suspension, while clear-before-deliver can lose the only wake on
1980        // crash.
1981        let messenger = self.agent.session_messenger().cloned();
1982        let admission = match messenger.as_ref() {
1983            Some(messenger) => match messenger.admit(bash_completion_envelope(&info)).await {
1984                Ok(admission) => Some(admission),
1985                Err(error) => {
1986                    tracing::warn!(
1987                        session_id = %info.session_id,
1988                        bash_id = %info.bash_id,
1989                        %error,
1990                        "background bash completion: durable SessionInbox admission failed; leaving wait armed"
1991                    );
1992                    return;
1993                }
1994            },
1995            None => {
1996                tracing::warn!(
1997                    session_id = %info.session_id,
1998                    bash_id = %info.bash_id,
1999                    "SessionMessenger unavailable; using compatibility injection before clearing wait"
2000                );
2001                match enqueue_bash_completion_injection(&self.persistence, &info).await {
2002                    Ok(Some(_)) => None,
2003                    Ok(None) => {
2004                        tracing::warn!(
2005                            session_id = %info.session_id,
2006                            "background bash compatibility target disappeared"
2007                        );
2008                        return;
2009                    }
2010                    Err(error) => {
2011                        tracing::warn!(
2012                            session_id = %info.session_id,
2013                            %error,
2014                            "background bash compatibility admission failed; leaving wait armed"
2015                        );
2016                        return;
2017                    }
2018                }
2019            }
2020        };
2021
2022        if delivery_plan == BashCompletionDeliveryPlan::DurableOnly {
2023            tracing::info!(
2024                session_id = %info.session_id,
2025                bash_id = %info.bash_id,
2026                "background bash completion is durable; sibling shells still run, so wait remains armed"
2027            );
2028            return;
2029        }
2030
2031        if let (Some(messenger), Some(admission)) = (messenger.as_ref(), admission.as_ref()) {
2032            if let Err(error) = messenger.prepare_activation(admission).await {
2033                tracing::warn!(
2034                    session_id = %info.session_id,
2035                    bash_id = %info.bash_id,
2036                    %error,
2037                    "background bash completion activation watermark failed; leaving wait armed"
2038                );
2039                return;
2040            }
2041        }
2042
2043        if delivery_plan == BashCompletionDeliveryPlan::ClearWaitThenActivate {
2044            tracing::info!(
2045                session_id = %info.session_id,
2046                bash_id = %info.bash_id,
2047                status = %info.status,
2048                "background bash completion: push-resuming suspended loop (event-driven)"
2049            );
2050            let mut resumable = session.clone();
2051            let mut runtime_state = read_runtime_state(&resumable);
2052            runtime_state.waiting_for_bash = None;
2053            runtime_state.status = AgentStatusState::Idle;
2054            runtime_state.suspension = None;
2055            write_runtime_state(&mut resumable, &runtime_state);
2056            resumable.metadata.remove("runtime.suspend_reason");
2057            resumable.updated_at = Utc::now();
2058            if let Err(error) = self.persistence.merge_save_runtime(&mut resumable).await {
2059                tracing::warn!(
2060                    session_id = %info.session_id,
2061                    %error,
2062                    "background bash completion is durable but wait-state clear failed; leaving activation to the wait backstop"
2063                );
2064                return;
2065            }
2066            self.sessions.insert(
2067                resumable.id.clone(),
2068                Arc::new(parking_lot::RwLock::new(resumable)),
2069            );
2070        }
2071
2072        // Stage 3: activation is allowed only after the durable wait-state
2073        // clear. A crash before here leaves either the wait backstop or startup
2074        // inbox reconciliation able to recover.
2075        let (Some(messenger), Some(admission)) = (messenger, admission) else {
2076            if delivery_plan == BashCompletionDeliveryPlan::ClearWaitThenActivate {
2077                let _ = self.resume_parent(info.session_id.clone()).await;
2078            }
2079            return;
2080        };
2081        match messenger.activate_prepared(&admission).await {
2082            Ok(receipt) => tracing::info!(
2083                session_id = %info.session_id,
2084                bash_id = %info.bash_id,
2085                status = %info.status,
2086                waiting,
2087                generation = receipt.delivery.generation,
2088                activation = ?receipt.activation,
2089                "background bash completion delivered through SessionMessenger"
2090            ),
2091            Err(error) => tracing::warn!(
2092                session_id = %info.session_id,
2093                bash_id = %info.bash_id,
2094                %error,
2095                "background bash completion: SessionMessenger delivery failed"
2096            ),
2097        }
2098    }
2099}
2100
2101impl BashCompletionSink for ChildCompletionCoordinator {
2102    fn on_bash_completed(&self, info: BashCompletionInfo) {
2103        // Best-effort, off the shell's completion-poll task: hand the delivery to
2104        // a detached task so the producer is never blocked (mirrors
2105        // `arrange_bash_self_resume`).
2106        let coordinator = Arc::new(self.clone());
2107        tokio::spawn(async move {
2108            coordinator.deliver_bash_completion(info).await;
2109        });
2110    }
2111}
2112
2113// ---------------------------------------------------------------------------
2114// Child-wait watchdog (issue #546)
2115// ---------------------------------------------------------------------------
2116
2117/// How often the child-wait watchdog sweeps suspended sessions.
2118const CHILD_WAIT_SWEEP_INTERVAL_SECS: u64 = 30;
2119/// Leave a freshly registered wait alone for this long so the event-driven
2120/// completion push always gets the first shot (and just-enqueued spawn jobs
2121/// have time to persist their running marker).
2122const CHILD_WAIT_REGISTRATION_GRACE_SECS: i64 = 60;
2123/// A waited child with NO live runner and a non-terminal index status is
2124/// declared dead once its control-plane has been quiet for this long.
2125const DEAD_CHILD_GRACE_SECS: i64 = 120;
2126/// Slack on top of the per-child liveness policy before a Running-but-frozen
2127/// runner entry (dead task) is force-finalized by the sweeper. The per-child
2128/// watchdog cancels at `max_idle`/`max_total` and the child then publishes its
2129/// own timeout; an entry frozen this far PAST those limits proves that
2130/// machinery is dead.
2131const STALE_RUNNER_SLACK_SECS: i64 = 600;
2132
2133/// Whether a waited child's index status means the sweeper must consider it
2134/// DEAD when nothing is driving it: non-terminal and not legitimately
2135/// suspended. `None` = never ran (created-but-never-started, or a spawn that
2136/// was lost before its running marker persisted).
2137fn is_dead_child_candidate_status(status: Option<&str>) -> bool {
2138    match status {
2139        // "suspended" children wait on a human / their own children / bash —
2140        // their wake has its own driver; never declare them dead here.
2141        Some(status) => !is_terminal_child_status(status) && status != "suspended",
2142        None => true,
2143    }
2144}
2145
2146/// Whether a reported completion's child id is genuinely a child of the parent
2147/// it claims (issue #546 read-side disclosure guard). `SubAgent.wait` ids are
2148/// model-provided, and the watchdog resolves an unowned id by publishing a
2149/// synthetic completion so the parent is unstranded — but the parent must NEVER
2150/// receive a FOREIGN session's content folded into its transcript.
2151/// `child_parent_linkage` is that session's own `parent_session_id` (`None`
2152/// when the session does not exist). Pure so the rule is unit-testable.
2153fn completion_child_is_owned(reported_parent: &str, child_parent_linkage: Option<&str>) -> bool {
2154    child_parent_linkage == Some(reported_parent)
2155}
2156
2157/// Pick which terminal child's completion to replay when the wait is already
2158/// satisfied but the parent is still suspended (lost wake). Prefer an
2159/// error-like child so a `FirstError` policy re-evaluates truthfully.
2160fn select_replay_child(terminal: &[(String, String)]) -> Option<&(String, String)> {
2161    terminal
2162        .iter()
2163        .find(|(_, status)| is_error_like(status))
2164        .or_else(|| terminal.last())
2165}
2166
2167fn child_wait_watchdog_resume_message(body: String) -> Message {
2168    let mut message = Message::user(body);
2169    message.metadata = Some(serde_json::json!({
2170        RUNTIME_RESUME_MESSAGE_HIDDEN_KEY: true,
2171        RUNTIME_RESUME_MESSAGE_KIND_KEY: "child_wait_watchdog_resume",
2172    }));
2173    message.never_compress = false;
2174    message
2175}
2176
2177fn empty_child_wait_message() -> Message {
2178    child_wait_watchdog_resume_message(
2179        "Runtime notification: this session was suspended waiting for child sessions, but the \
2180         wait tracked no children (internal inconsistency). The session has been resumed; use \
2181         SubAgent.list to inspect child state and continue the task."
2182            .to_string(),
2183    )
2184}
2185
2186fn child_wait_lease_expired_message(child_ids: &[String]) -> Message {
2187    child_wait_watchdog_resume_message(format!(
2188        "Runtime notification: the wait lease for child session(s) [{}] expired before they all \
2189         reported completion. They were NOT cancelled and may still be running or already \
2190         finished — verify their actual status with SubAgent.list / SubAgent.get before assuming \
2191         anything, then continue the task.",
2192        child_ids.join(", ")
2193    ))
2194}
2195
2196/// Child-wait watchdog (issue #546): the heartbeat backstop for parents
2197/// suspended on `waiting_for_children`.
2198///
2199/// The primary wake is the event-driven completion push (child terminal →
2200/// [`ChildCompletionHandler::on_child_completed`] → resume). This sweeper
2201/// exists because ANY break in that chain — a panicked child task, a dead
2202/// spawn scheduler, a process restart, a clobbered/exhausted resume, a wait
2203/// registered over an already-terminal child — previously stranded the parent
2204/// forever. It mirrors the bash backstop's philosophy: coarse, cheap, yields
2205/// to the push, and only acts when the durable state proves nothing else can.
2206///
2207/// All wake decisions funnel through the SAME machinery the push uses
2208/// (synthetic/replayed completions → `on_child_completed`, per-parent
2209/// serialization via [`session_resume_lock`]), so there is exactly one resume
2210/// implementation.
2211impl ChildCompletionCoordinator {
2212    /// Spawn the watchdog: one boot-time reconciliation pass, then a sweep
2213    /// every [`CHILD_WAIT_SWEEP_INTERVAL_SECS`]. Call once at server startup.
2214    pub fn spawn_child_wait_watchdog(self: &Arc<Self>) {
2215        let coordinator = Arc::clone(self);
2216        tokio::spawn(async move {
2217            use futures::FutureExt;
2218            if std::panic::AssertUnwindSafe(coordinator.reconcile_orphans_at_boot())
2219                .catch_unwind()
2220                .await
2221                .is_err()
2222            {
2223                tracing::error!("child-wait watchdog: boot reconciliation panicked");
2224            }
2225            let mut ticker = tokio::time::interval(std::time::Duration::from_secs(
2226                CHILD_WAIT_SWEEP_INTERVAL_SECS,
2227            ));
2228            ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
2229            // Skip the immediate tick (boot reconciliation just ran).
2230            ticker.tick().await;
2231            loop {
2232                ticker.tick().await;
2233                if std::panic::AssertUnwindSafe(coordinator.sweep_child_waits())
2234                    .catch_unwind()
2235                    .await
2236                    .is_err()
2237                {
2238                    tracing::error!("child-wait watchdog: sweep panicked; continuing");
2239                }
2240            }
2241        });
2242    }
2243
2244    /// One-shot startup reconciliation: a process restart kills every in-flight
2245    /// child task AND the in-memory bash backstop polls, but the durable state
2246    /// (child index status `running`, parents suspended on bash) survives —
2247    /// previously stranding those parents forever.
2248    async fn reconcile_orphans_at_boot(&self) {
2249        let cutoff = Utc::now();
2250
2251        // (1) Children left `running` by the previous process: no task in THIS
2252        // process is driving them, so no completion will ever fire. Mark them
2253        // terminal and wake their parents through the canonical path. (Root
2254        // sessions left `running` are user-visible and user-recoverable; only
2255        // children hold a suspended parent hostage.)
2256        let running = self
2257            .storage
2258            .list_sessions_by_run_status("running")
2259            .await
2260            .unwrap_or_default();
2261        for (child_id, parent_id) in running {
2262            let Some(parent_id) = parent_id else { continue };
2263            if self.runner_is_running(&child_id).await {
2264                continue;
2265            }
2266            let Some(control_plane) = self.load_control_plane(&child_id).await else {
2267                continue;
2268            };
2269            // A child that started AFTER boot is alive by definition — the
2270            // cutoff guards the tiny window where a fresh spawn races this scan.
2271            if control_plane.updated_at >= cutoff {
2272                continue;
2273            }
2274            tracing::warn!(
2275                child_session_id = %child_id,
2276                parent_session_id = %parent_id,
2277                "boot reconciliation: child was running when the process died; \
2278                 marking it error and waking the parent"
2279            );
2280            self.synthesize_child_completion(
2281                &parent_id,
2282                &child_id,
2283                "error",
2284                Some(
2285                    "orphaned by server restart: the process died while this child session \
2286                     was running"
2287                        .to_string(),
2288                ),
2289            )
2290            .await;
2291        }
2292
2293        // (2) Sessions suspended on `waiting_for_bash`: their backstop poll was
2294        // an in-memory task that died with the process. Re-arm it — with the
2295        // shell registry empty after a restart it resumes on its first check.
2296        let suspended = self
2297            .storage
2298            .list_sessions_by_run_status("suspended")
2299            .await
2300            .unwrap_or_default();
2301        for (session_id, _) in suspended {
2302            let Some(control_plane) = self.load_control_plane(&session_id).await else {
2303                continue;
2304            };
2305            if control_plane
2306                .metadata
2307                .get("runtime.suspend_reason")
2308                .map(String::as_str)
2309                != Some("waiting_for_bash")
2310            {
2311                continue;
2312            }
2313            if let Some(wait) = read_runtime_state(&control_plane).waiting_for_bash {
2314                tracing::warn!(
2315                    %session_id,
2316                    "boot reconciliation: re-arming bash self-resume backstop lost in restart"
2317                );
2318                let coordinator = self.clone();
2319                tokio::spawn(async move {
2320                    coordinator
2321                        .bash_self_resume(session_id, wait.bash_ids)
2322                        .await;
2323                });
2324            }
2325        }
2326    }
2327
2328    async fn runner_is_running(&self, session_id: &str) -> bool {
2329        let runners = self.agent_runners.read().await;
2330        runners
2331            .get(session_id)
2332            .is_some_and(|runner| matches!(runner.status, AgentStatus::Running))
2333    }
2334
2335    async fn load_control_plane(&self, session_id: &str) -> Option<Session> {
2336        match self.storage.load_runtime_control_plane(session_id).await {
2337            Ok(session) => session,
2338            Err(error) => {
2339                tracing::warn!(
2340                    %session_id,
2341                    %error,
2342                    "child-wait watchdog: failed to load session control plane"
2343                );
2344                None
2345            }
2346        }
2347    }
2348
2349    /// One sweep: inspect every session whose index status is `suspended`.
2350    /// Cheap in the common case — the candidate list is small and each check is
2351    /// a sidecar (control-plane) load.
2352    async fn sweep_child_waits(&self) {
2353        let suspended = match self.storage.list_sessions_by_run_status("suspended").await {
2354            Ok(entries) => entries,
2355            Err(error) => {
2356                tracing::warn!(%error, "child-wait watchdog: failed to list suspended sessions");
2357                return;
2358            }
2359        };
2360        for (session_id, _) in suspended {
2361            self.sweep_one_suspended_session(&session_id).await;
2362        }
2363    }
2364
2365    async fn sweep_one_suspended_session(&self, session_id: &str) {
2366        // A live runner means the session already resumed (the index status
2367        // only advances at its next terminal) — nothing to do.
2368        if self.runner_is_running(session_id).await {
2369            return;
2370        }
2371        let Some(session) = self.load_control_plane(session_id).await else {
2372            return;
2373        };
2374        let runtime_state = read_runtime_state(&session);
2375        let suspend_reason = session
2376            .metadata
2377            .get("runtime.suspend_reason")
2378            .map(String::as_str)
2379            .unwrap_or_default()
2380            .to_string();
2381        match (
2382            suspend_reason.as_str(),
2383            runtime_state.waiting_for_children.clone(),
2384        ) {
2385            // Human-gated or bash-owned waits: not ours to time out.
2386            ("waiting_for_bash", _)
2387            | ("awaiting_clarification", _)
2388            | ("awaiting_parent_approval", _) => {}
2389            (_, Some(wait)) => self.sweep_child_wait(session_id, wait).await,
2390            // Suspended with NO armed wait: either the coordinator cleared the
2391            // wait but its resume never spawned, or state is half-cleared.
2392            ("waiting_for_children", None) | ("", None) => {
2393                self.rescue_stranded_resume(session_id).await;
2394            }
2395            _ => {}
2396        }
2397    }
2398
2399    /// Evaluate one armed child wait against reality and act on what the
2400    /// durable state proves: dead children are synthesized terminal, an
2401    /// already-satisfied wait gets its lost wake replayed, an expired lease or
2402    /// empty wait force-resumes the parent.
2403    async fn sweep_child_wait(&self, parent_session_id: &str, wait: WaitingForChildrenState) {
2404        let now = Utc::now();
2405
2406        if wait.child_session_ids.is_empty() {
2407            tracing::warn!(
2408                %parent_session_id,
2409                "child-wait watchdog: wait armed over an empty child set; force-resuming"
2410            );
2411            self.force_resume_child_wait(parent_session_id, empty_child_wait_message())
2412                .await;
2413            return;
2414        }
2415
2416        // The 6h lease (previously written but never read). Expiry does not
2417        // kill children — child runners own child liveness; the parent is
2418        // resumed with a verify-don't-assume note.
2419        if wait.timeout_at.is_some_and(|deadline| now >= deadline) {
2420            tracing::warn!(
2421                %parent_session_id,
2422                "child-wait watchdog: wait lease expired; force-resuming parent"
2423            );
2424            self.force_resume_child_wait(
2425                parent_session_id,
2426                child_wait_lease_expired_message(&wait.child_session_ids),
2427            )
2428            .await;
2429            return;
2430        }
2431
2432        // Yield to the event-driven push on fresh waits.
2433        if now.signed_duration_since(wait.registered_at).num_seconds()
2434            < CHILD_WAIT_REGISTRATION_GRACE_SECS
2435        {
2436            return;
2437        }
2438
2439        let statuses: HashMap<String, Option<String>> = self
2440            .storage
2441            .list_child_run_statuses(parent_session_id)
2442            .await
2443            .unwrap_or_default()
2444            .into_iter()
2445            .collect();
2446
2447        struct DeadChild {
2448            child_id: String,
2449            status: String,
2450            reason: String,
2451            /// `false` = the id does not name a child of THIS parent (wait ids
2452            /// are model-provided and unvalidated): publish the wake but never
2453            /// persist onto / cancel / finalize the named session.
2454            owned: bool,
2455        }
2456
2457        let mut terminal: Vec<(String, String)> = Vec::new();
2458        let mut dead: Vec<DeadChild> = Vec::new();
2459        for child_id in &wait.child_session_ids {
2460            let status = statuses.get(child_id).and_then(|status| status.as_deref());
2461            if let Some(status) = status {
2462                if is_terminal_child_status(status) {
2463                    terminal.push((child_id.clone(), status.to_string()));
2464                    continue;
2465                }
2466            }
2467            if !is_dead_child_candidate_status(status) {
2468                continue;
2469            }
2470
2471            // Ownership check BEFORE anything destructive: `SubAgent.wait` ids
2472            // are model-provided and unvalidated, so an id absent from the
2473            // parent-scoped status map may name a REAL session in another tree
2474            // (a grandchild, a foreign root). Such a session must never be
2475            // mutated, cancelled, or finalized here — but the parent's bogus
2476            // wait entry still needs a synthetic completion to clear it.
2477            let control_plane = self.load_control_plane(child_id).await;
2478            let owned = control_plane
2479                .as_ref()
2480                .is_some_and(|cp| cp.parent_session_id.as_deref() == Some(parent_session_id));
2481            if !owned {
2482                dead.push(DeadChild {
2483                    child_id: child_id.clone(),
2484                    status: "error".to_string(),
2485                    reason: if control_plane.is_some() {
2486                        "waited-on session id is not a child of this session; clearing it \
2487                         from the wait without touching that session"
2488                            .to_string()
2489                    } else {
2490                        "waited-on child session does not exist".to_string()
2491                    },
2492                    owned: false,
2493                });
2494                continue;
2495            }
2496
2497            let runner = { self.agent_runners.read().await.get(child_id).cloned() };
2498            match runner {
2499                Some(runner) if matches!(runner.status, AgentStatus::Running) => {
2500                    // A live-looking runner: only intervene when it is frozen
2501                    // far PAST the per-child liveness limits — which proves the
2502                    // per-child watchdog machinery itself is dead (task
2503                    // panicked or lost), because it would have cancelled and
2504                    // published a timeout long before.
2505                    let last_activity = runner.last_event_at.unwrap_or(runner.started_at);
2506                    let idle_secs = now.signed_duration_since(last_activity).num_seconds();
2507                    let total_secs = now.signed_duration_since(runner.started_at).num_seconds();
2508                    let policy = match &control_plane {
2509                        Some(child) => {
2510                            crate::runtime::execution::spawn::watchdog_policy_for_session(child)
2511                        }
2512                        None => Default::default(),
2513                    };
2514                    let idle_limit = policy.max_idle_secs.saturating_add(STALE_RUNNER_SLACK_SECS);
2515                    let total_limit = policy
2516                        .max_total_secs
2517                        .saturating_add(STALE_RUNNER_SLACK_SECS);
2518                    if idle_secs >= idle_limit || total_secs >= total_limit {
2519                        runner.cancel_token.cancel();
2520                        dead.push(DeadChild {
2521                            child_id: child_id.clone(),
2522                            status: "timeout".to_string(),
2523                            reason: format!(
2524                                "child runner stalled: no events for {idle_secs}s \
2525                                 (limit {idle_limit}s including watchdog slack); \
2526                                 force-finalized by the child-wait watchdog"
2527                            ),
2528                            owned: true,
2529                        });
2530                    }
2531                }
2532                _ => {
2533                    // Nothing is driving this child, yet its index status will
2534                    // never advance by itself. The grace covers the enqueue →
2535                    // running-marker window and slow spawn queues.
2536                    let quiet_secs = control_plane
2537                        .as_ref()
2538                        .map(|child| now.signed_duration_since(child.updated_at).num_seconds())
2539                        .unwrap_or(i64::MAX);
2540                    if quiet_secs >= DEAD_CHILD_GRACE_SECS {
2541                        dead.push(DeadChild {
2542                            child_id: child_id.clone(),
2543                            status: "error".to_string(),
2544                            reason: format!(
2545                                "child runner lost (crashed task, dropped spawn job, or \
2546                                 process restart): index status {status:?} with no live \
2547                                 runner driving it"
2548                            ),
2549                            owned: true,
2550                        });
2551                    }
2552                }
2553            }
2554        }
2555
2556        if !dead.is_empty() {
2557            for entry in dead {
2558                tracing::warn!(
2559                    %parent_session_id,
2560                    child_session_id = %entry.child_id,
2561                    status = %entry.status,
2562                    reason = %entry.reason,
2563                    owned = entry.owned,
2564                    "child-wait watchdog: synthesizing terminal completion for dead child"
2565                );
2566                if entry.owned {
2567                    self.synthesize_child_completion(
2568                        parent_session_id,
2569                        &entry.child_id,
2570                        &entry.status,
2571                        Some(entry.reason),
2572                    )
2573                    .await;
2574                } else {
2575                    // Foreign / nonexistent id: wake the parent only.
2576                    self.publish_synthetic_completion(
2577                        parent_session_id,
2578                        &entry.child_id,
2579                        &entry.status,
2580                        Some(entry.reason),
2581                    )
2582                    .await;
2583                }
2584            }
2585            // The publishes above re-evaluate the wait policy themselves.
2586            return;
2587        }
2588
2589        // No dead children — but if the terminal set ALREADY satisfies the
2590        // policy, the original wake was lost (clobbered resume / retry budget
2591        // exhausted / completion raced the wait registration). Replay one real
2592        // completion through the canonical path; `on_child_completed` is
2593        // idempotent for an already-cleared wait.
2594        let terminal_ids: Vec<String> = terminal.iter().map(|(id, _)| id.clone()).collect();
2595        if let Some((child_id, status)) = select_replay_child(&terminal) {
2596            if wait_policy_satisfied(
2597                wait.wait_for,
2598                &wait.child_session_ids,
2599                &terminal_ids,
2600                child_id,
2601                status,
2602            ) {
2603                tracing::warn!(
2604                    %parent_session_id,
2605                    child_session_id = %child_id,
2606                    "child-wait watchdog: wait already satisfied but parent still suspended \
2607                     (lost wake); replaying the completion"
2608                );
2609                let error = self
2610                    .load_control_plane(child_id)
2611                    .await
2612                    .and_then(|child| child.last_run_error());
2613                self.publish_synthetic_completion(parent_session_id, child_id, status, error)
2614                    .await;
2615            }
2616        }
2617    }
2618
2619    /// Persist a synthesized terminal status on the child (so the index flips
2620    /// and nothing re-detects or re-suspends on it), finalize any lingering
2621    /// runner entry (so a future re-run can reserve), then publish through the
2622    /// canonical completion path — broadcast + `on_child_completed`, exactly
2623    /// like a real child terminal.
2624    async fn synthesize_child_completion(
2625        &self,
2626        parent_session_id: &str,
2627        child_session_id: &str,
2628        status: &str,
2629        error: Option<String>,
2630    ) {
2631        match self.storage.load_session(child_session_id).await {
2632            Ok(Some(mut child)) => {
2633                // Ownership guard (defense in depth — callers check too): only
2634                // a session that IS a child of this parent may be mutated. An
2635                // arbitrary session id named in a wait still wakes the parent
2636                // via the publish below, but its own state stays untouched.
2637                if child.parent_session_id.as_deref() != Some(parent_session_id) {
2638                    tracing::warn!(
2639                        %parent_session_id,
2640                        child_session_id = %child.id,
2641                        "child-wait watchdog: refusing to synthesize status onto a session \
2642                         that is not a child of this parent"
2643                    );
2644                    self.publish_synthetic_completion(
2645                        parent_session_id,
2646                        child_session_id,
2647                        status,
2648                        error,
2649                    )
2650                    .await;
2651                    return;
2652                }
2653                child.set_last_run_status(status);
2654                match &error {
2655                    Some(message) => child.set_last_run_error(message.clone()),
2656                    None => child.clear_last_run_error(),
2657                }
2658                child.updated_at = Utc::now();
2659                if let Err(save_error) = self.persistence.merge_save_runtime(&mut child).await {
2660                    tracing::warn!(
2661                        child_session_id = %child.id,
2662                        %save_error,
2663                        "child-wait watchdog: failed to persist synthesized terminal status"
2664                    );
2665                }
2666                self.sessions
2667                    .insert(child.id.clone(), Arc::new(parking_lot::RwLock::new(child)));
2668            }
2669            Ok(None) => {}
2670            Err(load_error) => {
2671                tracing::warn!(
2672                    %child_session_id,
2673                    %load_error,
2674                    "child-wait watchdog: failed to load child for synthesized terminal status"
2675                );
2676            }
2677        }
2678        finalize_runner(
2679            &self.agent_runners,
2680            child_session_id,
2681            &Err(bamboo_agent_core::AgentError::LLM(
2682                error
2683                    .clone()
2684                    .unwrap_or_else(|| format!("synthesized {status}")),
2685            )),
2686        )
2687        .await;
2688        self.publish_synthetic_completion(parent_session_id, child_session_id, status, error)
2689            .await;
2690    }
2691
2692    async fn publish_synthetic_completion(
2693        &self,
2694        parent_session_id: &str,
2695        child_session_id: &str,
2696        status: &str,
2697        error: Option<String>,
2698    ) {
2699        let publisher =
2700            crate::runtime::execution::session_events::ReplayableSessionEventPublisher::new(
2701                self.agent_runners.clone(),
2702                self.session_event_senders.clone(),
2703                self.account_feed_inbox.clone(),
2704            );
2705        let handler: Arc<dyn ChildCompletionHandler> = Arc::new(self.clone());
2706        crate::runtime::execution::spawn::publish_child_completion_parts(
2707            &publisher,
2708            Some(handler),
2709            parent_session_id.to_string(),
2710            child_session_id.to_string(),
2711            status.to_string(),
2712            error,
2713        )
2714        .await;
2715    }
2716
2717    /// A parent whose wait was already cleared (resume message appended) but
2718    /// whose resume never spawned — retry-budget exhaustion, root-tools not
2719    /// yet initialized, or a restart between clear and spawn. Detected by: no
2720    /// live runner, no armed wait, and a pending hidden runtime resume message
2721    /// as the LAST message. Resume is all that's left to do.
2722    async fn rescue_stranded_resume(&self, session_id: &str) {
2723        let Some(session) = self.load_session(session_id).await else {
2724            return;
2725        };
2726        let pending_runtime_resume = session.messages.last().is_some_and(|message| {
2727            matches!(message.role, Role::User)
2728                && message
2729                    .metadata
2730                    .as_ref()
2731                    .is_some_and(|meta| meta.get(RUNTIME_RESUME_MESSAGE_KIND_KEY).is_some())
2732        });
2733        if !pending_runtime_resume {
2734            return;
2735        }
2736        tracing::warn!(
2737            %session_id,
2738            "child-wait watchdog: stranded resume detected (wait cleared, resume never \
2739             spawned); resuming"
2740        );
2741        self.resume_parent(session_id.to_string()).await;
2742    }
2743
2744    /// Clear the parent's child wait, append `resume_message`, and drive the
2745    /// resume — with a bounded clobber-retry mirroring
2746    /// [`Self::perform_bash_resume`]: a suspending runner's one-shot finalize
2747    /// save can land after ours and revert the wait while dropping the
2748    /// message; we detect the re-armed wait and re-clear.
2749    async fn force_resume_child_wait(&self, session_id: &str, resume_message: Message) {
2750        const MAX_ATTEMPTS: u8 = 5;
2751        let lock = session_resume_lock(session_id);
2752        for attempt in 0..MAX_ATTEMPTS {
2753            if attempt > 0 {
2754                tokio::time::sleep(Duration::from_millis(200)).await;
2755            }
2756            {
2757                let _held = lock.lock().await;
2758                let Some(mut session) = self.load_session(session_id).await else {
2759                    return;
2760                };
2761                let mut runtime_state = read_runtime_state(&session);
2762                if runtime_state.waiting_for_children.is_none() {
2763                    // Another source already resumed this parent.
2764                    return;
2765                }
2766                runtime_state.waiting_for_children = None;
2767                runtime_state.status = AgentStatusState::Idle;
2768                runtime_state.suspension = None;
2769                write_runtime_state(&mut session, &runtime_state);
2770                session.metadata.remove("runtime.suspend_reason");
2771                session.add_message(resume_message.clone());
2772                session.updated_at = Utc::now();
2773                self.save_and_cache(&mut session).await;
2774            }
2775            let outcome = self.resume_parent(session_id.to_string()).await;
2776            match outcome {
2777                ResumeOutcome::Started { .. } | ResumeOutcome::NotFound => return,
2778                ResumeOutcome::Completed | ResumeOutcome::AlreadyRunning { .. } => {
2779                    // Only retry when the persisted wait was clobbered back to
2780                    // armed; if it stayed cleared with the message intact, the
2781                    // next sweep's stranded-resume rescue finishes the job.
2782                    let clobbered = self
2783                        .load_session(session_id)
2784                        .await
2785                        .map(|session| read_runtime_state(&session).waiting_for_children.is_some())
2786                        .unwrap_or(false);
2787                    if !clobbered {
2788                        return;
2789                    }
2790                }
2791            }
2792        }
2793        tracing::error!(
2794            %session_id,
2795            "child-wait watchdog: force-resume exhausted its clobber-retry budget"
2796        );
2797    }
2798}
2799
2800#[cfg(test)]
2801mod tests {
2802    use super::*;
2803    use bamboo_agent_core::Message;
2804    use bamboo_domain::SessionInboxPort;
2805    use futures::stream;
2806    use std::sync::atomic::{AtomicUsize, Ordering};
2807
2808    struct EmptyTools;
2809
2810    #[async_trait]
2811    impl bamboo_agent_core::tools::ToolExecutor for EmptyTools {
2812        async fn execute(
2813            &self,
2814            _call: &bamboo_agent_core::tools::ToolCall,
2815        ) -> Result<bamboo_agent_core::tools::ToolResult, bamboo_agent_core::tools::ToolError>
2816        {
2817            Err(bamboo_agent_core::tools::ToolError::NotFound(
2818                "no tools".to_string(),
2819            ))
2820        }
2821
2822        fn list_tools(&self) -> Vec<bamboo_agent_core::tools::ToolSchema> {
2823            Vec::new()
2824        }
2825    }
2826
2827    struct CompletedTestProvider;
2828
2829    #[async_trait]
2830    impl bamboo_llm::LLMProvider for CompletedTestProvider {
2831        async fn chat_stream(
2832            &self,
2833            _messages: &[Message],
2834            _tools: &[bamboo_agent_core::tools::ToolSchema],
2835            _max_output_tokens: Option<u32>,
2836            _model: &str,
2837        ) -> Result<bamboo_llm::LLMStream, bamboo_llm::LLMError> {
2838            let chunks: Vec<bamboo_llm::provider::Result<bamboo_llm::LLMChunk>> = vec![
2839                Ok(bamboo_llm::LLMChunk::Token("done".to_string())),
2840                Ok(bamboo_llm::LLMChunk::Done),
2841            ];
2842            Ok(Box::pin(stream::iter(chunks)))
2843        }
2844    }
2845
2846    struct CountingActivationSpawner {
2847        reservations: Arc<AtomicUsize>,
2848        launches: Arc<AtomicUsize>,
2849    }
2850
2851    #[async_trait]
2852    impl SessionActivationSpawner for CountingActivationSpawner {
2853        async fn reserve_activation(
2854            &self,
2855            target_session_id: &str,
2856            inbox_generation: u64,
2857        ) -> Result<SessionActivationReserveOutcome, bamboo_domain::SessionActivationError>
2858        {
2859            self.reservations.fetch_add(1, Ordering::SeqCst);
2860            let launches = self.launches.clone();
2861            Ok(SessionActivationReserveOutcome::Reserved(
2862                SessionActivationLaunch::new(
2863                    format!("{target_session_id}-{inbox_generation}"),
2864                    move || {
2865                        launches.fetch_add(1, Ordering::SeqCst);
2866                    },
2867                ),
2868            ))
2869        }
2870    }
2871
2872    async fn completion_inbox_fixture() -> (
2873        tempfile::TempDir,
2874        Arc<bamboo_storage::SessionStoreV2>,
2875        Arc<dyn SessionInboxPort>,
2876        Arc<ChildCompletionCoordinator>,
2877        Arc<AtomicUsize>,
2878        Arc<AtomicUsize>,
2879    ) {
2880        let temp = tempfile::tempdir().unwrap();
2881        let store = Arc::new(
2882            bamboo_storage::SessionStoreV2::new(temp.path().to_path_buf())
2883                .await
2884                .unwrap(),
2885        );
2886        let storage: Arc<dyn Storage> = store.clone();
2887        let locked = Arc::new(LockedSessionStore::new(storage.clone()));
2888        let inbox: Arc<dyn SessionInboxPort> = Arc::new(bamboo_storage::FileSessionInbox::new(
2889            store.clone(),
2890            bamboo_domain::SessionInboxLimits::default(),
2891        ));
2892        let router = crate::SessionActivationRouter::new();
2893        let messenger = Arc::new(crate::SessionMessenger::new(
2894            storage.clone(),
2895            inbox.clone(),
2896            router.clone(),
2897        ));
2898        let reservations = Arc::new(AtomicUsize::new(0));
2899        let launches = Arc::new(AtomicUsize::new(0));
2900        router
2901            .set_spawner(Arc::new(CountingActivationSpawner {
2902                reservations: reservations.clone(),
2903                launches: launches.clone(),
2904            }))
2905            .await;
2906        let provider: Arc<dyn bamboo_llm::LLMProvider> = Arc::new(CompletedTestProvider);
2907        let config = Arc::new(RwLock::new(Config::default()));
2908        let metrics = bamboo_metrics::MetricsCollector::spawn(
2909            Arc::new(bamboo_metrics::SqliteMetricsStorage::new(
2910                temp.path().join("metrics.db"),
2911            )),
2912            7,
2913        );
2914        let tools: Arc<dyn ToolExecutor> = Arc::new(EmptyTools);
2915        let agent = Arc::new(
2916            Agent::builder()
2917                .storage(storage.clone())
2918                .persistence(locked.clone())
2919                .session_inbox(inbox.clone())
2920                .activation_router(router)
2921                .session_messenger(messenger)
2922                .attachment_reader(store.clone())
2923                .skill_manager(Arc::new(bamboo_skills::SkillManager::new()))
2924                .metrics_collector(metrics)
2925                .config(config.clone())
2926                .provider(provider.clone())
2927                .default_tools(tools)
2928                .build()
2929                .unwrap(),
2930        );
2931        let mut providers = HashMap::new();
2932        providers.insert("test".to_string(), provider);
2933        let registry = Arc::new(ProviderRegistry::new(providers, "test".to_string()));
2934        let provider_router = Arc::new(ProviderModelRouter::new(registry.clone()));
2935        let coordinator = Arc::new(ChildCompletionCoordinator::new(
2936            storage,
2937            locked,
2938            Arc::new(dashmap::DashMap::new()),
2939            Arc::new(RwLock::new(HashMap::new())),
2940            Arc::new(RwLock::new(HashMap::new())),
2941            agent,
2942            config,
2943            registry,
2944            provider_router,
2945            temp.path().to_path_buf(),
2946            None,
2947        ));
2948        (temp, store, inbox, coordinator, reservations, launches)
2949    }
2950
2951    // ── child-wait watchdog pure helpers (issue #546) ────────────────────
2952
2953    #[test]
2954    fn dead_child_candidate_status_matrix() {
2955        // Never ran / lost before the running marker: dead candidate.
2956        assert!(is_dead_child_candidate_status(None));
2957        // Actively-reported non-terminal statuses: dead candidates when nothing
2958        // is driving them.
2959        assert!(is_dead_child_candidate_status(Some("running")));
2960        assert!(is_dead_child_candidate_status(Some("pending")));
2961        // Legitimately quiescent: waiting on a human / own children / bash.
2962        assert!(!is_dead_child_candidate_status(Some("suspended")));
2963        // Terminal statuses can never be "dead" — they are already done.
2964        for status in ["completed", "error", "timeout", "cancelled", "skipped"] {
2965            assert!(!is_dead_child_candidate_status(Some(status)), "{status}");
2966        }
2967    }
2968
2969    #[test]
2970    fn completion_child_ownership_gates_content_fold() {
2971        // Owned: the child's own parent linkage matches the reporting parent.
2972        assert!(completion_child_is_owned("parent-1", Some("parent-1")));
2973        // Foreign: a real session that belongs to a DIFFERENT parent — its
2974        // content must never be folded into parent-1's transcript.
2975        assert!(!completion_child_is_owned("parent-1", Some("parent-2")));
2976        // Root/unparented session, or a nonexistent id (linkage None).
2977        assert!(!completion_child_is_owned("parent-1", None));
2978    }
2979
2980    #[test]
2981    fn replay_child_prefers_error_like_for_first_error_policy() {
2982        let terminal = vec![
2983            ("c-ok".to_string(), "completed".to_string()),
2984            ("c-err".to_string(), "timeout".to_string()),
2985            ("c-late".to_string(), "completed".to_string()),
2986        ];
2987        let (id, status) = select_replay_child(&terminal).expect("non-empty");
2988        assert_eq!(id, "c-err");
2989        assert_eq!(status, "timeout");
2990
2991        let all_ok = vec![
2992            ("c-1".to_string(), "completed".to_string()),
2993            ("c-2".to_string(), "completed".to_string()),
2994        ];
2995        let (id, _) = select_replay_child(&all_ok).expect("non-empty");
2996        assert_eq!(id, "c-2");
2997
2998        assert!(select_replay_child(&[]).is_none());
2999    }
3000
3001    #[test]
3002    fn watchdog_resume_messages_are_hidden_runtime_messages() {
3003        for message in [
3004            empty_child_wait_message(),
3005            child_wait_lease_expired_message(&["c-1".to_string(), "c-2".to_string()]),
3006        ] {
3007            assert!(matches!(message.role, Role::User));
3008            let meta = message.metadata.expect("hidden runtime metadata");
3009            assert_eq!(meta[RUNTIME_RESUME_MESSAGE_HIDDEN_KEY], true);
3010            assert_eq!(
3011                meta[RUNTIME_RESUME_MESSAGE_KIND_KEY],
3012                "child_wait_watchdog_resume"
3013            );
3014        }
3015        let lease = child_wait_lease_expired_message(&["c-1".to_string()]);
3016        // The lease message must never claim the children finished.
3017        assert!(lease.content.contains("NOT cancelled"));
3018        assert!(lease.content.contains("c-1"));
3019    }
3020
3021    // ── on_child_completed terminality guard (issue #546) ────────────────
3022
3023    #[test]
3024    fn non_terminal_statuses_never_satisfy_wait_policies() {
3025        // The guard keys on `is_terminal_child_status`; "suspended" (and any
3026        // unknown non-terminal string) must not count toward any policy.
3027        assert!(!is_terminal_child_status("suspended"));
3028        assert!(!is_terminal_child_status("running"));
3029        assert!(!is_terminal_child_status("pending"));
3030    }
3031
3032    fn make_completion(status: &str) -> ChildCompletion {
3033        ChildCompletion {
3034            parent_session_id: "parent-1".to_string(),
3035            child_session_id: "child-1".to_string(),
3036            status: status.to_string(),
3037            error: None,
3038            completed_at: Utc::now(),
3039        }
3040    }
3041
3042    #[test]
3043    fn oversized_child_outcome_is_bounded_and_keeps_full_content_identity() {
3044        let mut completion = make_completion("completed");
3045        let wait_registered_at = Utc::now();
3046        let huge = format!("prefix-A-{}", "x".repeat(300 * 1024));
3047        let presentation = runtime_resume_message(&completion, 0, Some(&huge));
3048        let first = child_completion_envelope(
3049            &completion,
3050            wait_registered_at,
3051            Some(huge.clone()),
3052            &presentation,
3053        );
3054        assert!(
3055            serde_json::to_vec(&first).unwrap().len()
3056                < bamboo_domain::SessionInboxLimits::default().max_payload_bytes
3057        );
3058        let SessionMessageBody::ChildOutcome(outcome) = &first.body else {
3059            panic!("typed child outcome");
3060        };
3061        let stored = outcome.result.as_deref().unwrap();
3062        assert!(stored.contains("sha256="));
3063        assert!(stored.contains("SubAgent.get"));
3064        assert!(stored.len() < CHILD_COMPLETION_INLINE_FIELD_BYTES);
3065
3066        // Retry-only completion timestamps and provider presentation do not
3067        // change the logical id; full oversized content does.
3068        completion.completed_at += chrono::Duration::seconds(1);
3069        let exact_retry = child_completion_envelope(
3070            &completion,
3071            wait_registered_at,
3072            Some(huge.clone()),
3073            &runtime_resume_message(&completion, 9, Some(&huge)),
3074        );
3075        assert_eq!(exact_retry.id, first.id);
3076        let changed = format!("prefix-B-{}", "x".repeat(300 * 1024));
3077        let corrected = child_completion_envelope(
3078            &completion,
3079            wait_registered_at,
3080            Some(changed.clone()),
3081            &runtime_resume_message(&completion, 0, Some(&changed)),
3082        );
3083        assert_ne!(corrected.id, first.id);
3084    }
3085
3086    #[tokio::test]
3087    async fn oversized_child_completion_clears_wait_and_activates_exactly_once() {
3088        let (_temp, store, inbox, coordinator, reservations, launches) =
3089            completion_inbox_fixture().await;
3090        let parent_id = "oversized-parent";
3091        let child_id = "oversized-child";
3092        let now = Utc::now();
3093        let mut parent = Session::new(parent_id, "model");
3094        let mut parent_runtime = AgentRuntimeState::new("waiting-run");
3095        parent_runtime.status = AgentStatusState::Suspended;
3096        parent_runtime.waiting_for_children = Some(WaitingForChildrenState::for_children(
3097            vec![child_id.to_string()],
3098            ChildWaitPolicy::All,
3099            now,
3100        ));
3101        parent_runtime.suspension = Some(SuspensionState {
3102            reason: "waiting_for_children".to_string(),
3103            suspended_at: now,
3104            resumable: true,
3105            hook_point: Some("ChildCompletion".to_string()),
3106        });
3107        write_runtime_state(&mut parent, &parent_runtime);
3108        parent.metadata.insert(
3109            "runtime.suspend_reason".to_string(),
3110            "waiting_for_children".to_string(),
3111        );
3112        store.save_session(&parent).await.unwrap();
3113
3114        let mut child = Session::new_child(child_id, parent_id, "model", "Child");
3115        child.add_message(Message::assistant("z".repeat(300 * 1024), None));
3116        child.set_last_run_status("completed");
3117        store.save_session(&child).await.unwrap();
3118        let completion = ChildCompletion {
3119            parent_session_id: parent_id.to_string(),
3120            child_session_id: child_id.to_string(),
3121            status: "completed".to_string(),
3122            error: None,
3123            completed_at: Utc::now(),
3124        };
3125
3126        ChildCompletionHandler::on_child_completed(coordinator.as_ref(), completion.clone()).await;
3127        let durable_parent = store.load_session(parent_id).await.unwrap().unwrap();
3128        let durable_runtime = read_runtime_state(&durable_parent);
3129        assert!(durable_runtime.waiting_for_children.is_none());
3130        assert_eq!(durable_runtime.status, AgentStatusState::Idle);
3131        assert!(!durable_parent
3132            .metadata
3133            .contains_key("runtime.suspend_reason"));
3134        let backlog = inbox.inspect(parent_id).await.unwrap();
3135        assert_eq!(backlog.pending + backlog.claimed, 1);
3136        assert!(backlog.activation_pending());
3137        assert_eq!(reservations.load(Ordering::SeqCst), 1);
3138        assert_eq!(launches.load(Ordering::SeqCst), 1);
3139
3140        let claim = inbox.claim(parent_id, 1).await.unwrap().remove(0);
3141        assert!(
3142            serde_json::to_vec(&claim.envelope).unwrap().len()
3143                < bamboo_domain::SessionInboxLimits::default().max_payload_bytes
3144        );
3145        // Duplicate terminal notification after the wait was already cleared
3146        // cannot enqueue or activate a second outcome.
3147        ChildCompletionHandler::on_child_completed(coordinator.as_ref(), completion).await;
3148        assert_eq!(reservations.load(Ordering::SeqCst), 1);
3149        assert_eq!(launches.load(Ordering::SeqCst), 1);
3150        assert_eq!(inbox.inspect(parent_id).await.unwrap().claimed, 1);
3151    }
3152
3153    #[tokio::test]
3154    async fn latest_locked_activation_mutation_preserves_wait_armed_after_stale_load() {
3155        let temp = tempfile::tempdir().unwrap();
3156        let store = Arc::new(
3157            bamboo_storage::SessionStoreV2::new(temp.path().to_path_buf())
3158                .await
3159                .unwrap(),
3160        );
3161        let storage: Arc<dyn Storage> = store.clone();
3162        let locked = LockedSessionStore::new(storage);
3163        let mut session = Session::new("activation-stale-wait", "model");
3164        session.agent_runtime_state = Some(AgentRuntimeState::new("old-run"));
3165        store.save_session(&session).await.unwrap();
3166
3167        // The activation path has already loaded this stale, wait-free snapshot.
3168        let stale = store.load_session(&session.id).await.unwrap().unwrap();
3169        assert!(read_runtime_state(&stale).waiting_for_children.is_none());
3170
3171        locked
3172            .update_runtime_config(&session.id, |latest| {
3173                let mut state = read_runtime_state(latest);
3174                state.status = AgentStatusState::Suspended;
3175                state.waiting_for_children = Some(WaitingForChildrenState::for_children(
3176                    vec!["child-new".to_string()],
3177                    ChildWaitPolicy::All,
3178                    Utc::now(),
3179                ));
3180                state.suspension = Some(SuspensionState {
3181                    reason: "waiting_for_children".to_string(),
3182                    suspended_at: Utc::now(),
3183                    resumable: true,
3184                    hook_point: None,
3185                });
3186                write_runtime_state(latest, &state);
3187                latest.metadata.insert(
3188                    "runtime.suspend_reason".to_string(),
3189                    "waiting_for_children".to_string(),
3190                );
3191            })
3192            .await
3193            .unwrap();
3194
3195        let (prepared, ready) = prepare_session_inbox_activation(&locked, &session.id, false)
3196            .await
3197            .unwrap()
3198            .unwrap();
3199        assert!(!ready, "latest durable specific wait must block activation");
3200        let state = read_runtime_state(&prepared);
3201        assert!(state.waiting_for_children.is_some());
3202        assert_eq!(state.status, AgentStatusState::Suspended);
3203        assert_eq!(
3204            prepared
3205                .metadata
3206                .get("runtime.suspend_reason")
3207                .map(String::as_str),
3208            Some("waiting_for_children")
3209        );
3210    }
3211
3212    #[tokio::test]
3213    async fn partial_child_and_bash_backlogs_remain_inert_after_inbox_reopen() {
3214        let temp = tempfile::tempdir().unwrap();
3215        let store = Arc::new(
3216            bamboo_storage::SessionStoreV2::new(temp.path().to_path_buf())
3217                .await
3218                .unwrap(),
3219        );
3220        store
3221            .save_session(&Session::new("restart-child-parent", "model"))
3222            .await
3223            .unwrap();
3224        store
3225            .save_session(&Session::new("restart-bash-parent", "model"))
3226            .await
3227            .unwrap();
3228        let inbox = bamboo_storage::FileSessionInbox::new(
3229            store.clone(),
3230            bamboo_domain::SessionInboxLimits::default(),
3231        );
3232
3233        let completion = ChildCompletion {
3234            parent_session_id: "restart-child-parent".to_string(),
3235            child_session_id: "child-a".to_string(),
3236            status: "completed".to_string(),
3237            error: None,
3238            completed_at: Utc::now(),
3239        };
3240        let child_resume = runtime_resume_message(&completion, 1, Some("first child"));
3241        inbox
3242            .deliver(&child_completion_envelope(
3243                &completion,
3244                Utc::now(),
3245                Some("first child".to_string()),
3246                &child_resume,
3247            ))
3248            .await
3249            .unwrap();
3250        let bash = BashCompletionInfo {
3251            session_id: "restart-bash-parent".to_string(),
3252            bash_id: "bash-a".to_string(),
3253            command: "true".to_string(),
3254            exit_code: Some(0),
3255            status: "completed".to_string(),
3256            output_tail: String::new(),
3257        };
3258        inbox
3259            .deliver(&bash_completion_envelope(&bash))
3260            .await
3261            .unwrap();
3262
3263        let reopened = bamboo_storage::FileSessionInbox::new(
3264            store,
3265            bamboo_domain::SessionInboxLimits::default(),
3266        );
3267        for session_id in ["restart-child-parent", "restart-bash-parent"] {
3268            let backlog = reopened.inspect(session_id).await.unwrap();
3269            assert_eq!(backlog.pending, 1);
3270            assert_eq!(backlog.activation_generation, 0);
3271            assert!(
3272                !backlog.activation_pending(),
3273                "startup must not run a partial wait backlog for {session_id}"
3274            );
3275        }
3276    }
3277
3278    // ── ② derive completed children from the index ──────────────────────
3279
3280    struct StubChildIndex {
3281        children: Vec<(String, Option<String>)>,
3282    }
3283
3284    #[async_trait]
3285    impl Storage for StubChildIndex {
3286        async fn save_session(&self, _session: &Session) -> std::io::Result<()> {
3287            Ok(())
3288        }
3289        async fn load_session(&self, _id: &str) -> std::io::Result<Option<Session>> {
3290            Ok(None)
3291        }
3292        async fn delete_session(&self, _id: &str) -> std::io::Result<bool> {
3293            Ok(false)
3294        }
3295        async fn list_child_run_statuses(
3296            &self,
3297            _parent_session_id: &str,
3298        ) -> std::io::Result<Vec<(String, Option<String>)>> {
3299            Ok(self.children.clone())
3300        }
3301    }
3302
3303    #[tokio::test]
3304    async fn derive_completed_only_includes_terminal_children() {
3305        let storage: Arc<dyn Storage> = Arc::new(StubChildIndex {
3306            children: vec![
3307                ("a".into(), Some("completed".into())),
3308                ("b".into(), Some("running".into())),
3309                ("c".into(), Some("error".into())),
3310                ("d".into(), None),
3311            ],
3312        });
3313        let completed = derive_completed_child_ids(&storage, "parent-1", "b").await;
3314        // Terminal from index: a, c. Plus the just-completed child b folded in.
3315        assert_eq!(
3316            completed,
3317            vec!["a".to_string(), "b".to_string(), "c".to_string()]
3318        );
3319    }
3320
3321    #[tokio::test]
3322    async fn derive_completed_folds_in_just_completed_when_index_lags() {
3323        // Index hasn't caught up — reports the child as still running.
3324        let storage: Arc<dyn Storage> = Arc::new(StubChildIndex {
3325            children: vec![("only".into(), Some("running".into()))],
3326        });
3327        let completed = derive_completed_child_ids(&storage, "parent-1", "only").await;
3328        assert_eq!(completed, vec!["only".to_string()]);
3329    }
3330
3331    #[test]
3332    fn wait_policy_all_uses_derived_completed_set() {
3333        let waited = vec!["a".to_string(), "b".to_string()];
3334        assert!(!wait_policy_satisfied(
3335            ChildWaitPolicy::All,
3336            &waited,
3337            &["a".to_string()],
3338            "a",
3339            "completed"
3340        ));
3341        assert!(wait_policy_satisfied(
3342            ChildWaitPolicy::All,
3343            &waited,
3344            &["a".to_string(), "b".to_string()],
3345            "b",
3346            "completed"
3347        ));
3348    }
3349
3350    #[test]
3351    fn wait_policy_first_error_requires_tracked_membership() {
3352        let waited = vec!["a".to_string(), "b".to_string()];
3353        // An error from a TRACKED child resumes immediately.
3354        assert!(wait_policy_satisfied(
3355            ChildWaitPolicy::FirstError,
3356            &waited,
3357            &["a".to_string()],
3358            "a",
3359            "error"
3360        ));
3361        // An error-like completion from an UNTRACKED child (e.g. a zombie
3362        // task from an earlier run waking up late) must not resume the wait.
3363        assert!(!wait_policy_satisfied(
3364            ChildWaitPolicy::FirstError,
3365            &waited,
3366            &["a".to_string()],
3367            "stray-child",
3368            "timeout"
3369        ));
3370        // The all-complete fallback still applies regardless of the reporter.
3371        assert!(wait_policy_satisfied(
3372            ChildWaitPolicy::FirstError,
3373            &waited,
3374            &["a".to_string(), "b".to_string()],
3375            "stray-child",
3376            "completed"
3377        ));
3378    }
3379
3380    #[test]
3381    fn child_final_assistant_text_returns_last_assistant() {
3382        let mut session = Session::new("child-1", "gpt-4");
3383        session.messages.push(Message::user("hi"));
3384        session
3385            .messages
3386            .push(Message::assistant("first answer", None));
3387        session.messages.push(Message::user("again"));
3388        session
3389            .messages
3390            .push(Message::assistant("final answer", None));
3391
3392        assert_eq!(
3393            child_final_assistant_text(&session).as_deref(),
3394            Some("final answer")
3395        );
3396    }
3397
3398    #[test]
3399    fn child_final_assistant_text_returns_none_when_blank() {
3400        let mut session = Session::new("child-1", "gpt-4");
3401        session.messages.push(Message::assistant("   ", None));
3402        assert!(child_final_assistant_text(&session).is_none());
3403    }
3404
3405    #[test]
3406    fn child_final_assistant_text_returns_none_when_no_assistant() {
3407        let mut session = Session::new("child-1", "gpt-4");
3408        session.messages.push(Message::user("hi"));
3409        assert!(child_final_assistant_text(&session).is_none());
3410    }
3411
3412    #[test]
3413    fn runtime_resume_message_folds_full_response_without_truncation() {
3414        // A very long child final response is folded in verbatim (no 4000-char
3415        // cap, no truncation marker).
3416        let completion = make_completion("completed");
3417        let long: String = "a".repeat(10_000);
3418        let message = runtime_resume_message(&completion, 0, Some(&long));
3419        assert!(message.content.contains(&long));
3420        assert!(!message.content.contains("truncated"));
3421    }
3422
3423    #[test]
3424    fn runtime_resume_message_includes_child_response_when_provided() {
3425        let completion = make_completion("completed");
3426        let message = runtime_resume_message(&completion, 0, Some("the answer is 42"));
3427
3428        assert!(matches!(message.role, Role::User));
3429        // Folded child results are now compressible so the parent context can
3430        // reclaim them under compaction.
3431        assert!(!message.never_compress);
3432        assert!(message.content.contains("Child final response:"));
3433        assert!(message.content.contains("the answer is 42"));
3434
3435        let metadata = message.metadata.expect("metadata present");
3436        assert_eq!(
3437            metadata.get("hidden_from_ui").and_then(|v| v.as_bool()),
3438            Some(true)
3439        );
3440        assert_eq!(
3441            metadata.get("runtime_kind").and_then(|v| v.as_str()),
3442            Some("child_completion_resume")
3443        );
3444        assert_eq!(
3445            metadata
3446                .get("child_final_response_included")
3447                .and_then(|v| v.as_bool()),
3448            Some(true)
3449        );
3450    }
3451
3452    #[test]
3453    fn runtime_resume_message_falls_back_to_error_when_no_response() {
3454        let mut completion = make_completion("error");
3455        completion.error = Some("boom".to_string());
3456
3457        let message = runtime_resume_message(&completion, 1, None);
3458        assert!(message.content.contains("Child error:"));
3459        assert!(message.content.contains("boom"));
3460        let metadata = message.metadata.expect("metadata present");
3461        assert_eq!(
3462            metadata
3463                .get("child_final_response_included")
3464                .and_then(|v| v.as_bool()),
3465            Some(false)
3466        );
3467    }
3468
3469    #[test]
3470    fn runtime_resume_message_minimal_when_no_response_and_no_error() {
3471        let completion = make_completion("completed");
3472        let message = runtime_resume_message(&completion, 2, None);
3473        assert!(!message.content.contains("Child final response:"));
3474        assert!(!message.content.contains("Child error:"));
3475        assert!(message.content.contains("Resume the parent task"));
3476    }
3477
3478    #[test]
3479    fn read_config_snapshot_refreshes_cached_snapshot_from_live_config() {
3480        let runtime = tokio::runtime::Runtime::new().expect("runtime");
3481
3482        runtime.block_on(async {
3483            let config = Arc::new(RwLock::new(Config::default()));
3484            config.write().await.provider = "copilot".to_string();
3485            let cached_config = StdRwLock::new(Config::default());
3486
3487            let snapshot = read_config_snapshot(&config, &cached_config);
3488
3489            assert_eq!(snapshot.provider, "copilot");
3490            assert_eq!(
3491                cached_config.read().expect("cached snapshot lock").provider,
3492                "copilot"
3493            );
3494        });
3495    }
3496
3497    #[test]
3498    fn read_config_snapshot_uses_cached_snapshot_when_live_lock_is_busy() {
3499        let runtime = tokio::runtime::Runtime::new().expect("runtime");
3500
3501        runtime.block_on(async {
3502            let mut cached_snapshot = Config::default();
3503            cached_snapshot.provider = "cached-provider".to_string();
3504
3505            let config = Arc::new(RwLock::new(Config::default()));
3506            let cached_config = StdRwLock::new(cached_snapshot);
3507            let _write_guard = config.write().await;
3508
3509            let snapshot = read_config_snapshot(&config, &cached_config);
3510
3511            assert_eq!(snapshot.provider, "cached-provider");
3512        });
3513    }
3514
3515    // ── Bash self-resume (issue #84 Phase 2b): deadline message + clobber-retry ──
3516
3517    #[test]
3518    fn bash_completion_resume_message_normal_announces_completion() {
3519        let ids = vec!["bg-1".to_string(), "bg-2".to_string()];
3520        let message = bash_completion_resume_message(&ids, false);
3521        // Normal path: the shells genuinely finished.
3522        assert!(
3523            message.content.contains("have completed"),
3524            "normal resume message must announce completion: {}",
3525            message.content
3526        );
3527        // Hidden + compressible so the resume gate sees it but the UI hides it.
3528        let metadata = message.metadata.expect("metadata present");
3529        assert_eq!(
3530            metadata
3531                .get(RUNTIME_RESUME_MESSAGE_HIDDEN_KEY)
3532                .and_then(|v| v.as_bool()),
3533            Some(true),
3534            "resume message must be hidden from the UI"
3535        );
3536        assert_eq!(
3537            metadata
3538                .get(RUNTIME_RESUME_MESSAGE_KIND_KEY)
3539                .and_then(|v| v.as_str()),
3540            Some(BASH_COMPLETION_RESUME_KIND),
3541            "resume message must carry the bash-completion kind discriminant"
3542        );
3543    }
3544
3545    #[test]
3546    fn bash_completion_resume_message_deadline_does_not_claim_completion() {
3547        // The 6h+10m deadline force-breaks with shells STILL running. The message
3548        // must NOT say "have completed" — that would let the model assume success
3549        // on a false premise. It must direct the model to verify with BashOutput.
3550        let ids = vec!["bg-long".to_string()];
3551        let message = bash_completion_resume_message(&ids, true);
3552        assert!(
3553            !message.content.contains("have completed"),
3554            "deadline resume message must NOT claim the shells completed: {}",
3555            message.content
3556        );
3557        assert!(
3558            message.content.contains("may still be running"),
3559            "deadline resume message must warn shells may still be running: {}",
3560            message.content
3561        );
3562        assert!(
3563            message.content.contains("BashOutput"),
3564            "deadline resume message must direct verification via BashOutput: {}",
3565            message.content
3566        );
3567        // Same hidden/kind shape so the resume gate is satisfied identically.
3568        let metadata = message.metadata.expect("metadata present");
3569        assert_eq!(
3570            metadata
3571                .get(RUNTIME_RESUME_MESSAGE_KIND_KEY)
3572                .and_then(|v| v.as_str()),
3573            Some(BASH_COMPLETION_RESUME_KIND)
3574        );
3575    }
3576
3577    #[test]
3578    fn bash_resume_should_retry_matrix() {
3579        // The finalize-clobber retry predicate (issue #84 Phase 2b). Retry only
3580        // when the resume did NOT spawn (Completed / AlreadyRunning) AND the
3581        // persisted bash wait is still set on reload — the clobber signature.
3582
3583        // Started: the resume fired — never retry, regardless of persisted state.
3584        assert!(!bash_resume_should_retry(
3585            &ResumeOutcome::Started { run_id: "r".into() },
3586            true
3587        ));
3588        assert!(!bash_resume_should_retry(
3589            &ResumeOutcome::Started { run_id: "r".into() },
3590            false
3591        ));
3592
3593        // NotFound: session gone — never retry.
3594        assert!(!bash_resume_should_retry(&ResumeOutcome::NotFound, true));
3595        assert!(!bash_resume_should_retry(&ResumeOutcome::NotFound, false));
3596
3597        // Completed + persisted wait still set ⇒ finalize-clobber ⇒ retry.
3598        assert!(bash_resume_should_retry(&ResumeOutcome::Completed, true));
3599        // Completed + persisted wait cleared ⇒ handled (our message stuck, or a
3600        // concurrent resume finished) ⇒ stop.
3601        assert!(!bash_resume_should_retry(&ResumeOutcome::Completed, false));
3602
3603        // AlreadyRunning + persisted wait still set ⇒ clobbered while a runner is
3604        // (stale-)active ⇒ retry to re-establish the resume message.
3605        assert!(bash_resume_should_retry(
3606            &ResumeOutcome::AlreadyRunning { run_id: "r".into() },
3607            true
3608        ));
3609        // AlreadyRunning + wait cleared ⇒ a runner owns the session ⇒ stop.
3610        assert!(!bash_resume_should_retry(
3611            &ResumeOutcome::AlreadyRunning { run_id: "r".into() },
3612            false
3613        ));
3614    }
3615
3616    // ── bash completion injection body (Phase 2b follow-up) ──────────────
3617
3618    #[test]
3619    fn injection_body_includes_status_exit_command_and_tail() {
3620        let info = BashCompletionInfo {
3621            session_id: "s".into(),
3622            bash_id: "abc123".into(),
3623            command: "make build".into(),
3624            exit_code: Some(0),
3625            status: "completed".into(),
3626            output_tail: "BUILD OK".into(),
3627        };
3628        let body = bash_completion_injection_body(&info);
3629        assert!(body.contains("abc123"), "body: {body}");
3630        assert!(body.contains("make build"), "body: {body}");
3631        assert!(body.contains("completed"), "body: {body}");
3632        assert!(body.contains("exit code 0"), "body: {body}");
3633        assert!(body.contains("BUILD OK"), "body: {body}");
3634        // The model is pointed at BashOutput for the full log.
3635        assert!(body.contains("BashOutput"), "body: {body}");
3636        assert!(body.contains("bash_id=\"abc123\""), "body: {body}");
3637    }
3638
3639    #[test]
3640    fn injection_body_handles_no_output_and_signal_kill() {
3641        let info = BashCompletionInfo {
3642            session_id: "s".into(),
3643            bash_id: "xyz".into(),
3644            command: "sleep 99".into(),
3645            exit_code: None,
3646            status: "killed".into(),
3647            output_tail: String::new(),
3648        };
3649        let body = bash_completion_injection_body(&info);
3650        assert!(body.contains("killed"), "body: {body}");
3651        assert!(body.contains("none (signal/killed)"), "body: {body}");
3652        assert!(body.contains("no captured output"), "body: {body}");
3653        // No output tail section when there is nothing to show.
3654        assert!(!body.contains("Output tail:"), "body: {body}");
3655    }
3656
3657    #[test]
3658    fn background_completion_builds_post_tool_use_payload_and_feedback() {
3659        let mut info = BashCompletionInfo {
3660            session_id: "s".into(),
3661            bash_id: "bg-7".into(),
3662            command: "cargo test".into(),
3663            exit_code: Some(0),
3664            status: "completed".into(),
3665            output_tail: "test result: ok".into(),
3666        };
3667
3668        let payload = background_bash_post_tool_payload(&info);
3669        match payload {
3670            HookPayload::ToolResult {
3671                tool_name,
3672                tool_call_id,
3673                outcome,
3674            } => {
3675                assert_eq!(tool_name, "Bash");
3676                assert_eq!(tool_call_id, "bg-7");
3677                assert!(outcome.success);
3678                let response: serde_json::Value =
3679                    serde_json::from_str(outcome.result.as_deref().unwrap()).unwrap();
3680                assert_eq!(response["command"], "cargo test");
3681                assert_eq!(response["exit_code"], 0);
3682                assert_eq!(response["status"], "completed");
3683                assert_eq!(response["output_tail"], "test result: ok");
3684            }
3685            other => panic!("expected PostToolUse payload, got {other:?}"),
3686        }
3687
3688        append_background_bash_hook_feedback(
3689            &mut info,
3690            vec!["Run the formatter before continuing".to_string()],
3691        );
3692        assert!(info.output_tail.contains("<post_tool_use_feedback>"));
3693        assert!(info
3694            .output_tail
3695            .contains("Run the formatter before continuing"));
3696    }
3697
3698    #[tokio::test]
3699    async fn bash_completion_payload_identity_matches_file_inbox_idempotency() {
3700        let baseline = BashCompletionInfo {
3701            session_id: "session".into(),
3702            bash_id: "bg-7".into(),
3703            command: "cargo test".into(),
3704            exit_code: Some(0),
3705            status: "completed".into(),
3706            output_tail: "first tail".into(),
3707        };
3708        let mut retried = baseline.clone();
3709        retried.output_tail = "first tail\nlater bytes\n<hook feedback>".into();
3710        retried.status = "completed-after-hook".into();
3711        let baseline_envelope = bash_completion_envelope(&baseline);
3712        let exact_retry = bash_completion_envelope(&baseline);
3713        let corrected_envelope = bash_completion_envelope(&retried);
3714        assert_eq!(baseline_envelope.id, exact_retry.id);
3715        assert_ne!(
3716            baseline_envelope.id, corrected_envelope.id,
3717            "changed payload semantics must receive a distinct id"
3718        );
3719
3720        let mut other_shell = baseline.clone();
3721        other_shell.bash_id = "bg-8".into();
3722        assert_ne!(
3723            bash_completion_envelope(&baseline).id,
3724            bash_completion_envelope(&other_shell).id
3725        );
3726
3727        let temp = tempfile::tempdir().unwrap();
3728        let store = Arc::new(
3729            bamboo_storage::SessionStoreV2::new(temp.path().to_path_buf())
3730                .await
3731                .unwrap(),
3732        );
3733        store
3734            .save_session(&Session::new("session", "model"))
3735            .await
3736            .unwrap();
3737        let inbox = bamboo_storage::FileSessionInbox::new(
3738            store,
3739            bamboo_domain::SessionInboxLimits::default(),
3740        );
3741        let first = inbox.deliver(&baseline_envelope).await.unwrap();
3742        let duplicate = inbox.deliver(&exact_retry).await.unwrap();
3743        let corrected = inbox.deliver(&corrected_envelope).await.unwrap();
3744        assert_eq!(duplicate, first, "exact payload retry is idempotent");
3745        assert_ne!(corrected.id, first.id);
3746        assert_eq!(corrected.generation, first.generation + 1);
3747        assert_eq!(inbox.inspect("session").await.unwrap().pending, 2);
3748    }
3749
3750    #[cfg(unix)]
3751    #[tokio::test]
3752    async fn background_completion_fires_configured_post_tool_use_command() {
3753        use bamboo_config::{
3754            LifecycleHookGroup, LifecycleHookHandler, LifecycleHooksConfig,
3755            DEFAULT_LIFECYCLE_HOOK_TIMEOUT_MS,
3756        };
3757
3758        let dir = tempfile::tempdir().unwrap();
3759        let output = dir.path().join("background-post-tool.json");
3760        let command = format!(
3761            "cat > '{}'; printf '%s' '{{\"additional_context\":\"inspect the completed build log\"}}'",
3762            output.display()
3763        );
3764        let config = LifecycleHooksConfig {
3765            enabled: true,
3766            post_tool_use: vec![LifecycleHookGroup {
3767                enabled: true,
3768                matcher: Some("^Bash$".to_string()),
3769                hooks: vec![LifecycleHookHandler::command(
3770                    command,
3771                    DEFAULT_LIFECYCLE_HOOK_TIMEOUT_MS,
3772                )],
3773            }],
3774            ..Default::default()
3775        };
3776        let mut session = Session::new("session-bg-hook", "test-model");
3777        session.workspace = Some(dir.path().to_string_lossy().into_owned());
3778        let mut info = BashCompletionInfo {
3779            session_id: session.id.clone(),
3780            bash_id: "bg-9".into(),
3781            command: "cargo test".into(),
3782            exit_code: Some(0),
3783            status: "completed".into(),
3784            output_tail: "test result: ok".into(),
3785        };
3786
3787        assert!(run_background_bash_post_tool_hooks(&config, None, &session, &mut info).await);
3788        let envelope: serde_json::Value =
3789            serde_json::from_str(&std::fs::read_to_string(output).unwrap()).unwrap();
3790        assert_eq!(envelope["hook_event_name"], "PostToolUse");
3791        assert_eq!(envelope["tool_name"], "Bash");
3792        assert_eq!(envelope["payload"]["tool_call_id"], "bg-9");
3793        let response = envelope["tool_response"]["result"]
3794            .as_str()
3795            .map(serde_json::from_str::<serde_json::Value>)
3796            .transpose()
3797            .unwrap()
3798            .unwrap();
3799        assert_eq!(response["command"], "cargo test");
3800        assert_eq!(response["status"], "completed");
3801        assert!(info.output_tail.contains("inspect the completed build log"));
3802    }
3803
3804    async fn temp_store() -> (tempfile::TempDir, Arc<dyn Storage>, LockedSessionStore) {
3805        let temp = tempfile::tempdir().unwrap();
3806        let storage: Arc<dyn Storage> = Arc::new(
3807            bamboo_storage::v2::SessionStoreV2::new(temp.path().to_path_buf())
3808                .await
3809                .expect("storage init"),
3810        );
3811        let persistence = LockedSessionStore::new(storage.clone());
3812        (temp, storage, persistence)
3813    }
3814
3815    #[tokio::test]
3816    async fn enqueue_writes_pending_injection_and_preserves_messages() {
3817        let (_temp, storage, persistence) = temp_store().await;
3818
3819        let mut session = Session::new("sess-enq", "test-model");
3820        session.add_message(Message::user("do the build"));
3821        storage.save_session(&session).await.unwrap();
3822
3823        let info = BashCompletionInfo {
3824            session_id: "sess-enq".into(),
3825            bash_id: "sh-1".into(),
3826            command: "make".into(),
3827            exit_code: Some(0),
3828            status: "completed".into(),
3829            output_tail: "done".into(),
3830        };
3831        let saved = enqueue_bash_completion_injection(&persistence, &info)
3832            .await
3833            .expect("enqueue io ok")
3834            .expect("session exists");
3835
3836        let pending = saved
3837            .pending_injected_messages()
3838            .expect("pending injection present");
3839        assert_eq!(pending.len(), 1);
3840        let content = pending[0].get("content").and_then(|v| v.as_str()).unwrap();
3841        assert!(content.contains("sh-1"), "content: {content}");
3842        assert!(content.contains("make"), "content: {content}");
3843        assert!(content.contains("done"), "content: {content}");
3844        // The pre-existing conversation is untouched (no clobber).
3845        assert_eq!(saved.messages.len(), 1);
3846    }
3847
3848    #[tokio::test]
3849    async fn enqueue_returns_none_for_missing_session() {
3850        let (_temp, _storage, persistence) = temp_store().await;
3851        let info = BashCompletionInfo {
3852            session_id: "does-not-exist".into(),
3853            bash_id: "x".into(),
3854            command: "true".into(),
3855            exit_code: Some(0),
3856            status: "completed".into(),
3857            output_tail: String::new(),
3858        };
3859        let result = enqueue_bash_completion_injection(&persistence, &info)
3860            .await
3861            .expect("io ok");
3862        assert!(result.is_none(), "no session → nothing enqueued");
3863    }
3864
3865    // ── push-driven resume: the state transition + decision the push applies ──
3866
3867    /// A session suspended on `waiting_for_bash`, given the rich completion
3868    /// message, is transitioned to a resumable state: the wait is cleared, the
3869    /// runtime is Idle, the suspend-reason marker is gone, and the resume message
3870    /// is appended. This is exactly what the PUSH does to wake the loop
3871    /// event-driven (vs the old backstop poll).
3872    #[test]
3873    fn apply_bash_resume_transition_clears_wait_and_appends_message() {
3874        use bamboo_domain::session::runtime_state::WaitingForBashState;
3875
3876        let mut session = Session::new("sess-resume", "test-model");
3877        session.add_message(Message::user("kick off the build"));
3878        let mut rt = read_runtime_state(&session);
3879        rt.status = AgentStatusState::Running;
3880        rt.waiting_for_bash = Some(WaitingForBashState::for_bash(
3881            vec!["sh-1".into()],
3882            Utc::now(),
3883        ));
3884        write_runtime_state(&mut session, &rt);
3885        session.metadata.insert(
3886            "runtime.suspend_reason".to_string(),
3887            "waiting_for_bash".to_string(),
3888        );
3889
3890        let resume = bash_completion_resume_message(&["sh-1".to_string()], false);
3891        let did = apply_bash_resume_transition(&mut session, &resume);
3892
3893        assert!(did, "a suspended session must transition");
3894        let after = read_runtime_state(&session);
3895        assert!(
3896            after.waiting_for_bash.is_none(),
3897            "bash wait must be cleared"
3898        );
3899        assert_eq!(after.status, AgentStatusState::Idle, "runtime must be Idle");
3900        assert!(
3901            !session.metadata.contains_key("runtime.suspend_reason"),
3902            "suspend-reason marker must be removed"
3903        );
3904        assert_eq!(session.messages.len(), 2, "resume message must be appended");
3905        assert!(matches!(
3906            session.messages.last().map(|m| &m.role),
3907            Some(Role::User)
3908        ));
3909    }
3910
3911    /// The double-resume guard: a session NOT waiting on bash is a no-op — no
3912    /// message appended, nothing mutated. This is what makes the backstop poll
3913    /// harmlessly yield once the push has already resumed (and vice versa).
3914    #[test]
3915    fn apply_bash_resume_transition_noops_when_not_waiting() {
3916        let mut session = Session::new("sess-live", "test-model");
3917        session.add_message(Message::user("hi"));
3918
3919        let resume = bash_completion_resume_message(&["sh-1".to_string()], false);
3920        let did = apply_bash_resume_transition(&mut session, &resume);
3921
3922        assert!(!did, "a non-waiting session must not transition");
3923        assert_eq!(session.messages.len(), 1, "no resume message appended");
3924    }
3925
3926    /// The resume invariant: push-resume fires ONLY when the loop is suspended on
3927    /// bash AND every waited shell has finished. A still-running sibling shell
3928    /// keeps it on the enqueue path.
3929    #[test]
3930    fn bash_completion_should_resume_only_when_suspended_and_all_done() {
3931        assert!(bash_completion_should_resume(true, true));
3932        assert!(!bash_completion_should_resume(true, false)); // other shells still running
3933        assert!(!bash_completion_should_resume(false, true)); // live loop, not suspended
3934        assert!(!bash_completion_should_resume(false, false));
3935    }
3936
3937    #[test]
3938    fn two_shell_delivery_stages_backlog_before_one_final_activation() {
3939        let mut waiting = true;
3940        let mut durable_backlog = 0;
3941        let mut reservations = 0;
3942
3943        // First shell: its completion is durable, but a sibling still runs.
3944        durable_backlog += 1;
3945        let first = bash_completion_delivery_plan(waiting, false);
3946        assert_eq!(first, BashCompletionDeliveryPlan::DurableOnly);
3947        assert!(waiting);
3948        assert_eq!(durable_backlog, 1);
3949        assert_eq!(reservations, 0);
3950
3951        // Last shell: its own completion joins the same ordered backlog, then
3952        // the durable wait is cleared and exactly one activation is requested.
3953        durable_backlog += 1;
3954        let last = bash_completion_delivery_plan(waiting, true);
3955        assert_eq!(last, BashCompletionDeliveryPlan::ClearWaitThenActivate);
3956        waiting = false;
3957        reservations += 1;
3958        assert!(!waiting);
3959        assert_eq!(durable_backlog, 2);
3960        assert_eq!(reservations, 1);
3961    }
3962
3963    /// The push's resume message carries the shell's identity + status + output
3964    /// tail (so the model needs no `BashOutput` round-trip) and is tagged as a
3965    /// bash-completion resume so it satisfies the `has_pending_user_message` gate.
3966    #[test]
3967    fn bash_resume_message_from_info_carries_bashid_tail_and_kind() {
3968        let info = BashCompletionInfo {
3969            session_id: "s".into(),
3970            bash_id: "sh-42".into(),
3971            command: "cargo test".into(),
3972            exit_code: Some(0),
3973            status: "completed".into(),
3974            output_tail: "test result: ok".into(),
3975        };
3976        let msg = bash_resume_message_from_info(&info);
3977
3978        assert!(matches!(msg.role, Role::User));
3979        assert!(msg.content.contains("sh-42"), "content: {}", msg.content);
3980        assert!(
3981            msg.content.contains("cargo test"),
3982            "content: {}",
3983            msg.content
3984        );
3985        assert!(
3986            msg.content.contains("test result: ok"),
3987            "content: {}",
3988            msg.content
3989        );
3990        assert!(
3991            msg.content.contains("BashOutput"),
3992            "content: {}",
3993            msg.content
3994        );
3995        let meta = serde_json::to_string(&msg.metadata).unwrap();
3996        assert!(
3997            meta.contains(BASH_COMPLETION_RESUME_KIND),
3998            "resume message must be tagged as a bash-completion resume: {meta}"
3999        );
4000    }
4001}