Skip to main content

bamboo_engine/session_app/
child_completion_coordinator.rs

1//! Child-session completion coordinator.
2//!
3//! Receives terminal child runner notifications from `bamboo-engine`, updates
4//! durable parent wait state, and resumes the parent when the configured wait
5//! policy is satisfied.
6
7use std::collections::HashMap;
8use std::sync::{Arc, OnceLock, RwLock as StdRwLock};
9use std::time::Duration;
10
11use bamboo_domain::poison::PoisonRecover;
12use bamboo_domain::{AgentHookPoint, HookPayload, HookToolOutcome};
13
14use crate::execution::{
15    create_event_forwarder, finalize_runner, spawn_session_execution, try_reserve_runner,
16    AgentRunner, AgentStatus, ChildCompletion, ChildCompletionHandler, RunnerReservation,
17    SessionExecutionArgs,
18};
19use crate::runtime::config::{BashResumeHook, GuardianSpawner, BASH_COMPLETION_RESUME_KIND};
20use crate::runtime::guardian_state::{
21    parse_guardian_verdict, read_guardian_config, read_guardian_state, write_guardian_state,
22    GuardianVerdict,
23};
24use crate::Agent;
25use async_trait::async_trait;
26use bamboo_agent_core::storage::Storage;
27use bamboo_agent_core::tools::ToolExecutor;
28use bamboo_agent_core::{
29    AgentEvent, BashCompletionInfo, BashCompletionSink, Message, Role, Session,
30};
31use bamboo_domain::session::runtime_state::{
32    AgentRuntimeState, AgentStatusState, ChildWaitPolicy, SuspensionState, WaitingForChildrenState,
33};
34use bamboo_llm::{Config, ProviderModelRouter, ProviderRegistry};
35use bamboo_storage::LockedSessionStore;
36use chrono::Utc;
37use tokio::sync::{broadcast, RwLock};
38
39use crate::model_areas::resolve_global_area_models;
40use crate::model_config_helper::{
41    resolve_fast_model, resolve_gold_config, GOLD_CONFIG_METADATA_KEY,
42};
43use crate::session_app::provider_model::session_effective_model_ref;
44use crate::session_app::resume::{
45    resume_session_execution, ResumeExecutionPort, ResumeSpawnRequest,
46};
47use crate::session_app::types::{ResumeConfigSnapshot, ResumeOutcome};
48
49const AGENT_RUNTIME_STATE_METADATA_KEY: &str = "agent.runtime.state";
50const RUNTIME_RESUME_MESSAGE_HIDDEN_KEY: &str = "hidden_from_ui";
51const RUNTIME_RESUME_MESSAGE_KIND_KEY: &str = "runtime_kind";
52
53fn read_runtime_state(session: &Session) -> AgentRuntimeState {
54    session
55        .agent_runtime_state
56        .clone()
57        .or_else(|| {
58            session
59                .metadata
60                .get(AGENT_RUNTIME_STATE_METADATA_KEY)
61                .and_then(|raw| serde_json::from_str::<AgentRuntimeState>(raw).ok())
62        })
63        .unwrap_or_else(|| AgentRuntimeState::new(format!("{}-child-wait", session.id)))
64}
65
66fn write_runtime_state(session: &mut Session, runtime_state: &AgentRuntimeState) {
67    session.agent_runtime_state = Some(runtime_state.clone());
68    if let Ok(serialized) = serde_json::to_string(runtime_state) {
69        session
70            .metadata
71            .insert(AGENT_RUNTIME_STATE_METADATA_KEY.to_string(), serialized);
72    }
73}
74
75fn is_error_like(status: &str) -> bool {
76    matches!(status, "error" | "timeout" | "cancelled")
77}
78
79/// Terminal child run statuses, as mirrored into the session index.
80fn is_terminal_child_status(status: &str) -> bool {
81    matches!(
82        status,
83        "completed" | "error" | "timeout" | "cancelled" | "skipped"
84    )
85}
86
87/// Reconstruct the set of completed child session ids for a parent from the
88/// session index (the single source of truth), folding in the child whose
89/// completion event is being processed so a momentarily-lagging index can never
90/// stall the parent's resume.
91async fn derive_completed_child_ids(
92    storage: &Arc<dyn Storage>,
93    parent_session_id: &str,
94    just_completed_child_id: &str,
95) -> Vec<String> {
96    let mut completed: Vec<String> = storage
97        .list_child_run_statuses(parent_session_id)
98        .await
99        .unwrap_or_default()
100        .into_iter()
101        .filter(|(_, status)| status.as_deref().is_some_and(is_terminal_child_status))
102        .map(|(id, _)| id)
103        .collect();
104    if !completed.iter().any(|id| id == just_completed_child_id) {
105        completed.push(just_completed_child_id.to_string());
106    }
107    completed.sort();
108    completed.dedup();
109    completed
110}
111
112fn read_config_snapshot(config: &Arc<RwLock<Config>>, cached_config: &StdRwLock<Config>) -> Config {
113    if let Ok(config_guard) = config.try_read() {
114        let snapshot = config_guard.clone();
115
116        if let Ok(mut cached_guard) = cached_config.try_write() {
117            *cached_guard = snapshot.clone();
118        }
119
120        snapshot
121    } else {
122        cached_config
123            .try_read()
124            .map(|guard| guard.clone())
125            .unwrap_or_default()
126    }
127}
128
129/// Per-parent async locks that serialize concurrent `on_child_completed`
130/// invocations for the same parent session.
131///
132/// Race eliminated: when `wait_for=Any` and two child sessions complete
133/// simultaneously, both invocations load the parent with
134/// `waiting_for_children=Some` before either persists the cleared state, so
135/// both pass `wait_policy_satisfied`, both clear `waiting_for_children`, add a
136/// duplicate resume message, and call `resume_parent` — a double resume.
137/// Holding this per-parent `tokio::sync::Mutex` across the load-check-save
138/// critical section makes the second caller observe the already-cleared state.
139///
140/// The inner `std::sync::Mutex` guards only the brief HashMap lookup/insert
141/// (no await inside); the per-parent `tokio::sync::Mutex` is the one held
142/// across the async critical section. Entries accumulate but are small
143/// (`Arc<tokio::sync::Mutex<()>>` ≈ 24 bytes) and bounded by the number of
144/// distinct parent sessions.
145fn parent_locks() -> &'static std::sync::Mutex<HashMap<String, Arc<tokio::sync::Mutex<()>>>> {
146    static LOCKS: OnceLock<std::sync::Mutex<HashMap<String, Arc<tokio::sync::Mutex<()>>>>> =
147        OnceLock::new();
148    LOCKS.get_or_init(|| std::sync::Mutex::new(HashMap::new()))
149}
150
151/// Fetch (or create) the per-session async lock from [`parent_locks`]. Held
152/// across the load-check-clear-resume critical section so the three resume
153/// sources for one session — child completion, the loop-facing bash **push**
154/// ([`BashCompletionSink::on_bash_completed`]), and the bash **backstop** poll
155/// ([`ChildCompletionCoordinator::bash_self_resume`]) — can never double-resume.
156/// The inner sync `Mutex` guards only the brief map lookup (no await inside).
157fn session_resume_lock(session_id: &str) -> Arc<tokio::sync::Mutex<()>> {
158    let mut map = parent_locks().lock().recover_poison();
159    map.entry(session_id.to_string())
160        .or_insert_with(|| Arc::new(tokio::sync::Mutex::new(())))
161        .clone()
162}
163
164fn wait_policy_satisfied(
165    policy: ChildWaitPolicy,
166    wait_child_ids: &[String],
167    completed_child_ids: &[String],
168    latest_child_id: &str,
169    latest_status: &str,
170) -> bool {
171    if wait_child_ids.is_empty() {
172        return false;
173    }
174
175    match policy {
176        ChildWaitPolicy::All => wait_child_ids
177            .iter()
178            .all(|id| completed_child_ids.iter().any(|completed| completed == id)),
179        ChildWaitPolicy::Any => completed_child_ids
180            .iter()
181            .any(|id| wait_child_ids.iter().any(|wait_id| wait_id == id)),
182        ChildWaitPolicy::FirstError => {
183            // The error short-circuit only counts a completion from a child
184            // this wait actually tracks (issue #546): a stray/duplicate
185            // completion from an untracked child — e.g. a frozen runner's
186            // task waking up after the watchdog already synthesized its
187            // timeout, in a later run's wait — must not resume the parent.
188            (is_error_like(latest_status) && wait_child_ids.iter().any(|id| id == latest_child_id))
189                || wait_child_ids
190                    .iter()
191                    .all(|id| completed_child_ids.iter().any(|completed| completed == id))
192        }
193    }
194}
195
196/// Extract the child session's last assistant content, if any. Returns `None`
197/// when the child produced no assistant message (e.g. errored before the first
198/// model response, or only emitted tool messages).
199fn child_final_assistant_text(child: &Session) -> Option<String> {
200    child
201        .messages
202        .iter()
203        .rev()
204        .find(|message| matches!(message.role, Role::Assistant))
205        .map(|message| message.content.clone())
206        .filter(|content| !content.trim().is_empty())
207}
208
209fn runtime_resume_message(
210    completion: &ChildCompletion,
211    remaining_children: usize,
212    child_final_response: Option<&str>,
213) -> Message {
214    let mut body = format!(
215        "Runtime notification: child session `{}` finished with status `{}`. Remaining child sessions: {}.",
216        completion.child_session_id, completion.status, remaining_children
217    );
218
219    // Fold the child's full final response back into the parent — no
220    // truncation. Sub-agents are first-class agents whose complete conclusion
221    // should be available to the parent without an extra `SubAgent.get` round
222    // trip. The message is left compressible (see `never_compress` below) so a
223    // long transcript can still be reclaimed under parent compaction.
224    let final_response = child_final_response.map(str::to_string);
225    if let Some(response) = final_response.as_deref() {
226        body.push_str("\n\nChild final response:\n");
227        body.push_str(response);
228    } else if let Some(error) = completion.error.as_deref() {
229        if !error.is_empty() {
230            body.push_str("\n\nChild error:\n");
231            body.push_str(error);
232        }
233    }
234
235    body.push_str(
236        "\n\nResume the parent task using this child result and continue from the previous plan. \
237         If you need the full child transcript, call SubAgent.get(child_session_id).",
238    );
239
240    let mut message = Message::user(body);
241    message.metadata = Some(serde_json::json!({
242        RUNTIME_RESUME_MESSAGE_HIDDEN_KEY: true,
243        RUNTIME_RESUME_MESSAGE_KIND_KEY: "child_completion_resume",
244        "child_session_id": completion.child_session_id,
245        "child_status": completion.status,
246        "child_error": completion.error,
247        "completed_at": completion.completed_at,
248        "child_final_response_included": final_response.is_some(),
249    }));
250    // Allow parent-side compaction to reclaim this (now untruncated) message if
251    // the parent context grows — important once children nest and fold full
252    // results upward. The `SubAgent.get` hint preserves recoverability.
253    message.never_compress = false;
254    message
255}
256
257/// The hidden resume message for a completed **guardian** review: a directive,
258/// verdict-tailored note that carries the reviewer's findings straight into the
259/// parent (so it can act without a `SubAgent.get`), mirroring
260/// [`runtime_resume_message`]'s hidden/compressible shape.
261fn guardian_resume_message(completion: &ChildCompletion, verdict: &GuardianVerdict) -> Message {
262    let mut body = if verdict.approve {
263        String::from(
264            "Guardian review APPROVED: an independent reviewer verified the work and found no blocking issues. You may finalize the task.",
265        )
266    } else {
267        String::from(
268            "Guardian review REJECTED: an independent reviewer found issues. Address every finding below before completing — do NOT declare the task complete until they are resolved.",
269        )
270    };
271    if let Some(summary) = verdict.summary.as_deref().filter(|s| !s.trim().is_empty()) {
272        body.push_str("\n\nReviewer summary: ");
273        body.push_str(summary);
274    }
275    if !verdict.findings.is_empty() {
276        body.push_str("\n\nFindings:");
277        for (idx, finding) in verdict.findings.iter().enumerate() {
278            body.push_str(&format!("\n{}. {}", idx + 1, finding));
279        }
280    }
281    body.push_str(
282        "\n\nIf you need the full guardian transcript, call SubAgent.get(child_session_id).",
283    );
284
285    let mut message = Message::user(body);
286    message.metadata = Some(serde_json::json!({
287        RUNTIME_RESUME_MESSAGE_HIDDEN_KEY: true,
288        RUNTIME_RESUME_MESSAGE_KIND_KEY: "guardian_review_resume",
289        "child_session_id": completion.child_session_id,
290        "child_status": completion.status,
291        "guardian_approved": verdict.approve,
292        "completed_at": completion.completed_at,
293    }));
294    message.never_compress = false;
295    message
296}
297
298#[derive(Clone)]
299pub struct ChildCompletionCoordinator {
300    storage: Arc<dyn Storage>,
301    persistence: Arc<bamboo_storage::LockedSessionStore>,
302    sessions: crate::SessionCache,
303    agent_runners: Arc<RwLock<HashMap<String, AgentRunner>>>,
304    session_event_senders: Arc<RwLock<HashMap<String, broadcast::Sender<AgentEvent>>>>,
305    agent: Arc<Agent>,
306    config: Arc<RwLock<Config>>,
307    provider_registry: Arc<ProviderRegistry>,
308    provider_router: Arc<ProviderModelRouter>,
309    app_data_dir: std::path::PathBuf,
310    account_feed_inbox: Option<crate::execution::AccountFeedInbox>,
311    root_tools: Arc<RwLock<Option<Arc<dyn ToolExecutor>>>>,
312    /// Late-bound guardian reviewer spawner, set post-construction by the server
313    /// (mirrors `root_tools`). Re-injected into resumed runs so a guardian's
314    /// reject→fix verdict can be re-reviewed across the suspend/resume boundary.
315    guardian_spawner: Arc<RwLock<Option<Arc<dyn GuardianSpawner>>>>,
316}
317
318impl ChildCompletionCoordinator {
319    #[allow(clippy::too_many_arguments)]
320    pub fn new(
321        storage: Arc<dyn Storage>,
322        persistence: Arc<LockedSessionStore>,
323        sessions: crate::SessionCache,
324        agent_runners: Arc<RwLock<HashMap<String, AgentRunner>>>,
325        session_event_senders: Arc<RwLock<HashMap<String, broadcast::Sender<AgentEvent>>>>,
326        agent: Arc<Agent>,
327        config: Arc<RwLock<Config>>,
328        provider_registry: Arc<ProviderRegistry>,
329        provider_router: Arc<ProviderModelRouter>,
330        app_data_dir: std::path::PathBuf,
331        account_feed_inbox: Option<crate::execution::AccountFeedInbox>,
332    ) -> Self {
333        Self {
334            storage,
335            persistence,
336            sessions,
337            agent_runners,
338            session_event_senders,
339            agent,
340            config,
341            provider_registry,
342            provider_router,
343            app_data_dir,
344            account_feed_inbox,
345            root_tools: Arc::new(RwLock::new(None)),
346            guardian_spawner: Arc::new(RwLock::new(None)),
347        }
348    }
349
350    pub async fn set_root_tools(&self, tools: Arc<dyn ToolExecutor>) {
351        *self.root_tools.write().await = Some(tools);
352    }
353
354    /// Wire the guardian reviewer spawner (server-provided), so resumed runs can
355    /// re-spawn a guardian to re-review a fix after a reject verdict.
356    pub async fn set_guardian_spawner(&self, spawner: Arc<dyn GuardianSpawner>) {
357        *self.guardian_spawner.write().await = Some(spawner);
358    }
359
360    fn build_resume_config(
361        &self,
362        session: &Session,
363        config_snapshot: &Config,
364    ) -> ResumeConfigSnapshot {
365        crate::session_app::resolution::resolve_resume_config_snapshot(
366            config_snapshot,
367            &self.provider_registry,
368            session,
369            None,
370        )
371    }
372
373    /// Drive a parent-resume and return the final [`ResumeOutcome`] so callers
374    /// can distinguish a successful spawn (`Started`) from a gate-blocked
375    /// attempt (`Completed`). The bash self-resume poll task uses this to
376    /// detect the finalize-clobber case — its appended resume message was
377    /// reverted by the suspending runner's final `merge_save_runtime`, so the
378    /// resume port's `has_pending_user_message` gate fails and nothing spawns —
379    /// and retry the clear→append→resume (see [`Self::bash_self_resume`]).
380    async fn resume_parent(&self, parent_session_id: String) -> ResumeOutcome {
381        for attempt in 0..=5u8 {
382            if attempt > 0 {
383                tokio::time::sleep(Duration::from_millis(250 * attempt as u64)).await;
384            }
385
386            let Some(session) = self.load_session(&parent_session_id).await else {
387                tracing::warn!(%parent_session_id, "cannot resume parent after child completion: session not found");
388                return ResumeOutcome::NotFound;
389            };
390            let config_snapshot = self.config.read().await.clone();
391            let resume_config = self.build_resume_config(&session, &config_snapshot);
392            let outcome = resume_session_execution(self, &parent_session_id, resume_config).await;
393            tracing::info!(
394                %parent_session_id,
395                attempt,
396                outcome = outcome.as_str(),
397                "child completion requested parent resume"
398            );
399
400            if !matches!(outcome, ResumeOutcome::AlreadyRunning { .. }) {
401                return outcome;
402            }
403        }
404        // Exhausted the AlreadyRunning retry budget; surface the final state.
405        // The wait state was already cleared and the resume message persisted,
406        // so nothing event-driven will retry — the child-wait watchdog is the
407        // backstop that picks this stranded parent up (it resumes suspended
408        // sessions that hold a pending runtime resume message but no runner).
409        tracing::error!(
410            %parent_session_id,
411            "parent resume gave up after AlreadyRunning retry budget; \
412             relying on the child-wait watchdog backstop"
413        );
414        ResumeOutcome::AlreadyRunning {
415            run_id: String::new(),
416        }
417    }
418
419    async fn save_and_cache(&self, session: &mut Session) {
420        if let Err(error) = self.persistence.merge_save_runtime(session).await {
421            tracing::warn!(session_id = %session.id, %error, "failed to persist session");
422        }
423        self.sessions.insert(
424            session.id.clone(),
425            Arc::new(parking_lot::RwLock::new(session.clone())),
426        );
427    }
428}
429
430#[async_trait]
431impl ChildCompletionHandler for ChildCompletionCoordinator {
432    async fn on_child_completed(&self, completion: ChildCompletion) {
433        // Terminality guard: a child that reports a NON-terminal status (e.g.
434        // "suspended" — awaiting parent approval, its own bash wait, or its own
435        // grandchildren) is not done. It must never satisfy the parent's wait:
436        // `derive_completed_child_ids` folds the just-reported child in
437        // unconditionally, so without this guard a suspending child would
438        // resume the parent with a premature "finished with status
439        // `suspended`" message. The child will publish a real terminal
440        // completion when it later resumes and finishes.
441        if !is_terminal_child_status(&completion.status) {
442            tracing::info!(
443                parent_session_id = %completion.parent_session_id,
444                child_session_id = %completion.child_session_id,
445                status = %completion.status,
446                "non-terminal child status; leaving the parent wait armed"
447            );
448            return;
449        }
450
451        // Acquire the per-session async lock to eliminate the concurrent
452        // double-resume race (see `parent_locks` for the full scenario). The
453        // inner std::sync::Mutex is released immediately so no sync lock is
454        // held across the await that follows.
455        let per_parent = session_resume_lock(&completion.parent_session_id);
456        let _per_parent_guard = per_parent.lock().await;
457
458        let Some(mut parent) = self.load_session(&completion.parent_session_id).await else {
459            tracing::warn!(
460                parent_session_id = %completion.parent_session_id,
461                child_session_id = %completion.child_session_id,
462                "child completion received for missing parent"
463            );
464            return;
465        };
466
467        // A parent may itself be a child (nested sub-agents): the rest of this
468        // handler is kind-agnostic — it operates on `completion.parent_session_id`,
469        // inspects that session's own `waiting_for_children` runtime state, and
470        // resumes it. (Previously this bailed unless the parent was Root, which
471        // silently dropped grandchild completions.)
472        let mut runtime_state = read_runtime_state(&parent);
473
474        // Single source of truth: reconstruct the completed-child set from the
475        // session index rather than from a denormalized copy on the parent file.
476        let completed_child_ids = derive_completed_child_ids(
477            &self.storage,
478            &completion.parent_session_id,
479            &completion.child_session_id,
480        )
481        .await;
482
483        let mut should_resume = false;
484        let mut remaining_children = 0usize;
485        if let Some(wait) = runtime_state.waiting_for_children.clone() {
486            remaining_children = wait
487                .child_session_ids
488                .iter()
489                .filter(|id| !completed_child_ids.iter().any(|completed| completed == *id))
490                .count();
491            should_resume = wait_policy_satisfied(
492                wait.wait_for,
493                &wait.child_session_ids,
494                &completed_child_ids,
495                &completion.child_session_id,
496                &completion.status,
497            );
498            if should_resume {
499                runtime_state.waiting_for_children = None;
500                runtime_state.status = AgentStatusState::Idle;
501                runtime_state.suspension = None;
502            }
503        }
504
505        if should_resume {
506            parent.metadata.remove("runtime.suspend_reason");
507
508            // READ-SIDE OWNERSHIP GUARD (issue #546): `SubAgent.wait` ids are
509            // model-provided and unvalidated, and the watchdog unstrands a wait
510            // over a FOREIGN/unknown id by publishing a synthetic completion
511            // here. We must resume the parent (so it is not stranded) but MUST
512            // NOT fold that foreign session's transcript into the parent — that
513            // would be a cross-session disclosure primitive. Decide ownership
514            // from the child's OWN parent linkage (control-plane only, no
515            // messages loaded), and only load its full content when it is truly
516            // this parent's child. An unowned id resumes with the neutral/error
517            // message (`runtime_resume_message` falls back to `completion.error`
518            // when no child content is supplied).
519            let reported_child_owned = match self
520                .storage
521                .load_runtime_control_plane(&completion.child_session_id)
522                .await
523            {
524                Ok(Some(control_plane)) => completion_child_is_owned(
525                    &completion.parent_session_id,
526                    control_plane.parent_session_id.as_deref(),
527                ),
528                _ => false,
529            };
530
531            // Load the completed child once, ONLY when owned. The guardian
532            // branch inspects its subagent_type + final verdict; the generic
533            // path folds its final assistant content into the hidden resume
534            // message (avoiding an extra `SubAgent.get` round trip after resume).
535            let loaded_child = if reported_child_owned {
536                match self
537                    .storage
538                    .load_session(&completion.child_session_id)
539                    .await
540                {
541                    Ok(child) => child,
542                    Err(error) => {
543                        tracing::warn!(
544                            child_session_id = %completion.child_session_id,
545                            %error,
546                            "failed to load child session for runtime resume message"
547                        );
548                        None
549                    }
550                }
551            } else {
552                tracing::warn!(
553                    parent_session_id = %completion.parent_session_id,
554                    child_session_id = %completion.child_session_id,
555                    "completion child is not a child of this parent; resuming with a neutral \
556                     message and NOT folding its content"
557                );
558                None
559            };
560
561            // Guardian branch: a completing guardian reviewer that matches the
562            // parent's recorded review advances GuardianState (phase → Reviewed)
563            // and resumes with a verdict-tailored, findings-carrying message. Any
564            // id mismatch or unparseable verdict falls through to the generic
565            // resume, so the parent is never stranded.
566            let reviewed_round = runtime_state.round.current_round;
567            let guardian_resume = loaded_child.as_ref().and_then(|child| {
568                if child.subagent_type().as_deref() != Some("guardian") {
569                    return None;
570                }
571                let mut guardian_state = read_guardian_state(&parent)?;
572                if guardian_state.guardian_child_id.as_deref()
573                    != Some(completion.child_session_id.as_str())
574                {
575                    // A *different* guardian is legitimately still in flight —
576                    // leave its Pending state intact and use the generic resume.
577                    tracing::warn!(
578                        parent_session_id = %completion.parent_session_id,
579                        child_session_id = %completion.child_session_id,
580                        expected = ?guardian_state.guardian_child_id,
581                        "guardian completion does not match recorded guardian_child_id; using generic resume"
582                    );
583                    return None;
584                }
585                // This IS the guardian we dispatched, so we MUST advance the
586                // phase out of `Pending` — otherwise the next terminal gate's
587                // `Pending => return None` would let the run complete unreviewed.
588                // A reviewer that errored or produced unparseable output is
589                // treated as a SYNTHETIC REJECT (never a silent pass), so the
590                // budgeted re-review loop governs the outcome: fail-closed, but
591                // still bounded by `max_reviews`.
592                let verdict = child_final_assistant_text(child)
593                    .and_then(|text| match parse_guardian_verdict(&text) {
594                        Ok(verdict) => Some(verdict),
595                        Err(error) => {
596                            tracing::warn!(
597                                child_session_id = %completion.child_session_id,
598                                %error,
599                                "guardian verdict unparseable; recording a synthetic reject"
600                            );
601                            None
602                        }
603                    })
604                    .unwrap_or_else(|| {
605                        GuardianVerdict::rejected(vec![
606                            "The guardian reviewer did not return a usable verdict (it errored or \
607                             emitted unparseable output); the work has NOT been independently \
608                             verified."
609                                .to_string(),
610                        ])
611                    });
612                let approved = verdict.approve;
613                let message = guardian_resume_message(&completion, &verdict);
614                guardian_state.record_verdict(verdict, reviewed_round);
615                write_guardian_state(&mut parent, guardian_state);
616                tracing::info!(
617                    parent_session_id = %completion.parent_session_id,
618                    child_session_id = %completion.child_session_id,
619                    approved,
620                    "guardian verdict recorded; resuming parent"
621                );
622                Some(message)
623            });
624
625            let resume_message = guardian_resume.unwrap_or_else(|| {
626                runtime_resume_message(
627                    &completion,
628                    remaining_children,
629                    loaded_child
630                        .as_ref()
631                        .and_then(child_final_assistant_text)
632                        .as_deref(),
633                )
634            });
635            parent.add_message(resume_message);
636        } else if runtime_state.waiting_for_children.is_some() {
637            runtime_state.status = AgentStatusState::Suspended;
638            runtime_state.suspension = Some(SuspensionState {
639                reason: "waiting_for_children".to_string(),
640                suspended_at: Utc::now(),
641                resumable: true,
642                hook_point: Some("ChildCompletion".to_string()),
643            });
644        }
645
646        parent.updated_at = Utc::now();
647        write_runtime_state(&mut parent, &runtime_state);
648        self.save_and_cache(&mut parent).await;
649
650        // Capture before releasing the per-parent lock so the borrow checker
651        // is satisfied; `resume_parent` has its own retry loop and should not
652        // hold the per-parent lock (it would block other completions for the
653        // same parent, and the state is already durably settled above).
654        let resume_parent_id = parent.id.clone();
655        drop(_per_parent_guard);
656
657        if should_resume {
658            self.resume_parent(resume_parent_id).await;
659        }
660    }
661}
662
663#[async_trait]
664impl ResumeExecutionPort for ChildCompletionCoordinator {
665    async fn load_session(&self, session_id: &str) -> Option<Session> {
666        match self.storage.load_session(session_id).await {
667            Ok(Some(session)) => Some(session),
668            Ok(None) => self
669                .sessions
670                .get(session_id)
671                .map(|e| e.value().clone())
672                .map(|arc| arc.read().clone()),
673            Err(error) => {
674                tracing::warn!(%session_id, %error, "failed to load session from storage");
675                self.sessions
676                    .get(session_id)
677                    .map(|e| e.value().clone())
678                    .map(|arc| arc.read().clone())
679            }
680        }
681    }
682
683    async fn save_and_cache_session(&self, session: &mut Session) {
684        self.save_and_cache(session).await;
685    }
686
687    async fn try_reserve_runner(
688        &self,
689        session_id: &str,
690        event_sender: &broadcast::Sender<AgentEvent>,
691    ) -> Option<RunnerReservation> {
692        try_reserve_runner(
693            &self.agent_runners,
694            &self.session_event_senders,
695            session_id,
696            event_sender,
697        )
698        .await
699    }
700
701    async fn get_existing_runner_run_id(&self, session_id: &str) -> Option<String> {
702        let runners = self.agent_runners.read().await;
703        runners.get(session_id).map(|r| r.run_id.clone())
704    }
705
706    async fn get_or_create_event_sender(&self, session_id: &str) -> broadcast::Sender<AgentEvent> {
707        crate::execution::session_events::get_or_create_event_sender(
708            &self.session_event_senders,
709            session_id,
710        )
711        .await
712    }
713
714    async fn spawn_resume_execution(&self, request: ResumeSpawnRequest) {
715        let ResumeSpawnRequest {
716            session_id,
717            session,
718            cancel_token,
719            run_id: _,
720            event_sender,
721            config,
722        } = request;
723
724        let Some(root_tools) = self.root_tools.read().await.clone() else {
725            tracing::error!(%session_id, "cannot resume parent after child completion: root tool surface is not initialized");
726            return;
727        };
728
729        let model = session.model.clone();
730        let resolved_provider_name = session_effective_model_ref(&session)
731            .map(|model_ref| model_ref.provider)
732            .unwrap_or(config.provider_name);
733        let provider_override = session_effective_model_ref(&session)
734            .and_then(|model_ref| match self.provider_router.route(&model_ref) {
735                Ok(provider) => Some(provider),
736                Err(error) => {
737                    tracing::warn!(
738                        session_id = %session_id,
739                        provider = %model_ref.provider,
740                        model = %model_ref.model,
741                        error = %error,
742                        "failed to resolve provider override for child-completion parent resume; falling back to runtime provider"
743                    );
744                    None
745                }
746            });
747        let config_snapshot = self.config.read().await.clone();
748        let resolved_fast_provider = resolve_fast_model(
749            &config_snapshot,
750            &resolved_provider_name,
751            &self.provider_registry,
752        )
753        .map(|model| model.provider);
754        let reasoning_effort = session.reasoning_effort;
755        let reasoning_effort_source = session
756            .metadata
757            .get("reasoning_effort_source")
758            .cloned()
759            .unwrap_or_default();
760        let gold_config = resolve_gold_config(
761            &config_snapshot,
762            session
763                .metadata
764                .get(GOLD_CONFIG_METADATA_KEY)
765                .map(String::as_str),
766        )
767        .or(config.gold_config.clone());
768
769        let (mpsc_tx, _forwarder) = create_event_forwarder(
770            session_id.clone(),
771            event_sender,
772            self.agent_runners.clone(),
773            self.account_feed_inbox.clone(),
774        );
775
776        let config_handle = self.config.clone();
777        let cached_config = Arc::new(StdRwLock::new(config_snapshot.clone()));
778        let provider_registry = self.provider_registry.clone();
779        let provider_name_for_aux = resolved_provider_name.clone();
780        let auxiliary_model_resolver = std::sync::Arc::new(move || {
781            let config_snapshot = read_config_snapshot(&config_handle, cached_config.as_ref());
782            // Auxiliary models are global (config-derived), never session-bound.
783            let areas = resolve_global_area_models(
784                &config_snapshot,
785                &provider_name_for_aux,
786                &provider_registry,
787            );
788            crate::AuxiliaryModelConfig {
789                fast_model_name: areas.fast.as_ref().map(|m| m.model_name.clone()),
790                fast_model_provider: areas.fast.map(|m| m.provider),
791                background_model_name: areas.background.as_ref().map(|m| m.model_name.clone()),
792                planning_model_name: None,
793                search_model_name: None,
794                summarization_model_name: areas
795                    .summarization
796                    .as_ref()
797                    .map(|m| m.model_name.clone()),
798                background_model_provider: areas.background.map(|m| m.provider),
799                summarization_model_provider: areas.summarization.map(|m| m.provider),
800            }
801        });
802        let model_roster = crate::ModelRoster {
803            model: Some(model),
804            provider_name: Some(resolved_provider_name),
805            provider_type: config.provider_type.clone(),
806            fast: crate::RoleModel::from_parts(config.fast_model, resolved_fast_provider),
807            background: crate::RoleModel::from_parts(
808                config.background_model,
809                config.background_model_provider,
810            ),
811            summarization: crate::RoleModel::from_parts(
812                config.summarization_model,
813                config.summarization_model_provider,
814            ),
815        };
816
817        // Re-inject guardian state on resume so a reject→fix verdict can be
818        // re-reviewed: config from the session (persisted at first spawn),
819        // spawner from the coordinator-held handle. Absent guardian config this
820        // stays `None`, and the approve→complete path is unchanged.
821        let guardian_config = read_guardian_config(&session);
822        let guardian_spawner = self.guardian_spawner.read().await.clone();
823
824        spawn_session_execution(SessionExecutionArgs {
825            agent: self.agent.clone(),
826            session_id,
827            session,
828            tools_override: Some(root_tools),
829            provider_override,
830            model_roster,
831            reasoning_effort,
832            reasoning_effort_source,
833            auxiliary_model_resolver: Some(auxiliary_model_resolver),
834            // Resumed child runs keep the spawn-time disabled snapshot (#136 lives
835            // on the long-running main agent path; children are short-lived).
836            disabled_filter_resolver: None,
837            disabled_tools: Some(config.disabled_tools),
838            disabled_skill_ids: Some(config.disabled_skill_ids),
839            selected_skill_ids: None,
840            selected_skill_mode: None,
841            cancel_token,
842            mpsc_tx,
843            image_fallback: config.image_fallback,
844            gold_config,
845            guardian_config,
846            guardian_spawner,
847            bash_resume_hook: {
848                let hook: Arc<dyn BashResumeHook> = Arc::new(self.clone());
849                Some(hook)
850            },
851            bash_completion_sink: {
852                // Resumed runs keep the push wired too, so a background shell
853                // launched after resume still notifies the loop.
854                let sink: Arc<dyn BashCompletionSink> = Arc::new(self.clone());
855                Some(sink)
856            },
857            app_data_dir: Some(self.app_data_dir.clone()),
858            // Resume does not carry a fresh per-request override; the
859            // config-level default (issue #221) still applies.
860            run_budget: None,
861            runners: self.agent_runners.clone(),
862            sessions_cache: self.sessions.clone(),
863            on_complete: None,
864            // A resumed session that is itself a CHILD (nested sub-agents)
865            // must publish its terminal completion so ITS parent is woken
866            // in turn (issue #546).
867            child_completion_handler: Some(Arc::new(self.clone())),
868        });
869    }
870}
871
872/// Hidden resume message for a bash-completion self-resume (issue #84 Phase 2b).
873/// Mirrors [`runtime_resume_message`]'s hidden/compressible shape so the resume
874/// port's `has_pending_user_message` gate is satisfied.
875///
876/// `timed_out` selects the wording: the normal path (all shells finished)
877/// announces completion; the deadline path (the 6h+10m wait ceiling was hit with
878/// shells STILL running) must NOT claim the shells completed — it says they may
879/// still be running so the model verifies with BashOutput instead of assuming
880/// success on a false premise.
881fn bash_completion_resume_message(bash_ids: &[String], timed_out: bool) -> Message {
882    let body = if timed_out {
883        format!(
884            "Runtime notification: the background-Bash wait ceiling was reached while one or more \
885             shell(s) ({}) may still be running. The session is being resumed so it is not \
886             stranded; verify their actual status with BashOutput before assuming completion.",
887            bash_ids.join(", ")
888        )
889    } else {
890        format!(
891            "Runtime notification: all background Bash shell(s) ({}) have completed. \
892             Review their output with BashOutput and resume the task from where you left off.",
893            bash_ids.join(", ")
894        )
895    };
896    let mut message = Message::user(body);
897    message.metadata = Some(serde_json::json!({
898        RUNTIME_RESUME_MESSAGE_HIDDEN_KEY: true,
899        RUNTIME_RESUME_MESSAGE_KIND_KEY: BASH_COMPLETION_RESUME_KIND,
900    }));
901    message.never_compress = false;
902    message
903}
904
905/// Decide whether the bash self-resume should retry its clear→append→resume
906/// sequence after a resume attempt returned `outcome`, given that the persisted
907/// bash wait is (`true`) / is not (`false`) still set on reload.
908///
909/// Retry **only** when the resume did NOT spawn (`Completed` — no pending user
910/// message, i.e. our resume message was dropped — or `AlreadyRunning`) AND the
911/// persisted bash wait is still set: the signature of the finalize-clobber, where
912/// the suspending runner's one-shot final `merge_save_runtime` lands after our
913/// save and reverts `waiting_for_bash=Some` while dropping our resume message, so
914/// `has_pending_user_message` fails and nothing spawns. `Started` (resume fired)
915/// and `NotFound` (session gone) never retry. Pure helper so the clobber
916/// detection is unit-testable in isolation from async I/O.
917fn bash_resume_should_retry(outcome: &ResumeOutcome, persisted_waiting_for_bash: bool) -> bool {
918    match outcome {
919        ResumeOutcome::Started { .. } | ResumeOutcome::NotFound => false,
920        ResumeOutcome::Completed | ResumeOutcome::AlreadyRunning { .. } => {
921            persisted_waiting_for_bash
922        }
923    }
924}
925
926/// Whether a background-shell completion push should **resume** the owning loop
927/// (vs merely enqueue an injection). Resume only when the loop is actually
928/// suspended on a bash wait AND every shell it was waiting on has now finished —
929/// resuming while other waited shells are still running would drop them back into
930/// a foreground turn prematurely. The last shell to finish (or the backstop)
931/// drives the resume; earlier ones enqueue their notice. Pure so the invariant is
932/// unit-testable in isolation.
933fn bash_completion_should_resume(
934    loop_suspended_on_bash: bool,
935    all_waited_shells_done: bool,
936) -> bool {
937    loop_suspended_on_bash && all_waited_shells_done
938}
939
940fn background_bash_post_tool_payload(info: &BashCompletionInfo) -> HookPayload {
941    let success = info.status == "completed" && info.exit_code == Some(0);
942    let response = serde_json::json!({
943        "bash_id": info.bash_id,
944        "command": info.command,
945        "exit_code": info.exit_code,
946        "status": info.status,
947        "output_tail": info.output_tail,
948    });
949    HookPayload::ToolResult {
950        tool_name: "Bash".to_string(),
951        tool_call_id: info.bash_id.clone(),
952        outcome: HookToolOutcome {
953            success,
954            result: success.then(|| response.to_string()),
955            error: (!success).then(|| response.to_string()),
956            needs_human: false,
957            duration_ms: 0,
958        },
959    }
960}
961
962fn append_background_bash_hook_feedback(info: &mut BashCompletionInfo, feedback: Vec<String>) {
963    let feedback = feedback
964        .into_iter()
965        .map(|text| text.trim().to_string())
966        .filter(|text| !text.is_empty())
967        .collect::<Vec<_>>();
968    if feedback.is_empty() {
969        return;
970    }
971    if !info.output_tail.is_empty() {
972        info.output_tail.push_str("\n\n");
973    }
974    info.output_tail.push_str("<post_tool_use_feedback>\n");
975    info.output_tail.push_str(&feedback.join("\n"));
976    info.output_tail.push_str("\n</post_tool_use_feedback>");
977}
978
979async fn run_background_bash_post_tool_hooks(
980    config: &bamboo_config::LifecycleHooksConfig,
981    fallback_cwd: Option<std::path::PathBuf>,
982    session: &Session,
983    info: &mut BashCompletionInfo,
984) -> bool {
985    let runner = crate::HookRunner::new().with_lifecycle_config(config, fallback_cwd);
986    if !runner.has_hooks_for(AgentHookPoint::AfterToolExecution) {
987        return false;
988    }
989
990    let mut runtime_state = session
991        .agent_runtime_state
992        .clone()
993        .unwrap_or_else(|| AgentRuntimeState::new(&session.id));
994    let outcome = runner
995        .run_observer_hooks(
996            AgentHookPoint::AfterToolExecution,
997            &background_bash_post_tool_payload(info),
998            session,
999            &mut runtime_state,
1000            None,
1001        )
1002        .await;
1003    append_background_bash_hook_feedback(info, outcome.injected_contexts);
1004    true
1005}
1006
1007/// Apply the bash-resume state transition to a loaded session **in place**: clear
1008/// the `waiting_for_bash` wait, mark the runtime Idle, drop the suspension +
1009/// `runtime.suspend_reason`, and append `resume_message`. Returns `false` (a
1010/// no-op) when the session was not actually waiting on bash — the double-resume
1011/// guard shared by the push and the backstop. Pure (no I/O) so both
1012/// [`ChildCompletionCoordinator::perform_bash_resume`] and unit tests exercise the
1013/// exact same transition.
1014fn apply_bash_resume_transition(session: &mut Session, resume_message: &Message) -> bool {
1015    let mut runtime_state = read_runtime_state(session);
1016    if runtime_state.waiting_for_bash.is_none() {
1017        return false;
1018    }
1019    runtime_state.waiting_for_bash = None;
1020    runtime_state.status = AgentStatusState::Idle;
1021    runtime_state.suspension = None;
1022    write_runtime_state(session, &runtime_state);
1023    session.metadata.remove("runtime.suspend_reason");
1024    session.add_message(resume_message.clone());
1025    true
1026}
1027
1028/// Bash self-resume support (issue #84 Phase 2b; push follow-up).
1029impl ChildCompletionCoordinator {
1030    /// **Backstop** for a session suspended on `waiting_for_bash`. The primary,
1031    /// event-driven wake is the loop-facing push
1032    /// ([`BashCompletionSink::on_bash_completed`] → [`Self::deliver_bash_completion`]):
1033    /// the shell's completion task fires it the instant the process exits, and it
1034    /// resumes the loop directly. This task exists ONLY to catch a **lost push** —
1035    /// the completion landing in the window before the suspend was persisted (so
1036    /// the push saw no `waiting_for_bash` and only queued an injection), or a
1037    /// configuration with no sink wired — and to honour the wait ceiling.
1038    ///
1039    /// So it is deliberately NOT a hot spin: a coarse backoff (1 s → 30 s) that
1040    /// **yields to the push**. In the happy path the push has already cleared
1041    /// `waiting_for_bash` before the first check fires, so this returns after one
1042    /// cheap load with no registry polling at all. It only performs a resume when
1043    /// the shell(s) have finished but the loop is somehow still suspended, or the
1044    /// 6 h wait ceiling is reached.
1045    async fn bash_self_resume(&self, session_id: String, bash_ids: Vec<String>) {
1046        let mut delay = Duration::from_secs(1);
1047        let max_delay = Duration::from_secs(30);
1048        // Hard ceiling: the wait lease (6 h) + the registry GC TTL (5 min) +
1049        // margin. After this the shells are gone from the registry regardless,
1050        // so force-resume to avoid stranding the session on a GC edge case.
1051        let max_poll = Duration::from_secs(6 * 3600 + 600);
1052        let deadline = tokio::time::Instant::now() + max_poll;
1053
1054        loop {
1055            tokio::time::sleep(delay).await;
1056
1057            let Some(session) = self.load_session(&session_id).await else {
1058                tracing::info!(%session_id, "bash self-resume backstop: session gone; nothing to do");
1059                return;
1060            };
1061            if read_runtime_state(&session).waiting_for_bash.is_none() {
1062                // The push (or another path) already resumed. This is the common
1063                // case — the backstop yields silently after a single load.
1064                return;
1065            }
1066
1067            let still_running =
1068                bamboo_tools::tools::bash_runtime::running_shells_for_session(&session_id);
1069            let timed_out = tokio::time::Instant::now() >= deadline;
1070            if still_running.is_empty() || timed_out {
1071                // The shell(s) finished but the loop is still suspended → the push
1072                // was lost (pre-persist window / no sink), or the ceiling hit.
1073                // Resume under the shared per-session lock so we never race the
1074                // push or a concurrent child-completion resume.
1075                let guard = session_resume_lock(&session_id);
1076                let _held = guard.lock().await;
1077                tracing::warn!(
1078                    %session_id,
1079                    shell_count = bash_ids.len(),
1080                    timed_out,
1081                    "bash self-resume backstop engaged (push lost or wait ceiling reached)"
1082                );
1083                self.perform_bash_resume(
1084                    &session_id,
1085                    bash_completion_resume_message(&bash_ids, timed_out),
1086                )
1087                .await;
1088                return;
1089            }
1090
1091            delay = (delay * 2).min(max_delay);
1092        }
1093    }
1094
1095    /// Clear a session's `waiting_for_bash` state, append `resume_message`, and
1096    /// drive the parent resume — the shared clear→append→resume used by BOTH the
1097    /// event-driven push ([`Self::deliver_bash_completion`]) and the backstop poll
1098    /// ([`Self::bash_self_resume`]).
1099    ///
1100    /// **The caller MUST hold the [`session_resume_lock`] for `session_id`** so the
1101    /// load-check-clear-resume critical section is serialized against every other
1102    /// resume source (no double resume). No-op when the persisted wait was already
1103    /// cleared (another source handled it first).
1104    ///
1105    /// The clear→append→resume is a **bounded retry loop** that closes the
1106    /// finalize-clobber strand. The suspending runner's `finalize_task_context`
1107    /// runs a full `save_runtime_session` (same `merge_save_runtime`, which
1108    /// overwrites the whole `messages` array) that can land AFTER our save,
1109    /// reverting `waiting_for_bash=Some` and dropping our resume message, so
1110    /// `has_pending_user_message` fails and `resume_parent` returns `Completed`
1111    /// without spawning. We detect that (persisted wait still set after a
1112    /// non-`Started` outcome) and re-clear/re-append/re-resume. It converges
1113    /// because the runner's finalize persist is one-shot: once landed, our retry's
1114    /// save is the last writer, the message sticks, and resume fires.
1115    async fn perform_bash_resume(&self, session_id: &str, resume_message: Message) {
1116        let retry_backoff = Duration::from_millis(200);
1117        const MAX_RESUME_ATTEMPTS: u8 = 5;
1118        for attempt in 0..MAX_RESUME_ATTEMPTS {
1119            if attempt > 0 {
1120                tokio::time::sleep(retry_backoff).await;
1121            }
1122
1123            let Some(mut session) = self.load_session(session_id).await else {
1124                tracing::warn!(%session_id, "bash resume: session not found; nothing to resume");
1125                return;
1126            };
1127
1128            if !apply_bash_resume_transition(&mut session, &resume_message) {
1129                // Double-resume guard: the wait was already cleared by another
1130                // source (the push, the backstop, or a user-driven resume). Do
1131                // not append a duplicate message or request a redundant resume.
1132                tracing::info!(
1133                    %session_id, attempt,
1134                    "bash resume: persisted bash wait already cleared; nothing to resume"
1135                );
1136                return;
1137            }
1138            session.updated_at = Utc::now();
1139            self.save_and_cache(&mut session).await;
1140            tracing::info!(
1141                %session_id, attempt,
1142                "bash resume: cleared bash wait and appended resume message"
1143            );
1144
1145            let outcome = self.resume_parent(session_id.to_string()).await;
1146            match outcome {
1147                ResumeOutcome::Started { .. } => {
1148                    tracing::info!(%session_id, attempt, "bash resume: resume fired");
1149                    return;
1150                }
1151                ResumeOutcome::NotFound => {
1152                    tracing::warn!(%session_id, "bash resume: session vanished during resume");
1153                    return;
1154                }
1155                _ => {
1156                    // Completed (no pending user message ⇒ our resume message was
1157                    // dropped by the runner's finalize persist) or AlreadyRunning.
1158                    // Decide via the persisted bash wait: still set ⇒
1159                    // finalize-clobber ⇒ retry; cleared ⇒ the session is being
1160                    // handled (by us or a concurrent resume) ⇒ stop.
1161                    let clobbered = match self.load_session(session_id).await {
1162                        Some(reloaded) => read_runtime_state(&reloaded).waiting_for_bash.is_some(),
1163                        None => {
1164                            tracing::warn!(%session_id, "bash resume: session vanished after resume");
1165                            return;
1166                        }
1167                    };
1168                    if bash_resume_should_retry(&outcome, clobbered) {
1169                        tracing::warn!(
1170                            %session_id, attempt,
1171                            outcome = outcome.as_str(),
1172                            "bash resume: persisted wait still set after resume (finalize-clobber); retrying"
1173                        );
1174                        continue;
1175                    }
1176                    tracing::info!(
1177                        %session_id, attempt,
1178                        outcome = outcome.as_str(),
1179                        "bash resume: wait cleared and resume handled; stopping"
1180                    );
1181                    return;
1182                }
1183            }
1184        }
1185
1186        tracing::warn!(
1187            %session_id,
1188            attempts = MAX_RESUME_ATTEMPTS,
1189            "bash resume: exhausted clobber-retry budget without confirming resume; giving up"
1190        );
1191    }
1192}
1193
1194impl BashResumeHook for ChildCompletionCoordinator {
1195    fn arrange_bash_self_resume(&self, session_id: String, bash_ids: Vec<String>) {
1196        let coordinator = Arc::new(self.clone());
1197        tokio::spawn(async move {
1198            coordinator.bash_self_resume(session_id, bash_ids).await;
1199        });
1200    }
1201}
1202
1203/// Build the injected user-message body for a completed background shell — a
1204/// concise notice plus a bounded output tail — so the model can act on the
1205/// result without a mandatory `BashOutput` round-trip (issue #84 Phase 2b
1206/// follow-up).
1207fn bash_completion_injection_body(info: &BashCompletionInfo) -> String {
1208    let exit = match info.exit_code {
1209        Some(code) => code.to_string(),
1210        None => "none (signal/killed)".to_string(),
1211    };
1212    let mut body = format!(
1213        "Runtime notification: background shell `{}` (`{}`) finished — status {}, exit code {}.",
1214        info.bash_id, info.command, info.status, exit
1215    );
1216    if info.output_tail.trim().is_empty() {
1217        body.push_str(" It produced no captured output.");
1218    } else {
1219        body.push_str("\n\nOutput tail:\n");
1220        body.push_str(&info.output_tail);
1221    }
1222    body.push_str(&format!(
1223        "\n\nUse BashOutput with bash_id=\"{}\" for the full output, then continue the task.",
1224        info.bash_id
1225    ));
1226    body
1227}
1228
1229/// Loop-facing background-Bash completion delivery (issue #84 Phase 2b
1230/// follow-up). Pushes a completed shell's result into its owning session's loop,
1231/// mirroring how a sub-agent completion reaches its parent — but via the
1232/// running-loop channel, which children never exercise (a parent waiting on
1233/// children is always suspended when one completes; bash is the first completion
1234/// source that can land on a *live, iterating* loop).
1235///
1236/// The push enqueues onto `pending_injected_messages`, the same round-boundary
1237/// steering channel `send_message` uses. That covers every reachable loop state:
1238/// an actively-looping session drains it at its next round
1239/// (`merge_pending_injected_messages`); a session suspended on `waiting_for_bash`
1240/// drains it when the durable end-of-turn poll backstop (`bash_resume_hook`)
1241/// resumes it at round 0. A wired-sink session is never idle-with-a-running-shell
1242/// (ending a turn with one suspends), so no separate idle-wake path is needed —
1243/// keeping the push a pure latency optimization that never races the backstop.
1244/// Enqueue a completed shell's summary as a pending injected message on the
1245/// owning session. Race-safe: `update_runtime_config` loads the freshest session
1246/// under the per-session lock and re-saves, so it can never revert a message the
1247/// live loop appended concurrently — unlike `merge_save_runtime` (which writes
1248/// the caller's whole `messages` snapshot verbatim). Free fn so it is unit-
1249/// testable without constructing a full coordinator. Returns the saved session,
1250/// or `None` if the owning session no longer exists.
1251async fn enqueue_bash_completion_injection(
1252    persistence: &LockedSessionStore,
1253    info: &BashCompletionInfo,
1254) -> std::io::Result<Option<Session>> {
1255    let body = bash_completion_injection_body(info);
1256    let queued = serde_json::json!({
1257        "content": body,
1258        "created_at": Utc::now(),
1259    });
1260    persistence
1261        .update_runtime_config(&info.session_id, move |session| {
1262            let mut pending = session.pending_injected_messages().unwrap_or_default();
1263            pending.push(queued);
1264            session.set_pending_injected_messages(pending);
1265        })
1266        .await
1267}
1268
1269/// Build the hidden, compressible resume message for a completed background
1270/// shell — the same rich notice body used for a live-loop injection
1271/// ([`bash_completion_injection_body`]), but tagged as a resume message so it
1272/// satisfies the `has_pending_user_message` gate that lets a suspended session
1273/// spawn. This is what the **push** appends when it wakes a suspended loop, so
1274/// the model gets the shell's status + output tail in one shot without a
1275/// separate `BashOutput` round-trip.
1276fn bash_resume_message_from_info(info: &BashCompletionInfo) -> Message {
1277    let mut message = Message::user(bash_completion_injection_body(info));
1278    message.metadata = Some(serde_json::json!({
1279        RUNTIME_RESUME_MESSAGE_HIDDEN_KEY: true,
1280        RUNTIME_RESUME_MESSAGE_KIND_KEY: BASH_COMPLETION_RESUME_KIND,
1281    }));
1282    message.never_compress = false;
1283    message
1284}
1285
1286impl ChildCompletionCoordinator {
1287    /// Loop-facing delivery of a completed background shell. Two paths, chosen
1288    /// under the per-session resume lock so we never race the backstop poll or a
1289    /// concurrent child-completion resume:
1290    ///
1291    /// - **Suspended loop** (the model ended its turn with the shell running, so
1292    ///   `waiting_for_bash` is set) AND every waited shell has now finished →
1293    ///   **resume the loop directly**, event-driven, appending the rich completion
1294    ///   notice as the resume message. This is the push's whole point: no polling.
1295    /// - Otherwise (a live/iterating loop, or a suspend still waiting on OTHER
1296    ///   shells) → **enqueue** the notice as a pending injected message, drained at
1297    ///   the next round boundary (a live loop) or folded into the eventual resume
1298    ///   when the last shell finishes.
1299    async fn deliver_bash_completion(&self, mut info: BashCompletionInfo) {
1300        // A background shell completes outside the originating tool call, so
1301        // fire its PostToolUse seam here before routing the completion into the
1302        // next round/resume message. Do not hold the resume lock while an
1303        // arbitrary command hook runs: its configured timeout must never block
1304        // the independent liveness backstop.
1305        if let Some(hook_session) = self.load_session(&info.session_id).await {
1306            let config_snapshot = self.config.read().await.clone();
1307            let _ = run_background_bash_post_tool_hooks(
1308                &config_snapshot.lifecycle_hooks,
1309                Some(self.app_data_dir.clone()),
1310                &hook_session,
1311                &mut info,
1312            )
1313            .await;
1314        }
1315
1316        let guard = session_resume_lock(&info.session_id);
1317        let _held = guard.lock().await;
1318
1319        let Some(session) = self.load_session(&info.session_id).await else {
1320            tracing::warn!(
1321                session_id = %info.session_id,
1322                bash_id = %info.bash_id,
1323                "background bash completion: owning session not found; nothing to notify"
1324            );
1325            return;
1326        };
1327
1328        let waiting = read_runtime_state(&session).waiting_for_bash.is_some();
1329        // The producer flips the shell's `running` flag false BEFORE firing this
1330        // push, so a now-empty per-session registry means every shell the loop was
1331        // waiting on has finished — safe to resume. If OTHER waited shells are
1332        // still running, fall through to the enqueue path and let the last one (or
1333        // the backstop) drive the resume.
1334        let all_shells_done =
1335            bamboo_tools::tools::bash_runtime::running_shells_for_session(&info.session_id)
1336                .is_empty();
1337
1338        if bash_completion_should_resume(waiting, all_shells_done) {
1339            tracing::info!(
1340                session_id = %info.session_id,
1341                bash_id = %info.bash_id,
1342                status = %info.status,
1343                "background bash completion: push-resuming suspended loop (event-driven)"
1344            );
1345            self.perform_bash_resume(&info.session_id, bash_resume_message_from_info(&info))
1346                .await;
1347            return;
1348        }
1349
1350        match enqueue_bash_completion_injection(&self.persistence, &info).await {
1351            Ok(Some(_)) => tracing::info!(
1352                session_id = %info.session_id,
1353                bash_id = %info.bash_id,
1354                status = %info.status,
1355                waiting,
1356                "background bash completion queued for injection at the next round boundary"
1357            ),
1358            Ok(None) => tracing::warn!(
1359                session_id = %info.session_id,
1360                bash_id = %info.bash_id,
1361                "background bash completion: owning session not found; nothing to notify"
1362            ),
1363            Err(error) => tracing::warn!(
1364                session_id = %info.session_id,
1365                bash_id = %info.bash_id,
1366                %error,
1367                "background bash completion: failed to queue injection"
1368            ),
1369        }
1370    }
1371}
1372
1373impl BashCompletionSink for ChildCompletionCoordinator {
1374    fn on_bash_completed(&self, info: BashCompletionInfo) {
1375        // Best-effort, off the shell's completion-poll task: hand the delivery to
1376        // a detached task so the producer is never blocked (mirrors
1377        // `arrange_bash_self_resume`).
1378        let coordinator = Arc::new(self.clone());
1379        tokio::spawn(async move {
1380            coordinator.deliver_bash_completion(info).await;
1381        });
1382    }
1383}
1384
1385// ---------------------------------------------------------------------------
1386// Child-wait watchdog (issue #546)
1387// ---------------------------------------------------------------------------
1388
1389/// How often the child-wait watchdog sweeps suspended sessions.
1390const CHILD_WAIT_SWEEP_INTERVAL_SECS: u64 = 30;
1391/// Leave a freshly registered wait alone for this long so the event-driven
1392/// completion push always gets the first shot (and just-enqueued spawn jobs
1393/// have time to persist their running marker).
1394const CHILD_WAIT_REGISTRATION_GRACE_SECS: i64 = 60;
1395/// A waited child with NO live runner and a non-terminal index status is
1396/// declared dead once its control-plane has been quiet for this long.
1397const DEAD_CHILD_GRACE_SECS: i64 = 120;
1398/// Slack on top of the per-child liveness policy before a Running-but-frozen
1399/// runner entry (dead task) is force-finalized by the sweeper. The per-child
1400/// watchdog cancels at `max_idle`/`max_total` and the child then publishes its
1401/// own timeout; an entry frozen this far PAST those limits proves that
1402/// machinery is dead.
1403const STALE_RUNNER_SLACK_SECS: i64 = 600;
1404
1405/// Whether a waited child's index status means the sweeper must consider it
1406/// DEAD when nothing is driving it: non-terminal and not legitimately
1407/// suspended. `None` = never ran (created-but-never-started, or a spawn that
1408/// was lost before its running marker persisted).
1409fn is_dead_child_candidate_status(status: Option<&str>) -> bool {
1410    match status {
1411        // "suspended" children wait on a human / their own children / bash —
1412        // their wake has its own driver; never declare them dead here.
1413        Some(status) => !is_terminal_child_status(status) && status != "suspended",
1414        None => true,
1415    }
1416}
1417
1418/// Whether a reported completion's child id is genuinely a child of the parent
1419/// it claims (issue #546 read-side disclosure guard). `SubAgent.wait` ids are
1420/// model-provided, and the watchdog resolves an unowned id by publishing a
1421/// synthetic completion so the parent is unstranded — but the parent must NEVER
1422/// receive a FOREIGN session's content folded into its transcript.
1423/// `child_parent_linkage` is that session's own `parent_session_id` (`None`
1424/// when the session does not exist). Pure so the rule is unit-testable.
1425fn completion_child_is_owned(reported_parent: &str, child_parent_linkage: Option<&str>) -> bool {
1426    child_parent_linkage == Some(reported_parent)
1427}
1428
1429/// Pick which terminal child's completion to replay when the wait is already
1430/// satisfied but the parent is still suspended (lost wake). Prefer an
1431/// error-like child so a `FirstError` policy re-evaluates truthfully.
1432fn select_replay_child(terminal: &[(String, String)]) -> Option<&(String, String)> {
1433    terminal
1434        .iter()
1435        .find(|(_, status)| is_error_like(status))
1436        .or_else(|| terminal.last())
1437}
1438
1439fn child_wait_watchdog_resume_message(body: String) -> Message {
1440    let mut message = Message::user(body);
1441    message.metadata = Some(serde_json::json!({
1442        RUNTIME_RESUME_MESSAGE_HIDDEN_KEY: true,
1443        RUNTIME_RESUME_MESSAGE_KIND_KEY: "child_wait_watchdog_resume",
1444    }));
1445    message.never_compress = false;
1446    message
1447}
1448
1449fn empty_child_wait_message() -> Message {
1450    child_wait_watchdog_resume_message(
1451        "Runtime notification: this session was suspended waiting for child sessions, but the \
1452         wait tracked no children (internal inconsistency). The session has been resumed; use \
1453         SubAgent.list to inspect child state and continue the task."
1454            .to_string(),
1455    )
1456}
1457
1458fn child_wait_lease_expired_message(child_ids: &[String]) -> Message {
1459    child_wait_watchdog_resume_message(format!(
1460        "Runtime notification: the wait lease for child session(s) [{}] expired before they all \
1461         reported completion. They were NOT cancelled and may still be running or already \
1462         finished — verify their actual status with SubAgent.list / SubAgent.get before assuming \
1463         anything, then continue the task.",
1464        child_ids.join(", ")
1465    ))
1466}
1467
1468/// Child-wait watchdog (issue #546): the heartbeat backstop for parents
1469/// suspended on `waiting_for_children`.
1470///
1471/// The primary wake is the event-driven completion push (child terminal →
1472/// [`ChildCompletionHandler::on_child_completed`] → resume). This sweeper
1473/// exists because ANY break in that chain — a panicked child task, a dead
1474/// spawn scheduler, a process restart, a clobbered/exhausted resume, a wait
1475/// registered over an already-terminal child — previously stranded the parent
1476/// forever. It mirrors the bash backstop's philosophy: coarse, cheap, yields
1477/// to the push, and only acts when the durable state proves nothing else can.
1478///
1479/// All wake decisions funnel through the SAME machinery the push uses
1480/// (synthetic/replayed completions → `on_child_completed`, per-parent
1481/// serialization via [`session_resume_lock`]), so there is exactly one resume
1482/// implementation.
1483impl ChildCompletionCoordinator {
1484    /// Spawn the watchdog: one boot-time reconciliation pass, then a sweep
1485    /// every [`CHILD_WAIT_SWEEP_INTERVAL_SECS`]. Call once at server startup.
1486    pub fn spawn_child_wait_watchdog(self: &Arc<Self>) {
1487        let coordinator = Arc::clone(self);
1488        tokio::spawn(async move {
1489            use futures::FutureExt;
1490            if std::panic::AssertUnwindSafe(coordinator.reconcile_orphans_at_boot())
1491                .catch_unwind()
1492                .await
1493                .is_err()
1494            {
1495                tracing::error!("child-wait watchdog: boot reconciliation panicked");
1496            }
1497            let mut ticker = tokio::time::interval(std::time::Duration::from_secs(
1498                CHILD_WAIT_SWEEP_INTERVAL_SECS,
1499            ));
1500            ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
1501            // Skip the immediate tick (boot reconciliation just ran).
1502            ticker.tick().await;
1503            loop {
1504                ticker.tick().await;
1505                if std::panic::AssertUnwindSafe(coordinator.sweep_child_waits())
1506                    .catch_unwind()
1507                    .await
1508                    .is_err()
1509                {
1510                    tracing::error!("child-wait watchdog: sweep panicked; continuing");
1511                }
1512            }
1513        });
1514    }
1515
1516    /// One-shot startup reconciliation: a process restart kills every in-flight
1517    /// child task AND the in-memory bash backstop polls, but the durable state
1518    /// (child index status `running`, parents suspended on bash) survives —
1519    /// previously stranding those parents forever.
1520    async fn reconcile_orphans_at_boot(&self) {
1521        let cutoff = Utc::now();
1522
1523        // (1) Children left `running` by the previous process: no task in THIS
1524        // process is driving them, so no completion will ever fire. Mark them
1525        // terminal and wake their parents through the canonical path. (Root
1526        // sessions left `running` are user-visible and user-recoverable; only
1527        // children hold a suspended parent hostage.)
1528        let running = self
1529            .storage
1530            .list_sessions_by_run_status("running")
1531            .await
1532            .unwrap_or_default();
1533        for (child_id, parent_id) in running {
1534            let Some(parent_id) = parent_id else { continue };
1535            if self.runner_is_running(&child_id).await {
1536                continue;
1537            }
1538            let Some(control_plane) = self.load_control_plane(&child_id).await else {
1539                continue;
1540            };
1541            // A child that started AFTER boot is alive by definition — the
1542            // cutoff guards the tiny window where a fresh spawn races this scan.
1543            if control_plane.updated_at >= cutoff {
1544                continue;
1545            }
1546            tracing::warn!(
1547                child_session_id = %child_id,
1548                parent_session_id = %parent_id,
1549                "boot reconciliation: child was running when the process died; \
1550                 marking it error and waking the parent"
1551            );
1552            self.synthesize_child_completion(
1553                &parent_id,
1554                &child_id,
1555                "error",
1556                Some(
1557                    "orphaned by server restart: the process died while this child session \
1558                     was running"
1559                        .to_string(),
1560                ),
1561            )
1562            .await;
1563        }
1564
1565        // (2) Sessions suspended on `waiting_for_bash`: their backstop poll was
1566        // an in-memory task that died with the process. Re-arm it — with the
1567        // shell registry empty after a restart it resumes on its first check.
1568        let suspended = self
1569            .storage
1570            .list_sessions_by_run_status("suspended")
1571            .await
1572            .unwrap_or_default();
1573        for (session_id, _) in suspended {
1574            let Some(control_plane) = self.load_control_plane(&session_id).await else {
1575                continue;
1576            };
1577            if control_plane
1578                .metadata
1579                .get("runtime.suspend_reason")
1580                .map(String::as_str)
1581                != Some("waiting_for_bash")
1582            {
1583                continue;
1584            }
1585            if let Some(wait) = read_runtime_state(&control_plane).waiting_for_bash {
1586                tracing::warn!(
1587                    %session_id,
1588                    "boot reconciliation: re-arming bash self-resume backstop lost in restart"
1589                );
1590                let coordinator = self.clone();
1591                tokio::spawn(async move {
1592                    coordinator
1593                        .bash_self_resume(session_id, wait.bash_ids)
1594                        .await;
1595                });
1596            }
1597        }
1598    }
1599
1600    async fn runner_is_running(&self, session_id: &str) -> bool {
1601        let runners = self.agent_runners.read().await;
1602        runners
1603            .get(session_id)
1604            .is_some_and(|runner| matches!(runner.status, AgentStatus::Running))
1605    }
1606
1607    async fn load_control_plane(&self, session_id: &str) -> Option<Session> {
1608        match self.storage.load_runtime_control_plane(session_id).await {
1609            Ok(session) => session,
1610            Err(error) => {
1611                tracing::warn!(
1612                    %session_id,
1613                    %error,
1614                    "child-wait watchdog: failed to load session control plane"
1615                );
1616                None
1617            }
1618        }
1619    }
1620
1621    /// One sweep: inspect every session whose index status is `suspended`.
1622    /// Cheap in the common case — the candidate list is small and each check is
1623    /// a sidecar (control-plane) load.
1624    async fn sweep_child_waits(&self) {
1625        let suspended = match self.storage.list_sessions_by_run_status("suspended").await {
1626            Ok(entries) => entries,
1627            Err(error) => {
1628                tracing::warn!(%error, "child-wait watchdog: failed to list suspended sessions");
1629                return;
1630            }
1631        };
1632        for (session_id, _) in suspended {
1633            self.sweep_one_suspended_session(&session_id).await;
1634        }
1635    }
1636
1637    async fn sweep_one_suspended_session(&self, session_id: &str) {
1638        // A live runner means the session already resumed (the index status
1639        // only advances at its next terminal) — nothing to do.
1640        if self.runner_is_running(session_id).await {
1641            return;
1642        }
1643        let Some(session) = self.load_control_plane(session_id).await else {
1644            return;
1645        };
1646        let runtime_state = read_runtime_state(&session);
1647        let suspend_reason = session
1648            .metadata
1649            .get("runtime.suspend_reason")
1650            .map(String::as_str)
1651            .unwrap_or_default()
1652            .to_string();
1653        match (
1654            suspend_reason.as_str(),
1655            runtime_state.waiting_for_children.clone(),
1656        ) {
1657            // Human-gated or bash-owned waits: not ours to time out.
1658            ("waiting_for_bash", _)
1659            | ("awaiting_clarification", _)
1660            | ("awaiting_parent_approval", _) => {}
1661            (_, Some(wait)) => self.sweep_child_wait(session_id, wait).await,
1662            // Suspended with NO armed wait: either the coordinator cleared the
1663            // wait but its resume never spawned, or state is half-cleared.
1664            ("waiting_for_children", None) | ("", None) => {
1665                self.rescue_stranded_resume(session_id).await;
1666            }
1667            _ => {}
1668        }
1669    }
1670
1671    /// Evaluate one armed child wait against reality and act on what the
1672    /// durable state proves: dead children are synthesized terminal, an
1673    /// already-satisfied wait gets its lost wake replayed, an expired lease or
1674    /// empty wait force-resumes the parent.
1675    async fn sweep_child_wait(&self, parent_session_id: &str, wait: WaitingForChildrenState) {
1676        let now = Utc::now();
1677
1678        if wait.child_session_ids.is_empty() {
1679            tracing::warn!(
1680                %parent_session_id,
1681                "child-wait watchdog: wait armed over an empty child set; force-resuming"
1682            );
1683            self.force_resume_child_wait(parent_session_id, empty_child_wait_message())
1684                .await;
1685            return;
1686        }
1687
1688        // The 6h lease (previously written but never read). Expiry does not
1689        // kill children — child runners own child liveness; the parent is
1690        // resumed with a verify-don't-assume note.
1691        if wait.timeout_at.is_some_and(|deadline| now >= deadline) {
1692            tracing::warn!(
1693                %parent_session_id,
1694                "child-wait watchdog: wait lease expired; force-resuming parent"
1695            );
1696            self.force_resume_child_wait(
1697                parent_session_id,
1698                child_wait_lease_expired_message(&wait.child_session_ids),
1699            )
1700            .await;
1701            return;
1702        }
1703
1704        // Yield to the event-driven push on fresh waits.
1705        if now.signed_duration_since(wait.registered_at).num_seconds()
1706            < CHILD_WAIT_REGISTRATION_GRACE_SECS
1707        {
1708            return;
1709        }
1710
1711        let statuses: HashMap<String, Option<String>> = self
1712            .storage
1713            .list_child_run_statuses(parent_session_id)
1714            .await
1715            .unwrap_or_default()
1716            .into_iter()
1717            .collect();
1718
1719        struct DeadChild {
1720            child_id: String,
1721            status: String,
1722            reason: String,
1723            /// `false` = the id does not name a child of THIS parent (wait ids
1724            /// are model-provided and unvalidated): publish the wake but never
1725            /// persist onto / cancel / finalize the named session.
1726            owned: bool,
1727        }
1728
1729        let mut terminal: Vec<(String, String)> = Vec::new();
1730        let mut dead: Vec<DeadChild> = Vec::new();
1731        for child_id in &wait.child_session_ids {
1732            let status = statuses.get(child_id).and_then(|status| status.as_deref());
1733            if let Some(status) = status {
1734                if is_terminal_child_status(status) {
1735                    terminal.push((child_id.clone(), status.to_string()));
1736                    continue;
1737                }
1738            }
1739            if !is_dead_child_candidate_status(status) {
1740                continue;
1741            }
1742
1743            // Ownership check BEFORE anything destructive: `SubAgent.wait` ids
1744            // are model-provided and unvalidated, so an id absent from the
1745            // parent-scoped status map may name a REAL session in another tree
1746            // (a grandchild, a foreign root). Such a session must never be
1747            // mutated, cancelled, or finalized here — but the parent's bogus
1748            // wait entry still needs a synthetic completion to clear it.
1749            let control_plane = self.load_control_plane(child_id).await;
1750            let owned = control_plane
1751                .as_ref()
1752                .is_some_and(|cp| cp.parent_session_id.as_deref() == Some(parent_session_id));
1753            if !owned {
1754                dead.push(DeadChild {
1755                    child_id: child_id.clone(),
1756                    status: "error".to_string(),
1757                    reason: if control_plane.is_some() {
1758                        "waited-on session id is not a child of this session; clearing it \
1759                         from the wait without touching that session"
1760                            .to_string()
1761                    } else {
1762                        "waited-on child session does not exist".to_string()
1763                    },
1764                    owned: false,
1765                });
1766                continue;
1767            }
1768
1769            let runner = { self.agent_runners.read().await.get(child_id).cloned() };
1770            match runner {
1771                Some(runner) if matches!(runner.status, AgentStatus::Running) => {
1772                    // A live-looking runner: only intervene when it is frozen
1773                    // far PAST the per-child liveness limits — which proves the
1774                    // per-child watchdog machinery itself is dead (task
1775                    // panicked or lost), because it would have cancelled and
1776                    // published a timeout long before.
1777                    let last_activity = runner.last_event_at.unwrap_or(runner.started_at);
1778                    let idle_secs = now.signed_duration_since(last_activity).num_seconds();
1779                    let total_secs = now.signed_duration_since(runner.started_at).num_seconds();
1780                    let policy = match &control_plane {
1781                        Some(child) => {
1782                            crate::runtime::execution::spawn::watchdog_policy_for_session(child)
1783                        }
1784                        None => Default::default(),
1785                    };
1786                    let idle_limit = policy.max_idle_secs.saturating_add(STALE_RUNNER_SLACK_SECS);
1787                    let total_limit = policy
1788                        .max_total_secs
1789                        .saturating_add(STALE_RUNNER_SLACK_SECS);
1790                    if idle_secs >= idle_limit || total_secs >= total_limit {
1791                        runner.cancel_token.cancel();
1792                        dead.push(DeadChild {
1793                            child_id: child_id.clone(),
1794                            status: "timeout".to_string(),
1795                            reason: format!(
1796                                "child runner stalled: no events for {idle_secs}s \
1797                                 (limit {idle_limit}s including watchdog slack); \
1798                                 force-finalized by the child-wait watchdog"
1799                            ),
1800                            owned: true,
1801                        });
1802                    }
1803                }
1804                _ => {
1805                    // Nothing is driving this child, yet its index status will
1806                    // never advance by itself. The grace covers the enqueue →
1807                    // running-marker window and slow spawn queues.
1808                    let quiet_secs = control_plane
1809                        .as_ref()
1810                        .map(|child| now.signed_duration_since(child.updated_at).num_seconds())
1811                        .unwrap_or(i64::MAX);
1812                    if quiet_secs >= DEAD_CHILD_GRACE_SECS {
1813                        dead.push(DeadChild {
1814                            child_id: child_id.clone(),
1815                            status: "error".to_string(),
1816                            reason: format!(
1817                                "child runner lost (crashed task, dropped spawn job, or \
1818                                 process restart): index status {status:?} with no live \
1819                                 runner driving it"
1820                            ),
1821                            owned: true,
1822                        });
1823                    }
1824                }
1825            }
1826        }
1827
1828        if !dead.is_empty() {
1829            for entry in dead {
1830                tracing::warn!(
1831                    %parent_session_id,
1832                    child_session_id = %entry.child_id,
1833                    status = %entry.status,
1834                    reason = %entry.reason,
1835                    owned = entry.owned,
1836                    "child-wait watchdog: synthesizing terminal completion for dead child"
1837                );
1838                if entry.owned {
1839                    self.synthesize_child_completion(
1840                        parent_session_id,
1841                        &entry.child_id,
1842                        &entry.status,
1843                        Some(entry.reason),
1844                    )
1845                    .await;
1846                } else {
1847                    // Foreign / nonexistent id: wake the parent only.
1848                    self.publish_synthetic_completion(
1849                        parent_session_id,
1850                        &entry.child_id,
1851                        &entry.status,
1852                        Some(entry.reason),
1853                    )
1854                    .await;
1855                }
1856            }
1857            // The publishes above re-evaluate the wait policy themselves.
1858            return;
1859        }
1860
1861        // No dead children — but if the terminal set ALREADY satisfies the
1862        // policy, the original wake was lost (clobbered resume / retry budget
1863        // exhausted / completion raced the wait registration). Replay one real
1864        // completion through the canonical path; `on_child_completed` is
1865        // idempotent for an already-cleared wait.
1866        let terminal_ids: Vec<String> = terminal.iter().map(|(id, _)| id.clone()).collect();
1867        if let Some((child_id, status)) = select_replay_child(&terminal) {
1868            if wait_policy_satisfied(
1869                wait.wait_for,
1870                &wait.child_session_ids,
1871                &terminal_ids,
1872                child_id,
1873                status,
1874            ) {
1875                tracing::warn!(
1876                    %parent_session_id,
1877                    child_session_id = %child_id,
1878                    "child-wait watchdog: wait already satisfied but parent still suspended \
1879                     (lost wake); replaying the completion"
1880                );
1881                let error = self
1882                    .load_control_plane(child_id)
1883                    .await
1884                    .and_then(|child| child.last_run_error());
1885                self.publish_synthetic_completion(parent_session_id, child_id, status, error)
1886                    .await;
1887            }
1888        }
1889    }
1890
1891    /// Persist a synthesized terminal status on the child (so the index flips
1892    /// and nothing re-detects or re-suspends on it), finalize any lingering
1893    /// runner entry (so a future re-run can reserve), then publish through the
1894    /// canonical completion path — broadcast + `on_child_completed`, exactly
1895    /// like a real child terminal.
1896    async fn synthesize_child_completion(
1897        &self,
1898        parent_session_id: &str,
1899        child_session_id: &str,
1900        status: &str,
1901        error: Option<String>,
1902    ) {
1903        match self.storage.load_session(child_session_id).await {
1904            Ok(Some(mut child)) => {
1905                // Ownership guard (defense in depth — callers check too): only
1906                // a session that IS a child of this parent may be mutated. An
1907                // arbitrary session id named in a wait still wakes the parent
1908                // via the publish below, but its own state stays untouched.
1909                if child.parent_session_id.as_deref() != Some(parent_session_id) {
1910                    tracing::warn!(
1911                        %parent_session_id,
1912                        child_session_id = %child.id,
1913                        "child-wait watchdog: refusing to synthesize status onto a session \
1914                         that is not a child of this parent"
1915                    );
1916                    self.publish_synthetic_completion(
1917                        parent_session_id,
1918                        child_session_id,
1919                        status,
1920                        error,
1921                    )
1922                    .await;
1923                    return;
1924                }
1925                child.set_last_run_status(status);
1926                match &error {
1927                    Some(message) => child.set_last_run_error(message.clone()),
1928                    None => child.clear_last_run_error(),
1929                }
1930                child.updated_at = Utc::now();
1931                if let Err(save_error) = self.persistence.merge_save_runtime(&mut child).await {
1932                    tracing::warn!(
1933                        child_session_id = %child.id,
1934                        %save_error,
1935                        "child-wait watchdog: failed to persist synthesized terminal status"
1936                    );
1937                }
1938                self.sessions
1939                    .insert(child.id.clone(), Arc::new(parking_lot::RwLock::new(child)));
1940            }
1941            Ok(None) => {}
1942            Err(load_error) => {
1943                tracing::warn!(
1944                    %child_session_id,
1945                    %load_error,
1946                    "child-wait watchdog: failed to load child for synthesized terminal status"
1947                );
1948            }
1949        }
1950        finalize_runner(
1951            &self.agent_runners,
1952            child_session_id,
1953            &Err(bamboo_agent_core::AgentError::LLM(
1954                error
1955                    .clone()
1956                    .unwrap_or_else(|| format!("synthesized {status}")),
1957            )),
1958        )
1959        .await;
1960        self.publish_synthetic_completion(parent_session_id, child_session_id, status, error)
1961            .await;
1962    }
1963
1964    async fn publish_synthetic_completion(
1965        &self,
1966        parent_session_id: &str,
1967        child_session_id: &str,
1968        status: &str,
1969        error: Option<String>,
1970    ) {
1971        let parent_tx = crate::execution::session_events::get_or_create_event_sender(
1972            &self.session_event_senders,
1973            parent_session_id,
1974        )
1975        .await;
1976        let handler: Arc<dyn ChildCompletionHandler> = Arc::new(self.clone());
1977        crate::runtime::execution::spawn::publish_child_completion_parts(
1978            &parent_tx,
1979            Some(handler),
1980            parent_session_id.to_string(),
1981            child_session_id.to_string(),
1982            status.to_string(),
1983            error,
1984        )
1985        .await;
1986    }
1987
1988    /// A parent whose wait was already cleared (resume message appended) but
1989    /// whose resume never spawned — retry-budget exhaustion, root-tools not
1990    /// yet initialized, or a restart between clear and spawn. Detected by: no
1991    /// live runner, no armed wait, and a pending hidden runtime resume message
1992    /// as the LAST message. Resume is all that's left to do.
1993    async fn rescue_stranded_resume(&self, session_id: &str) {
1994        let Some(session) = self.load_session(session_id).await else {
1995            return;
1996        };
1997        let pending_runtime_resume = session.messages.last().is_some_and(|message| {
1998            matches!(message.role, Role::User)
1999                && message
2000                    .metadata
2001                    .as_ref()
2002                    .is_some_and(|meta| meta.get(RUNTIME_RESUME_MESSAGE_KIND_KEY).is_some())
2003        });
2004        if !pending_runtime_resume {
2005            return;
2006        }
2007        tracing::warn!(
2008            %session_id,
2009            "child-wait watchdog: stranded resume detected (wait cleared, resume never \
2010             spawned); resuming"
2011        );
2012        self.resume_parent(session_id.to_string()).await;
2013    }
2014
2015    /// Clear the parent's child wait, append `resume_message`, and drive the
2016    /// resume — with a bounded clobber-retry mirroring
2017    /// [`Self::perform_bash_resume`]: a suspending runner's one-shot finalize
2018    /// save can land after ours and revert the wait while dropping the
2019    /// message; we detect the re-armed wait and re-clear.
2020    async fn force_resume_child_wait(&self, session_id: &str, resume_message: Message) {
2021        const MAX_ATTEMPTS: u8 = 5;
2022        let lock = session_resume_lock(session_id);
2023        for attempt in 0..MAX_ATTEMPTS {
2024            if attempt > 0 {
2025                tokio::time::sleep(Duration::from_millis(200)).await;
2026            }
2027            {
2028                let _held = lock.lock().await;
2029                let Some(mut session) = self.load_session(session_id).await else {
2030                    return;
2031                };
2032                let mut runtime_state = read_runtime_state(&session);
2033                if runtime_state.waiting_for_children.is_none() {
2034                    // Another source already resumed this parent.
2035                    return;
2036                }
2037                runtime_state.waiting_for_children = None;
2038                runtime_state.status = AgentStatusState::Idle;
2039                runtime_state.suspension = None;
2040                write_runtime_state(&mut session, &runtime_state);
2041                session.metadata.remove("runtime.suspend_reason");
2042                session.add_message(resume_message.clone());
2043                session.updated_at = Utc::now();
2044                self.save_and_cache(&mut session).await;
2045            }
2046            let outcome = self.resume_parent(session_id.to_string()).await;
2047            match outcome {
2048                ResumeOutcome::Started { .. } | ResumeOutcome::NotFound => return,
2049                ResumeOutcome::Completed | ResumeOutcome::AlreadyRunning { .. } => {
2050                    // Only retry when the persisted wait was clobbered back to
2051                    // armed; if it stayed cleared with the message intact, the
2052                    // next sweep's stranded-resume rescue finishes the job.
2053                    let clobbered = self
2054                        .load_session(session_id)
2055                        .await
2056                        .map(|session| read_runtime_state(&session).waiting_for_children.is_some())
2057                        .unwrap_or(false);
2058                    if !clobbered {
2059                        return;
2060                    }
2061                }
2062            }
2063        }
2064        tracing::error!(
2065            %session_id,
2066            "child-wait watchdog: force-resume exhausted its clobber-retry budget"
2067        );
2068    }
2069}
2070
2071#[cfg(test)]
2072mod tests {
2073    use super::*;
2074    use bamboo_agent_core::Message;
2075
2076    // ── child-wait watchdog pure helpers (issue #546) ────────────────────
2077
2078    #[test]
2079    fn dead_child_candidate_status_matrix() {
2080        // Never ran / lost before the running marker: dead candidate.
2081        assert!(is_dead_child_candidate_status(None));
2082        // Actively-reported non-terminal statuses: dead candidates when nothing
2083        // is driving them.
2084        assert!(is_dead_child_candidate_status(Some("running")));
2085        assert!(is_dead_child_candidate_status(Some("pending")));
2086        // Legitimately quiescent: waiting on a human / own children / bash.
2087        assert!(!is_dead_child_candidate_status(Some("suspended")));
2088        // Terminal statuses can never be "dead" — they are already done.
2089        for status in ["completed", "error", "timeout", "cancelled", "skipped"] {
2090            assert!(!is_dead_child_candidate_status(Some(status)), "{status}");
2091        }
2092    }
2093
2094    #[test]
2095    fn completion_child_ownership_gates_content_fold() {
2096        // Owned: the child's own parent linkage matches the reporting parent.
2097        assert!(completion_child_is_owned("parent-1", Some("parent-1")));
2098        // Foreign: a real session that belongs to a DIFFERENT parent — its
2099        // content must never be folded into parent-1's transcript.
2100        assert!(!completion_child_is_owned("parent-1", Some("parent-2")));
2101        // Root/unparented session, or a nonexistent id (linkage None).
2102        assert!(!completion_child_is_owned("parent-1", None));
2103    }
2104
2105    #[test]
2106    fn replay_child_prefers_error_like_for_first_error_policy() {
2107        let terminal = vec![
2108            ("c-ok".to_string(), "completed".to_string()),
2109            ("c-err".to_string(), "timeout".to_string()),
2110            ("c-late".to_string(), "completed".to_string()),
2111        ];
2112        let (id, status) = select_replay_child(&terminal).expect("non-empty");
2113        assert_eq!(id, "c-err");
2114        assert_eq!(status, "timeout");
2115
2116        let all_ok = vec![
2117            ("c-1".to_string(), "completed".to_string()),
2118            ("c-2".to_string(), "completed".to_string()),
2119        ];
2120        let (id, _) = select_replay_child(&all_ok).expect("non-empty");
2121        assert_eq!(id, "c-2");
2122
2123        assert!(select_replay_child(&[]).is_none());
2124    }
2125
2126    #[test]
2127    fn watchdog_resume_messages_are_hidden_runtime_messages() {
2128        for message in [
2129            empty_child_wait_message(),
2130            child_wait_lease_expired_message(&["c-1".to_string(), "c-2".to_string()]),
2131        ] {
2132            assert!(matches!(message.role, Role::User));
2133            let meta = message.metadata.expect("hidden runtime metadata");
2134            assert_eq!(meta[RUNTIME_RESUME_MESSAGE_HIDDEN_KEY], true);
2135            assert_eq!(
2136                meta[RUNTIME_RESUME_MESSAGE_KIND_KEY],
2137                "child_wait_watchdog_resume"
2138            );
2139        }
2140        let lease = child_wait_lease_expired_message(&["c-1".to_string()]);
2141        // The lease message must never claim the children finished.
2142        assert!(lease.content.contains("NOT cancelled"));
2143        assert!(lease.content.contains("c-1"));
2144    }
2145
2146    // ── on_child_completed terminality guard (issue #546) ────────────────
2147
2148    #[test]
2149    fn non_terminal_statuses_never_satisfy_wait_policies() {
2150        // The guard keys on `is_terminal_child_status`; "suspended" (and any
2151        // unknown non-terminal string) must not count toward any policy.
2152        assert!(!is_terminal_child_status("suspended"));
2153        assert!(!is_terminal_child_status("running"));
2154        assert!(!is_terminal_child_status("pending"));
2155    }
2156
2157    fn make_completion(status: &str) -> ChildCompletion {
2158        ChildCompletion {
2159            parent_session_id: "parent-1".to_string(),
2160            child_session_id: "child-1".to_string(),
2161            status: status.to_string(),
2162            error: None,
2163            completed_at: Utc::now(),
2164        }
2165    }
2166
2167    // ── ② derive completed children from the index ──────────────────────
2168
2169    struct StubChildIndex {
2170        children: Vec<(String, Option<String>)>,
2171    }
2172
2173    #[async_trait]
2174    impl Storage for StubChildIndex {
2175        async fn save_session(&self, _session: &Session) -> std::io::Result<()> {
2176            Ok(())
2177        }
2178        async fn load_session(&self, _id: &str) -> std::io::Result<Option<Session>> {
2179            Ok(None)
2180        }
2181        async fn delete_session(&self, _id: &str) -> std::io::Result<bool> {
2182            Ok(false)
2183        }
2184        async fn list_child_run_statuses(
2185            &self,
2186            _parent_session_id: &str,
2187        ) -> std::io::Result<Vec<(String, Option<String>)>> {
2188            Ok(self.children.clone())
2189        }
2190    }
2191
2192    #[tokio::test]
2193    async fn derive_completed_only_includes_terminal_children() {
2194        let storage: Arc<dyn Storage> = Arc::new(StubChildIndex {
2195            children: vec![
2196                ("a".into(), Some("completed".into())),
2197                ("b".into(), Some("running".into())),
2198                ("c".into(), Some("error".into())),
2199                ("d".into(), None),
2200            ],
2201        });
2202        let completed = derive_completed_child_ids(&storage, "parent-1", "b").await;
2203        // Terminal from index: a, c. Plus the just-completed child b folded in.
2204        assert_eq!(
2205            completed,
2206            vec!["a".to_string(), "b".to_string(), "c".to_string()]
2207        );
2208    }
2209
2210    #[tokio::test]
2211    async fn derive_completed_folds_in_just_completed_when_index_lags() {
2212        // Index hasn't caught up — reports the child as still running.
2213        let storage: Arc<dyn Storage> = Arc::new(StubChildIndex {
2214            children: vec![("only".into(), Some("running".into()))],
2215        });
2216        let completed = derive_completed_child_ids(&storage, "parent-1", "only").await;
2217        assert_eq!(completed, vec!["only".to_string()]);
2218    }
2219
2220    #[test]
2221    fn wait_policy_all_uses_derived_completed_set() {
2222        let waited = vec!["a".to_string(), "b".to_string()];
2223        assert!(!wait_policy_satisfied(
2224            ChildWaitPolicy::All,
2225            &waited,
2226            &["a".to_string()],
2227            "a",
2228            "completed"
2229        ));
2230        assert!(wait_policy_satisfied(
2231            ChildWaitPolicy::All,
2232            &waited,
2233            &["a".to_string(), "b".to_string()],
2234            "b",
2235            "completed"
2236        ));
2237    }
2238
2239    #[test]
2240    fn wait_policy_first_error_requires_tracked_membership() {
2241        let waited = vec!["a".to_string(), "b".to_string()];
2242        // An error from a TRACKED child resumes immediately.
2243        assert!(wait_policy_satisfied(
2244            ChildWaitPolicy::FirstError,
2245            &waited,
2246            &["a".to_string()],
2247            "a",
2248            "error"
2249        ));
2250        // An error-like completion from an UNTRACKED child (e.g. a zombie
2251        // task from an earlier run waking up late) must not resume the wait.
2252        assert!(!wait_policy_satisfied(
2253            ChildWaitPolicy::FirstError,
2254            &waited,
2255            &["a".to_string()],
2256            "stray-child",
2257            "timeout"
2258        ));
2259        // The all-complete fallback still applies regardless of the reporter.
2260        assert!(wait_policy_satisfied(
2261            ChildWaitPolicy::FirstError,
2262            &waited,
2263            &["a".to_string(), "b".to_string()],
2264            "stray-child",
2265            "completed"
2266        ));
2267    }
2268
2269    #[test]
2270    fn child_final_assistant_text_returns_last_assistant() {
2271        let mut session = Session::new("child-1", "gpt-4");
2272        session.messages.push(Message::user("hi"));
2273        session
2274            .messages
2275            .push(Message::assistant("first answer", None));
2276        session.messages.push(Message::user("again"));
2277        session
2278            .messages
2279            .push(Message::assistant("final answer", None));
2280
2281        assert_eq!(
2282            child_final_assistant_text(&session).as_deref(),
2283            Some("final answer")
2284        );
2285    }
2286
2287    #[test]
2288    fn child_final_assistant_text_returns_none_when_blank() {
2289        let mut session = Session::new("child-1", "gpt-4");
2290        session.messages.push(Message::assistant("   ", None));
2291        assert!(child_final_assistant_text(&session).is_none());
2292    }
2293
2294    #[test]
2295    fn child_final_assistant_text_returns_none_when_no_assistant() {
2296        let mut session = Session::new("child-1", "gpt-4");
2297        session.messages.push(Message::user("hi"));
2298        assert!(child_final_assistant_text(&session).is_none());
2299    }
2300
2301    #[test]
2302    fn runtime_resume_message_folds_full_response_without_truncation() {
2303        // A very long child final response is folded in verbatim (no 4000-char
2304        // cap, no truncation marker).
2305        let completion = make_completion("completed");
2306        let long: String = "a".repeat(10_000);
2307        let message = runtime_resume_message(&completion, 0, Some(&long));
2308        assert!(message.content.contains(&long));
2309        assert!(!message.content.contains("truncated"));
2310    }
2311
2312    #[test]
2313    fn runtime_resume_message_includes_child_response_when_provided() {
2314        let completion = make_completion("completed");
2315        let message = runtime_resume_message(&completion, 0, Some("the answer is 42"));
2316
2317        assert!(matches!(message.role, Role::User));
2318        // Folded child results are now compressible so the parent context can
2319        // reclaim them under compaction.
2320        assert!(!message.never_compress);
2321        assert!(message.content.contains("Child final response:"));
2322        assert!(message.content.contains("the answer is 42"));
2323
2324        let metadata = message.metadata.expect("metadata present");
2325        assert_eq!(
2326            metadata.get("hidden_from_ui").and_then(|v| v.as_bool()),
2327            Some(true)
2328        );
2329        assert_eq!(
2330            metadata.get("runtime_kind").and_then(|v| v.as_str()),
2331            Some("child_completion_resume")
2332        );
2333        assert_eq!(
2334            metadata
2335                .get("child_final_response_included")
2336                .and_then(|v| v.as_bool()),
2337            Some(true)
2338        );
2339    }
2340
2341    #[test]
2342    fn runtime_resume_message_falls_back_to_error_when_no_response() {
2343        let mut completion = make_completion("error");
2344        completion.error = Some("boom".to_string());
2345
2346        let message = runtime_resume_message(&completion, 1, None);
2347        assert!(message.content.contains("Child error:"));
2348        assert!(message.content.contains("boom"));
2349        let metadata = message.metadata.expect("metadata present");
2350        assert_eq!(
2351            metadata
2352                .get("child_final_response_included")
2353                .and_then(|v| v.as_bool()),
2354            Some(false)
2355        );
2356    }
2357
2358    #[test]
2359    fn runtime_resume_message_minimal_when_no_response_and_no_error() {
2360        let completion = make_completion("completed");
2361        let message = runtime_resume_message(&completion, 2, None);
2362        assert!(!message.content.contains("Child final response:"));
2363        assert!(!message.content.contains("Child error:"));
2364        assert!(message.content.contains("Resume the parent task"));
2365    }
2366
2367    #[test]
2368    fn read_config_snapshot_refreshes_cached_snapshot_from_live_config() {
2369        let runtime = tokio::runtime::Runtime::new().expect("runtime");
2370
2371        runtime.block_on(async {
2372            let config = Arc::new(RwLock::new(Config::default()));
2373            config.write().await.provider = "copilot".to_string();
2374            let cached_config = StdRwLock::new(Config::default());
2375
2376            let snapshot = read_config_snapshot(&config, &cached_config);
2377
2378            assert_eq!(snapshot.provider, "copilot");
2379            assert_eq!(
2380                cached_config.read().expect("cached snapshot lock").provider,
2381                "copilot"
2382            );
2383        });
2384    }
2385
2386    #[test]
2387    fn read_config_snapshot_uses_cached_snapshot_when_live_lock_is_busy() {
2388        let runtime = tokio::runtime::Runtime::new().expect("runtime");
2389
2390        runtime.block_on(async {
2391            let mut cached_snapshot = Config::default();
2392            cached_snapshot.provider = "cached-provider".to_string();
2393
2394            let config = Arc::new(RwLock::new(Config::default()));
2395            let cached_config = StdRwLock::new(cached_snapshot);
2396            let _write_guard = config.write().await;
2397
2398            let snapshot = read_config_snapshot(&config, &cached_config);
2399
2400            assert_eq!(snapshot.provider, "cached-provider");
2401        });
2402    }
2403
2404    // ── Bash self-resume (issue #84 Phase 2b): deadline message + clobber-retry ──
2405
2406    #[test]
2407    fn bash_completion_resume_message_normal_announces_completion() {
2408        let ids = vec!["bg-1".to_string(), "bg-2".to_string()];
2409        let message = bash_completion_resume_message(&ids, false);
2410        // Normal path: the shells genuinely finished.
2411        assert!(
2412            message.content.contains("have completed"),
2413            "normal resume message must announce completion: {}",
2414            message.content
2415        );
2416        // Hidden + compressible so the resume gate sees it but the UI hides it.
2417        let metadata = message.metadata.expect("metadata present");
2418        assert_eq!(
2419            metadata
2420                .get(RUNTIME_RESUME_MESSAGE_HIDDEN_KEY)
2421                .and_then(|v| v.as_bool()),
2422            Some(true),
2423            "resume message must be hidden from the UI"
2424        );
2425        assert_eq!(
2426            metadata
2427                .get(RUNTIME_RESUME_MESSAGE_KIND_KEY)
2428                .and_then(|v| v.as_str()),
2429            Some(BASH_COMPLETION_RESUME_KIND),
2430            "resume message must carry the bash-completion kind discriminant"
2431        );
2432    }
2433
2434    #[test]
2435    fn bash_completion_resume_message_deadline_does_not_claim_completion() {
2436        // The 6h+10m deadline force-breaks with shells STILL running. The message
2437        // must NOT say "have completed" — that would let the model assume success
2438        // on a false premise. It must direct the model to verify with BashOutput.
2439        let ids = vec!["bg-long".to_string()];
2440        let message = bash_completion_resume_message(&ids, true);
2441        assert!(
2442            !message.content.contains("have completed"),
2443            "deadline resume message must NOT claim the shells completed: {}",
2444            message.content
2445        );
2446        assert!(
2447            message.content.contains("may still be running"),
2448            "deadline resume message must warn shells may still be running: {}",
2449            message.content
2450        );
2451        assert!(
2452            message.content.contains("BashOutput"),
2453            "deadline resume message must direct verification via BashOutput: {}",
2454            message.content
2455        );
2456        // Same hidden/kind shape so the resume gate is satisfied identically.
2457        let metadata = message.metadata.expect("metadata present");
2458        assert_eq!(
2459            metadata
2460                .get(RUNTIME_RESUME_MESSAGE_KIND_KEY)
2461                .and_then(|v| v.as_str()),
2462            Some(BASH_COMPLETION_RESUME_KIND)
2463        );
2464    }
2465
2466    #[test]
2467    fn bash_resume_should_retry_matrix() {
2468        // The finalize-clobber retry predicate (issue #84 Phase 2b). Retry only
2469        // when the resume did NOT spawn (Completed / AlreadyRunning) AND the
2470        // persisted bash wait is still set on reload — the clobber signature.
2471
2472        // Started: the resume fired — never retry, regardless of persisted state.
2473        assert!(!bash_resume_should_retry(
2474            &ResumeOutcome::Started { run_id: "r".into() },
2475            true
2476        ));
2477        assert!(!bash_resume_should_retry(
2478            &ResumeOutcome::Started { run_id: "r".into() },
2479            false
2480        ));
2481
2482        // NotFound: session gone — never retry.
2483        assert!(!bash_resume_should_retry(&ResumeOutcome::NotFound, true));
2484        assert!(!bash_resume_should_retry(&ResumeOutcome::NotFound, false));
2485
2486        // Completed + persisted wait still set ⇒ finalize-clobber ⇒ retry.
2487        assert!(bash_resume_should_retry(&ResumeOutcome::Completed, true));
2488        // Completed + persisted wait cleared ⇒ handled (our message stuck, or a
2489        // concurrent resume finished) ⇒ stop.
2490        assert!(!bash_resume_should_retry(&ResumeOutcome::Completed, false));
2491
2492        // AlreadyRunning + persisted wait still set ⇒ clobbered while a runner is
2493        // (stale-)active ⇒ retry to re-establish the resume message.
2494        assert!(bash_resume_should_retry(
2495            &ResumeOutcome::AlreadyRunning { run_id: "r".into() },
2496            true
2497        ));
2498        // AlreadyRunning + wait cleared ⇒ a runner owns the session ⇒ stop.
2499        assert!(!bash_resume_should_retry(
2500            &ResumeOutcome::AlreadyRunning { run_id: "r".into() },
2501            false
2502        ));
2503    }
2504
2505    // ── bash completion injection body (Phase 2b follow-up) ──────────────
2506
2507    #[test]
2508    fn injection_body_includes_status_exit_command_and_tail() {
2509        let info = BashCompletionInfo {
2510            session_id: "s".into(),
2511            bash_id: "abc123".into(),
2512            command: "make build".into(),
2513            exit_code: Some(0),
2514            status: "completed".into(),
2515            output_tail: "BUILD OK".into(),
2516        };
2517        let body = bash_completion_injection_body(&info);
2518        assert!(body.contains("abc123"), "body: {body}");
2519        assert!(body.contains("make build"), "body: {body}");
2520        assert!(body.contains("completed"), "body: {body}");
2521        assert!(body.contains("exit code 0"), "body: {body}");
2522        assert!(body.contains("BUILD OK"), "body: {body}");
2523        // The model is pointed at BashOutput for the full log.
2524        assert!(body.contains("BashOutput"), "body: {body}");
2525        assert!(body.contains("bash_id=\"abc123\""), "body: {body}");
2526    }
2527
2528    #[test]
2529    fn injection_body_handles_no_output_and_signal_kill() {
2530        let info = BashCompletionInfo {
2531            session_id: "s".into(),
2532            bash_id: "xyz".into(),
2533            command: "sleep 99".into(),
2534            exit_code: None,
2535            status: "killed".into(),
2536            output_tail: String::new(),
2537        };
2538        let body = bash_completion_injection_body(&info);
2539        assert!(body.contains("killed"), "body: {body}");
2540        assert!(body.contains("none (signal/killed)"), "body: {body}");
2541        assert!(body.contains("no captured output"), "body: {body}");
2542        // No output tail section when there is nothing to show.
2543        assert!(!body.contains("Output tail:"), "body: {body}");
2544    }
2545
2546    #[test]
2547    fn background_completion_builds_post_tool_use_payload_and_feedback() {
2548        let mut info = BashCompletionInfo {
2549            session_id: "s".into(),
2550            bash_id: "bg-7".into(),
2551            command: "cargo test".into(),
2552            exit_code: Some(0),
2553            status: "completed".into(),
2554            output_tail: "test result: ok".into(),
2555        };
2556
2557        let payload = background_bash_post_tool_payload(&info);
2558        match payload {
2559            HookPayload::ToolResult {
2560                tool_name,
2561                tool_call_id,
2562                outcome,
2563            } => {
2564                assert_eq!(tool_name, "Bash");
2565                assert_eq!(tool_call_id, "bg-7");
2566                assert!(outcome.success);
2567                let response: serde_json::Value =
2568                    serde_json::from_str(outcome.result.as_deref().unwrap()).unwrap();
2569                assert_eq!(response["command"], "cargo test");
2570                assert_eq!(response["exit_code"], 0);
2571                assert_eq!(response["status"], "completed");
2572                assert_eq!(response["output_tail"], "test result: ok");
2573            }
2574            other => panic!("expected PostToolUse payload, got {other:?}"),
2575        }
2576
2577        append_background_bash_hook_feedback(
2578            &mut info,
2579            vec!["Run the formatter before continuing".to_string()],
2580        );
2581        assert!(info.output_tail.contains("<post_tool_use_feedback>"));
2582        assert!(info
2583            .output_tail
2584            .contains("Run the formatter before continuing"));
2585    }
2586
2587    #[cfg(unix)]
2588    #[tokio::test]
2589    async fn background_completion_fires_configured_post_tool_use_command() {
2590        use bamboo_config::{
2591            LifecycleHookCommand, LifecycleHookGroup, LifecycleHookType, LifecycleHooksConfig,
2592            DEFAULT_LIFECYCLE_HOOK_TIMEOUT_MS,
2593        };
2594
2595        let dir = tempfile::tempdir().unwrap();
2596        let output = dir.path().join("background-post-tool.json");
2597        let command = format!(
2598            "cat > '{}'; printf '%s' '{{\"additional_context\":\"inspect the completed build log\"}}'",
2599            output.display()
2600        );
2601        let config = LifecycleHooksConfig {
2602            enabled: true,
2603            post_tool_use: vec![LifecycleHookGroup {
2604                enabled: true,
2605                matcher: Some("^Bash$".to_string()),
2606                hooks: vec![LifecycleHookCommand {
2607                    hook_type: LifecycleHookType::Command,
2608                    command,
2609                    timeout_ms: DEFAULT_LIFECYCLE_HOOK_TIMEOUT_MS,
2610                }],
2611            }],
2612            ..Default::default()
2613        };
2614        let mut session = Session::new("session-bg-hook", "test-model");
2615        session.workspace = Some(dir.path().to_string_lossy().into_owned());
2616        let mut info = BashCompletionInfo {
2617            session_id: session.id.clone(),
2618            bash_id: "bg-9".into(),
2619            command: "cargo test".into(),
2620            exit_code: Some(0),
2621            status: "completed".into(),
2622            output_tail: "test result: ok".into(),
2623        };
2624
2625        assert!(run_background_bash_post_tool_hooks(&config, None, &session, &mut info).await);
2626        let envelope: serde_json::Value =
2627            serde_json::from_str(&std::fs::read_to_string(output).unwrap()).unwrap();
2628        assert_eq!(envelope["hook_event_name"], "PostToolUse");
2629        assert_eq!(envelope["tool_name"], "Bash");
2630        assert_eq!(envelope["payload"]["tool_call_id"], "bg-9");
2631        let response = envelope["tool_response"]["result"]
2632            .as_str()
2633            .map(serde_json::from_str::<serde_json::Value>)
2634            .transpose()
2635            .unwrap()
2636            .unwrap();
2637        assert_eq!(response["command"], "cargo test");
2638        assert_eq!(response["status"], "completed");
2639        assert!(info.output_tail.contains("inspect the completed build log"));
2640    }
2641
2642    async fn temp_store() -> (tempfile::TempDir, Arc<dyn Storage>, LockedSessionStore) {
2643        let temp = tempfile::tempdir().unwrap();
2644        let storage: Arc<dyn Storage> = Arc::new(
2645            bamboo_storage::v2::SessionStoreV2::new(temp.path().to_path_buf())
2646                .await
2647                .expect("storage init"),
2648        );
2649        let persistence = LockedSessionStore::new(storage.clone());
2650        (temp, storage, persistence)
2651    }
2652
2653    #[tokio::test]
2654    async fn enqueue_writes_pending_injection_and_preserves_messages() {
2655        let (_temp, storage, persistence) = temp_store().await;
2656
2657        let mut session = Session::new("sess-enq", "test-model");
2658        session.add_message(Message::user("do the build"));
2659        storage.save_session(&session).await.unwrap();
2660
2661        let info = BashCompletionInfo {
2662            session_id: "sess-enq".into(),
2663            bash_id: "sh-1".into(),
2664            command: "make".into(),
2665            exit_code: Some(0),
2666            status: "completed".into(),
2667            output_tail: "done".into(),
2668        };
2669        let saved = enqueue_bash_completion_injection(&persistence, &info)
2670            .await
2671            .expect("enqueue io ok")
2672            .expect("session exists");
2673
2674        let pending = saved
2675            .pending_injected_messages()
2676            .expect("pending injection present");
2677        assert_eq!(pending.len(), 1);
2678        let content = pending[0].get("content").and_then(|v| v.as_str()).unwrap();
2679        assert!(content.contains("sh-1"), "content: {content}");
2680        assert!(content.contains("make"), "content: {content}");
2681        assert!(content.contains("done"), "content: {content}");
2682        // The pre-existing conversation is untouched (no clobber).
2683        assert_eq!(saved.messages.len(), 1);
2684    }
2685
2686    #[tokio::test]
2687    async fn enqueue_returns_none_for_missing_session() {
2688        let (_temp, _storage, persistence) = temp_store().await;
2689        let info = BashCompletionInfo {
2690            session_id: "does-not-exist".into(),
2691            bash_id: "x".into(),
2692            command: "true".into(),
2693            exit_code: Some(0),
2694            status: "completed".into(),
2695            output_tail: String::new(),
2696        };
2697        let result = enqueue_bash_completion_injection(&persistence, &info)
2698            .await
2699            .expect("io ok");
2700        assert!(result.is_none(), "no session → nothing enqueued");
2701    }
2702
2703    // ── push-driven resume: the state transition + decision the push applies ──
2704
2705    /// A session suspended on `waiting_for_bash`, given the rich completion
2706    /// message, is transitioned to a resumable state: the wait is cleared, the
2707    /// runtime is Idle, the suspend-reason marker is gone, and the resume message
2708    /// is appended. This is exactly what the PUSH does to wake the loop
2709    /// event-driven (vs the old backstop poll).
2710    #[test]
2711    fn apply_bash_resume_transition_clears_wait_and_appends_message() {
2712        use bamboo_domain::session::runtime_state::WaitingForBashState;
2713
2714        let mut session = Session::new("sess-resume", "test-model");
2715        session.add_message(Message::user("kick off the build"));
2716        let mut rt = read_runtime_state(&session);
2717        rt.status = AgentStatusState::Running;
2718        rt.waiting_for_bash = Some(WaitingForBashState::for_bash(
2719            vec!["sh-1".into()],
2720            Utc::now(),
2721        ));
2722        write_runtime_state(&mut session, &rt);
2723        session.metadata.insert(
2724            "runtime.suspend_reason".to_string(),
2725            "waiting_for_bash".to_string(),
2726        );
2727
2728        let resume = bash_completion_resume_message(&["sh-1".to_string()], false);
2729        let did = apply_bash_resume_transition(&mut session, &resume);
2730
2731        assert!(did, "a suspended session must transition");
2732        let after = read_runtime_state(&session);
2733        assert!(
2734            after.waiting_for_bash.is_none(),
2735            "bash wait must be cleared"
2736        );
2737        assert_eq!(after.status, AgentStatusState::Idle, "runtime must be Idle");
2738        assert!(
2739            !session.metadata.contains_key("runtime.suspend_reason"),
2740            "suspend-reason marker must be removed"
2741        );
2742        assert_eq!(session.messages.len(), 2, "resume message must be appended");
2743        assert!(matches!(
2744            session.messages.last().map(|m| &m.role),
2745            Some(Role::User)
2746        ));
2747    }
2748
2749    /// The double-resume guard: a session NOT waiting on bash is a no-op — no
2750    /// message appended, nothing mutated. This is what makes the backstop poll
2751    /// harmlessly yield once the push has already resumed (and vice versa).
2752    #[test]
2753    fn apply_bash_resume_transition_noops_when_not_waiting() {
2754        let mut session = Session::new("sess-live", "test-model");
2755        session.add_message(Message::user("hi"));
2756
2757        let resume = bash_completion_resume_message(&["sh-1".to_string()], false);
2758        let did = apply_bash_resume_transition(&mut session, &resume);
2759
2760        assert!(!did, "a non-waiting session must not transition");
2761        assert_eq!(session.messages.len(), 1, "no resume message appended");
2762    }
2763
2764    /// The resume invariant: push-resume fires ONLY when the loop is suspended on
2765    /// bash AND every waited shell has finished. A still-running sibling shell
2766    /// keeps it on the enqueue path.
2767    #[test]
2768    fn bash_completion_should_resume_only_when_suspended_and_all_done() {
2769        assert!(bash_completion_should_resume(true, true));
2770        assert!(!bash_completion_should_resume(true, false)); // other shells still running
2771        assert!(!bash_completion_should_resume(false, true)); // live loop, not suspended
2772        assert!(!bash_completion_should_resume(false, false));
2773    }
2774
2775    /// The push's resume message carries the shell's identity + status + output
2776    /// tail (so the model needs no `BashOutput` round-trip) and is tagged as a
2777    /// bash-completion resume so it satisfies the `has_pending_user_message` gate.
2778    #[test]
2779    fn bash_resume_message_from_info_carries_bashid_tail_and_kind() {
2780        let info = BashCompletionInfo {
2781            session_id: "s".into(),
2782            bash_id: "sh-42".into(),
2783            command: "cargo test".into(),
2784            exit_code: Some(0),
2785            status: "completed".into(),
2786            output_tail: "test result: ok".into(),
2787        };
2788        let msg = bash_resume_message_from_info(&info);
2789
2790        assert!(matches!(msg.role, Role::User));
2791        assert!(msg.content.contains("sh-42"), "content: {}", msg.content);
2792        assert!(
2793            msg.content.contains("cargo test"),
2794            "content: {}",
2795            msg.content
2796        );
2797        assert!(
2798            msg.content.contains("test result: ok"),
2799            "content: {}",
2800            msg.content
2801        );
2802        assert!(
2803            msg.content.contains("BashOutput"),
2804            "content: {}",
2805            msg.content
2806        );
2807        let meta = serde_json::to_string(&msg.metadata).unwrap();
2808        assert!(
2809            meta.contains(BASH_COMPLETION_RESUME_KIND),
2810            "resume message must be tagged as a bash-completion resume: {meta}"
2811        );
2812    }
2813}