Skip to main content

bamboo_engine/session_app/
child_completion_coordinator.rs

1//! Child-session completion coordinator.
2//!
3//! Receives terminal child runner notifications from `bamboo-engine`, updates
4//! durable parent wait state, and resumes the parent when the configured wait
5//! policy is satisfied.
6
7use std::collections::HashMap;
8use std::sync::{Arc, OnceLock, RwLock as StdRwLock};
9use std::time::Duration;
10
11use bamboo_domain::poison::PoisonRecover;
12
13use crate::execution::{
14    create_event_forwarder, spawn_session_execution, try_reserve_runner, AgentRunner,
15    ChildCompletion, ChildCompletionHandler, RunnerReservation, SessionExecutionArgs,
16};
17use crate::runtime::config::{BashResumeHook, GuardianSpawner, BASH_COMPLETION_RESUME_KIND};
18use crate::runtime::guardian_state::{
19    parse_guardian_verdict, read_guardian_config, read_guardian_state, write_guardian_state,
20    GuardianVerdict,
21};
22use crate::Agent;
23use async_trait::async_trait;
24use bamboo_agent_core::storage::Storage;
25use bamboo_agent_core::tools::ToolExecutor;
26use bamboo_agent_core::{
27    AgentEvent, BashCompletionInfo, BashCompletionSink, Message, Role, Session,
28};
29use bamboo_domain::session::runtime_state::{
30    AgentRuntimeState, AgentStatusState, ChildWaitPolicy, SuspensionState,
31};
32use bamboo_llm::{Config, ProviderModelRouter, ProviderRegistry};
33use bamboo_storage::LockedSessionStore;
34use chrono::Utc;
35use tokio::sync::{broadcast, RwLock};
36
37use crate::model_areas::resolve_global_area_models;
38use crate::model_config_helper::{
39    resolve_fast_model, resolve_gold_config, GOLD_CONFIG_METADATA_KEY,
40};
41use crate::session_app::provider_model::session_effective_model_ref;
42use crate::session_app::resume::{
43    resume_session_execution, ResumeExecutionPort, ResumeSpawnRequest,
44};
45use crate::session_app::types::{ResumeConfigSnapshot, ResumeOutcome};
46
47const AGENT_RUNTIME_STATE_METADATA_KEY: &str = "agent.runtime.state";
48const RUNTIME_RESUME_MESSAGE_HIDDEN_KEY: &str = "hidden_from_ui";
49const RUNTIME_RESUME_MESSAGE_KIND_KEY: &str = "runtime_kind";
50
51fn read_runtime_state(session: &Session) -> AgentRuntimeState {
52    session
53        .agent_runtime_state
54        .clone()
55        .or_else(|| {
56            session
57                .metadata
58                .get(AGENT_RUNTIME_STATE_METADATA_KEY)
59                .and_then(|raw| serde_json::from_str::<AgentRuntimeState>(raw).ok())
60        })
61        .unwrap_or_else(|| AgentRuntimeState::new(format!("{}-child-wait", session.id)))
62}
63
64fn write_runtime_state(session: &mut Session, runtime_state: &AgentRuntimeState) {
65    session.agent_runtime_state = Some(runtime_state.clone());
66    if let Ok(serialized) = serde_json::to_string(runtime_state) {
67        session
68            .metadata
69            .insert(AGENT_RUNTIME_STATE_METADATA_KEY.to_string(), serialized);
70    }
71}
72
73fn is_error_like(status: &str) -> bool {
74    matches!(status, "error" | "timeout" | "cancelled")
75}
76
77/// Terminal child run statuses, as mirrored into the session index.
78fn is_terminal_child_status(status: &str) -> bool {
79    matches!(
80        status,
81        "completed" | "error" | "timeout" | "cancelled" | "skipped"
82    )
83}
84
85/// Reconstruct the set of completed child session ids for a parent from the
86/// session index (the single source of truth), folding in the child whose
87/// completion event is being processed so a momentarily-lagging index can never
88/// stall the parent's resume.
89async fn derive_completed_child_ids(
90    storage: &Arc<dyn Storage>,
91    parent_session_id: &str,
92    just_completed_child_id: &str,
93) -> Vec<String> {
94    let mut completed: Vec<String> = storage
95        .list_child_run_statuses(parent_session_id)
96        .await
97        .unwrap_or_default()
98        .into_iter()
99        .filter(|(_, status)| status.as_deref().is_some_and(is_terminal_child_status))
100        .map(|(id, _)| id)
101        .collect();
102    if !completed.iter().any(|id| id == just_completed_child_id) {
103        completed.push(just_completed_child_id.to_string());
104    }
105    completed.sort();
106    completed.dedup();
107    completed
108}
109
110fn read_config_snapshot(config: &Arc<RwLock<Config>>, cached_config: &StdRwLock<Config>) -> Config {
111    if let Ok(config_guard) = config.try_read() {
112        let snapshot = config_guard.clone();
113
114        if let Ok(mut cached_guard) = cached_config.try_write() {
115            *cached_guard = snapshot.clone();
116        }
117
118        snapshot
119    } else {
120        cached_config
121            .try_read()
122            .map(|guard| guard.clone())
123            .unwrap_or_default()
124    }
125}
126
127/// Per-parent async locks that serialize concurrent `on_child_completed`
128/// invocations for the same parent session.
129///
130/// Race eliminated: when `wait_for=Any` and two child sessions complete
131/// simultaneously, both invocations load the parent with
132/// `waiting_for_children=Some` before either persists the cleared state, so
133/// both pass `wait_policy_satisfied`, both clear `waiting_for_children`, add a
134/// duplicate resume message, and call `resume_parent` — a double resume.
135/// Holding this per-parent `tokio::sync::Mutex` across the load-check-save
136/// critical section makes the second caller observe the already-cleared state.
137///
138/// The inner `std::sync::Mutex` guards only the brief HashMap lookup/insert
139/// (no await inside); the per-parent `tokio::sync::Mutex` is the one held
140/// across the async critical section. Entries accumulate but are small
141/// (`Arc<tokio::sync::Mutex<()>>` ≈ 24 bytes) and bounded by the number of
142/// distinct parent sessions.
143fn parent_locks() -> &'static std::sync::Mutex<HashMap<String, Arc<tokio::sync::Mutex<()>>>> {
144    static LOCKS: OnceLock<std::sync::Mutex<HashMap<String, Arc<tokio::sync::Mutex<()>>>>> =
145        OnceLock::new();
146    LOCKS.get_or_init(|| std::sync::Mutex::new(HashMap::new()))
147}
148
149/// Fetch (or create) the per-session async lock from [`parent_locks`]. Held
150/// across the load-check-clear-resume critical section so the three resume
151/// sources for one session — child completion, the loop-facing bash **push**
152/// ([`BashCompletionSink::on_bash_completed`]), and the bash **backstop** poll
153/// ([`ChildCompletionCoordinator::bash_self_resume`]) — can never double-resume.
154/// The inner sync `Mutex` guards only the brief map lookup (no await inside).
155fn session_resume_lock(session_id: &str) -> Arc<tokio::sync::Mutex<()>> {
156    let mut map = parent_locks().lock().recover_poison();
157    map.entry(session_id.to_string())
158        .or_insert_with(|| Arc::new(tokio::sync::Mutex::new(())))
159        .clone()
160}
161
162fn wait_policy_satisfied(
163    policy: ChildWaitPolicy,
164    wait_child_ids: &[String],
165    completed_child_ids: &[String],
166    latest_status: &str,
167) -> bool {
168    if wait_child_ids.is_empty() {
169        return false;
170    }
171
172    match policy {
173        ChildWaitPolicy::All => wait_child_ids
174            .iter()
175            .all(|id| completed_child_ids.iter().any(|completed| completed == id)),
176        ChildWaitPolicy::Any => completed_child_ids
177            .iter()
178            .any(|id| wait_child_ids.iter().any(|wait_id| wait_id == id)),
179        ChildWaitPolicy::FirstError => {
180            is_error_like(latest_status)
181                || wait_child_ids
182                    .iter()
183                    .all(|id| completed_child_ids.iter().any(|completed| completed == id))
184        }
185    }
186}
187
188/// Extract the child session's last assistant content, if any. Returns `None`
189/// when the child produced no assistant message (e.g. errored before the first
190/// model response, or only emitted tool messages).
191fn child_final_assistant_text(child: &Session) -> Option<String> {
192    child
193        .messages
194        .iter()
195        .rev()
196        .find(|message| matches!(message.role, Role::Assistant))
197        .map(|message| message.content.clone())
198        .filter(|content| !content.trim().is_empty())
199}
200
201fn runtime_resume_message(
202    completion: &ChildCompletion,
203    remaining_children: usize,
204    child_final_response: Option<&str>,
205) -> Message {
206    let mut body = format!(
207        "Runtime notification: child session `{}` finished with status `{}`. Remaining child sessions: {}.",
208        completion.child_session_id, completion.status, remaining_children
209    );
210
211    // Fold the child's full final response back into the parent — no
212    // truncation. Sub-agents are first-class agents whose complete conclusion
213    // should be available to the parent without an extra `SubAgent.get` round
214    // trip. The message is left compressible (see `never_compress` below) so a
215    // long transcript can still be reclaimed under parent compaction.
216    let final_response = child_final_response.map(str::to_string);
217    if let Some(response) = final_response.as_deref() {
218        body.push_str("\n\nChild final response:\n");
219        body.push_str(response);
220    } else if let Some(error) = completion.error.as_deref() {
221        if !error.is_empty() {
222            body.push_str("\n\nChild error:\n");
223            body.push_str(error);
224        }
225    }
226
227    body.push_str(
228        "\n\nResume the parent task using this child result and continue from the previous plan. \
229         If you need the full child transcript, call SubAgent.get(child_session_id).",
230    );
231
232    let mut message = Message::user(body);
233    message.metadata = Some(serde_json::json!({
234        RUNTIME_RESUME_MESSAGE_HIDDEN_KEY: true,
235        RUNTIME_RESUME_MESSAGE_KIND_KEY: "child_completion_resume",
236        "child_session_id": completion.child_session_id,
237        "child_status": completion.status,
238        "child_error": completion.error,
239        "completed_at": completion.completed_at,
240        "child_final_response_included": final_response.is_some(),
241    }));
242    // Allow parent-side compaction to reclaim this (now untruncated) message if
243    // the parent context grows — important once children nest and fold full
244    // results upward. The `SubAgent.get` hint preserves recoverability.
245    message.never_compress = false;
246    message
247}
248
249/// The hidden resume message for a completed **guardian** review: a directive,
250/// verdict-tailored note that carries the reviewer's findings straight into the
251/// parent (so it can act without a `SubAgent.get`), mirroring
252/// [`runtime_resume_message`]'s hidden/compressible shape.
253fn guardian_resume_message(completion: &ChildCompletion, verdict: &GuardianVerdict) -> Message {
254    let mut body = if verdict.approve {
255        String::from(
256            "Guardian review APPROVED: an independent reviewer verified the work and found no blocking issues. You may finalize the task.",
257        )
258    } else {
259        String::from(
260            "Guardian review REJECTED: an independent reviewer found issues. Address every finding below before completing — do NOT declare the task complete until they are resolved.",
261        )
262    };
263    if let Some(summary) = verdict.summary.as_deref().filter(|s| !s.trim().is_empty()) {
264        body.push_str("\n\nReviewer summary: ");
265        body.push_str(summary);
266    }
267    if !verdict.findings.is_empty() {
268        body.push_str("\n\nFindings:");
269        for (idx, finding) in verdict.findings.iter().enumerate() {
270            body.push_str(&format!("\n{}. {}", idx + 1, finding));
271        }
272    }
273    body.push_str(
274        "\n\nIf you need the full guardian transcript, call SubAgent.get(child_session_id).",
275    );
276
277    let mut message = Message::user(body);
278    message.metadata = Some(serde_json::json!({
279        RUNTIME_RESUME_MESSAGE_HIDDEN_KEY: true,
280        RUNTIME_RESUME_MESSAGE_KIND_KEY: "guardian_review_resume",
281        "child_session_id": completion.child_session_id,
282        "child_status": completion.status,
283        "guardian_approved": verdict.approve,
284        "completed_at": completion.completed_at,
285    }));
286    message.never_compress = false;
287    message
288}
289
290#[derive(Clone)]
291pub struct ChildCompletionCoordinator {
292    storage: Arc<dyn Storage>,
293    persistence: Arc<bamboo_storage::LockedSessionStore>,
294    sessions: crate::SessionCache,
295    agent_runners: Arc<RwLock<HashMap<String, AgentRunner>>>,
296    session_event_senders: Arc<RwLock<HashMap<String, broadcast::Sender<AgentEvent>>>>,
297    agent: Arc<Agent>,
298    config: Arc<RwLock<Config>>,
299    provider_registry: Arc<ProviderRegistry>,
300    provider_router: Arc<ProviderModelRouter>,
301    app_data_dir: std::path::PathBuf,
302    account_feed_inbox: Option<crate::execution::AccountFeedInbox>,
303    root_tools: Arc<RwLock<Option<Arc<dyn ToolExecutor>>>>,
304    /// Late-bound guardian reviewer spawner, set post-construction by the server
305    /// (mirrors `root_tools`). Re-injected into resumed runs so a guardian's
306    /// reject→fix verdict can be re-reviewed across the suspend/resume boundary.
307    guardian_spawner: Arc<RwLock<Option<Arc<dyn GuardianSpawner>>>>,
308}
309
310impl ChildCompletionCoordinator {
311    #[allow(clippy::too_many_arguments)]
312    pub fn new(
313        storage: Arc<dyn Storage>,
314        persistence: Arc<LockedSessionStore>,
315        sessions: crate::SessionCache,
316        agent_runners: Arc<RwLock<HashMap<String, AgentRunner>>>,
317        session_event_senders: Arc<RwLock<HashMap<String, broadcast::Sender<AgentEvent>>>>,
318        agent: Arc<Agent>,
319        config: Arc<RwLock<Config>>,
320        provider_registry: Arc<ProviderRegistry>,
321        provider_router: Arc<ProviderModelRouter>,
322        app_data_dir: std::path::PathBuf,
323        account_feed_inbox: Option<crate::execution::AccountFeedInbox>,
324    ) -> Self {
325        Self {
326            storage,
327            persistence,
328            sessions,
329            agent_runners,
330            session_event_senders,
331            agent,
332            config,
333            provider_registry,
334            provider_router,
335            app_data_dir,
336            account_feed_inbox,
337            root_tools: Arc::new(RwLock::new(None)),
338            guardian_spawner: Arc::new(RwLock::new(None)),
339        }
340    }
341
342    pub async fn set_root_tools(&self, tools: Arc<dyn ToolExecutor>) {
343        *self.root_tools.write().await = Some(tools);
344    }
345
346    /// Wire the guardian reviewer spawner (server-provided), so resumed runs can
347    /// re-spawn a guardian to re-review a fix after a reject verdict.
348    pub async fn set_guardian_spawner(&self, spawner: Arc<dyn GuardianSpawner>) {
349        *self.guardian_spawner.write().await = Some(spawner);
350    }
351
352    fn build_resume_config(
353        &self,
354        session: &Session,
355        config_snapshot: &Config,
356    ) -> ResumeConfigSnapshot {
357        crate::session_app::resolution::resolve_resume_config_snapshot(
358            config_snapshot,
359            &self.provider_registry,
360            session,
361            None,
362        )
363    }
364
365    /// Drive a parent-resume and return the final [`ResumeOutcome`] so callers
366    /// can distinguish a successful spawn (`Started`) from a gate-blocked
367    /// attempt (`Completed`). The bash self-resume poll task uses this to
368    /// detect the finalize-clobber case — its appended resume message was
369    /// reverted by the suspending runner's final `merge_save_runtime`, so the
370    /// resume port's `has_pending_user_message` gate fails and nothing spawns —
371    /// and retry the clear→append→resume (see [`Self::bash_self_resume`]).
372    async fn resume_parent(&self, parent_session_id: String) -> ResumeOutcome {
373        for attempt in 0..=5u8 {
374            if attempt > 0 {
375                tokio::time::sleep(Duration::from_millis(250 * attempt as u64)).await;
376            }
377
378            let Some(session) = self.load_session(&parent_session_id).await else {
379                tracing::warn!(%parent_session_id, "cannot resume parent after child completion: session not found");
380                return ResumeOutcome::NotFound;
381            };
382            let config_snapshot = self.config.read().await.clone();
383            let resume_config = self.build_resume_config(&session, &config_snapshot);
384            let outcome = resume_session_execution(self, &parent_session_id, resume_config).await;
385            tracing::info!(
386                %parent_session_id,
387                attempt,
388                outcome = outcome.as_str(),
389                "child completion requested parent resume"
390            );
391
392            if !matches!(outcome, ResumeOutcome::AlreadyRunning { .. }) {
393                return outcome;
394            }
395        }
396        // Exhausted the AlreadyRunning retry budget; surface the final state.
397        ResumeOutcome::AlreadyRunning {
398            run_id: String::new(),
399        }
400    }
401
402    async fn save_and_cache(&self, session: &mut Session) {
403        if let Err(error) = self.persistence.merge_save_runtime(session).await {
404            tracing::warn!(session_id = %session.id, %error, "failed to persist session");
405        }
406        self.sessions.insert(
407            session.id.clone(),
408            Arc::new(parking_lot::RwLock::new(session.clone())),
409        );
410    }
411}
412
413#[async_trait]
414impl ChildCompletionHandler for ChildCompletionCoordinator {
415    async fn on_child_completed(&self, completion: ChildCompletion) {
416        // Acquire the per-session async lock to eliminate the concurrent
417        // double-resume race (see `parent_locks` for the full scenario). The
418        // inner std::sync::Mutex is released immediately so no sync lock is
419        // held across the await that follows.
420        let per_parent = session_resume_lock(&completion.parent_session_id);
421        let _per_parent_guard = per_parent.lock().await;
422
423        let Some(mut parent) = self.load_session(&completion.parent_session_id).await else {
424            tracing::warn!(
425                parent_session_id = %completion.parent_session_id,
426                child_session_id = %completion.child_session_id,
427                "child completion received for missing parent"
428            );
429            return;
430        };
431
432        // A parent may itself be a child (nested sub-agents): the rest of this
433        // handler is kind-agnostic — it operates on `completion.parent_session_id`,
434        // inspects that session's own `waiting_for_children` runtime state, and
435        // resumes it. (Previously this bailed unless the parent was Root, which
436        // silently dropped grandchild completions.)
437        let mut runtime_state = read_runtime_state(&parent);
438
439        // Single source of truth: reconstruct the completed-child set from the
440        // session index rather than from a denormalized copy on the parent file.
441        let completed_child_ids = derive_completed_child_ids(
442            &self.storage,
443            &completion.parent_session_id,
444            &completion.child_session_id,
445        )
446        .await;
447
448        let mut should_resume = false;
449        let mut remaining_children = 0usize;
450        if let Some(wait) = runtime_state.waiting_for_children.clone() {
451            remaining_children = wait
452                .child_session_ids
453                .iter()
454                .filter(|id| !completed_child_ids.iter().any(|completed| completed == *id))
455                .count();
456            should_resume = wait_policy_satisfied(
457                wait.wait_for,
458                &wait.child_session_ids,
459                &completed_child_ids,
460                &completion.status,
461            );
462            if should_resume {
463                runtime_state.waiting_for_children = None;
464                runtime_state.status = AgentStatusState::Idle;
465                runtime_state.suspension = None;
466            }
467        }
468
469        if should_resume {
470            parent.metadata.remove("runtime.suspend_reason");
471            // Load the completed child once. The guardian branch inspects its
472            // subagent_type + final verdict; the generic path folds its final
473            // assistant content into the hidden resume message (avoiding an extra
474            // `SubAgent.get` round trip after resume).
475            let loaded_child = match self
476                .storage
477                .load_session(&completion.child_session_id)
478                .await
479            {
480                Ok(child) => child,
481                Err(error) => {
482                    tracing::warn!(
483                        child_session_id = %completion.child_session_id,
484                        %error,
485                        "failed to load child session for runtime resume message"
486                    );
487                    None
488                }
489            };
490
491            // Guardian branch: a completing guardian reviewer that matches the
492            // parent's recorded review advances GuardianState (phase → Reviewed)
493            // and resumes with a verdict-tailored, findings-carrying message. Any
494            // id mismatch or unparseable verdict falls through to the generic
495            // resume, so the parent is never stranded.
496            let reviewed_round = runtime_state.round.current_round;
497            let guardian_resume = loaded_child.as_ref().and_then(|child| {
498                if child.subagent_type().as_deref() != Some("guardian") {
499                    return None;
500                }
501                let mut guardian_state = read_guardian_state(&parent)?;
502                if guardian_state.guardian_child_id.as_deref()
503                    != Some(completion.child_session_id.as_str())
504                {
505                    // A *different* guardian is legitimately still in flight —
506                    // leave its Pending state intact and use the generic resume.
507                    tracing::warn!(
508                        parent_session_id = %completion.parent_session_id,
509                        child_session_id = %completion.child_session_id,
510                        expected = ?guardian_state.guardian_child_id,
511                        "guardian completion does not match recorded guardian_child_id; using generic resume"
512                    );
513                    return None;
514                }
515                // This IS the guardian we dispatched, so we MUST advance the
516                // phase out of `Pending` — otherwise the next terminal gate's
517                // `Pending => return None` would let the run complete unreviewed.
518                // A reviewer that errored or produced unparseable output is
519                // treated as a SYNTHETIC REJECT (never a silent pass), so the
520                // budgeted re-review loop governs the outcome: fail-closed, but
521                // still bounded by `max_reviews`.
522                let verdict = child_final_assistant_text(child)
523                    .and_then(|text| match parse_guardian_verdict(&text) {
524                        Ok(verdict) => Some(verdict),
525                        Err(error) => {
526                            tracing::warn!(
527                                child_session_id = %completion.child_session_id,
528                                %error,
529                                "guardian verdict unparseable; recording a synthetic reject"
530                            );
531                            None
532                        }
533                    })
534                    .unwrap_or_else(|| {
535                        GuardianVerdict::rejected(vec![
536                            "The guardian reviewer did not return a usable verdict (it errored or \
537                             emitted unparseable output); the work has NOT been independently \
538                             verified."
539                                .to_string(),
540                        ])
541                    });
542                let approved = verdict.approve;
543                let message = guardian_resume_message(&completion, &verdict);
544                guardian_state.record_verdict(verdict, reviewed_round);
545                write_guardian_state(&mut parent, guardian_state);
546                tracing::info!(
547                    parent_session_id = %completion.parent_session_id,
548                    child_session_id = %completion.child_session_id,
549                    approved,
550                    "guardian verdict recorded; resuming parent"
551                );
552                Some(message)
553            });
554
555            let resume_message = guardian_resume.unwrap_or_else(|| {
556                runtime_resume_message(
557                    &completion,
558                    remaining_children,
559                    loaded_child
560                        .as_ref()
561                        .and_then(child_final_assistant_text)
562                        .as_deref(),
563                )
564            });
565            parent.add_message(resume_message);
566        } else if runtime_state.waiting_for_children.is_some() {
567            runtime_state.status = AgentStatusState::Suspended;
568            runtime_state.suspension = Some(SuspensionState {
569                reason: "waiting_for_children".to_string(),
570                suspended_at: Utc::now(),
571                resumable: true,
572                hook_point: Some("ChildCompletion".to_string()),
573            });
574        }
575
576        parent.updated_at = Utc::now();
577        write_runtime_state(&mut parent, &runtime_state);
578        self.save_and_cache(&mut parent).await;
579
580        // Capture before releasing the per-parent lock so the borrow checker
581        // is satisfied; `resume_parent` has its own retry loop and should not
582        // hold the per-parent lock (it would block other completions for the
583        // same parent, and the state is already durably settled above).
584        let resume_parent_id = parent.id.clone();
585        drop(_per_parent_guard);
586
587        if should_resume {
588            self.resume_parent(resume_parent_id).await;
589        }
590    }
591}
592
593#[async_trait]
594impl ResumeExecutionPort for ChildCompletionCoordinator {
595    async fn load_session(&self, session_id: &str) -> Option<Session> {
596        match self.storage.load_session(session_id).await {
597            Ok(Some(session)) => Some(session),
598            Ok(None) => self
599                .sessions
600                .get(session_id)
601                .map(|e| e.value().clone())
602                .map(|arc| arc.read().clone()),
603            Err(error) => {
604                tracing::warn!(%session_id, %error, "failed to load session from storage");
605                self.sessions
606                    .get(session_id)
607                    .map(|e| e.value().clone())
608                    .map(|arc| arc.read().clone())
609            }
610        }
611    }
612
613    async fn save_and_cache_session(&self, session: &mut Session) {
614        self.save_and_cache(session).await;
615    }
616
617    async fn try_reserve_runner(
618        &self,
619        session_id: &str,
620        event_sender: &broadcast::Sender<AgentEvent>,
621    ) -> Option<RunnerReservation> {
622        try_reserve_runner(
623            &self.agent_runners,
624            &self.session_event_senders,
625            session_id,
626            event_sender,
627        )
628        .await
629    }
630
631    async fn get_existing_runner_run_id(&self, session_id: &str) -> Option<String> {
632        let runners = self.agent_runners.read().await;
633        runners.get(session_id).map(|r| r.run_id.clone())
634    }
635
636    async fn get_or_create_event_sender(&self, session_id: &str) -> broadcast::Sender<AgentEvent> {
637        crate::execution::session_events::get_or_create_event_sender(
638            &self.session_event_senders,
639            session_id,
640        )
641        .await
642    }
643
644    async fn spawn_resume_execution(&self, request: ResumeSpawnRequest) {
645        let ResumeSpawnRequest {
646            session_id,
647            session,
648            cancel_token,
649            run_id: _,
650            event_sender,
651            config,
652        } = request;
653
654        let Some(root_tools) = self.root_tools.read().await.clone() else {
655            tracing::error!(%session_id, "cannot resume parent after child completion: root tool surface is not initialized");
656            return;
657        };
658
659        let model = session.model.clone();
660        let resolved_provider_name = session_effective_model_ref(&session)
661            .map(|model_ref| model_ref.provider)
662            .unwrap_or(config.provider_name);
663        let provider_override = session_effective_model_ref(&session)
664            .and_then(|model_ref| match self.provider_router.route(&model_ref) {
665                Ok(provider) => Some(provider),
666                Err(error) => {
667                    tracing::warn!(
668                        session_id = %session_id,
669                        provider = %model_ref.provider,
670                        model = %model_ref.model,
671                        error = %error,
672                        "failed to resolve provider override for child-completion parent resume; falling back to runtime provider"
673                    );
674                    None
675                }
676            });
677        let config_snapshot = self.config.read().await.clone();
678        let resolved_fast_provider = resolve_fast_model(
679            &config_snapshot,
680            &resolved_provider_name,
681            &self.provider_registry,
682        )
683        .map(|model| model.provider);
684        let reasoning_effort = session.reasoning_effort;
685        let reasoning_effort_source = session
686            .metadata
687            .get("reasoning_effort_source")
688            .cloned()
689            .unwrap_or_default();
690        let gold_config = resolve_gold_config(
691            &config_snapshot,
692            session
693                .metadata
694                .get(GOLD_CONFIG_METADATA_KEY)
695                .map(String::as_str),
696        )
697        .or(config.gold_config.clone());
698
699        let (mpsc_tx, _forwarder) = create_event_forwarder(
700            session_id.clone(),
701            event_sender,
702            self.agent_runners.clone(),
703            self.account_feed_inbox.clone(),
704        );
705
706        let config_handle = self.config.clone();
707        let cached_config = Arc::new(StdRwLock::new(config_snapshot.clone()));
708        let provider_registry = self.provider_registry.clone();
709        let provider_name_for_aux = resolved_provider_name.clone();
710        let auxiliary_model_resolver = std::sync::Arc::new(move || {
711            let config_snapshot = read_config_snapshot(&config_handle, cached_config.as_ref());
712            // Auxiliary models are global (config-derived), never session-bound.
713            let areas = resolve_global_area_models(
714                &config_snapshot,
715                &provider_name_for_aux,
716                &provider_registry,
717            );
718            crate::AuxiliaryModelConfig {
719                fast_model_name: areas.fast.as_ref().map(|m| m.model_name.clone()),
720                fast_model_provider: areas.fast.map(|m| m.provider),
721                background_model_name: areas.background.as_ref().map(|m| m.model_name.clone()),
722                planning_model_name: None,
723                search_model_name: None,
724                summarization_model_name: areas
725                    .summarization
726                    .as_ref()
727                    .map(|m| m.model_name.clone()),
728                background_model_provider: areas.background.map(|m| m.provider),
729                summarization_model_provider: areas.summarization.map(|m| m.provider),
730            }
731        });
732        let model_roster = crate::ModelRoster {
733            model: Some(model),
734            provider_name: Some(resolved_provider_name),
735            provider_type: config.provider_type.clone(),
736            fast: crate::RoleModel::from_parts(config.fast_model, resolved_fast_provider),
737            background: crate::RoleModel::from_parts(
738                config.background_model,
739                config.background_model_provider,
740            ),
741            summarization: crate::RoleModel::from_parts(
742                config.summarization_model,
743                config.summarization_model_provider,
744            ),
745        };
746
747        // Re-inject guardian state on resume so a reject→fix verdict can be
748        // re-reviewed: config from the session (persisted at first spawn),
749        // spawner from the coordinator-held handle. Absent guardian config this
750        // stays `None`, and the approve→complete path is unchanged.
751        let guardian_config = read_guardian_config(&session);
752        let guardian_spawner = self.guardian_spawner.read().await.clone();
753
754        spawn_session_execution(SessionExecutionArgs {
755            agent: self.agent.clone(),
756            session_id,
757            session,
758            tools_override: Some(root_tools),
759            provider_override,
760            model_roster,
761            reasoning_effort,
762            reasoning_effort_source,
763            auxiliary_model_resolver: Some(auxiliary_model_resolver),
764            // Resumed child runs keep the spawn-time disabled snapshot (#136 lives
765            // on the long-running main agent path; children are short-lived).
766            disabled_filter_resolver: None,
767            disabled_tools: Some(config.disabled_tools),
768            disabled_skill_ids: Some(config.disabled_skill_ids),
769            selected_skill_ids: None,
770            selected_skill_mode: None,
771            cancel_token,
772            mpsc_tx,
773            image_fallback: config.image_fallback,
774            gold_config,
775            guardian_config,
776            guardian_spawner,
777            bash_resume_hook: {
778                let hook: Arc<dyn BashResumeHook> = Arc::new(self.clone());
779                Some(hook)
780            },
781            bash_completion_sink: {
782                // Resumed runs keep the push wired too, so a background shell
783                // launched after resume still notifies the loop.
784                let sink: Arc<dyn BashCompletionSink> = Arc::new(self.clone());
785                Some(sink)
786            },
787            app_data_dir: Some(self.app_data_dir.clone()),
788            runners: self.agent_runners.clone(),
789            sessions_cache: self.sessions.clone(),
790            on_complete: None,
791        });
792    }
793}
794
795/// Hidden resume message for a bash-completion self-resume (issue #84 Phase 2b).
796/// Mirrors [`runtime_resume_message`]'s hidden/compressible shape so the resume
797/// port's `has_pending_user_message` gate is satisfied.
798///
799/// `timed_out` selects the wording: the normal path (all shells finished)
800/// announces completion; the deadline path (the 6h+10m wait ceiling was hit with
801/// shells STILL running) must NOT claim the shells completed — it says they may
802/// still be running so the model verifies with BashOutput instead of assuming
803/// success on a false premise.
804fn bash_completion_resume_message(bash_ids: &[String], timed_out: bool) -> Message {
805    let body = if timed_out {
806        format!(
807            "Runtime notification: the background-Bash wait ceiling was reached while one or more \
808             shell(s) ({}) may still be running. The session is being resumed so it is not \
809             stranded; verify their actual status with BashOutput before assuming completion.",
810            bash_ids.join(", ")
811        )
812    } else {
813        format!(
814            "Runtime notification: all background Bash shell(s) ({}) have completed. \
815             Review their output with BashOutput and resume the task from where you left off.",
816            bash_ids.join(", ")
817        )
818    };
819    let mut message = Message::user(body);
820    message.metadata = Some(serde_json::json!({
821        RUNTIME_RESUME_MESSAGE_HIDDEN_KEY: true,
822        RUNTIME_RESUME_MESSAGE_KIND_KEY: BASH_COMPLETION_RESUME_KIND,
823    }));
824    message.never_compress = false;
825    message
826}
827
828/// Decide whether the bash self-resume should retry its clear→append→resume
829/// sequence after a resume attempt returned `outcome`, given that the persisted
830/// bash wait is (`true`) / is not (`false`) still set on reload.
831///
832/// Retry **only** when the resume did NOT spawn (`Completed` — no pending user
833/// message, i.e. our resume message was dropped — or `AlreadyRunning`) AND the
834/// persisted bash wait is still set: the signature of the finalize-clobber, where
835/// the suspending runner's one-shot final `merge_save_runtime` lands after our
836/// save and reverts `waiting_for_bash=Some` while dropping our resume message, so
837/// `has_pending_user_message` fails and nothing spawns. `Started` (resume fired)
838/// and `NotFound` (session gone) never retry. Pure helper so the clobber
839/// detection is unit-testable in isolation from async I/O.
840fn bash_resume_should_retry(outcome: &ResumeOutcome, persisted_waiting_for_bash: bool) -> bool {
841    match outcome {
842        ResumeOutcome::Started { .. } | ResumeOutcome::NotFound => false,
843        ResumeOutcome::Completed | ResumeOutcome::AlreadyRunning { .. } => {
844            persisted_waiting_for_bash
845        }
846    }
847}
848
849/// Whether a background-shell completion push should **resume** the owning loop
850/// (vs merely enqueue an injection). Resume only when the loop is actually
851/// suspended on a bash wait AND every shell it was waiting on has now finished —
852/// resuming while other waited shells are still running would drop them back into
853/// a foreground turn prematurely. The last shell to finish (or the backstop)
854/// drives the resume; earlier ones enqueue their notice. Pure so the invariant is
855/// unit-testable in isolation.
856fn bash_completion_should_resume(
857    loop_suspended_on_bash: bool,
858    all_waited_shells_done: bool,
859) -> bool {
860    loop_suspended_on_bash && all_waited_shells_done
861}
862
863/// Apply the bash-resume state transition to a loaded session **in place**: clear
864/// the `waiting_for_bash` wait, mark the runtime Idle, drop the suspension +
865/// `runtime.suspend_reason`, and append `resume_message`. Returns `false` (a
866/// no-op) when the session was not actually waiting on bash — the double-resume
867/// guard shared by the push and the backstop. Pure (no I/O) so both
868/// [`ChildCompletionCoordinator::perform_bash_resume`] and unit tests exercise the
869/// exact same transition.
870fn apply_bash_resume_transition(session: &mut Session, resume_message: &Message) -> bool {
871    let mut runtime_state = read_runtime_state(session);
872    if runtime_state.waiting_for_bash.is_none() {
873        return false;
874    }
875    runtime_state.waiting_for_bash = None;
876    runtime_state.status = AgentStatusState::Idle;
877    runtime_state.suspension = None;
878    write_runtime_state(session, &runtime_state);
879    session.metadata.remove("runtime.suspend_reason");
880    session.add_message(resume_message.clone());
881    true
882}
883
884/// Bash self-resume support (issue #84 Phase 2b; push follow-up).
885impl ChildCompletionCoordinator {
886    /// **Backstop** for a session suspended on `waiting_for_bash`. The primary,
887    /// event-driven wake is the loop-facing push
888    /// ([`BashCompletionSink::on_bash_completed`] → [`Self::deliver_bash_completion`]):
889    /// the shell's completion task fires it the instant the process exits, and it
890    /// resumes the loop directly. This task exists ONLY to catch a **lost push** —
891    /// the completion landing in the window before the suspend was persisted (so
892    /// the push saw no `waiting_for_bash` and only queued an injection), or a
893    /// configuration with no sink wired — and to honour the wait ceiling.
894    ///
895    /// So it is deliberately NOT a hot spin: a coarse backoff (1 s → 30 s) that
896    /// **yields to the push**. In the happy path the push has already cleared
897    /// `waiting_for_bash` before the first check fires, so this returns after one
898    /// cheap load with no registry polling at all. It only performs a resume when
899    /// the shell(s) have finished but the loop is somehow still suspended, or the
900    /// 6 h wait ceiling is reached.
901    async fn bash_self_resume(&self, session_id: String, bash_ids: Vec<String>) {
902        let mut delay = Duration::from_secs(1);
903        let max_delay = Duration::from_secs(30);
904        // Hard ceiling: the wait lease (6 h) + the registry GC TTL (5 min) +
905        // margin. After this the shells are gone from the registry regardless,
906        // so force-resume to avoid stranding the session on a GC edge case.
907        let max_poll = Duration::from_secs(6 * 3600 + 600);
908        let deadline = tokio::time::Instant::now() + max_poll;
909
910        loop {
911            tokio::time::sleep(delay).await;
912
913            let Some(session) = self.load_session(&session_id).await else {
914                tracing::info!(%session_id, "bash self-resume backstop: session gone; nothing to do");
915                return;
916            };
917            if read_runtime_state(&session).waiting_for_bash.is_none() {
918                // The push (or another path) already resumed. This is the common
919                // case — the backstop yields silently after a single load.
920                return;
921            }
922
923            let still_running =
924                bamboo_tools::tools::bash_runtime::running_shells_for_session(&session_id);
925            let timed_out = tokio::time::Instant::now() >= deadline;
926            if still_running.is_empty() || timed_out {
927                // The shell(s) finished but the loop is still suspended → the push
928                // was lost (pre-persist window / no sink), or the ceiling hit.
929                // Resume under the shared per-session lock so we never race the
930                // push or a concurrent child-completion resume.
931                let guard = session_resume_lock(&session_id);
932                let _held = guard.lock().await;
933                tracing::warn!(
934                    %session_id,
935                    shell_count = bash_ids.len(),
936                    timed_out,
937                    "bash self-resume backstop engaged (push lost or wait ceiling reached)"
938                );
939                self.perform_bash_resume(
940                    &session_id,
941                    bash_completion_resume_message(&bash_ids, timed_out),
942                )
943                .await;
944                return;
945            }
946
947            delay = (delay * 2).min(max_delay);
948        }
949    }
950
951    /// Clear a session's `waiting_for_bash` state, append `resume_message`, and
952    /// drive the parent resume — the shared clear→append→resume used by BOTH the
953    /// event-driven push ([`Self::deliver_bash_completion`]) and the backstop poll
954    /// ([`Self::bash_self_resume`]).
955    ///
956    /// **The caller MUST hold the [`session_resume_lock`] for `session_id`** so the
957    /// load-check-clear-resume critical section is serialized against every other
958    /// resume source (no double resume). No-op when the persisted wait was already
959    /// cleared (another source handled it first).
960    ///
961    /// The clear→append→resume is a **bounded retry loop** that closes the
962    /// finalize-clobber strand. The suspending runner's `finalize_task_context`
963    /// runs a full `save_runtime_session` (same `merge_save_runtime`, which
964    /// overwrites the whole `messages` array) that can land AFTER our save,
965    /// reverting `waiting_for_bash=Some` and dropping our resume message, so
966    /// `has_pending_user_message` fails and `resume_parent` returns `Completed`
967    /// without spawning. We detect that (persisted wait still set after a
968    /// non-`Started` outcome) and re-clear/re-append/re-resume. It converges
969    /// because the runner's finalize persist is one-shot: once landed, our retry's
970    /// save is the last writer, the message sticks, and resume fires.
971    async fn perform_bash_resume(&self, session_id: &str, resume_message: Message) {
972        let retry_backoff = Duration::from_millis(200);
973        const MAX_RESUME_ATTEMPTS: u8 = 5;
974        for attempt in 0..MAX_RESUME_ATTEMPTS {
975            if attempt > 0 {
976                tokio::time::sleep(retry_backoff).await;
977            }
978
979            let Some(mut session) = self.load_session(session_id).await else {
980                tracing::warn!(%session_id, "bash resume: session not found; nothing to resume");
981                return;
982            };
983
984            if !apply_bash_resume_transition(&mut session, &resume_message) {
985                // Double-resume guard: the wait was already cleared by another
986                // source (the push, the backstop, or a user-driven resume). Do
987                // not append a duplicate message or request a redundant resume.
988                tracing::info!(
989                    %session_id, attempt,
990                    "bash resume: persisted bash wait already cleared; nothing to resume"
991                );
992                return;
993            }
994            session.updated_at = Utc::now();
995            self.save_and_cache(&mut session).await;
996            tracing::info!(
997                %session_id, attempt,
998                "bash resume: cleared bash wait and appended resume message"
999            );
1000
1001            let outcome = self.resume_parent(session_id.to_string()).await;
1002            match outcome {
1003                ResumeOutcome::Started { .. } => {
1004                    tracing::info!(%session_id, attempt, "bash resume: resume fired");
1005                    return;
1006                }
1007                ResumeOutcome::NotFound => {
1008                    tracing::warn!(%session_id, "bash resume: session vanished during resume");
1009                    return;
1010                }
1011                _ => {
1012                    // Completed (no pending user message ⇒ our resume message was
1013                    // dropped by the runner's finalize persist) or AlreadyRunning.
1014                    // Decide via the persisted bash wait: still set ⇒
1015                    // finalize-clobber ⇒ retry; cleared ⇒ the session is being
1016                    // handled (by us or a concurrent resume) ⇒ stop.
1017                    let clobbered = match self.load_session(session_id).await {
1018                        Some(reloaded) => read_runtime_state(&reloaded).waiting_for_bash.is_some(),
1019                        None => {
1020                            tracing::warn!(%session_id, "bash resume: session vanished after resume");
1021                            return;
1022                        }
1023                    };
1024                    if bash_resume_should_retry(&outcome, clobbered) {
1025                        tracing::warn!(
1026                            %session_id, attempt,
1027                            outcome = outcome.as_str(),
1028                            "bash resume: persisted wait still set after resume (finalize-clobber); retrying"
1029                        );
1030                        continue;
1031                    }
1032                    tracing::info!(
1033                        %session_id, attempt,
1034                        outcome = outcome.as_str(),
1035                        "bash resume: wait cleared and resume handled; stopping"
1036                    );
1037                    return;
1038                }
1039            }
1040        }
1041
1042        tracing::warn!(
1043            %session_id,
1044            attempts = MAX_RESUME_ATTEMPTS,
1045            "bash resume: exhausted clobber-retry budget without confirming resume; giving up"
1046        );
1047    }
1048}
1049
1050impl BashResumeHook for ChildCompletionCoordinator {
1051    fn arrange_bash_self_resume(&self, session_id: String, bash_ids: Vec<String>) {
1052        let coordinator = Arc::new(self.clone());
1053        tokio::spawn(async move {
1054            coordinator.bash_self_resume(session_id, bash_ids).await;
1055        });
1056    }
1057}
1058
1059/// Build the injected user-message body for a completed background shell — a
1060/// concise notice plus a bounded output tail — so the model can act on the
1061/// result without a mandatory `BashOutput` round-trip (issue #84 Phase 2b
1062/// follow-up).
1063fn bash_completion_injection_body(info: &BashCompletionInfo) -> String {
1064    let exit = match info.exit_code {
1065        Some(code) => code.to_string(),
1066        None => "none (signal/killed)".to_string(),
1067    };
1068    let mut body = format!(
1069        "Runtime notification: background shell `{}` (`{}`) finished — status {}, exit code {}.",
1070        info.bash_id, info.command, info.status, exit
1071    );
1072    if info.output_tail.trim().is_empty() {
1073        body.push_str(" It produced no captured output.");
1074    } else {
1075        body.push_str("\n\nOutput tail:\n");
1076        body.push_str(&info.output_tail);
1077    }
1078    body.push_str(&format!(
1079        "\n\nUse BashOutput with bash_id=\"{}\" for the full output, then continue the task.",
1080        info.bash_id
1081    ));
1082    body
1083}
1084
1085/// Loop-facing background-Bash completion delivery (issue #84 Phase 2b
1086/// follow-up). Pushes a completed shell's result into its owning session's loop,
1087/// mirroring how a sub-agent completion reaches its parent — but via the
1088/// running-loop channel, which children never exercise (a parent waiting on
1089/// children is always suspended when one completes; bash is the first completion
1090/// source that can land on a *live, iterating* loop).
1091///
1092/// The push enqueues onto `pending_injected_messages`, the same round-boundary
1093/// steering channel `send_message` uses. That covers every reachable loop state:
1094/// an actively-looping session drains it at its next round
1095/// (`merge_pending_injected_messages`); a session suspended on `waiting_for_bash`
1096/// drains it when the durable end-of-turn poll backstop (`bash_resume_hook`)
1097/// resumes it at round 0. A wired-sink session is never idle-with-a-running-shell
1098/// (ending a turn with one suspends), so no separate idle-wake path is needed —
1099/// keeping the push a pure latency optimization that never races the backstop.
1100/// Enqueue a completed shell's summary as a pending injected message on the
1101/// owning session. Race-safe: `update_runtime_config` loads the freshest session
1102/// under the per-session lock and re-saves, so it can never revert a message the
1103/// live loop appended concurrently — unlike `merge_save_runtime` (which writes
1104/// the caller's whole `messages` snapshot verbatim). Free fn so it is unit-
1105/// testable without constructing a full coordinator. Returns the saved session,
1106/// or `None` if the owning session no longer exists.
1107async fn enqueue_bash_completion_injection(
1108    persistence: &LockedSessionStore,
1109    info: &BashCompletionInfo,
1110) -> std::io::Result<Option<Session>> {
1111    let body = bash_completion_injection_body(info);
1112    let queued = serde_json::json!({
1113        "content": body,
1114        "created_at": Utc::now(),
1115    });
1116    persistence
1117        .update_runtime_config(&info.session_id, move |session| {
1118            let mut pending = session.pending_injected_messages().unwrap_or_default();
1119            pending.push(queued);
1120            session.set_pending_injected_messages(pending);
1121        })
1122        .await
1123}
1124
1125/// Build the hidden, compressible resume message for a completed background
1126/// shell — the same rich notice body used for a live-loop injection
1127/// ([`bash_completion_injection_body`]), but tagged as a resume message so it
1128/// satisfies the `has_pending_user_message` gate that lets a suspended session
1129/// spawn. This is what the **push** appends when it wakes a suspended loop, so
1130/// the model gets the shell's status + output tail in one shot without a
1131/// separate `BashOutput` round-trip.
1132fn bash_resume_message_from_info(info: &BashCompletionInfo) -> Message {
1133    let mut message = Message::user(bash_completion_injection_body(info));
1134    message.metadata = Some(serde_json::json!({
1135        RUNTIME_RESUME_MESSAGE_HIDDEN_KEY: true,
1136        RUNTIME_RESUME_MESSAGE_KIND_KEY: BASH_COMPLETION_RESUME_KIND,
1137    }));
1138    message.never_compress = false;
1139    message
1140}
1141
1142impl ChildCompletionCoordinator {
1143    /// Loop-facing delivery of a completed background shell. Two paths, chosen
1144    /// under the per-session resume lock so we never race the backstop poll or a
1145    /// concurrent child-completion resume:
1146    ///
1147    /// - **Suspended loop** (the model ended its turn with the shell running, so
1148    ///   `waiting_for_bash` is set) AND every waited shell has now finished →
1149    ///   **resume the loop directly**, event-driven, appending the rich completion
1150    ///   notice as the resume message. This is the push's whole point: no polling.
1151    /// - Otherwise (a live/iterating loop, or a suspend still waiting on OTHER
1152    ///   shells) → **enqueue** the notice as a pending injected message, drained at
1153    ///   the next round boundary (a live loop) or folded into the eventual resume
1154    ///   when the last shell finishes.
1155    async fn deliver_bash_completion(&self, info: BashCompletionInfo) {
1156        let guard = session_resume_lock(&info.session_id);
1157        let _held = guard.lock().await;
1158
1159        let Some(session) = self.load_session(&info.session_id).await else {
1160            tracing::warn!(
1161                session_id = %info.session_id,
1162                bash_id = %info.bash_id,
1163                "background bash completion: owning session not found; nothing to notify"
1164            );
1165            return;
1166        };
1167
1168        let waiting = read_runtime_state(&session).waiting_for_bash.is_some();
1169        // The producer flips the shell's `running` flag false BEFORE firing this
1170        // push, so a now-empty per-session registry means every shell the loop was
1171        // waiting on has finished — safe to resume. If OTHER waited shells are
1172        // still running, fall through to the enqueue path and let the last one (or
1173        // the backstop) drive the resume.
1174        let all_shells_done =
1175            bamboo_tools::tools::bash_runtime::running_shells_for_session(&info.session_id)
1176                .is_empty();
1177
1178        if bash_completion_should_resume(waiting, all_shells_done) {
1179            tracing::info!(
1180                session_id = %info.session_id,
1181                bash_id = %info.bash_id,
1182                status = %info.status,
1183                "background bash completion: push-resuming suspended loop (event-driven)"
1184            );
1185            self.perform_bash_resume(&info.session_id, bash_resume_message_from_info(&info))
1186                .await;
1187            return;
1188        }
1189
1190        match enqueue_bash_completion_injection(&self.persistence, &info).await {
1191            Ok(Some(_)) => tracing::info!(
1192                session_id = %info.session_id,
1193                bash_id = %info.bash_id,
1194                status = %info.status,
1195                waiting,
1196                "background bash completion queued for injection at the next round boundary"
1197            ),
1198            Ok(None) => tracing::warn!(
1199                session_id = %info.session_id,
1200                bash_id = %info.bash_id,
1201                "background bash completion: owning session not found; nothing to notify"
1202            ),
1203            Err(error) => tracing::warn!(
1204                session_id = %info.session_id,
1205                bash_id = %info.bash_id,
1206                %error,
1207                "background bash completion: failed to queue injection"
1208            ),
1209        }
1210    }
1211}
1212
1213impl BashCompletionSink for ChildCompletionCoordinator {
1214    fn on_bash_completed(&self, info: BashCompletionInfo) {
1215        // Best-effort, off the shell's completion-poll task: hand the delivery to
1216        // a detached task so the producer is never blocked (mirrors
1217        // `arrange_bash_self_resume`).
1218        let coordinator = Arc::new(self.clone());
1219        tokio::spawn(async move {
1220            coordinator.deliver_bash_completion(info).await;
1221        });
1222    }
1223}
1224
1225#[cfg(test)]
1226mod tests {
1227    use super::*;
1228    use bamboo_agent_core::Message;
1229
1230    fn make_completion(status: &str) -> ChildCompletion {
1231        ChildCompletion {
1232            parent_session_id: "parent-1".to_string(),
1233            child_session_id: "child-1".to_string(),
1234            status: status.to_string(),
1235            error: None,
1236            completed_at: Utc::now(),
1237        }
1238    }
1239
1240    // ── ② derive completed children from the index ──────────────────────
1241
1242    struct StubChildIndex {
1243        children: Vec<(String, Option<String>)>,
1244    }
1245
1246    #[async_trait]
1247    impl Storage for StubChildIndex {
1248        async fn save_session(&self, _session: &Session) -> std::io::Result<()> {
1249            Ok(())
1250        }
1251        async fn load_session(&self, _id: &str) -> std::io::Result<Option<Session>> {
1252            Ok(None)
1253        }
1254        async fn delete_session(&self, _id: &str) -> std::io::Result<bool> {
1255            Ok(false)
1256        }
1257        async fn list_child_run_statuses(
1258            &self,
1259            _parent_session_id: &str,
1260        ) -> std::io::Result<Vec<(String, Option<String>)>> {
1261            Ok(self.children.clone())
1262        }
1263    }
1264
1265    #[tokio::test]
1266    async fn derive_completed_only_includes_terminal_children() {
1267        let storage: Arc<dyn Storage> = Arc::new(StubChildIndex {
1268            children: vec![
1269                ("a".into(), Some("completed".into())),
1270                ("b".into(), Some("running".into())),
1271                ("c".into(), Some("error".into())),
1272                ("d".into(), None),
1273            ],
1274        });
1275        let completed = derive_completed_child_ids(&storage, "parent-1", "b").await;
1276        // Terminal from index: a, c. Plus the just-completed child b folded in.
1277        assert_eq!(
1278            completed,
1279            vec!["a".to_string(), "b".to_string(), "c".to_string()]
1280        );
1281    }
1282
1283    #[tokio::test]
1284    async fn derive_completed_folds_in_just_completed_when_index_lags() {
1285        // Index hasn't caught up — reports the child as still running.
1286        let storage: Arc<dyn Storage> = Arc::new(StubChildIndex {
1287            children: vec![("only".into(), Some("running".into()))],
1288        });
1289        let completed = derive_completed_child_ids(&storage, "parent-1", "only").await;
1290        assert_eq!(completed, vec!["only".to_string()]);
1291    }
1292
1293    #[test]
1294    fn wait_policy_all_uses_derived_completed_set() {
1295        let waited = vec!["a".to_string(), "b".to_string()];
1296        assert!(!wait_policy_satisfied(
1297            ChildWaitPolicy::All,
1298            &waited,
1299            &["a".to_string()],
1300            "completed"
1301        ));
1302        assert!(wait_policy_satisfied(
1303            ChildWaitPolicy::All,
1304            &waited,
1305            &["a".to_string(), "b".to_string()],
1306            "completed"
1307        ));
1308    }
1309
1310    #[test]
1311    fn child_final_assistant_text_returns_last_assistant() {
1312        let mut session = Session::new("child-1", "gpt-4");
1313        session.messages.push(Message::user("hi"));
1314        session
1315            .messages
1316            .push(Message::assistant("first answer", None));
1317        session.messages.push(Message::user("again"));
1318        session
1319            .messages
1320            .push(Message::assistant("final answer", None));
1321
1322        assert_eq!(
1323            child_final_assistant_text(&session).as_deref(),
1324            Some("final answer")
1325        );
1326    }
1327
1328    #[test]
1329    fn child_final_assistant_text_returns_none_when_blank() {
1330        let mut session = Session::new("child-1", "gpt-4");
1331        session.messages.push(Message::assistant("   ", None));
1332        assert!(child_final_assistant_text(&session).is_none());
1333    }
1334
1335    #[test]
1336    fn child_final_assistant_text_returns_none_when_no_assistant() {
1337        let mut session = Session::new("child-1", "gpt-4");
1338        session.messages.push(Message::user("hi"));
1339        assert!(child_final_assistant_text(&session).is_none());
1340    }
1341
1342    #[test]
1343    fn runtime_resume_message_folds_full_response_without_truncation() {
1344        // A very long child final response is folded in verbatim (no 4000-char
1345        // cap, no truncation marker).
1346        let completion = make_completion("completed");
1347        let long: String = "a".repeat(10_000);
1348        let message = runtime_resume_message(&completion, 0, Some(&long));
1349        assert!(message.content.contains(&long));
1350        assert!(!message.content.contains("truncated"));
1351    }
1352
1353    #[test]
1354    fn runtime_resume_message_includes_child_response_when_provided() {
1355        let completion = make_completion("completed");
1356        let message = runtime_resume_message(&completion, 0, Some("the answer is 42"));
1357
1358        assert!(matches!(message.role, Role::User));
1359        // Folded child results are now compressible so the parent context can
1360        // reclaim them under compaction.
1361        assert!(!message.never_compress);
1362        assert!(message.content.contains("Child final response:"));
1363        assert!(message.content.contains("the answer is 42"));
1364
1365        let metadata = message.metadata.expect("metadata present");
1366        assert_eq!(
1367            metadata.get("hidden_from_ui").and_then(|v| v.as_bool()),
1368            Some(true)
1369        );
1370        assert_eq!(
1371            metadata.get("runtime_kind").and_then(|v| v.as_str()),
1372            Some("child_completion_resume")
1373        );
1374        assert_eq!(
1375            metadata
1376                .get("child_final_response_included")
1377                .and_then(|v| v.as_bool()),
1378            Some(true)
1379        );
1380    }
1381
1382    #[test]
1383    fn runtime_resume_message_falls_back_to_error_when_no_response() {
1384        let mut completion = make_completion("error");
1385        completion.error = Some("boom".to_string());
1386
1387        let message = runtime_resume_message(&completion, 1, None);
1388        assert!(message.content.contains("Child error:"));
1389        assert!(message.content.contains("boom"));
1390        let metadata = message.metadata.expect("metadata present");
1391        assert_eq!(
1392            metadata
1393                .get("child_final_response_included")
1394                .and_then(|v| v.as_bool()),
1395            Some(false)
1396        );
1397    }
1398
1399    #[test]
1400    fn runtime_resume_message_minimal_when_no_response_and_no_error() {
1401        let completion = make_completion("completed");
1402        let message = runtime_resume_message(&completion, 2, None);
1403        assert!(!message.content.contains("Child final response:"));
1404        assert!(!message.content.contains("Child error:"));
1405        assert!(message.content.contains("Resume the parent task"));
1406    }
1407
1408    #[test]
1409    fn read_config_snapshot_refreshes_cached_snapshot_from_live_config() {
1410        let runtime = tokio::runtime::Runtime::new().expect("runtime");
1411
1412        runtime.block_on(async {
1413            let config = Arc::new(RwLock::new(Config::default()));
1414            config.write().await.provider = "copilot".to_string();
1415            let cached_config = StdRwLock::new(Config::default());
1416
1417            let snapshot = read_config_snapshot(&config, &cached_config);
1418
1419            assert_eq!(snapshot.provider, "copilot");
1420            assert_eq!(
1421                cached_config.read().expect("cached snapshot lock").provider,
1422                "copilot"
1423            );
1424        });
1425    }
1426
1427    #[test]
1428    fn read_config_snapshot_uses_cached_snapshot_when_live_lock_is_busy() {
1429        let runtime = tokio::runtime::Runtime::new().expect("runtime");
1430
1431        runtime.block_on(async {
1432            let cached_snapshot = Config {
1433                provider: "cached-provider".to_string(),
1434                ..Default::default()
1435            };
1436
1437            let config = Arc::new(RwLock::new(Config::default()));
1438            let cached_config = StdRwLock::new(cached_snapshot);
1439            let _write_guard = config.write().await;
1440
1441            let snapshot = read_config_snapshot(&config, &cached_config);
1442
1443            assert_eq!(snapshot.provider, "cached-provider");
1444        });
1445    }
1446
1447    // ── Bash self-resume (issue #84 Phase 2b): deadline message + clobber-retry ──
1448
1449    #[test]
1450    fn bash_completion_resume_message_normal_announces_completion() {
1451        let ids = vec!["bg-1".to_string(), "bg-2".to_string()];
1452        let message = bash_completion_resume_message(&ids, false);
1453        // Normal path: the shells genuinely finished.
1454        assert!(
1455            message.content.contains("have completed"),
1456            "normal resume message must announce completion: {}",
1457            message.content
1458        );
1459        // Hidden + compressible so the resume gate sees it but the UI hides it.
1460        let metadata = message.metadata.expect("metadata present");
1461        assert_eq!(
1462            metadata
1463                .get(RUNTIME_RESUME_MESSAGE_HIDDEN_KEY)
1464                .and_then(|v| v.as_bool()),
1465            Some(true),
1466            "resume message must be hidden from the UI"
1467        );
1468        assert_eq!(
1469            metadata
1470                .get(RUNTIME_RESUME_MESSAGE_KIND_KEY)
1471                .and_then(|v| v.as_str()),
1472            Some(BASH_COMPLETION_RESUME_KIND),
1473            "resume message must carry the bash-completion kind discriminant"
1474        );
1475    }
1476
1477    #[test]
1478    fn bash_completion_resume_message_deadline_does_not_claim_completion() {
1479        // The 6h+10m deadline force-breaks with shells STILL running. The message
1480        // must NOT say "have completed" — that would let the model assume success
1481        // on a false premise. It must direct the model to verify with BashOutput.
1482        let ids = vec!["bg-long".to_string()];
1483        let message = bash_completion_resume_message(&ids, true);
1484        assert!(
1485            !message.content.contains("have completed"),
1486            "deadline resume message must NOT claim the shells completed: {}",
1487            message.content
1488        );
1489        assert!(
1490            message.content.contains("may still be running"),
1491            "deadline resume message must warn shells may still be running: {}",
1492            message.content
1493        );
1494        assert!(
1495            message.content.contains("BashOutput"),
1496            "deadline resume message must direct verification via BashOutput: {}",
1497            message.content
1498        );
1499        // Same hidden/kind shape so the resume gate is satisfied identically.
1500        let metadata = message.metadata.expect("metadata present");
1501        assert_eq!(
1502            metadata
1503                .get(RUNTIME_RESUME_MESSAGE_KIND_KEY)
1504                .and_then(|v| v.as_str()),
1505            Some(BASH_COMPLETION_RESUME_KIND)
1506        );
1507    }
1508
1509    #[test]
1510    fn bash_resume_should_retry_matrix() {
1511        // The finalize-clobber retry predicate (issue #84 Phase 2b). Retry only
1512        // when the resume did NOT spawn (Completed / AlreadyRunning) AND the
1513        // persisted bash wait is still set on reload — the clobber signature.
1514
1515        // Started: the resume fired — never retry, regardless of persisted state.
1516        assert!(!bash_resume_should_retry(
1517            &ResumeOutcome::Started { run_id: "r".into() },
1518            true
1519        ));
1520        assert!(!bash_resume_should_retry(
1521            &ResumeOutcome::Started { run_id: "r".into() },
1522            false
1523        ));
1524
1525        // NotFound: session gone — never retry.
1526        assert!(!bash_resume_should_retry(&ResumeOutcome::NotFound, true));
1527        assert!(!bash_resume_should_retry(&ResumeOutcome::NotFound, false));
1528
1529        // Completed + persisted wait still set ⇒ finalize-clobber ⇒ retry.
1530        assert!(bash_resume_should_retry(&ResumeOutcome::Completed, true));
1531        // Completed + persisted wait cleared ⇒ handled (our message stuck, or a
1532        // concurrent resume finished) ⇒ stop.
1533        assert!(!bash_resume_should_retry(&ResumeOutcome::Completed, false));
1534
1535        // AlreadyRunning + persisted wait still set ⇒ clobbered while a runner is
1536        // (stale-)active ⇒ retry to re-establish the resume message.
1537        assert!(bash_resume_should_retry(
1538            &ResumeOutcome::AlreadyRunning { run_id: "r".into() },
1539            true
1540        ));
1541        // AlreadyRunning + wait cleared ⇒ a runner owns the session ⇒ stop.
1542        assert!(!bash_resume_should_retry(
1543            &ResumeOutcome::AlreadyRunning { run_id: "r".into() },
1544            false
1545        ));
1546    }
1547
1548    // ── bash completion injection body (Phase 2b follow-up) ──────────────
1549
1550    #[test]
1551    fn injection_body_includes_status_exit_command_and_tail() {
1552        let info = BashCompletionInfo {
1553            session_id: "s".into(),
1554            bash_id: "abc123".into(),
1555            command: "make build".into(),
1556            exit_code: Some(0),
1557            status: "completed".into(),
1558            output_tail: "BUILD OK".into(),
1559        };
1560        let body = bash_completion_injection_body(&info);
1561        assert!(body.contains("abc123"), "body: {body}");
1562        assert!(body.contains("make build"), "body: {body}");
1563        assert!(body.contains("completed"), "body: {body}");
1564        assert!(body.contains("exit code 0"), "body: {body}");
1565        assert!(body.contains("BUILD OK"), "body: {body}");
1566        // The model is pointed at BashOutput for the full log.
1567        assert!(body.contains("BashOutput"), "body: {body}");
1568        assert!(body.contains("bash_id=\"abc123\""), "body: {body}");
1569    }
1570
1571    #[test]
1572    fn injection_body_handles_no_output_and_signal_kill() {
1573        let info = BashCompletionInfo {
1574            session_id: "s".into(),
1575            bash_id: "xyz".into(),
1576            command: "sleep 99".into(),
1577            exit_code: None,
1578            status: "killed".into(),
1579            output_tail: String::new(),
1580        };
1581        let body = bash_completion_injection_body(&info);
1582        assert!(body.contains("killed"), "body: {body}");
1583        assert!(body.contains("none (signal/killed)"), "body: {body}");
1584        assert!(body.contains("no captured output"), "body: {body}");
1585        // No output tail section when there is nothing to show.
1586        assert!(!body.contains("Output tail:"), "body: {body}");
1587    }
1588
1589    async fn temp_store() -> (tempfile::TempDir, Arc<dyn Storage>, LockedSessionStore) {
1590        let temp = tempfile::tempdir().unwrap();
1591        let storage: Arc<dyn Storage> = Arc::new(
1592            bamboo_storage::v2::SessionStoreV2::new(temp.path().to_path_buf())
1593                .await
1594                .expect("storage init"),
1595        );
1596        let persistence = LockedSessionStore::new(storage.clone());
1597        (temp, storage, persistence)
1598    }
1599
1600    #[tokio::test]
1601    async fn enqueue_writes_pending_injection_and_preserves_messages() {
1602        let (_temp, storage, persistence) = temp_store().await;
1603
1604        let mut session = Session::new("sess-enq", "test-model");
1605        session.add_message(Message::user("do the build"));
1606        storage.save_session(&session).await.unwrap();
1607
1608        let info = BashCompletionInfo {
1609            session_id: "sess-enq".into(),
1610            bash_id: "sh-1".into(),
1611            command: "make".into(),
1612            exit_code: Some(0),
1613            status: "completed".into(),
1614            output_tail: "done".into(),
1615        };
1616        let saved = enqueue_bash_completion_injection(&persistence, &info)
1617            .await
1618            .expect("enqueue io ok")
1619            .expect("session exists");
1620
1621        let pending = saved
1622            .pending_injected_messages()
1623            .expect("pending injection present");
1624        assert_eq!(pending.len(), 1);
1625        let content = pending[0].get("content").and_then(|v| v.as_str()).unwrap();
1626        assert!(content.contains("sh-1"), "content: {content}");
1627        assert!(content.contains("make"), "content: {content}");
1628        assert!(content.contains("done"), "content: {content}");
1629        // The pre-existing conversation is untouched (no clobber).
1630        assert_eq!(saved.messages.len(), 1);
1631    }
1632
1633    #[tokio::test]
1634    async fn enqueue_returns_none_for_missing_session() {
1635        let (_temp, _storage, persistence) = temp_store().await;
1636        let info = BashCompletionInfo {
1637            session_id: "does-not-exist".into(),
1638            bash_id: "x".into(),
1639            command: "true".into(),
1640            exit_code: Some(0),
1641            status: "completed".into(),
1642            output_tail: String::new(),
1643        };
1644        let result = enqueue_bash_completion_injection(&persistence, &info)
1645            .await
1646            .expect("io ok");
1647        assert!(result.is_none(), "no session → nothing enqueued");
1648    }
1649
1650    // ── push-driven resume: the state transition + decision the push applies ──
1651
1652    /// A session suspended on `waiting_for_bash`, given the rich completion
1653    /// message, is transitioned to a resumable state: the wait is cleared, the
1654    /// runtime is Idle, the suspend-reason marker is gone, and the resume message
1655    /// is appended. This is exactly what the PUSH does to wake the loop
1656    /// event-driven (vs the old backstop poll).
1657    #[test]
1658    fn apply_bash_resume_transition_clears_wait_and_appends_message() {
1659        use bamboo_domain::session::runtime_state::WaitingForBashState;
1660
1661        let mut session = Session::new("sess-resume", "test-model");
1662        session.add_message(Message::user("kick off the build"));
1663        let mut rt = read_runtime_state(&session);
1664        rt.status = AgentStatusState::Running;
1665        rt.waiting_for_bash = Some(WaitingForBashState::for_bash(
1666            vec!["sh-1".into()],
1667            Utc::now(),
1668        ));
1669        write_runtime_state(&mut session, &rt);
1670        session.metadata.insert(
1671            "runtime.suspend_reason".to_string(),
1672            "waiting_for_bash".to_string(),
1673        );
1674
1675        let resume = bash_completion_resume_message(&["sh-1".to_string()], false);
1676        let did = apply_bash_resume_transition(&mut session, &resume);
1677
1678        assert!(did, "a suspended session must transition");
1679        let after = read_runtime_state(&session);
1680        assert!(
1681            after.waiting_for_bash.is_none(),
1682            "bash wait must be cleared"
1683        );
1684        assert_eq!(after.status, AgentStatusState::Idle, "runtime must be Idle");
1685        assert!(
1686            !session.metadata.contains_key("runtime.suspend_reason"),
1687            "suspend-reason marker must be removed"
1688        );
1689        assert_eq!(session.messages.len(), 2, "resume message must be appended");
1690        assert!(matches!(
1691            session.messages.last().map(|m| &m.role),
1692            Some(Role::User)
1693        ));
1694    }
1695
1696    /// The double-resume guard: a session NOT waiting on bash is a no-op — no
1697    /// message appended, nothing mutated. This is what makes the backstop poll
1698    /// harmlessly yield once the push has already resumed (and vice versa).
1699    #[test]
1700    fn apply_bash_resume_transition_noops_when_not_waiting() {
1701        let mut session = Session::new("sess-live", "test-model");
1702        session.add_message(Message::user("hi"));
1703
1704        let resume = bash_completion_resume_message(&["sh-1".to_string()], false);
1705        let did = apply_bash_resume_transition(&mut session, &resume);
1706
1707        assert!(!did, "a non-waiting session must not transition");
1708        assert_eq!(session.messages.len(), 1, "no resume message appended");
1709    }
1710
1711    /// The resume invariant: push-resume fires ONLY when the loop is suspended on
1712    /// bash AND every waited shell has finished. A still-running sibling shell
1713    /// keeps it on the enqueue path.
1714    #[test]
1715    fn bash_completion_should_resume_only_when_suspended_and_all_done() {
1716        assert!(bash_completion_should_resume(true, true));
1717        assert!(!bash_completion_should_resume(true, false)); // other shells still running
1718        assert!(!bash_completion_should_resume(false, true)); // live loop, not suspended
1719        assert!(!bash_completion_should_resume(false, false));
1720    }
1721
1722    /// The push's resume message carries the shell's identity + status + output
1723    /// tail (so the model needs no `BashOutput` round-trip) and is tagged as a
1724    /// bash-completion resume so it satisfies the `has_pending_user_message` gate.
1725    #[test]
1726    fn bash_resume_message_from_info_carries_bashid_tail_and_kind() {
1727        let info = BashCompletionInfo {
1728            session_id: "s".into(),
1729            bash_id: "sh-42".into(),
1730            command: "cargo test".into(),
1731            exit_code: Some(0),
1732            status: "completed".into(),
1733            output_tail: "test result: ok".into(),
1734        };
1735        let msg = bash_resume_message_from_info(&info);
1736
1737        assert!(matches!(msg.role, Role::User));
1738        assert!(msg.content.contains("sh-42"), "content: {}", msg.content);
1739        assert!(
1740            msg.content.contains("cargo test"),
1741            "content: {}",
1742            msg.content
1743        );
1744        assert!(
1745            msg.content.contains("test result: ok"),
1746            "content: {}",
1747            msg.content
1748        );
1749        assert!(
1750            msg.content.contains("BashOutput"),
1751            "content: {}",
1752            msg.content
1753        );
1754        let meta = serde_json::to_string(&msg.metadata).unwrap();
1755        assert!(
1756            meta.contains(BASH_COMPLETION_RESUME_KIND),
1757            "resume message must be tagged as a bash-completion resume: {meta}"
1758        );
1759    }
1760}