Skip to main content

bamboo_engine/session_app/
child_completion_coordinator.rs

1//! Child-session completion coordinator.
2//!
3//! Receives terminal child runner notifications from `bamboo-engine`, updates
4//! durable parent wait state, and resumes the parent when the configured wait
5//! policy is satisfied.
6
7use std::collections::HashMap;
8use std::sync::{Arc, OnceLock, RwLock as StdRwLock};
9use std::time::Duration;
10
11use bamboo_domain::poison::PoisonRecover;
12
13use crate::execution::{
14    create_event_forwarder, spawn_session_execution, try_reserve_runner, AgentRunner,
15    ChildCompletion, ChildCompletionHandler, RunnerReservation, SessionExecutionArgs,
16};
17use crate::runtime::config::{BashResumeHook, GuardianSpawner, BASH_COMPLETION_RESUME_KIND};
18use crate::runtime::guardian_state::{
19    parse_guardian_verdict, read_guardian_config, read_guardian_state, write_guardian_state,
20    GuardianVerdict,
21};
22use crate::Agent;
23use async_trait::async_trait;
24use bamboo_agent_core::storage::Storage;
25use bamboo_agent_core::tools::ToolExecutor;
26use bamboo_agent_core::{
27    AgentEvent, BashCompletionInfo, BashCompletionSink, Message, Role, Session,
28};
29use bamboo_domain::session::runtime_state::{
30    AgentRuntimeState, AgentStatusState, ChildWaitPolicy, SuspensionState,
31};
32use bamboo_llm::{Config, ProviderModelRouter, ProviderRegistry};
33use bamboo_storage::LockedSessionStore;
34use chrono::Utc;
35use tokio::sync::{broadcast, RwLock};
36
37use crate::model_areas::resolve_global_area_models;
38use crate::model_config_helper::{
39    resolve_fast_model, resolve_gold_config, GOLD_CONFIG_METADATA_KEY,
40};
41use crate::session_app::provider_model::session_effective_model_ref;
42use crate::session_app::resume::{
43    resume_session_execution, ResumeExecutionPort, ResumeSpawnRequest,
44};
45use crate::session_app::types::{ResumeConfigSnapshot, ResumeOutcome};
46
47const AGENT_RUNTIME_STATE_METADATA_KEY: &str = "agent.runtime.state";
48const RUNTIME_RESUME_MESSAGE_HIDDEN_KEY: &str = "hidden_from_ui";
49const RUNTIME_RESUME_MESSAGE_KIND_KEY: &str = "runtime_kind";
50
51fn read_runtime_state(session: &Session) -> AgentRuntimeState {
52    session
53        .agent_runtime_state
54        .clone()
55        .or_else(|| {
56            session
57                .metadata
58                .get(AGENT_RUNTIME_STATE_METADATA_KEY)
59                .and_then(|raw| serde_json::from_str::<AgentRuntimeState>(raw).ok())
60        })
61        .unwrap_or_else(|| AgentRuntimeState::new(format!("{}-child-wait", session.id)))
62}
63
64fn write_runtime_state(session: &mut Session, runtime_state: &AgentRuntimeState) {
65    session.agent_runtime_state = Some(runtime_state.clone());
66    if let Ok(serialized) = serde_json::to_string(runtime_state) {
67        session
68            .metadata
69            .insert(AGENT_RUNTIME_STATE_METADATA_KEY.to_string(), serialized);
70    }
71}
72
73fn is_error_like(status: &str) -> bool {
74    matches!(status, "error" | "timeout" | "cancelled")
75}
76
77/// Terminal child run statuses, as mirrored into the session index.
78fn is_terminal_child_status(status: &str) -> bool {
79    matches!(
80        status,
81        "completed" | "error" | "timeout" | "cancelled" | "skipped"
82    )
83}
84
85/// Reconstruct the set of completed child session ids for a parent from the
86/// session index (the single source of truth), folding in the child whose
87/// completion event is being processed so a momentarily-lagging index can never
88/// stall the parent's resume.
89async fn derive_completed_child_ids(
90    storage: &Arc<dyn Storage>,
91    parent_session_id: &str,
92    just_completed_child_id: &str,
93) -> Vec<String> {
94    let mut completed: Vec<String> = storage
95        .list_child_run_statuses(parent_session_id)
96        .await
97        .unwrap_or_default()
98        .into_iter()
99        .filter(|(_, status)| status.as_deref().is_some_and(is_terminal_child_status))
100        .map(|(id, _)| id)
101        .collect();
102    if !completed.iter().any(|id| id == just_completed_child_id) {
103        completed.push(just_completed_child_id.to_string());
104    }
105    completed.sort();
106    completed.dedup();
107    completed
108}
109
110fn read_config_snapshot(config: &Arc<RwLock<Config>>, cached_config: &StdRwLock<Config>) -> Config {
111    if let Ok(config_guard) = config.try_read() {
112        let snapshot = config_guard.clone();
113
114        if let Ok(mut cached_guard) = cached_config.try_write() {
115            *cached_guard = snapshot.clone();
116        }
117
118        snapshot
119    } else {
120        cached_config
121            .try_read()
122            .map(|guard| guard.clone())
123            .unwrap_or_default()
124    }
125}
126
127/// Per-parent async locks that serialize concurrent `on_child_completed`
128/// invocations for the same parent session.
129///
130/// Race eliminated: when `wait_for=Any` and two child sessions complete
131/// simultaneously, both invocations load the parent with
132/// `waiting_for_children=Some` before either persists the cleared state, so
133/// both pass `wait_policy_satisfied`, both clear `waiting_for_children`, add a
134/// duplicate resume message, and call `resume_parent` — a double resume.
135/// Holding this per-parent `tokio::sync::Mutex` across the load-check-save
136/// critical section makes the second caller observe the already-cleared state.
137///
138/// The inner `std::sync::Mutex` guards only the brief HashMap lookup/insert
139/// (no await inside); the per-parent `tokio::sync::Mutex` is the one held
140/// across the async critical section. Entries accumulate but are small
141/// (`Arc<tokio::sync::Mutex<()>>` ≈ 24 bytes) and bounded by the number of
142/// distinct parent sessions.
143fn parent_locks() -> &'static std::sync::Mutex<HashMap<String, Arc<tokio::sync::Mutex<()>>>> {
144    static LOCKS: OnceLock<std::sync::Mutex<HashMap<String, Arc<tokio::sync::Mutex<()>>>>> =
145        OnceLock::new();
146    LOCKS.get_or_init(|| std::sync::Mutex::new(HashMap::new()))
147}
148
149/// Fetch (or create) the per-session async lock from [`parent_locks`]. Held
150/// across the load-check-clear-resume critical section so the three resume
151/// sources for one session — child completion, the loop-facing bash **push**
152/// ([`BashCompletionSink::on_bash_completed`]), and the bash **backstop** poll
153/// ([`ChildCompletionCoordinator::bash_self_resume`]) — can never double-resume.
154/// The inner sync `Mutex` guards only the brief map lookup (no await inside).
155fn session_resume_lock(session_id: &str) -> Arc<tokio::sync::Mutex<()>> {
156    let mut map = parent_locks().lock().recover_poison();
157    map.entry(session_id.to_string())
158        .or_insert_with(|| Arc::new(tokio::sync::Mutex::new(())))
159        .clone()
160}
161
162fn wait_policy_satisfied(
163    policy: ChildWaitPolicy,
164    wait_child_ids: &[String],
165    completed_child_ids: &[String],
166    latest_status: &str,
167) -> bool {
168    if wait_child_ids.is_empty() {
169        return false;
170    }
171
172    match policy {
173        ChildWaitPolicy::All => wait_child_ids
174            .iter()
175            .all(|id| completed_child_ids.iter().any(|completed| completed == id)),
176        ChildWaitPolicy::Any => completed_child_ids
177            .iter()
178            .any(|id| wait_child_ids.iter().any(|wait_id| wait_id == id)),
179        ChildWaitPolicy::FirstError => {
180            is_error_like(latest_status)
181                || wait_child_ids
182                    .iter()
183                    .all(|id| completed_child_ids.iter().any(|completed| completed == id))
184        }
185    }
186}
187
188/// Extract the child session's last assistant content, if any. Returns `None`
189/// when the child produced no assistant message (e.g. errored before the first
190/// model response, or only emitted tool messages).
191fn child_final_assistant_text(child: &Session) -> Option<String> {
192    child
193        .messages
194        .iter()
195        .rev()
196        .find(|message| matches!(message.role, Role::Assistant))
197        .map(|message| message.content.clone())
198        .filter(|content| !content.trim().is_empty())
199}
200
201fn runtime_resume_message(
202    completion: &ChildCompletion,
203    remaining_children: usize,
204    child_final_response: Option<&str>,
205) -> Message {
206    let mut body = format!(
207        "Runtime notification: child session `{}` finished with status `{}`. Remaining child sessions: {}.",
208        completion.child_session_id, completion.status, remaining_children
209    );
210
211    // Fold the child's full final response back into the parent — no
212    // truncation. Sub-agents are first-class agents whose complete conclusion
213    // should be available to the parent without an extra `SubAgent.get` round
214    // trip. The message is left compressible (see `never_compress` below) so a
215    // long transcript can still be reclaimed under parent compaction.
216    let final_response = child_final_response.map(str::to_string);
217    if let Some(response) = final_response.as_deref() {
218        body.push_str("\n\nChild final response:\n");
219        body.push_str(response);
220    } else if let Some(error) = completion.error.as_deref() {
221        if !error.is_empty() {
222            body.push_str("\n\nChild error:\n");
223            body.push_str(error);
224        }
225    }
226
227    body.push_str(
228        "\n\nResume the parent task using this child result and continue from the previous plan. \
229         If you need the full child transcript, call SubAgent.get(child_session_id).",
230    );
231
232    let mut message = Message::user(body);
233    message.metadata = Some(serde_json::json!({
234        RUNTIME_RESUME_MESSAGE_HIDDEN_KEY: true,
235        RUNTIME_RESUME_MESSAGE_KIND_KEY: "child_completion_resume",
236        "child_session_id": completion.child_session_id,
237        "child_status": completion.status,
238        "child_error": completion.error,
239        "completed_at": completion.completed_at,
240        "child_final_response_included": final_response.is_some(),
241    }));
242    // Allow parent-side compaction to reclaim this (now untruncated) message if
243    // the parent context grows — important once children nest and fold full
244    // results upward. The `SubAgent.get` hint preserves recoverability.
245    message.never_compress = false;
246    message
247}
248
249/// The hidden resume message for a completed **guardian** review: a directive,
250/// verdict-tailored note that carries the reviewer's findings straight into the
251/// parent (so it can act without a `SubAgent.get`), mirroring
252/// [`runtime_resume_message`]'s hidden/compressible shape.
253fn guardian_resume_message(completion: &ChildCompletion, verdict: &GuardianVerdict) -> Message {
254    let mut body = if verdict.approve {
255        String::from(
256            "Guardian review APPROVED: an independent reviewer verified the work and found no blocking issues. You may finalize the task.",
257        )
258    } else {
259        String::from(
260            "Guardian review REJECTED: an independent reviewer found issues. Address every finding below before completing — do NOT declare the task complete until they are resolved.",
261        )
262    };
263    if let Some(summary) = verdict.summary.as_deref().filter(|s| !s.trim().is_empty()) {
264        body.push_str("\n\nReviewer summary: ");
265        body.push_str(summary);
266    }
267    if !verdict.findings.is_empty() {
268        body.push_str("\n\nFindings:");
269        for (idx, finding) in verdict.findings.iter().enumerate() {
270            body.push_str(&format!("\n{}. {}", idx + 1, finding));
271        }
272    }
273    body.push_str(
274        "\n\nIf you need the full guardian transcript, call SubAgent.get(child_session_id).",
275    );
276
277    let mut message = Message::user(body);
278    message.metadata = Some(serde_json::json!({
279        RUNTIME_RESUME_MESSAGE_HIDDEN_KEY: true,
280        RUNTIME_RESUME_MESSAGE_KIND_KEY: "guardian_review_resume",
281        "child_session_id": completion.child_session_id,
282        "child_status": completion.status,
283        "guardian_approved": verdict.approve,
284        "completed_at": completion.completed_at,
285    }));
286    message.never_compress = false;
287    message
288}
289
290#[derive(Clone)]
291pub struct ChildCompletionCoordinator {
292    storage: Arc<dyn Storage>,
293    persistence: Arc<bamboo_storage::LockedSessionStore>,
294    sessions: crate::SessionCache,
295    agent_runners: Arc<RwLock<HashMap<String, AgentRunner>>>,
296    session_event_senders: Arc<RwLock<HashMap<String, broadcast::Sender<AgentEvent>>>>,
297    agent: Arc<Agent>,
298    config: Arc<RwLock<Config>>,
299    provider_registry: Arc<ProviderRegistry>,
300    provider_router: Arc<ProviderModelRouter>,
301    app_data_dir: std::path::PathBuf,
302    account_feed_inbox: Option<crate::execution::AccountFeedInbox>,
303    root_tools: Arc<RwLock<Option<Arc<dyn ToolExecutor>>>>,
304    /// Late-bound guardian reviewer spawner, set post-construction by the server
305    /// (mirrors `root_tools`). Re-injected into resumed runs so a guardian's
306    /// reject→fix verdict can be re-reviewed across the suspend/resume boundary.
307    guardian_spawner: Arc<RwLock<Option<Arc<dyn GuardianSpawner>>>>,
308}
309
310impl ChildCompletionCoordinator {
311    #[allow(clippy::too_many_arguments)]
312    pub fn new(
313        storage: Arc<dyn Storage>,
314        persistence: Arc<LockedSessionStore>,
315        sessions: crate::SessionCache,
316        agent_runners: Arc<RwLock<HashMap<String, AgentRunner>>>,
317        session_event_senders: Arc<RwLock<HashMap<String, broadcast::Sender<AgentEvent>>>>,
318        agent: Arc<Agent>,
319        config: Arc<RwLock<Config>>,
320        provider_registry: Arc<ProviderRegistry>,
321        provider_router: Arc<ProviderModelRouter>,
322        app_data_dir: std::path::PathBuf,
323        account_feed_inbox: Option<crate::execution::AccountFeedInbox>,
324    ) -> Self {
325        Self {
326            storage,
327            persistence,
328            sessions,
329            agent_runners,
330            session_event_senders,
331            agent,
332            config,
333            provider_registry,
334            provider_router,
335            app_data_dir,
336            account_feed_inbox,
337            root_tools: Arc::new(RwLock::new(None)),
338            guardian_spawner: Arc::new(RwLock::new(None)),
339        }
340    }
341
342    pub async fn set_root_tools(&self, tools: Arc<dyn ToolExecutor>) {
343        *self.root_tools.write().await = Some(tools);
344    }
345
346    /// Wire the guardian reviewer spawner (server-provided), so resumed runs can
347    /// re-spawn a guardian to re-review a fix after a reject verdict.
348    pub async fn set_guardian_spawner(&self, spawner: Arc<dyn GuardianSpawner>) {
349        *self.guardian_spawner.write().await = Some(spawner);
350    }
351
352    fn build_resume_config(
353        &self,
354        session: &Session,
355        config_snapshot: &Config,
356    ) -> ResumeConfigSnapshot {
357        crate::session_app::resolution::resolve_resume_config_snapshot(
358            config_snapshot,
359            &self.provider_registry,
360            session,
361            None,
362        )
363    }
364
365    /// Drive a parent-resume and return the final [`ResumeOutcome`] so callers
366    /// can distinguish a successful spawn (`Started`) from a gate-blocked
367    /// attempt (`Completed`). The bash self-resume poll task uses this to
368    /// detect the finalize-clobber case — its appended resume message was
369    /// reverted by the suspending runner's final `merge_save_runtime`, so the
370    /// resume port's `has_pending_user_message` gate fails and nothing spawns —
371    /// and retry the clear→append→resume (see [`Self::bash_self_resume`]).
372    async fn resume_parent(&self, parent_session_id: String) -> ResumeOutcome {
373        for attempt in 0..=5u8 {
374            if attempt > 0 {
375                tokio::time::sleep(Duration::from_millis(250 * attempt as u64)).await;
376            }
377
378            let Some(session) = self.load_session(&parent_session_id).await else {
379                tracing::warn!(%parent_session_id, "cannot resume parent after child completion: session not found");
380                return ResumeOutcome::NotFound;
381            };
382            let config_snapshot = self.config.read().await.clone();
383            let resume_config = self.build_resume_config(&session, &config_snapshot);
384            let outcome = resume_session_execution(self, &parent_session_id, resume_config).await;
385            tracing::info!(
386                %parent_session_id,
387                attempt,
388                outcome = outcome.as_str(),
389                "child completion requested parent resume"
390            );
391
392            if !matches!(outcome, ResumeOutcome::AlreadyRunning { .. }) {
393                return outcome;
394            }
395        }
396        // Exhausted the AlreadyRunning retry budget; surface the final state.
397        ResumeOutcome::AlreadyRunning {
398            run_id: String::new(),
399        }
400    }
401
402    async fn save_and_cache(&self, session: &mut Session) {
403        if let Err(error) = self.persistence.merge_save_runtime(session).await {
404            tracing::warn!(session_id = %session.id, %error, "failed to persist session");
405        }
406        self.sessions.insert(
407            session.id.clone(),
408            Arc::new(parking_lot::RwLock::new(session.clone())),
409        );
410    }
411}
412
413#[async_trait]
414impl ChildCompletionHandler for ChildCompletionCoordinator {
415    async fn on_child_completed(&self, completion: ChildCompletion) {
416        // Acquire the per-session async lock to eliminate the concurrent
417        // double-resume race (see `parent_locks` for the full scenario). The
418        // inner std::sync::Mutex is released immediately so no sync lock is
419        // held across the await that follows.
420        let per_parent = session_resume_lock(&completion.parent_session_id);
421        let _per_parent_guard = per_parent.lock().await;
422
423        let Some(mut parent) = self.load_session(&completion.parent_session_id).await else {
424            tracing::warn!(
425                parent_session_id = %completion.parent_session_id,
426                child_session_id = %completion.child_session_id,
427                "child completion received for missing parent"
428            );
429            return;
430        };
431
432        // A parent may itself be a child (nested sub-agents): the rest of this
433        // handler is kind-agnostic — it operates on `completion.parent_session_id`,
434        // inspects that session's own `waiting_for_children` runtime state, and
435        // resumes it. (Previously this bailed unless the parent was Root, which
436        // silently dropped grandchild completions.)
437        let mut runtime_state = read_runtime_state(&parent);
438
439        // Single source of truth: reconstruct the completed-child set from the
440        // session index rather than from a denormalized copy on the parent file.
441        let completed_child_ids = derive_completed_child_ids(
442            &self.storage,
443            &completion.parent_session_id,
444            &completion.child_session_id,
445        )
446        .await;
447
448        let mut should_resume = false;
449        let mut remaining_children = 0usize;
450        if let Some(wait) = runtime_state.waiting_for_children.clone() {
451            remaining_children = wait
452                .child_session_ids
453                .iter()
454                .filter(|id| !completed_child_ids.iter().any(|completed| completed == *id))
455                .count();
456            should_resume = wait_policy_satisfied(
457                wait.wait_for,
458                &wait.child_session_ids,
459                &completed_child_ids,
460                &completion.status,
461            );
462            if should_resume {
463                runtime_state.waiting_for_children = None;
464                runtime_state.status = AgentStatusState::Idle;
465                runtime_state.suspension = None;
466            }
467        }
468
469        if should_resume {
470            parent.metadata.remove("runtime.suspend_reason");
471            // Load the completed child once. The guardian branch inspects its
472            // subagent_type + final verdict; the generic path folds its final
473            // assistant content into the hidden resume message (avoiding an extra
474            // `SubAgent.get` round trip after resume).
475            let loaded_child = match self
476                .storage
477                .load_session(&completion.child_session_id)
478                .await
479            {
480                Ok(child) => child,
481                Err(error) => {
482                    tracing::warn!(
483                        child_session_id = %completion.child_session_id,
484                        %error,
485                        "failed to load child session for runtime resume message"
486                    );
487                    None
488                }
489            };
490
491            // Guardian branch: a completing guardian reviewer that matches the
492            // parent's recorded review advances GuardianState (phase → Reviewed)
493            // and resumes with a verdict-tailored, findings-carrying message. Any
494            // id mismatch or unparseable verdict falls through to the generic
495            // resume, so the parent is never stranded.
496            let reviewed_round = runtime_state.round.current_round;
497            let guardian_resume = loaded_child.as_ref().and_then(|child| {
498                if child.subagent_type().as_deref() != Some("guardian") {
499                    return None;
500                }
501                let mut guardian_state = read_guardian_state(&parent)?;
502                if guardian_state.guardian_child_id.as_deref()
503                    != Some(completion.child_session_id.as_str())
504                {
505                    // A *different* guardian is legitimately still in flight —
506                    // leave its Pending state intact and use the generic resume.
507                    tracing::warn!(
508                        parent_session_id = %completion.parent_session_id,
509                        child_session_id = %completion.child_session_id,
510                        expected = ?guardian_state.guardian_child_id,
511                        "guardian completion does not match recorded guardian_child_id; using generic resume"
512                    );
513                    return None;
514                }
515                // This IS the guardian we dispatched, so we MUST advance the
516                // phase out of `Pending` — otherwise the next terminal gate's
517                // `Pending => return None` would let the run complete unreviewed.
518                // A reviewer that errored or produced unparseable output is
519                // treated as a SYNTHETIC REJECT (never a silent pass), so the
520                // budgeted re-review loop governs the outcome: fail-closed, but
521                // still bounded by `max_reviews`.
522                let verdict = child_final_assistant_text(child)
523                    .and_then(|text| match parse_guardian_verdict(&text) {
524                        Ok(verdict) => Some(verdict),
525                        Err(error) => {
526                            tracing::warn!(
527                                child_session_id = %completion.child_session_id,
528                                %error,
529                                "guardian verdict unparseable; recording a synthetic reject"
530                            );
531                            None
532                        }
533                    })
534                    .unwrap_or_else(|| {
535                        GuardianVerdict::rejected(vec![
536                            "The guardian reviewer did not return a usable verdict (it errored or \
537                             emitted unparseable output); the work has NOT been independently \
538                             verified."
539                                .to_string(),
540                        ])
541                    });
542                let approved = verdict.approve;
543                let message = guardian_resume_message(&completion, &verdict);
544                guardian_state.record_verdict(verdict, reviewed_round);
545                write_guardian_state(&mut parent, guardian_state);
546                tracing::info!(
547                    parent_session_id = %completion.parent_session_id,
548                    child_session_id = %completion.child_session_id,
549                    approved,
550                    "guardian verdict recorded; resuming parent"
551                );
552                Some(message)
553            });
554
555            let resume_message = guardian_resume.unwrap_or_else(|| {
556                runtime_resume_message(
557                    &completion,
558                    remaining_children,
559                    loaded_child
560                        .as_ref()
561                        .and_then(child_final_assistant_text)
562                        .as_deref(),
563                )
564            });
565            parent.add_message(resume_message);
566        } else if runtime_state.waiting_for_children.is_some() {
567            runtime_state.status = AgentStatusState::Suspended;
568            runtime_state.suspension = Some(SuspensionState {
569                reason: "waiting_for_children".to_string(),
570                suspended_at: Utc::now(),
571                resumable: true,
572                hook_point: Some("ChildCompletion".to_string()),
573            });
574        }
575
576        parent.updated_at = Utc::now();
577        write_runtime_state(&mut parent, &runtime_state);
578        self.save_and_cache(&mut parent).await;
579
580        // Capture before releasing the per-parent lock so the borrow checker
581        // is satisfied; `resume_parent` has its own retry loop and should not
582        // hold the per-parent lock (it would block other completions for the
583        // same parent, and the state is already durably settled above).
584        let resume_parent_id = parent.id.clone();
585        drop(_per_parent_guard);
586
587        if should_resume {
588            self.resume_parent(resume_parent_id).await;
589        }
590    }
591}
592
593#[async_trait]
594impl ResumeExecutionPort for ChildCompletionCoordinator {
595    async fn load_session(&self, session_id: &str) -> Option<Session> {
596        match self.storage.load_session(session_id).await {
597            Ok(Some(session)) => Some(session),
598            Ok(None) => self
599                .sessions
600                .get(session_id)
601                .map(|e| e.value().clone())
602                .map(|arc| arc.read().clone()),
603            Err(error) => {
604                tracing::warn!(%session_id, %error, "failed to load session from storage");
605                self.sessions
606                    .get(session_id)
607                    .map(|e| e.value().clone())
608                    .map(|arc| arc.read().clone())
609            }
610        }
611    }
612
613    async fn save_and_cache_session(&self, session: &mut Session) {
614        self.save_and_cache(session).await;
615    }
616
617    async fn try_reserve_runner(
618        &self,
619        session_id: &str,
620        event_sender: &broadcast::Sender<AgentEvent>,
621    ) -> Option<RunnerReservation> {
622        try_reserve_runner(&self.agent_runners, session_id, event_sender).await
623    }
624
625    async fn get_existing_runner_run_id(&self, session_id: &str) -> Option<String> {
626        let runners = self.agent_runners.read().await;
627        runners.get(session_id).map(|r| r.run_id.clone())
628    }
629
630    async fn get_or_create_event_sender(&self, session_id: &str) -> broadcast::Sender<AgentEvent> {
631        crate::execution::session_events::get_or_create_event_sender(
632            &self.session_event_senders,
633            session_id,
634        )
635        .await
636    }
637
638    async fn spawn_resume_execution(&self, request: ResumeSpawnRequest) {
639        let ResumeSpawnRequest {
640            session_id,
641            session,
642            cancel_token,
643            run_id: _,
644            event_sender,
645            config,
646        } = request;
647
648        let Some(root_tools) = self.root_tools.read().await.clone() else {
649            tracing::error!(%session_id, "cannot resume parent after child completion: root tool surface is not initialized");
650            return;
651        };
652
653        let model = session.model.clone();
654        let resolved_provider_name = session_effective_model_ref(&session)
655            .map(|model_ref| model_ref.provider)
656            .unwrap_or(config.provider_name);
657        let provider_override = session_effective_model_ref(&session)
658            .and_then(|model_ref| match self.provider_router.route(&model_ref) {
659                Ok(provider) => Some(provider),
660                Err(error) => {
661                    tracing::warn!(
662                        session_id = %session_id,
663                        provider = %model_ref.provider,
664                        model = %model_ref.model,
665                        error = %error,
666                        "failed to resolve provider override for child-completion parent resume; falling back to runtime provider"
667                    );
668                    None
669                }
670            });
671        let config_snapshot = self.config.read().await.clone();
672        let resolved_fast_provider = resolve_fast_model(
673            &config_snapshot,
674            &resolved_provider_name,
675            &self.provider_registry,
676        )
677        .map(|model| model.provider);
678        let reasoning_effort = session.reasoning_effort;
679        let reasoning_effort_source = session
680            .metadata
681            .get("reasoning_effort_source")
682            .cloned()
683            .unwrap_or_default();
684        let gold_config = resolve_gold_config(
685            &config_snapshot,
686            session
687                .metadata
688                .get(GOLD_CONFIG_METADATA_KEY)
689                .map(String::as_str),
690        )
691        .or(config.gold_config.clone());
692
693        let (mpsc_tx, _forwarder) = create_event_forwarder(
694            session_id.clone(),
695            event_sender,
696            self.agent_runners.clone(),
697            self.account_feed_inbox.clone(),
698        );
699
700        let config_handle = self.config.clone();
701        let cached_config = Arc::new(StdRwLock::new(config_snapshot.clone()));
702        let provider_registry = self.provider_registry.clone();
703        let provider_name_for_aux = resolved_provider_name.clone();
704        let auxiliary_model_resolver = std::sync::Arc::new(move || {
705            let config_snapshot = read_config_snapshot(&config_handle, cached_config.as_ref());
706            // Auxiliary models are global (config-derived), never session-bound.
707            let areas = resolve_global_area_models(
708                &config_snapshot,
709                &provider_name_for_aux,
710                &provider_registry,
711            );
712            crate::AuxiliaryModelConfig {
713                fast_model_name: areas.fast.as_ref().map(|m| m.model_name.clone()),
714                fast_model_provider: areas.fast.map(|m| m.provider),
715                background_model_name: areas.background.as_ref().map(|m| m.model_name.clone()),
716                planning_model_name: None,
717                search_model_name: None,
718                summarization_model_name: areas
719                    .summarization
720                    .as_ref()
721                    .map(|m| m.model_name.clone()),
722                background_model_provider: areas.background.map(|m| m.provider),
723                summarization_model_provider: areas.summarization.map(|m| m.provider),
724            }
725        });
726        let model_roster = crate::ModelRoster {
727            model: Some(model),
728            provider_name: Some(resolved_provider_name),
729            provider_type: config.provider_type.clone(),
730            fast: crate::RoleModel::from_parts(config.fast_model, resolved_fast_provider),
731            background: crate::RoleModel::from_parts(
732                config.background_model,
733                config.background_model_provider,
734            ),
735            summarization: crate::RoleModel::from_parts(
736                config.summarization_model,
737                config.summarization_model_provider,
738            ),
739        };
740
741        // Re-inject guardian state on resume so a reject→fix verdict can be
742        // re-reviewed: config from the session (persisted at first spawn),
743        // spawner from the coordinator-held handle. Absent guardian config this
744        // stays `None`, and the approve→complete path is unchanged.
745        let guardian_config = read_guardian_config(&session);
746        let guardian_spawner = self.guardian_spawner.read().await.clone();
747
748        spawn_session_execution(SessionExecutionArgs {
749            agent: self.agent.clone(),
750            session_id,
751            session,
752            tools_override: Some(root_tools),
753            provider_override,
754            model_roster,
755            reasoning_effort,
756            reasoning_effort_source,
757            auxiliary_model_resolver: Some(auxiliary_model_resolver),
758            // Resumed child runs keep the spawn-time disabled snapshot (#136 lives
759            // on the long-running main agent path; children are short-lived).
760            disabled_filter_resolver: None,
761            disabled_tools: Some(config.disabled_tools),
762            disabled_skill_ids: Some(config.disabled_skill_ids),
763            selected_skill_ids: None,
764            selected_skill_mode: None,
765            cancel_token,
766            mpsc_tx,
767            image_fallback: config.image_fallback,
768            gold_config,
769            guardian_config,
770            guardian_spawner,
771            bash_resume_hook: {
772                let hook: Arc<dyn BashResumeHook> = Arc::new(self.clone());
773                Some(hook)
774            },
775            bash_completion_sink: {
776                // Resumed runs keep the push wired too, so a background shell
777                // launched after resume still notifies the loop.
778                let sink: Arc<dyn BashCompletionSink> = Arc::new(self.clone());
779                Some(sink)
780            },
781            app_data_dir: Some(self.app_data_dir.clone()),
782            runners: self.agent_runners.clone(),
783            sessions_cache: self.sessions.clone(),
784            on_complete: None,
785        });
786    }
787}
788
789/// Hidden resume message for a bash-completion self-resume (issue #84 Phase 2b).
790/// Mirrors [`runtime_resume_message`]'s hidden/compressible shape so the resume
791/// port's `has_pending_user_message` gate is satisfied.
792///
793/// `timed_out` selects the wording: the normal path (all shells finished)
794/// announces completion; the deadline path (the 6h+10m wait ceiling was hit with
795/// shells STILL running) must NOT claim the shells completed — it says they may
796/// still be running so the model verifies with BashOutput instead of assuming
797/// success on a false premise.
798fn bash_completion_resume_message(bash_ids: &[String], timed_out: bool) -> Message {
799    let body = if timed_out {
800        format!(
801            "Runtime notification: the background-Bash wait ceiling was reached while one or more \
802             shell(s) ({}) may still be running. The session is being resumed so it is not \
803             stranded; verify their actual status with BashOutput before assuming completion.",
804            bash_ids.join(", ")
805        )
806    } else {
807        format!(
808            "Runtime notification: all background Bash shell(s) ({}) have completed. \
809             Review their output with BashOutput and resume the task from where you left off.",
810            bash_ids.join(", ")
811        )
812    };
813    let mut message = Message::user(body);
814    message.metadata = Some(serde_json::json!({
815        RUNTIME_RESUME_MESSAGE_HIDDEN_KEY: true,
816        RUNTIME_RESUME_MESSAGE_KIND_KEY: BASH_COMPLETION_RESUME_KIND,
817    }));
818    message.never_compress = false;
819    message
820}
821
822/// Decide whether the bash self-resume should retry its clear→append→resume
823/// sequence after a resume attempt returned `outcome`, given that the persisted
824/// bash wait is (`true`) / is not (`false`) still set on reload.
825///
826/// Retry **only** when the resume did NOT spawn (`Completed` — no pending user
827/// message, i.e. our resume message was dropped — or `AlreadyRunning`) AND the
828/// persisted bash wait is still set: the signature of the finalize-clobber, where
829/// the suspending runner's one-shot final `merge_save_runtime` lands after our
830/// save and reverts `waiting_for_bash=Some` while dropping our resume message, so
831/// `has_pending_user_message` fails and nothing spawns. `Started` (resume fired)
832/// and `NotFound` (session gone) never retry. Pure helper so the clobber
833/// detection is unit-testable in isolation from async I/O.
834fn bash_resume_should_retry(outcome: &ResumeOutcome, persisted_waiting_for_bash: bool) -> bool {
835    match outcome {
836        ResumeOutcome::Started { .. } | ResumeOutcome::NotFound => false,
837        ResumeOutcome::Completed | ResumeOutcome::AlreadyRunning { .. } => {
838            persisted_waiting_for_bash
839        }
840    }
841}
842
843/// Whether a background-shell completion push should **resume** the owning loop
844/// (vs merely enqueue an injection). Resume only when the loop is actually
845/// suspended on a bash wait AND every shell it was waiting on has now finished —
846/// resuming while other waited shells are still running would drop them back into
847/// a foreground turn prematurely. The last shell to finish (or the backstop)
848/// drives the resume; earlier ones enqueue their notice. Pure so the invariant is
849/// unit-testable in isolation.
850fn bash_completion_should_resume(
851    loop_suspended_on_bash: bool,
852    all_waited_shells_done: bool,
853) -> bool {
854    loop_suspended_on_bash && all_waited_shells_done
855}
856
857/// Apply the bash-resume state transition to a loaded session **in place**: clear
858/// the `waiting_for_bash` wait, mark the runtime Idle, drop the suspension +
859/// `runtime.suspend_reason`, and append `resume_message`. Returns `false` (a
860/// no-op) when the session was not actually waiting on bash — the double-resume
861/// guard shared by the push and the backstop. Pure (no I/O) so both
862/// [`ChildCompletionCoordinator::perform_bash_resume`] and unit tests exercise the
863/// exact same transition.
864fn apply_bash_resume_transition(session: &mut Session, resume_message: &Message) -> bool {
865    let mut runtime_state = read_runtime_state(session);
866    if runtime_state.waiting_for_bash.is_none() {
867        return false;
868    }
869    runtime_state.waiting_for_bash = None;
870    runtime_state.status = AgentStatusState::Idle;
871    runtime_state.suspension = None;
872    write_runtime_state(session, &runtime_state);
873    session.metadata.remove("runtime.suspend_reason");
874    session.add_message(resume_message.clone());
875    true
876}
877
878/// Bash self-resume support (issue #84 Phase 2b; push follow-up).
879impl ChildCompletionCoordinator {
880    /// **Backstop** for a session suspended on `waiting_for_bash`. The primary,
881    /// event-driven wake is the loop-facing push
882    /// ([`BashCompletionSink::on_bash_completed`] → [`Self::deliver_bash_completion`]):
883    /// the shell's completion task fires it the instant the process exits, and it
884    /// resumes the loop directly. This task exists ONLY to catch a **lost push** —
885    /// the completion landing in the window before the suspend was persisted (so
886    /// the push saw no `waiting_for_bash` and only queued an injection), or a
887    /// configuration with no sink wired — and to honour the wait ceiling.
888    ///
889    /// So it is deliberately NOT a hot spin: a coarse backoff (1 s → 30 s) that
890    /// **yields to the push**. In the happy path the push has already cleared
891    /// `waiting_for_bash` before the first check fires, so this returns after one
892    /// cheap load with no registry polling at all. It only performs a resume when
893    /// the shell(s) have finished but the loop is somehow still suspended, or the
894    /// 6 h wait ceiling is reached.
895    async fn bash_self_resume(&self, session_id: String, bash_ids: Vec<String>) {
896        let mut delay = Duration::from_secs(1);
897        let max_delay = Duration::from_secs(30);
898        // Hard ceiling: the wait lease (6 h) + the registry GC TTL (5 min) +
899        // margin. After this the shells are gone from the registry regardless,
900        // so force-resume to avoid stranding the session on a GC edge case.
901        let max_poll = Duration::from_secs(6 * 3600 + 600);
902        let deadline = tokio::time::Instant::now() + max_poll;
903
904        loop {
905            tokio::time::sleep(delay).await;
906
907            let Some(session) = self.load_session(&session_id).await else {
908                tracing::info!(%session_id, "bash self-resume backstop: session gone; nothing to do");
909                return;
910            };
911            if read_runtime_state(&session).waiting_for_bash.is_none() {
912                // The push (or another path) already resumed. This is the common
913                // case — the backstop yields silently after a single load.
914                return;
915            }
916
917            let still_running =
918                bamboo_tools::tools::bash_runtime::running_shells_for_session(&session_id);
919            let timed_out = tokio::time::Instant::now() >= deadline;
920            if still_running.is_empty() || timed_out {
921                // The shell(s) finished but the loop is still suspended → the push
922                // was lost (pre-persist window / no sink), or the ceiling hit.
923                // Resume under the shared per-session lock so we never race the
924                // push or a concurrent child-completion resume.
925                let guard = session_resume_lock(&session_id);
926                let _held = guard.lock().await;
927                tracing::warn!(
928                    %session_id,
929                    shell_count = bash_ids.len(),
930                    timed_out,
931                    "bash self-resume backstop engaged (push lost or wait ceiling reached)"
932                );
933                self.perform_bash_resume(
934                    &session_id,
935                    bash_completion_resume_message(&bash_ids, timed_out),
936                )
937                .await;
938                return;
939            }
940
941            delay = (delay * 2).min(max_delay);
942        }
943    }
944
945    /// Clear a session's `waiting_for_bash` state, append `resume_message`, and
946    /// drive the parent resume — the shared clear→append→resume used by BOTH the
947    /// event-driven push ([`Self::deliver_bash_completion`]) and the backstop poll
948    /// ([`Self::bash_self_resume`]).
949    ///
950    /// **The caller MUST hold the [`session_resume_lock`] for `session_id`** so the
951    /// load-check-clear-resume critical section is serialized against every other
952    /// resume source (no double resume). No-op when the persisted wait was already
953    /// cleared (another source handled it first).
954    ///
955    /// The clear→append→resume is a **bounded retry loop** that closes the
956    /// finalize-clobber strand. The suspending runner's `finalize_task_context`
957    /// runs a full `save_runtime_session` (same `merge_save_runtime`, which
958    /// overwrites the whole `messages` array) that can land AFTER our save,
959    /// reverting `waiting_for_bash=Some` and dropping our resume message, so
960    /// `has_pending_user_message` fails and `resume_parent` returns `Completed`
961    /// without spawning. We detect that (persisted wait still set after a
962    /// non-`Started` outcome) and re-clear/re-append/re-resume. It converges
963    /// because the runner's finalize persist is one-shot: once landed, our retry's
964    /// save is the last writer, the message sticks, and resume fires.
965    async fn perform_bash_resume(&self, session_id: &str, resume_message: Message) {
966        let retry_backoff = Duration::from_millis(200);
967        const MAX_RESUME_ATTEMPTS: u8 = 5;
968        for attempt in 0..MAX_RESUME_ATTEMPTS {
969            if attempt > 0 {
970                tokio::time::sleep(retry_backoff).await;
971            }
972
973            let Some(mut session) = self.load_session(session_id).await else {
974                tracing::warn!(%session_id, "bash resume: session not found; nothing to resume");
975                return;
976            };
977
978            if !apply_bash_resume_transition(&mut session, &resume_message) {
979                // Double-resume guard: the wait was already cleared by another
980                // source (the push, the backstop, or a user-driven resume). Do
981                // not append a duplicate message or request a redundant resume.
982                tracing::info!(
983                    %session_id, attempt,
984                    "bash resume: persisted bash wait already cleared; nothing to resume"
985                );
986                return;
987            }
988            session.updated_at = Utc::now();
989            self.save_and_cache(&mut session).await;
990            tracing::info!(
991                %session_id, attempt,
992                "bash resume: cleared bash wait and appended resume message"
993            );
994
995            let outcome = self.resume_parent(session_id.to_string()).await;
996            match outcome {
997                ResumeOutcome::Started { .. } => {
998                    tracing::info!(%session_id, attempt, "bash resume: resume fired");
999                    return;
1000                }
1001                ResumeOutcome::NotFound => {
1002                    tracing::warn!(%session_id, "bash resume: session vanished during resume");
1003                    return;
1004                }
1005                _ => {
1006                    // Completed (no pending user message ⇒ our resume message was
1007                    // dropped by the runner's finalize persist) or AlreadyRunning.
1008                    // Decide via the persisted bash wait: still set ⇒
1009                    // finalize-clobber ⇒ retry; cleared ⇒ the session is being
1010                    // handled (by us or a concurrent resume) ⇒ stop.
1011                    let clobbered = match self.load_session(session_id).await {
1012                        Some(reloaded) => read_runtime_state(&reloaded).waiting_for_bash.is_some(),
1013                        None => {
1014                            tracing::warn!(%session_id, "bash resume: session vanished after resume");
1015                            return;
1016                        }
1017                    };
1018                    if bash_resume_should_retry(&outcome, clobbered) {
1019                        tracing::warn!(
1020                            %session_id, attempt,
1021                            outcome = outcome.as_str(),
1022                            "bash resume: persisted wait still set after resume (finalize-clobber); retrying"
1023                        );
1024                        continue;
1025                    }
1026                    tracing::info!(
1027                        %session_id, attempt,
1028                        outcome = outcome.as_str(),
1029                        "bash resume: wait cleared and resume handled; stopping"
1030                    );
1031                    return;
1032                }
1033            }
1034        }
1035
1036        tracing::warn!(
1037            %session_id,
1038            attempts = MAX_RESUME_ATTEMPTS,
1039            "bash resume: exhausted clobber-retry budget without confirming resume; giving up"
1040        );
1041    }
1042}
1043
1044impl BashResumeHook for ChildCompletionCoordinator {
1045    fn arrange_bash_self_resume(&self, session_id: String, bash_ids: Vec<String>) {
1046        let coordinator = Arc::new(self.clone());
1047        tokio::spawn(async move {
1048            coordinator.bash_self_resume(session_id, bash_ids).await;
1049        });
1050    }
1051}
1052
1053/// Build the injected user-message body for a completed background shell — a
1054/// concise notice plus a bounded output tail — so the model can act on the
1055/// result without a mandatory `BashOutput` round-trip (issue #84 Phase 2b
1056/// follow-up).
1057fn bash_completion_injection_body(info: &BashCompletionInfo) -> String {
1058    let exit = match info.exit_code {
1059        Some(code) => code.to_string(),
1060        None => "none (signal/killed)".to_string(),
1061    };
1062    let mut body = format!(
1063        "Runtime notification: background shell `{}` (`{}`) finished — status {}, exit code {}.",
1064        info.bash_id, info.command, info.status, exit
1065    );
1066    if info.output_tail.trim().is_empty() {
1067        body.push_str(" It produced no captured output.");
1068    } else {
1069        body.push_str("\n\nOutput tail:\n");
1070        body.push_str(&info.output_tail);
1071    }
1072    body.push_str(&format!(
1073        "\n\nUse BashOutput with bash_id=\"{}\" for the full output, then continue the task.",
1074        info.bash_id
1075    ));
1076    body
1077}
1078
1079/// Loop-facing background-Bash completion delivery (issue #84 Phase 2b
1080/// follow-up). Pushes a completed shell's result into its owning session's loop,
1081/// mirroring how a sub-agent completion reaches its parent — but via the
1082/// running-loop channel, which children never exercise (a parent waiting on
1083/// children is always suspended when one completes; bash is the first completion
1084/// source that can land on a *live, iterating* loop).
1085///
1086/// The push enqueues onto `pending_injected_messages`, the same round-boundary
1087/// steering channel `send_message` uses. That covers every reachable loop state:
1088/// an actively-looping session drains it at its next round
1089/// (`merge_pending_injected_messages`); a session suspended on `waiting_for_bash`
1090/// drains it when the durable end-of-turn poll backstop (`bash_resume_hook`)
1091/// resumes it at round 0. A wired-sink session is never idle-with-a-running-shell
1092/// (ending a turn with one suspends), so no separate idle-wake path is needed —
1093/// keeping the push a pure latency optimization that never races the backstop.
1094/// Enqueue a completed shell's summary as a pending injected message on the
1095/// owning session. Race-safe: `update_runtime_config` loads the freshest session
1096/// under the per-session lock and re-saves, so it can never revert a message the
1097/// live loop appended concurrently — unlike `merge_save_runtime` (which writes
1098/// the caller's whole `messages` snapshot verbatim). Free fn so it is unit-
1099/// testable without constructing a full coordinator. Returns the saved session,
1100/// or `None` if the owning session no longer exists.
1101async fn enqueue_bash_completion_injection(
1102    persistence: &LockedSessionStore,
1103    info: &BashCompletionInfo,
1104) -> std::io::Result<Option<Session>> {
1105    let body = bash_completion_injection_body(info);
1106    let queued = serde_json::json!({
1107        "content": body,
1108        "created_at": Utc::now(),
1109    });
1110    persistence
1111        .update_runtime_config(&info.session_id, move |session| {
1112            let mut pending = session.pending_injected_messages().unwrap_or_default();
1113            pending.push(queued);
1114            session.set_pending_injected_messages(pending);
1115        })
1116        .await
1117}
1118
1119/// Build the hidden, compressible resume message for a completed background
1120/// shell — the same rich notice body used for a live-loop injection
1121/// ([`bash_completion_injection_body`]), but tagged as a resume message so it
1122/// satisfies the `has_pending_user_message` gate that lets a suspended session
1123/// spawn. This is what the **push** appends when it wakes a suspended loop, so
1124/// the model gets the shell's status + output tail in one shot without a
1125/// separate `BashOutput` round-trip.
1126fn bash_resume_message_from_info(info: &BashCompletionInfo) -> Message {
1127    let mut message = Message::user(bash_completion_injection_body(info));
1128    message.metadata = Some(serde_json::json!({
1129        RUNTIME_RESUME_MESSAGE_HIDDEN_KEY: true,
1130        RUNTIME_RESUME_MESSAGE_KIND_KEY: BASH_COMPLETION_RESUME_KIND,
1131    }));
1132    message.never_compress = false;
1133    message
1134}
1135
1136impl ChildCompletionCoordinator {
1137    /// Loop-facing delivery of a completed background shell. Two paths, chosen
1138    /// under the per-session resume lock so we never race the backstop poll or a
1139    /// concurrent child-completion resume:
1140    ///
1141    /// - **Suspended loop** (the model ended its turn with the shell running, so
1142    ///   `waiting_for_bash` is set) AND every waited shell has now finished →
1143    ///   **resume the loop directly**, event-driven, appending the rich completion
1144    ///   notice as the resume message. This is the push's whole point: no polling.
1145    /// - Otherwise (a live/iterating loop, or a suspend still waiting on OTHER
1146    ///   shells) → **enqueue** the notice as a pending injected message, drained at
1147    ///   the next round boundary (a live loop) or folded into the eventual resume
1148    ///   when the last shell finishes.
1149    async fn deliver_bash_completion(&self, info: BashCompletionInfo) {
1150        let guard = session_resume_lock(&info.session_id);
1151        let _held = guard.lock().await;
1152
1153        let Some(session) = self.load_session(&info.session_id).await else {
1154            tracing::warn!(
1155                session_id = %info.session_id,
1156                bash_id = %info.bash_id,
1157                "background bash completion: owning session not found; nothing to notify"
1158            );
1159            return;
1160        };
1161
1162        let waiting = read_runtime_state(&session).waiting_for_bash.is_some();
1163        // The producer flips the shell's `running` flag false BEFORE firing this
1164        // push, so a now-empty per-session registry means every shell the loop was
1165        // waiting on has finished — safe to resume. If OTHER waited shells are
1166        // still running, fall through to the enqueue path and let the last one (or
1167        // the backstop) drive the resume.
1168        let all_shells_done =
1169            bamboo_tools::tools::bash_runtime::running_shells_for_session(&info.session_id)
1170                .is_empty();
1171
1172        if bash_completion_should_resume(waiting, all_shells_done) {
1173            tracing::info!(
1174                session_id = %info.session_id,
1175                bash_id = %info.bash_id,
1176                status = %info.status,
1177                "background bash completion: push-resuming suspended loop (event-driven)"
1178            );
1179            self.perform_bash_resume(&info.session_id, bash_resume_message_from_info(&info))
1180                .await;
1181            return;
1182        }
1183
1184        match enqueue_bash_completion_injection(&self.persistence, &info).await {
1185            Ok(Some(_)) => tracing::info!(
1186                session_id = %info.session_id,
1187                bash_id = %info.bash_id,
1188                status = %info.status,
1189                waiting,
1190                "background bash completion queued for injection at the next round boundary"
1191            ),
1192            Ok(None) => tracing::warn!(
1193                session_id = %info.session_id,
1194                bash_id = %info.bash_id,
1195                "background bash completion: owning session not found; nothing to notify"
1196            ),
1197            Err(error) => tracing::warn!(
1198                session_id = %info.session_id,
1199                bash_id = %info.bash_id,
1200                %error,
1201                "background bash completion: failed to queue injection"
1202            ),
1203        }
1204    }
1205}
1206
1207impl BashCompletionSink for ChildCompletionCoordinator {
1208    fn on_bash_completed(&self, info: BashCompletionInfo) {
1209        // Best-effort, off the shell's completion-poll task: hand the delivery to
1210        // a detached task so the producer is never blocked (mirrors
1211        // `arrange_bash_self_resume`).
1212        let coordinator = Arc::new(self.clone());
1213        tokio::spawn(async move {
1214            coordinator.deliver_bash_completion(info).await;
1215        });
1216    }
1217}
1218
1219#[cfg(test)]
1220mod tests {
1221    use super::*;
1222    use bamboo_agent_core::Message;
1223
1224    fn make_completion(status: &str) -> ChildCompletion {
1225        ChildCompletion {
1226            parent_session_id: "parent-1".to_string(),
1227            child_session_id: "child-1".to_string(),
1228            status: status.to_string(),
1229            error: None,
1230            completed_at: Utc::now(),
1231        }
1232    }
1233
1234    // ── ② derive completed children from the index ──────────────────────
1235
1236    struct StubChildIndex {
1237        children: Vec<(String, Option<String>)>,
1238    }
1239
1240    #[async_trait]
1241    impl Storage for StubChildIndex {
1242        async fn save_session(&self, _session: &Session) -> std::io::Result<()> {
1243            Ok(())
1244        }
1245        async fn load_session(&self, _id: &str) -> std::io::Result<Option<Session>> {
1246            Ok(None)
1247        }
1248        async fn delete_session(&self, _id: &str) -> std::io::Result<bool> {
1249            Ok(false)
1250        }
1251        async fn list_child_run_statuses(
1252            &self,
1253            _parent_session_id: &str,
1254        ) -> std::io::Result<Vec<(String, Option<String>)>> {
1255            Ok(self.children.clone())
1256        }
1257    }
1258
1259    #[tokio::test]
1260    async fn derive_completed_only_includes_terminal_children() {
1261        let storage: Arc<dyn Storage> = Arc::new(StubChildIndex {
1262            children: vec![
1263                ("a".into(), Some("completed".into())),
1264                ("b".into(), Some("running".into())),
1265                ("c".into(), Some("error".into())),
1266                ("d".into(), None),
1267            ],
1268        });
1269        let completed = derive_completed_child_ids(&storage, "parent-1", "b").await;
1270        // Terminal from index: a, c. Plus the just-completed child b folded in.
1271        assert_eq!(
1272            completed,
1273            vec!["a".to_string(), "b".to_string(), "c".to_string()]
1274        );
1275    }
1276
1277    #[tokio::test]
1278    async fn derive_completed_folds_in_just_completed_when_index_lags() {
1279        // Index hasn't caught up — reports the child as still running.
1280        let storage: Arc<dyn Storage> = Arc::new(StubChildIndex {
1281            children: vec![("only".into(), Some("running".into()))],
1282        });
1283        let completed = derive_completed_child_ids(&storage, "parent-1", "only").await;
1284        assert_eq!(completed, vec!["only".to_string()]);
1285    }
1286
1287    #[test]
1288    fn wait_policy_all_uses_derived_completed_set() {
1289        let waited = vec!["a".to_string(), "b".to_string()];
1290        assert!(!wait_policy_satisfied(
1291            ChildWaitPolicy::All,
1292            &waited,
1293            &["a".to_string()],
1294            "completed"
1295        ));
1296        assert!(wait_policy_satisfied(
1297            ChildWaitPolicy::All,
1298            &waited,
1299            &["a".to_string(), "b".to_string()],
1300            "completed"
1301        ));
1302    }
1303
1304    #[test]
1305    fn child_final_assistant_text_returns_last_assistant() {
1306        let mut session = Session::new("child-1", "gpt-4");
1307        session.messages.push(Message::user("hi"));
1308        session
1309            .messages
1310            .push(Message::assistant("first answer", None));
1311        session.messages.push(Message::user("again"));
1312        session
1313            .messages
1314            .push(Message::assistant("final answer", None));
1315
1316        assert_eq!(
1317            child_final_assistant_text(&session).as_deref(),
1318            Some("final answer")
1319        );
1320    }
1321
1322    #[test]
1323    fn child_final_assistant_text_returns_none_when_blank() {
1324        let mut session = Session::new("child-1", "gpt-4");
1325        session.messages.push(Message::assistant("   ", None));
1326        assert!(child_final_assistant_text(&session).is_none());
1327    }
1328
1329    #[test]
1330    fn child_final_assistant_text_returns_none_when_no_assistant() {
1331        let mut session = Session::new("child-1", "gpt-4");
1332        session.messages.push(Message::user("hi"));
1333        assert!(child_final_assistant_text(&session).is_none());
1334    }
1335
1336    #[test]
1337    fn runtime_resume_message_folds_full_response_without_truncation() {
1338        // A very long child final response is folded in verbatim (no 4000-char
1339        // cap, no truncation marker).
1340        let completion = make_completion("completed");
1341        let long: String = "a".repeat(10_000);
1342        let message = runtime_resume_message(&completion, 0, Some(&long));
1343        assert!(message.content.contains(&long));
1344        assert!(!message.content.contains("truncated"));
1345    }
1346
1347    #[test]
1348    fn runtime_resume_message_includes_child_response_when_provided() {
1349        let completion = make_completion("completed");
1350        let message = runtime_resume_message(&completion, 0, Some("the answer is 42"));
1351
1352        assert!(matches!(message.role, Role::User));
1353        // Folded child results are now compressible so the parent context can
1354        // reclaim them under compaction.
1355        assert!(!message.never_compress);
1356        assert!(message.content.contains("Child final response:"));
1357        assert!(message.content.contains("the answer is 42"));
1358
1359        let metadata = message.metadata.expect("metadata present");
1360        assert_eq!(
1361            metadata.get("hidden_from_ui").and_then(|v| v.as_bool()),
1362            Some(true)
1363        );
1364        assert_eq!(
1365            metadata.get("runtime_kind").and_then(|v| v.as_str()),
1366            Some("child_completion_resume")
1367        );
1368        assert_eq!(
1369            metadata
1370                .get("child_final_response_included")
1371                .and_then(|v| v.as_bool()),
1372            Some(true)
1373        );
1374    }
1375
1376    #[test]
1377    fn runtime_resume_message_falls_back_to_error_when_no_response() {
1378        let mut completion = make_completion("error");
1379        completion.error = Some("boom".to_string());
1380
1381        let message = runtime_resume_message(&completion, 1, None);
1382        assert!(message.content.contains("Child error:"));
1383        assert!(message.content.contains("boom"));
1384        let metadata = message.metadata.expect("metadata present");
1385        assert_eq!(
1386            metadata
1387                .get("child_final_response_included")
1388                .and_then(|v| v.as_bool()),
1389            Some(false)
1390        );
1391    }
1392
1393    #[test]
1394    fn runtime_resume_message_minimal_when_no_response_and_no_error() {
1395        let completion = make_completion("completed");
1396        let message = runtime_resume_message(&completion, 2, None);
1397        assert!(!message.content.contains("Child final response:"));
1398        assert!(!message.content.contains("Child error:"));
1399        assert!(message.content.contains("Resume the parent task"));
1400    }
1401
1402    #[test]
1403    fn read_config_snapshot_refreshes_cached_snapshot_from_live_config() {
1404        let runtime = tokio::runtime::Runtime::new().expect("runtime");
1405
1406        runtime.block_on(async {
1407            let config = Arc::new(RwLock::new(Config::default()));
1408            config.write().await.provider = "copilot".to_string();
1409            let cached_config = StdRwLock::new(Config::default());
1410
1411            let snapshot = read_config_snapshot(&config, &cached_config);
1412
1413            assert_eq!(snapshot.provider, "copilot");
1414            assert_eq!(
1415                cached_config.read().expect("cached snapshot lock").provider,
1416                "copilot"
1417            );
1418        });
1419    }
1420
1421    #[test]
1422    fn read_config_snapshot_uses_cached_snapshot_when_live_lock_is_busy() {
1423        let runtime = tokio::runtime::Runtime::new().expect("runtime");
1424
1425        runtime.block_on(async {
1426            let cached_snapshot = Config {
1427                provider: "cached-provider".to_string(),
1428                ..Default::default()
1429            };
1430
1431            let config = Arc::new(RwLock::new(Config::default()));
1432            let cached_config = StdRwLock::new(cached_snapshot);
1433            let _write_guard = config.write().await;
1434
1435            let snapshot = read_config_snapshot(&config, &cached_config);
1436
1437            assert_eq!(snapshot.provider, "cached-provider");
1438        });
1439    }
1440
1441    // ── Bash self-resume (issue #84 Phase 2b): deadline message + clobber-retry ──
1442
1443    #[test]
1444    fn bash_completion_resume_message_normal_announces_completion() {
1445        let ids = vec!["bg-1".to_string(), "bg-2".to_string()];
1446        let message = bash_completion_resume_message(&ids, false);
1447        // Normal path: the shells genuinely finished.
1448        assert!(
1449            message.content.contains("have completed"),
1450            "normal resume message must announce completion: {}",
1451            message.content
1452        );
1453        // Hidden + compressible so the resume gate sees it but the UI hides it.
1454        let metadata = message.metadata.expect("metadata present");
1455        assert_eq!(
1456            metadata
1457                .get(RUNTIME_RESUME_MESSAGE_HIDDEN_KEY)
1458                .and_then(|v| v.as_bool()),
1459            Some(true),
1460            "resume message must be hidden from the UI"
1461        );
1462        assert_eq!(
1463            metadata
1464                .get(RUNTIME_RESUME_MESSAGE_KIND_KEY)
1465                .and_then(|v| v.as_str()),
1466            Some(BASH_COMPLETION_RESUME_KIND),
1467            "resume message must carry the bash-completion kind discriminant"
1468        );
1469    }
1470
1471    #[test]
1472    fn bash_completion_resume_message_deadline_does_not_claim_completion() {
1473        // The 6h+10m deadline force-breaks with shells STILL running. The message
1474        // must NOT say "have completed" — that would let the model assume success
1475        // on a false premise. It must direct the model to verify with BashOutput.
1476        let ids = vec!["bg-long".to_string()];
1477        let message = bash_completion_resume_message(&ids, true);
1478        assert!(
1479            !message.content.contains("have completed"),
1480            "deadline resume message must NOT claim the shells completed: {}",
1481            message.content
1482        );
1483        assert!(
1484            message.content.contains("may still be running"),
1485            "deadline resume message must warn shells may still be running: {}",
1486            message.content
1487        );
1488        assert!(
1489            message.content.contains("BashOutput"),
1490            "deadline resume message must direct verification via BashOutput: {}",
1491            message.content
1492        );
1493        // Same hidden/kind shape so the resume gate is satisfied identically.
1494        let metadata = message.metadata.expect("metadata present");
1495        assert_eq!(
1496            metadata
1497                .get(RUNTIME_RESUME_MESSAGE_KIND_KEY)
1498                .and_then(|v| v.as_str()),
1499            Some(BASH_COMPLETION_RESUME_KIND)
1500        );
1501    }
1502
1503    #[test]
1504    fn bash_resume_should_retry_matrix() {
1505        // The finalize-clobber retry predicate (issue #84 Phase 2b). Retry only
1506        // when the resume did NOT spawn (Completed / AlreadyRunning) AND the
1507        // persisted bash wait is still set on reload — the clobber signature.
1508
1509        // Started: the resume fired — never retry, regardless of persisted state.
1510        assert!(!bash_resume_should_retry(
1511            &ResumeOutcome::Started { run_id: "r".into() },
1512            true
1513        ));
1514        assert!(!bash_resume_should_retry(
1515            &ResumeOutcome::Started { run_id: "r".into() },
1516            false
1517        ));
1518
1519        // NotFound: session gone — never retry.
1520        assert!(!bash_resume_should_retry(&ResumeOutcome::NotFound, true));
1521        assert!(!bash_resume_should_retry(&ResumeOutcome::NotFound, false));
1522
1523        // Completed + persisted wait still set ⇒ finalize-clobber ⇒ retry.
1524        assert!(bash_resume_should_retry(&ResumeOutcome::Completed, true));
1525        // Completed + persisted wait cleared ⇒ handled (our message stuck, or a
1526        // concurrent resume finished) ⇒ stop.
1527        assert!(!bash_resume_should_retry(&ResumeOutcome::Completed, false));
1528
1529        // AlreadyRunning + persisted wait still set ⇒ clobbered while a runner is
1530        // (stale-)active ⇒ retry to re-establish the resume message.
1531        assert!(bash_resume_should_retry(
1532            &ResumeOutcome::AlreadyRunning { run_id: "r".into() },
1533            true
1534        ));
1535        // AlreadyRunning + wait cleared ⇒ a runner owns the session ⇒ stop.
1536        assert!(!bash_resume_should_retry(
1537            &ResumeOutcome::AlreadyRunning { run_id: "r".into() },
1538            false
1539        ));
1540    }
1541
1542    // ── bash completion injection body (Phase 2b follow-up) ──────────────
1543
1544    #[test]
1545    fn injection_body_includes_status_exit_command_and_tail() {
1546        let info = BashCompletionInfo {
1547            session_id: "s".into(),
1548            bash_id: "abc123".into(),
1549            command: "make build".into(),
1550            exit_code: Some(0),
1551            status: "completed".into(),
1552            output_tail: "BUILD OK".into(),
1553        };
1554        let body = bash_completion_injection_body(&info);
1555        assert!(body.contains("abc123"), "body: {body}");
1556        assert!(body.contains("make build"), "body: {body}");
1557        assert!(body.contains("completed"), "body: {body}");
1558        assert!(body.contains("exit code 0"), "body: {body}");
1559        assert!(body.contains("BUILD OK"), "body: {body}");
1560        // The model is pointed at BashOutput for the full log.
1561        assert!(body.contains("BashOutput"), "body: {body}");
1562        assert!(body.contains("bash_id=\"abc123\""), "body: {body}");
1563    }
1564
1565    #[test]
1566    fn injection_body_handles_no_output_and_signal_kill() {
1567        let info = BashCompletionInfo {
1568            session_id: "s".into(),
1569            bash_id: "xyz".into(),
1570            command: "sleep 99".into(),
1571            exit_code: None,
1572            status: "killed".into(),
1573            output_tail: String::new(),
1574        };
1575        let body = bash_completion_injection_body(&info);
1576        assert!(body.contains("killed"), "body: {body}");
1577        assert!(body.contains("none (signal/killed)"), "body: {body}");
1578        assert!(body.contains("no captured output"), "body: {body}");
1579        // No output tail section when there is nothing to show.
1580        assert!(!body.contains("Output tail:"), "body: {body}");
1581    }
1582
1583    async fn temp_store() -> (tempfile::TempDir, Arc<dyn Storage>, LockedSessionStore) {
1584        let temp = tempfile::tempdir().unwrap();
1585        let storage: Arc<dyn Storage> = Arc::new(
1586            bamboo_storage::v2::SessionStoreV2::new(temp.path().to_path_buf())
1587                .await
1588                .expect("storage init"),
1589        );
1590        let persistence = LockedSessionStore::new(storage.clone());
1591        (temp, storage, persistence)
1592    }
1593
1594    #[tokio::test]
1595    async fn enqueue_writes_pending_injection_and_preserves_messages() {
1596        let (_temp, storage, persistence) = temp_store().await;
1597
1598        let mut session = Session::new("sess-enq", "test-model");
1599        session.add_message(Message::user("do the build"));
1600        storage.save_session(&session).await.unwrap();
1601
1602        let info = BashCompletionInfo {
1603            session_id: "sess-enq".into(),
1604            bash_id: "sh-1".into(),
1605            command: "make".into(),
1606            exit_code: Some(0),
1607            status: "completed".into(),
1608            output_tail: "done".into(),
1609        };
1610        let saved = enqueue_bash_completion_injection(&persistence, &info)
1611            .await
1612            .expect("enqueue io ok")
1613            .expect("session exists");
1614
1615        let pending = saved
1616            .pending_injected_messages()
1617            .expect("pending injection present");
1618        assert_eq!(pending.len(), 1);
1619        let content = pending[0].get("content").and_then(|v| v.as_str()).unwrap();
1620        assert!(content.contains("sh-1"), "content: {content}");
1621        assert!(content.contains("make"), "content: {content}");
1622        assert!(content.contains("done"), "content: {content}");
1623        // The pre-existing conversation is untouched (no clobber).
1624        assert_eq!(saved.messages.len(), 1);
1625    }
1626
1627    #[tokio::test]
1628    async fn enqueue_returns_none_for_missing_session() {
1629        let (_temp, _storage, persistence) = temp_store().await;
1630        let info = BashCompletionInfo {
1631            session_id: "does-not-exist".into(),
1632            bash_id: "x".into(),
1633            command: "true".into(),
1634            exit_code: Some(0),
1635            status: "completed".into(),
1636            output_tail: String::new(),
1637        };
1638        let result = enqueue_bash_completion_injection(&persistence, &info)
1639            .await
1640            .expect("io ok");
1641        assert!(result.is_none(), "no session → nothing enqueued");
1642    }
1643
1644    // ── push-driven resume: the state transition + decision the push applies ──
1645
1646    /// A session suspended on `waiting_for_bash`, given the rich completion
1647    /// message, is transitioned to a resumable state: the wait is cleared, the
1648    /// runtime is Idle, the suspend-reason marker is gone, and the resume message
1649    /// is appended. This is exactly what the PUSH does to wake the loop
1650    /// event-driven (vs the old backstop poll).
1651    #[test]
1652    fn apply_bash_resume_transition_clears_wait_and_appends_message() {
1653        use bamboo_domain::session::runtime_state::WaitingForBashState;
1654
1655        let mut session = Session::new("sess-resume", "test-model");
1656        session.add_message(Message::user("kick off the build"));
1657        let mut rt = read_runtime_state(&session);
1658        rt.status = AgentStatusState::Running;
1659        rt.waiting_for_bash = Some(WaitingForBashState::for_bash(
1660            vec!["sh-1".into()],
1661            Utc::now(),
1662        ));
1663        write_runtime_state(&mut session, &rt);
1664        session.metadata.insert(
1665            "runtime.suspend_reason".to_string(),
1666            "waiting_for_bash".to_string(),
1667        );
1668
1669        let resume = bash_completion_resume_message(&["sh-1".to_string()], false);
1670        let did = apply_bash_resume_transition(&mut session, &resume);
1671
1672        assert!(did, "a suspended session must transition");
1673        let after = read_runtime_state(&session);
1674        assert!(
1675            after.waiting_for_bash.is_none(),
1676            "bash wait must be cleared"
1677        );
1678        assert_eq!(after.status, AgentStatusState::Idle, "runtime must be Idle");
1679        assert!(
1680            !session.metadata.contains_key("runtime.suspend_reason"),
1681            "suspend-reason marker must be removed"
1682        );
1683        assert_eq!(session.messages.len(), 2, "resume message must be appended");
1684        assert!(matches!(
1685            session.messages.last().map(|m| &m.role),
1686            Some(Role::User)
1687        ));
1688    }
1689
1690    /// The double-resume guard: a session NOT waiting on bash is a no-op — no
1691    /// message appended, nothing mutated. This is what makes the backstop poll
1692    /// harmlessly yield once the push has already resumed (and vice versa).
1693    #[test]
1694    fn apply_bash_resume_transition_noops_when_not_waiting() {
1695        let mut session = Session::new("sess-live", "test-model");
1696        session.add_message(Message::user("hi"));
1697
1698        let resume = bash_completion_resume_message(&["sh-1".to_string()], false);
1699        let did = apply_bash_resume_transition(&mut session, &resume);
1700
1701        assert!(!did, "a non-waiting session must not transition");
1702        assert_eq!(session.messages.len(), 1, "no resume message appended");
1703    }
1704
1705    /// The resume invariant: push-resume fires ONLY when the loop is suspended on
1706    /// bash AND every waited shell has finished. A still-running sibling shell
1707    /// keeps it on the enqueue path.
1708    #[test]
1709    fn bash_completion_should_resume_only_when_suspended_and_all_done() {
1710        assert!(bash_completion_should_resume(true, true));
1711        assert!(!bash_completion_should_resume(true, false)); // other shells still running
1712        assert!(!bash_completion_should_resume(false, true)); // live loop, not suspended
1713        assert!(!bash_completion_should_resume(false, false));
1714    }
1715
1716    /// The push's resume message carries the shell's identity + status + output
1717    /// tail (so the model needs no `BashOutput` round-trip) and is tagged as a
1718    /// bash-completion resume so it satisfies the `has_pending_user_message` gate.
1719    #[test]
1720    fn bash_resume_message_from_info_carries_bashid_tail_and_kind() {
1721        let info = BashCompletionInfo {
1722            session_id: "s".into(),
1723            bash_id: "sh-42".into(),
1724            command: "cargo test".into(),
1725            exit_code: Some(0),
1726            status: "completed".into(),
1727            output_tail: "test result: ok".into(),
1728        };
1729        let msg = bash_resume_message_from_info(&info);
1730
1731        assert!(matches!(msg.role, Role::User));
1732        assert!(msg.content.contains("sh-42"), "content: {}", msg.content);
1733        assert!(
1734            msg.content.contains("cargo test"),
1735            "content: {}",
1736            msg.content
1737        );
1738        assert!(
1739            msg.content.contains("test result: ok"),
1740            "content: {}",
1741            msg.content
1742        );
1743        assert!(
1744            msg.content.contains("BashOutput"),
1745            "content: {}",
1746            msg.content
1747        );
1748        let meta = serde_json::to_string(&msg.metadata).unwrap();
1749        assert!(
1750            meta.contains(BASH_COMPLETION_RESUME_KIND),
1751            "resume message must be tagged as a bash-completion resume: {meta}"
1752        );
1753    }
1754}