Skip to main content

bamboo_engine/session_app/
child_completion_coordinator.rs

1//! Child-session completion coordinator.
2//!
3//! Receives terminal child runner notifications from `bamboo-engine`, updates
4//! durable parent wait state, and resumes the parent when the configured wait
5//! policy is satisfied.
6
7use std::collections::HashMap;
8use std::sync::{Arc, OnceLock, RwLock as StdRwLock};
9use std::time::Duration;
10
11use bamboo_domain::poison::PoisonRecover;
12
13use crate::execution::{
14    create_event_forwarder, spawn_session_execution, try_reserve_runner, AgentRunner,
15    ChildCompletion, ChildCompletionHandler, RunnerReservation, SessionExecutionArgs,
16};
17use crate::runtime::config::{BashResumeHook, GuardianSpawner, BASH_COMPLETION_RESUME_KIND};
18use crate::runtime::guardian_state::{
19    parse_guardian_verdict, read_guardian_config, read_guardian_state, write_guardian_state,
20    GuardianVerdict,
21};
22use crate::Agent;
23use async_trait::async_trait;
24use bamboo_agent_core::storage::Storage;
25use bamboo_agent_core::tools::ToolExecutor;
26use bamboo_agent_core::{
27    AgentEvent, BashCompletionInfo, BashCompletionSink, Message, Role, Session,
28};
29use bamboo_domain::session::runtime_state::{
30    AgentRuntimeState, AgentStatusState, ChildWaitPolicy, SuspensionState,
31};
32use bamboo_llm::{Config, ProviderModelRouter, ProviderRegistry};
33use bamboo_storage::LockedSessionStore;
34use chrono::Utc;
35use tokio::sync::{broadcast, RwLock};
36
37use crate::model_areas::resolve_global_area_models;
38use crate::model_config_helper::{
39    resolve_fast_model, resolve_gold_config, GOLD_CONFIG_METADATA_KEY,
40};
41use crate::session_app::provider_model::session_effective_model_ref;
42use crate::session_app::resume::{
43    resume_session_execution, ResumeExecutionPort, ResumeSpawnRequest,
44};
45use crate::session_app::types::{ResumeConfigSnapshot, ResumeOutcome};
46
47const AGENT_RUNTIME_STATE_METADATA_KEY: &str = "agent.runtime.state";
48const RUNTIME_RESUME_MESSAGE_HIDDEN_KEY: &str = "hidden_from_ui";
49const RUNTIME_RESUME_MESSAGE_KIND_KEY: &str = "runtime_kind";
50
51fn read_runtime_state(session: &Session) -> AgentRuntimeState {
52    session
53        .agent_runtime_state
54        .clone()
55        .or_else(|| {
56            session
57                .metadata
58                .get(AGENT_RUNTIME_STATE_METADATA_KEY)
59                .and_then(|raw| serde_json::from_str::<AgentRuntimeState>(raw).ok())
60        })
61        .unwrap_or_else(|| AgentRuntimeState::new(format!("{}-child-wait", session.id)))
62}
63
64fn write_runtime_state(session: &mut Session, runtime_state: &AgentRuntimeState) {
65    session.agent_runtime_state = Some(runtime_state.clone());
66    if let Ok(serialized) = serde_json::to_string(runtime_state) {
67        session
68            .metadata
69            .insert(AGENT_RUNTIME_STATE_METADATA_KEY.to_string(), serialized);
70    }
71}
72
73fn is_error_like(status: &str) -> bool {
74    matches!(status, "error" | "timeout" | "cancelled")
75}
76
77/// Terminal child run statuses, as mirrored into the session index.
78fn is_terminal_child_status(status: &str) -> bool {
79    matches!(
80        status,
81        "completed" | "error" | "timeout" | "cancelled" | "skipped"
82    )
83}
84
85/// Reconstruct the set of completed child session ids for a parent from the
86/// session index (the single source of truth), folding in the child whose
87/// completion event is being processed so a momentarily-lagging index can never
88/// stall the parent's resume.
89async fn derive_completed_child_ids(
90    storage: &Arc<dyn Storage>,
91    parent_session_id: &str,
92    just_completed_child_id: &str,
93) -> Vec<String> {
94    let mut completed: Vec<String> = storage
95        .list_child_run_statuses(parent_session_id)
96        .await
97        .unwrap_or_default()
98        .into_iter()
99        .filter(|(_, status)| status.as_deref().is_some_and(is_terminal_child_status))
100        .map(|(id, _)| id)
101        .collect();
102    if !completed.iter().any(|id| id == just_completed_child_id) {
103        completed.push(just_completed_child_id.to_string());
104    }
105    completed.sort();
106    completed.dedup();
107    completed
108}
109
110fn read_config_snapshot(config: &Arc<RwLock<Config>>, cached_config: &StdRwLock<Config>) -> Config {
111    if let Ok(config_guard) = config.try_read() {
112        let snapshot = config_guard.clone();
113
114        if let Ok(mut cached_guard) = cached_config.try_write() {
115            *cached_guard = snapshot.clone();
116        }
117
118        snapshot
119    } else {
120        cached_config
121            .try_read()
122            .map(|guard| guard.clone())
123            .unwrap_or_default()
124    }
125}
126
127/// Per-parent async locks that serialize concurrent `on_child_completed`
128/// invocations for the same parent session.
129///
130/// Race eliminated: when `wait_for=Any` and two child sessions complete
131/// simultaneously, both invocations load the parent with
132/// `waiting_for_children=Some` before either persists the cleared state, so
133/// both pass `wait_policy_satisfied`, both clear `waiting_for_children`, add a
134/// duplicate resume message, and call `resume_parent` — a double resume.
135/// Holding this per-parent `tokio::sync::Mutex` across the load-check-save
136/// critical section makes the second caller observe the already-cleared state.
137///
138/// The inner `std::sync::Mutex` guards only the brief HashMap lookup/insert
139/// (no await inside); the per-parent `tokio::sync::Mutex` is the one held
140/// across the async critical section. Entries accumulate but are small
141/// (`Arc<tokio::sync::Mutex<()>>` ≈ 24 bytes) and bounded by the number of
142/// distinct parent sessions.
143fn parent_locks() -> &'static std::sync::Mutex<HashMap<String, Arc<tokio::sync::Mutex<()>>>> {
144    static LOCKS: OnceLock<std::sync::Mutex<HashMap<String, Arc<tokio::sync::Mutex<()>>>>> =
145        OnceLock::new();
146    LOCKS.get_or_init(|| std::sync::Mutex::new(HashMap::new()))
147}
148
149/// Fetch (or create) the per-session async lock from [`parent_locks`]. Held
150/// across the load-check-clear-resume critical section so the three resume
151/// sources for one session — child completion, the loop-facing bash **push**
152/// ([`BashCompletionSink::on_bash_completed`]), and the bash **backstop** poll
153/// ([`ChildCompletionCoordinator::bash_self_resume`]) — can never double-resume.
154/// The inner sync `Mutex` guards only the brief map lookup (no await inside).
155fn session_resume_lock(session_id: &str) -> Arc<tokio::sync::Mutex<()>> {
156    let mut map = parent_locks().lock().recover_poison();
157    map.entry(session_id.to_string())
158        .or_insert_with(|| Arc::new(tokio::sync::Mutex::new(())))
159        .clone()
160}
161
162fn wait_policy_satisfied(
163    policy: ChildWaitPolicy,
164    wait_child_ids: &[String],
165    completed_child_ids: &[String],
166    latest_status: &str,
167) -> bool {
168    if wait_child_ids.is_empty() {
169        return false;
170    }
171
172    match policy {
173        ChildWaitPolicy::All => wait_child_ids
174            .iter()
175            .all(|id| completed_child_ids.iter().any(|completed| completed == id)),
176        ChildWaitPolicy::Any => completed_child_ids
177            .iter()
178            .any(|id| wait_child_ids.iter().any(|wait_id| wait_id == id)),
179        ChildWaitPolicy::FirstError => {
180            is_error_like(latest_status)
181                || wait_child_ids
182                    .iter()
183                    .all(|id| completed_child_ids.iter().any(|completed| completed == id))
184        }
185    }
186}
187
188/// Extract the child session's last assistant content, if any. Returns `None`
189/// when the child produced no assistant message (e.g. errored before the first
190/// model response, or only emitted tool messages).
191fn child_final_assistant_text(child: &Session) -> Option<String> {
192    child
193        .messages
194        .iter()
195        .rev()
196        .find(|message| matches!(message.role, Role::Assistant))
197        .map(|message| message.content.clone())
198        .filter(|content| !content.trim().is_empty())
199}
200
201fn runtime_resume_message(
202    completion: &ChildCompletion,
203    remaining_children: usize,
204    child_final_response: Option<&str>,
205) -> Message {
206    let mut body = format!(
207        "Runtime notification: child session `{}` finished with status `{}`. Remaining child sessions: {}.",
208        completion.child_session_id, completion.status, remaining_children
209    );
210
211    // Fold the child's full final response back into the parent — no
212    // truncation. Sub-agents are first-class agents whose complete conclusion
213    // should be available to the parent without an extra `SubAgent.get` round
214    // trip. The message is left compressible (see `never_compress` below) so a
215    // long transcript can still be reclaimed under parent compaction.
216    let final_response = child_final_response.map(str::to_string);
217    if let Some(response) = final_response.as_deref() {
218        body.push_str("\n\nChild final response:\n");
219        body.push_str(response);
220    } else if let Some(error) = completion.error.as_deref() {
221        if !error.is_empty() {
222            body.push_str("\n\nChild error:\n");
223            body.push_str(error);
224        }
225    }
226
227    body.push_str(
228        "\n\nResume the parent task using this child result and continue from the previous plan. \
229         If you need the full child transcript, call SubAgent.get(child_session_id).",
230    );
231
232    let mut message = Message::user(body);
233    message.metadata = Some(serde_json::json!({
234        RUNTIME_RESUME_MESSAGE_HIDDEN_KEY: true,
235        RUNTIME_RESUME_MESSAGE_KIND_KEY: "child_completion_resume",
236        "child_session_id": completion.child_session_id,
237        "child_status": completion.status,
238        "child_error": completion.error,
239        "completed_at": completion.completed_at,
240        "child_final_response_included": final_response.is_some(),
241    }));
242    // Allow parent-side compaction to reclaim this (now untruncated) message if
243    // the parent context grows — important once children nest and fold full
244    // results upward. The `SubAgent.get` hint preserves recoverability.
245    message.never_compress = false;
246    message
247}
248
249/// The hidden resume message for a completed **guardian** review: a directive,
250/// verdict-tailored note that carries the reviewer's findings straight into the
251/// parent (so it can act without a `SubAgent.get`), mirroring
252/// [`runtime_resume_message`]'s hidden/compressible shape.
253fn guardian_resume_message(completion: &ChildCompletion, verdict: &GuardianVerdict) -> Message {
254    let mut body = if verdict.approve {
255        String::from(
256            "Guardian review APPROVED: an independent reviewer verified the work and found no blocking issues. You may finalize the task.",
257        )
258    } else {
259        String::from(
260            "Guardian review REJECTED: an independent reviewer found issues. Address every finding below before completing — do NOT declare the task complete until they are resolved.",
261        )
262    };
263    if let Some(summary) = verdict.summary.as_deref().filter(|s| !s.trim().is_empty()) {
264        body.push_str("\n\nReviewer summary: ");
265        body.push_str(summary);
266    }
267    if !verdict.findings.is_empty() {
268        body.push_str("\n\nFindings:");
269        for (idx, finding) in verdict.findings.iter().enumerate() {
270            body.push_str(&format!("\n{}. {}", idx + 1, finding));
271        }
272    }
273    body.push_str(
274        "\n\nIf you need the full guardian transcript, call SubAgent.get(child_session_id).",
275    );
276
277    let mut message = Message::user(body);
278    message.metadata = Some(serde_json::json!({
279        RUNTIME_RESUME_MESSAGE_HIDDEN_KEY: true,
280        RUNTIME_RESUME_MESSAGE_KIND_KEY: "guardian_review_resume",
281        "child_session_id": completion.child_session_id,
282        "child_status": completion.status,
283        "guardian_approved": verdict.approve,
284        "completed_at": completion.completed_at,
285    }));
286    message.never_compress = false;
287    message
288}
289
290#[derive(Clone)]
291pub struct ChildCompletionCoordinator {
292    storage: Arc<dyn Storage>,
293    persistence: Arc<bamboo_storage::LockedSessionStore>,
294    sessions: crate::SessionCache,
295    agent_runners: Arc<RwLock<HashMap<String, AgentRunner>>>,
296    session_event_senders: Arc<RwLock<HashMap<String, broadcast::Sender<AgentEvent>>>>,
297    agent: Arc<Agent>,
298    config: Arc<RwLock<Config>>,
299    provider_registry: Arc<ProviderRegistry>,
300    provider_router: Arc<ProviderModelRouter>,
301    app_data_dir: std::path::PathBuf,
302    account_feed_inbox: Option<crate::execution::AccountFeedInbox>,
303    root_tools: Arc<RwLock<Option<Arc<dyn ToolExecutor>>>>,
304    /// Late-bound guardian reviewer spawner, set post-construction by the server
305    /// (mirrors `root_tools`). Re-injected into resumed runs so a guardian's
306    /// reject→fix verdict can be re-reviewed across the suspend/resume boundary.
307    guardian_spawner: Arc<RwLock<Option<Arc<dyn GuardianSpawner>>>>,
308}
309
310impl ChildCompletionCoordinator {
311    #[allow(clippy::too_many_arguments)]
312    pub fn new(
313        storage: Arc<dyn Storage>,
314        persistence: Arc<LockedSessionStore>,
315        sessions: crate::SessionCache,
316        agent_runners: Arc<RwLock<HashMap<String, AgentRunner>>>,
317        session_event_senders: Arc<RwLock<HashMap<String, broadcast::Sender<AgentEvent>>>>,
318        agent: Arc<Agent>,
319        config: Arc<RwLock<Config>>,
320        provider_registry: Arc<ProviderRegistry>,
321        provider_router: Arc<ProviderModelRouter>,
322        app_data_dir: std::path::PathBuf,
323        account_feed_inbox: Option<crate::execution::AccountFeedInbox>,
324    ) -> Self {
325        Self {
326            storage,
327            persistence,
328            sessions,
329            agent_runners,
330            session_event_senders,
331            agent,
332            config,
333            provider_registry,
334            provider_router,
335            app_data_dir,
336            account_feed_inbox,
337            root_tools: Arc::new(RwLock::new(None)),
338            guardian_spawner: Arc::new(RwLock::new(None)),
339        }
340    }
341
342    pub async fn set_root_tools(&self, tools: Arc<dyn ToolExecutor>) {
343        *self.root_tools.write().await = Some(tools);
344    }
345
346    /// Wire the guardian reviewer spawner (server-provided), so resumed runs can
347    /// re-spawn a guardian to re-review a fix after a reject verdict.
348    pub async fn set_guardian_spawner(&self, spawner: Arc<dyn GuardianSpawner>) {
349        *self.guardian_spawner.write().await = Some(spawner);
350    }
351
352    fn build_resume_config(
353        &self,
354        session: &Session,
355        config_snapshot: &Config,
356    ) -> ResumeConfigSnapshot {
357        crate::session_app::resolution::resolve_resume_config_snapshot(
358            config_snapshot,
359            &self.provider_registry,
360            session,
361            None,
362        )
363    }
364
365    /// Drive a parent-resume and return the final [`ResumeOutcome`] so callers
366    /// can distinguish a successful spawn (`Started`) from a gate-blocked
367    /// attempt (`Completed`). The bash self-resume poll task uses this to
368    /// detect the finalize-clobber case — its appended resume message was
369    /// reverted by the suspending runner's final `merge_save_runtime`, so the
370    /// resume port's `has_pending_user_message` gate fails and nothing spawns —
371    /// and retry the clear→append→resume (see [`Self::bash_self_resume`]).
372    async fn resume_parent(&self, parent_session_id: String) -> ResumeOutcome {
373        for attempt in 0..=5u8 {
374            if attempt > 0 {
375                tokio::time::sleep(Duration::from_millis(250 * attempt as u64)).await;
376            }
377
378            let Some(session) = self.load_session(&parent_session_id).await else {
379                tracing::warn!(%parent_session_id, "cannot resume parent after child completion: session not found");
380                return ResumeOutcome::NotFound;
381            };
382            let config_snapshot = self.config.read().await.clone();
383            let resume_config = self.build_resume_config(&session, &config_snapshot);
384            let outcome = resume_session_execution(self, &parent_session_id, resume_config).await;
385            tracing::info!(
386                %parent_session_id,
387                attempt,
388                outcome = outcome.as_str(),
389                "child completion requested parent resume"
390            );
391
392            if !matches!(outcome, ResumeOutcome::AlreadyRunning { .. }) {
393                return outcome;
394            }
395        }
396        // Exhausted the AlreadyRunning retry budget; surface the final state.
397        ResumeOutcome::AlreadyRunning {
398            run_id: String::new(),
399        }
400    }
401
402    async fn save_and_cache(&self, session: &mut Session) {
403        if let Err(error) = self.persistence.merge_save_runtime(session).await {
404            tracing::warn!(session_id = %session.id, %error, "failed to persist session");
405        }
406        self.sessions.insert(
407            session.id.clone(),
408            Arc::new(parking_lot::RwLock::new(session.clone())),
409        );
410    }
411}
412
413#[async_trait]
414impl ChildCompletionHandler for ChildCompletionCoordinator {
415    async fn on_child_completed(&self, completion: ChildCompletion) {
416        // Acquire the per-session async lock to eliminate the concurrent
417        // double-resume race (see `parent_locks` for the full scenario). The
418        // inner std::sync::Mutex is released immediately so no sync lock is
419        // held across the await that follows.
420        let per_parent = session_resume_lock(&completion.parent_session_id);
421        let _per_parent_guard = per_parent.lock().await;
422
423        let Some(mut parent) = self.load_session(&completion.parent_session_id).await else {
424            tracing::warn!(
425                parent_session_id = %completion.parent_session_id,
426                child_session_id = %completion.child_session_id,
427                "child completion received for missing parent"
428            );
429            return;
430        };
431
432        // A parent may itself be a child (nested sub-agents): the rest of this
433        // handler is kind-agnostic — it operates on `completion.parent_session_id`,
434        // inspects that session's own `waiting_for_children` runtime state, and
435        // resumes it. (Previously this bailed unless the parent was Root, which
436        // silently dropped grandchild completions.)
437        let mut runtime_state = read_runtime_state(&parent);
438
439        // Single source of truth: reconstruct the completed-child set from the
440        // session index rather than from a denormalized copy on the parent file.
441        let completed_child_ids = derive_completed_child_ids(
442            &self.storage,
443            &completion.parent_session_id,
444            &completion.child_session_id,
445        )
446        .await;
447
448        let mut should_resume = false;
449        let mut remaining_children = 0usize;
450        if let Some(wait) = runtime_state.waiting_for_children.clone() {
451            remaining_children = wait
452                .child_session_ids
453                .iter()
454                .filter(|id| !completed_child_ids.iter().any(|completed| completed == *id))
455                .count();
456            should_resume = wait_policy_satisfied(
457                wait.wait_for,
458                &wait.child_session_ids,
459                &completed_child_ids,
460                &completion.status,
461            );
462            if should_resume {
463                runtime_state.waiting_for_children = None;
464                runtime_state.status = AgentStatusState::Idle;
465                runtime_state.suspension = None;
466            }
467        }
468
469        if should_resume {
470            parent.metadata.remove("runtime.suspend_reason");
471            // Load the completed child once. The guardian branch inspects its
472            // subagent_type + final verdict; the generic path folds its final
473            // assistant content into the hidden resume message (avoiding an extra
474            // `SubAgent.get` round trip after resume).
475            let loaded_child = match self
476                .storage
477                .load_session(&completion.child_session_id)
478                .await
479            {
480                Ok(child) => child,
481                Err(error) => {
482                    tracing::warn!(
483                        child_session_id = %completion.child_session_id,
484                        %error,
485                        "failed to load child session for runtime resume message"
486                    );
487                    None
488                }
489            };
490
491            // Guardian branch: a completing guardian reviewer that matches the
492            // parent's recorded review advances GuardianState (phase → Reviewed)
493            // and resumes with a verdict-tailored, findings-carrying message. Any
494            // id mismatch or unparseable verdict falls through to the generic
495            // resume, so the parent is never stranded.
496            let reviewed_round = runtime_state.round.current_round;
497            let guardian_resume = loaded_child.as_ref().and_then(|child| {
498                if child.subagent_type().as_deref() != Some("guardian") {
499                    return None;
500                }
501                let mut guardian_state = read_guardian_state(&parent)?;
502                if guardian_state.guardian_child_id.as_deref()
503                    != Some(completion.child_session_id.as_str())
504                {
505                    // A *different* guardian is legitimately still in flight —
506                    // leave its Pending state intact and use the generic resume.
507                    tracing::warn!(
508                        parent_session_id = %completion.parent_session_id,
509                        child_session_id = %completion.child_session_id,
510                        expected = ?guardian_state.guardian_child_id,
511                        "guardian completion does not match recorded guardian_child_id; using generic resume"
512                    );
513                    return None;
514                }
515                // This IS the guardian we dispatched, so we MUST advance the
516                // phase out of `Pending` — otherwise the next terminal gate's
517                // `Pending => return None` would let the run complete unreviewed.
518                // A reviewer that errored or produced unparseable output is
519                // treated as a SYNTHETIC REJECT (never a silent pass), so the
520                // budgeted re-review loop governs the outcome: fail-closed, but
521                // still bounded by `max_reviews`.
522                let verdict = child_final_assistant_text(child)
523                    .and_then(|text| match parse_guardian_verdict(&text) {
524                        Ok(verdict) => Some(verdict),
525                        Err(error) => {
526                            tracing::warn!(
527                                child_session_id = %completion.child_session_id,
528                                %error,
529                                "guardian verdict unparseable; recording a synthetic reject"
530                            );
531                            None
532                        }
533                    })
534                    .unwrap_or_else(|| {
535                        GuardianVerdict::rejected(vec![
536                            "The guardian reviewer did not return a usable verdict (it errored or \
537                             emitted unparseable output); the work has NOT been independently \
538                             verified."
539                                .to_string(),
540                        ])
541                    });
542                let approved = verdict.approve;
543                let message = guardian_resume_message(&completion, &verdict);
544                guardian_state.record_verdict(verdict, reviewed_round);
545                write_guardian_state(&mut parent, guardian_state);
546                tracing::info!(
547                    parent_session_id = %completion.parent_session_id,
548                    child_session_id = %completion.child_session_id,
549                    approved,
550                    "guardian verdict recorded; resuming parent"
551                );
552                Some(message)
553            });
554
555            let resume_message = guardian_resume.unwrap_or_else(|| {
556                runtime_resume_message(
557                    &completion,
558                    remaining_children,
559                    loaded_child
560                        .as_ref()
561                        .and_then(child_final_assistant_text)
562                        .as_deref(),
563                )
564            });
565            parent.add_message(resume_message);
566        } else if runtime_state.waiting_for_children.is_some() {
567            runtime_state.status = AgentStatusState::Suspended;
568            runtime_state.suspension = Some(SuspensionState {
569                reason: "waiting_for_children".to_string(),
570                suspended_at: Utc::now(),
571                resumable: true,
572                hook_point: Some("ChildCompletion".to_string()),
573            });
574        }
575
576        parent.updated_at = Utc::now();
577        write_runtime_state(&mut parent, &runtime_state);
578        self.save_and_cache(&mut parent).await;
579
580        // Capture before releasing the per-parent lock so the borrow checker
581        // is satisfied; `resume_parent` has its own retry loop and should not
582        // hold the per-parent lock (it would block other completions for the
583        // same parent, and the state is already durably settled above).
584        let resume_parent_id = parent.id.clone();
585        drop(_per_parent_guard);
586
587        if should_resume {
588            self.resume_parent(resume_parent_id).await;
589        }
590    }
591}
592
593#[async_trait]
594impl ResumeExecutionPort for ChildCompletionCoordinator {
595    async fn load_session(&self, session_id: &str) -> Option<Session> {
596        match self.storage.load_session(session_id).await {
597            Ok(Some(session)) => Some(session),
598            Ok(None) => self
599                .sessions
600                .get(session_id)
601                .map(|e| e.value().clone())
602                .map(|arc| arc.read().clone()),
603            Err(error) => {
604                tracing::warn!(%session_id, %error, "failed to load session from storage");
605                self.sessions
606                    .get(session_id)
607                    .map(|e| e.value().clone())
608                    .map(|arc| arc.read().clone())
609            }
610        }
611    }
612
613    async fn save_and_cache_session(&self, session: &mut Session) {
614        self.save_and_cache(session).await;
615    }
616
617    async fn try_reserve_runner(
618        &self,
619        session_id: &str,
620        event_sender: &broadcast::Sender<AgentEvent>,
621    ) -> Option<RunnerReservation> {
622        try_reserve_runner(&self.agent_runners, session_id, event_sender).await
623    }
624
625    async fn get_existing_runner_run_id(&self, session_id: &str) -> Option<String> {
626        let runners = self.agent_runners.read().await;
627        runners.get(session_id).map(|r| r.run_id.clone())
628    }
629
630    async fn get_or_create_event_sender(&self, session_id: &str) -> broadcast::Sender<AgentEvent> {
631        crate::execution::session_events::get_or_create_event_sender(
632            &self.session_event_senders,
633            session_id,
634        )
635        .await
636    }
637
638    async fn spawn_resume_execution(&self, request: ResumeSpawnRequest) {
639        let ResumeSpawnRequest {
640            session_id,
641            session,
642            cancel_token,
643            run_id: _,
644            event_sender,
645            config,
646        } = request;
647
648        let Some(root_tools) = self.root_tools.read().await.clone() else {
649            tracing::error!(%session_id, "cannot resume parent after child completion: root tool surface is not initialized");
650            return;
651        };
652
653        let model = session.model.clone();
654        let resolved_provider_name = session_effective_model_ref(&session)
655            .map(|model_ref| model_ref.provider)
656            .unwrap_or(config.provider_name);
657        let provider_override = session_effective_model_ref(&session)
658            .and_then(|model_ref| match self.provider_router.route(&model_ref) {
659                Ok(provider) => Some(provider),
660                Err(error) => {
661                    tracing::warn!(
662                        session_id = %session_id,
663                        provider = %model_ref.provider,
664                        model = %model_ref.model,
665                        error = %error,
666                        "failed to resolve provider override for child-completion parent resume; falling back to runtime provider"
667                    );
668                    None
669                }
670            });
671        let config_snapshot = self.config.read().await.clone();
672        let resolved_fast_provider = resolve_fast_model(
673            &config_snapshot,
674            &resolved_provider_name,
675            &self.provider_registry,
676        )
677        .map(|model| model.provider);
678        let reasoning_effort = session.reasoning_effort;
679        let reasoning_effort_source = session
680            .metadata
681            .get("reasoning_effort_source")
682            .cloned()
683            .unwrap_or_default();
684        let gold_config = resolve_gold_config(
685            &config_snapshot,
686            session
687                .metadata
688                .get(GOLD_CONFIG_METADATA_KEY)
689                .map(String::as_str),
690        )
691        .or(config.gold_config.clone());
692
693        let (mpsc_tx, _forwarder) = create_event_forwarder(
694            session_id.clone(),
695            event_sender,
696            self.agent_runners.clone(),
697            self.account_feed_inbox.clone(),
698        );
699
700        let config_handle = self.config.clone();
701        let cached_config = Arc::new(StdRwLock::new(config_snapshot.clone()));
702        let provider_registry = self.provider_registry.clone();
703        let provider_name_for_aux = resolved_provider_name.clone();
704        let auxiliary_model_resolver = std::sync::Arc::new(move || {
705            let config_snapshot = read_config_snapshot(&config_handle, cached_config.as_ref());
706            // Auxiliary models are global (config-derived), never session-bound.
707            let areas = resolve_global_area_models(
708                &config_snapshot,
709                &provider_name_for_aux,
710                &provider_registry,
711            );
712            crate::AuxiliaryModelConfig {
713                fast_model_name: areas.fast.as_ref().map(|m| m.model_name.clone()),
714                fast_model_provider: areas.fast.map(|m| m.provider),
715                background_model_name: areas.background.as_ref().map(|m| m.model_name.clone()),
716                planning_model_name: None,
717                search_model_name: None,
718                summarization_model_name: areas
719                    .summarization
720                    .as_ref()
721                    .map(|m| m.model_name.clone()),
722                background_model_provider: areas.background.map(|m| m.provider),
723                summarization_model_provider: areas.summarization.map(|m| m.provider),
724            }
725        });
726        let model_roster = crate::ModelRoster {
727            model: Some(model),
728            provider_name: Some(resolved_provider_name),
729            provider_type: config.provider_type.clone(),
730            fast: crate::RoleModel::from_parts(config.fast_model, resolved_fast_provider),
731            background: crate::RoleModel::from_parts(
732                config.background_model,
733                config.background_model_provider,
734            ),
735            summarization: crate::RoleModel::from_parts(
736                config.summarization_model,
737                config.summarization_model_provider,
738            ),
739        };
740
741        // Re-inject guardian state on resume so a reject→fix verdict can be
742        // re-reviewed: config from the session (persisted at first spawn),
743        // spawner from the coordinator-held handle. Absent guardian config this
744        // stays `None`, and the approve→complete path is unchanged.
745        let guardian_config = read_guardian_config(&session);
746        let guardian_spawner = self.guardian_spawner.read().await.clone();
747
748        spawn_session_execution(SessionExecutionArgs {
749            agent: self.agent.clone(),
750            session_id,
751            session,
752            tools_override: Some(root_tools),
753            provider_override,
754            model_roster,
755            reasoning_effort,
756            reasoning_effort_source,
757            auxiliary_model_resolver: Some(auxiliary_model_resolver),
758            // Resumed child runs keep the spawn-time disabled snapshot (#136 lives
759            // on the long-running main agent path; children are short-lived).
760            disabled_filter_resolver: None,
761            disabled_tools: Some(config.disabled_tools),
762            disabled_skill_ids: Some(config.disabled_skill_ids),
763            selected_skill_ids: None,
764            selected_skill_mode: None,
765            cancel_token,
766            mpsc_tx,
767            image_fallback: config.image_fallback,
768            gold_config,
769            guardian_config,
770            guardian_spawner,
771            bash_resume_hook: {
772                let hook: Arc<dyn BashResumeHook> = Arc::new(self.clone());
773                Some(hook)
774            },
775            bash_completion_sink: {
776                // Resumed runs keep the push wired too, so a background shell
777                // launched after resume still notifies the loop.
778                let sink: Arc<dyn BashCompletionSink> = Arc::new(self.clone());
779                Some(sink)
780            },
781            app_data_dir: Some(self.app_data_dir.clone()),
782            runners: self.agent_runners.clone(),
783            sessions_cache: self.sessions.clone(),
784            on_complete: None,
785        });
786    }
787}
788
789/// Hidden resume message for a bash-completion self-resume (issue #84 Phase 2b).
790/// Mirrors [`runtime_resume_message`]'s hidden/compressible shape so the resume
791/// port's `has_pending_user_message` gate is satisfied.
792///
793/// `timed_out` selects the wording: the normal path (all shells finished)
794/// announces completion; the deadline path (the 6h+10m wait ceiling was hit with
795/// shells STILL running) must NOT claim the shells completed — it says they may
796/// still be running so the model verifies with BashOutput instead of assuming
797/// success on a false premise.
798fn bash_completion_resume_message(bash_ids: &[String], timed_out: bool) -> Message {
799    let body = if timed_out {
800        format!(
801            "Runtime notification: the background-Bash wait ceiling was reached while one or more \
802             shell(s) ({}) may still be running. The session is being resumed so it is not \
803             stranded; verify their actual status with BashOutput before assuming completion.",
804            bash_ids.join(", ")
805        )
806    } else {
807        format!(
808            "Runtime notification: all background Bash shell(s) ({}) have completed. \
809             Review their output with BashOutput and resume the task from where you left off.",
810            bash_ids.join(", ")
811        )
812    };
813    let mut message = Message::user(body);
814    message.metadata = Some(serde_json::json!({
815        RUNTIME_RESUME_MESSAGE_HIDDEN_KEY: true,
816        RUNTIME_RESUME_MESSAGE_KIND_KEY: BASH_COMPLETION_RESUME_KIND,
817    }));
818    message.never_compress = false;
819    message
820}
821
822/// Decide whether the bash self-resume should retry its clear→append→resume
823/// sequence after a resume attempt returned `outcome`, given that the persisted
824/// bash wait is (`true`) / is not (`false`) still set on reload.
825///
826/// Retry **only** when the resume did NOT spawn (`Completed` — no pending user
827/// message, i.e. our resume message was dropped — or `AlreadyRunning`) AND the
828/// persisted bash wait is still set: the signature of the finalize-clobber, where
829/// the suspending runner's one-shot final `merge_save_runtime` lands after our
830/// save and reverts `waiting_for_bash=Some` while dropping our resume message, so
831/// `has_pending_user_message` fails and nothing spawns. `Started` (resume fired)
832/// and `NotFound` (session gone) never retry. Pure helper so the clobber
833/// detection is unit-testable in isolation from async I/O.
834fn bash_resume_should_retry(outcome: &ResumeOutcome, persisted_waiting_for_bash: bool) -> bool {
835    match outcome {
836        ResumeOutcome::Started { .. } | ResumeOutcome::NotFound => false,
837        ResumeOutcome::Completed | ResumeOutcome::AlreadyRunning { .. } => {
838            persisted_waiting_for_bash
839        }
840    }
841}
842
843/// Whether a background-shell completion push should **resume** the owning loop
844/// (vs merely enqueue an injection). Resume only when the loop is actually
845/// suspended on a bash wait AND every shell it was waiting on has now finished —
846/// resuming while other waited shells are still running would drop them back into
847/// a foreground turn prematurely. The last shell to finish (or the backstop)
848/// drives the resume; earlier ones enqueue their notice. Pure so the invariant is
849/// unit-testable in isolation.
850fn bash_completion_should_resume(loop_suspended_on_bash: bool, all_waited_shells_done: bool) -> bool {
851    loop_suspended_on_bash && all_waited_shells_done
852}
853
854/// Apply the bash-resume state transition to a loaded session **in place**: clear
855/// the `waiting_for_bash` wait, mark the runtime Idle, drop the suspension +
856/// `runtime.suspend_reason`, and append `resume_message`. Returns `false` (a
857/// no-op) when the session was not actually waiting on bash — the double-resume
858/// guard shared by the push and the backstop. Pure (no I/O) so both
859/// [`ChildCompletionCoordinator::perform_bash_resume`] and unit tests exercise the
860/// exact same transition.
861fn apply_bash_resume_transition(session: &mut Session, resume_message: &Message) -> bool {
862    let mut runtime_state = read_runtime_state(session);
863    if runtime_state.waiting_for_bash.is_none() {
864        return false;
865    }
866    runtime_state.waiting_for_bash = None;
867    runtime_state.status = AgentStatusState::Idle;
868    runtime_state.suspension = None;
869    write_runtime_state(session, &runtime_state);
870    session.metadata.remove("runtime.suspend_reason");
871    session.add_message(resume_message.clone());
872    true
873}
874
875/// Bash self-resume support (issue #84 Phase 2b; push follow-up).
876impl ChildCompletionCoordinator {
877    /// **Backstop** for a session suspended on `waiting_for_bash`. The primary,
878    /// event-driven wake is the loop-facing push
879    /// ([`BashCompletionSink::on_bash_completed`] → [`Self::deliver_bash_completion`]):
880    /// the shell's completion task fires it the instant the process exits, and it
881    /// resumes the loop directly. This task exists ONLY to catch a **lost push** —
882    /// the completion landing in the window before the suspend was persisted (so
883    /// the push saw no `waiting_for_bash` and only queued an injection), or a
884    /// configuration with no sink wired — and to honour the wait ceiling.
885    ///
886    /// So it is deliberately NOT a hot spin: a coarse backoff (1 s → 30 s) that
887    /// **yields to the push**. In the happy path the push has already cleared
888    /// `waiting_for_bash` before the first check fires, so this returns after one
889    /// cheap load with no registry polling at all. It only performs a resume when
890    /// the shell(s) have finished but the loop is somehow still suspended, or the
891    /// 6 h wait ceiling is reached.
892    async fn bash_self_resume(&self, session_id: String, bash_ids: Vec<String>) {
893        let mut delay = Duration::from_secs(1);
894        let max_delay = Duration::from_secs(30);
895        // Hard ceiling: the wait lease (6 h) + the registry GC TTL (5 min) +
896        // margin. After this the shells are gone from the registry regardless,
897        // so force-resume to avoid stranding the session on a GC edge case.
898        let max_poll = Duration::from_secs(6 * 3600 + 600);
899        let deadline = tokio::time::Instant::now() + max_poll;
900
901        loop {
902            tokio::time::sleep(delay).await;
903
904            let Some(session) = self.load_session(&session_id).await else {
905                tracing::info!(%session_id, "bash self-resume backstop: session gone; nothing to do");
906                return;
907            };
908            if read_runtime_state(&session).waiting_for_bash.is_none() {
909                // The push (or another path) already resumed. This is the common
910                // case — the backstop yields silently after a single load.
911                return;
912            }
913
914            let still_running =
915                bamboo_tools::tools::bash_runtime::running_shells_for_session(&session_id);
916            let timed_out = tokio::time::Instant::now() >= deadline;
917            if still_running.is_empty() || timed_out {
918                // The shell(s) finished but the loop is still suspended → the push
919                // was lost (pre-persist window / no sink), or the ceiling hit.
920                // Resume under the shared per-session lock so we never race the
921                // push or a concurrent child-completion resume.
922                let guard = session_resume_lock(&session_id);
923                let _held = guard.lock().await;
924                tracing::warn!(
925                    %session_id,
926                    shell_count = bash_ids.len(),
927                    timed_out,
928                    "bash self-resume backstop engaged (push lost or wait ceiling reached)"
929                );
930                self.perform_bash_resume(
931                    &session_id,
932                    bash_completion_resume_message(&bash_ids, timed_out),
933                )
934                .await;
935                return;
936            }
937
938            delay = (delay * 2).min(max_delay);
939        }
940    }
941
942    /// Clear a session's `waiting_for_bash` state, append `resume_message`, and
943    /// drive the parent resume — the shared clear→append→resume used by BOTH the
944    /// event-driven push ([`Self::deliver_bash_completion`]) and the backstop poll
945    /// ([`Self::bash_self_resume`]).
946    ///
947    /// **The caller MUST hold the [`session_resume_lock`] for `session_id`** so the
948    /// load-check-clear-resume critical section is serialized against every other
949    /// resume source (no double resume). No-op when the persisted wait was already
950    /// cleared (another source handled it first).
951    ///
952    /// The clear→append→resume is a **bounded retry loop** that closes the
953    /// finalize-clobber strand. The suspending runner's `finalize_task_context`
954    /// runs a full `save_runtime_session` (same `merge_save_runtime`, which
955    /// overwrites the whole `messages` array) that can land AFTER our save,
956    /// reverting `waiting_for_bash=Some` and dropping our resume message, so
957    /// `has_pending_user_message` fails and `resume_parent` returns `Completed`
958    /// without spawning. We detect that (persisted wait still set after a
959    /// non-`Started` outcome) and re-clear/re-append/re-resume. It converges
960    /// because the runner's finalize persist is one-shot: once landed, our retry's
961    /// save is the last writer, the message sticks, and resume fires.
962    async fn perform_bash_resume(&self, session_id: &str, resume_message: Message) {
963        let retry_backoff = Duration::from_millis(200);
964        const MAX_RESUME_ATTEMPTS: u8 = 5;
965        for attempt in 0..MAX_RESUME_ATTEMPTS {
966            if attempt > 0 {
967                tokio::time::sleep(retry_backoff).await;
968            }
969
970            let Some(mut session) = self.load_session(session_id).await else {
971                tracing::warn!(%session_id, "bash resume: session not found; nothing to resume");
972                return;
973            };
974
975            if !apply_bash_resume_transition(&mut session, &resume_message) {
976                // Double-resume guard: the wait was already cleared by another
977                // source (the push, the backstop, or a user-driven resume). Do
978                // not append a duplicate message or request a redundant resume.
979                tracing::info!(
980                    %session_id, attempt,
981                    "bash resume: persisted bash wait already cleared; nothing to resume"
982                );
983                return;
984            }
985            session.updated_at = Utc::now();
986            self.save_and_cache(&mut session).await;
987            tracing::info!(
988                %session_id, attempt,
989                "bash resume: cleared bash wait and appended resume message"
990            );
991
992            let outcome = self.resume_parent(session_id.to_string()).await;
993            match outcome {
994                ResumeOutcome::Started { .. } => {
995                    tracing::info!(%session_id, attempt, "bash resume: resume fired");
996                    return;
997                }
998                ResumeOutcome::NotFound => {
999                    tracing::warn!(%session_id, "bash resume: session vanished during resume");
1000                    return;
1001                }
1002                _ => {
1003                    // Completed (no pending user message ⇒ our resume message was
1004                    // dropped by the runner's finalize persist) or AlreadyRunning.
1005                    // Decide via the persisted bash wait: still set ⇒
1006                    // finalize-clobber ⇒ retry; cleared ⇒ the session is being
1007                    // handled (by us or a concurrent resume) ⇒ stop.
1008                    let clobbered = match self.load_session(session_id).await {
1009                        Some(reloaded) => read_runtime_state(&reloaded).waiting_for_bash.is_some(),
1010                        None => {
1011                            tracing::warn!(%session_id, "bash resume: session vanished after resume");
1012                            return;
1013                        }
1014                    };
1015                    if bash_resume_should_retry(&outcome, clobbered) {
1016                        tracing::warn!(
1017                            %session_id, attempt,
1018                            outcome = outcome.as_str(),
1019                            "bash resume: persisted wait still set after resume (finalize-clobber); retrying"
1020                        );
1021                        continue;
1022                    }
1023                    tracing::info!(
1024                        %session_id, attempt,
1025                        outcome = outcome.as_str(),
1026                        "bash resume: wait cleared and resume handled; stopping"
1027                    );
1028                    return;
1029                }
1030            }
1031        }
1032
1033        tracing::warn!(
1034            %session_id,
1035            attempts = MAX_RESUME_ATTEMPTS,
1036            "bash resume: exhausted clobber-retry budget without confirming resume; giving up"
1037        );
1038    }
1039}
1040
1041impl BashResumeHook for ChildCompletionCoordinator {
1042    fn arrange_bash_self_resume(&self, session_id: String, bash_ids: Vec<String>) {
1043        let coordinator = Arc::new(self.clone());
1044        tokio::spawn(async move {
1045            coordinator.bash_self_resume(session_id, bash_ids).await;
1046        });
1047    }
1048}
1049
1050/// Build the injected user-message body for a completed background shell — a
1051/// concise notice plus a bounded output tail — so the model can act on the
1052/// result without a mandatory `BashOutput` round-trip (issue #84 Phase 2b
1053/// follow-up).
1054fn bash_completion_injection_body(info: &BashCompletionInfo) -> String {
1055    let exit = match info.exit_code {
1056        Some(code) => code.to_string(),
1057        None => "none (signal/killed)".to_string(),
1058    };
1059    let mut body = format!(
1060        "Runtime notification: background shell `{}` (`{}`) finished — status {}, exit code {}.",
1061        info.bash_id, info.command, info.status, exit
1062    );
1063    if info.output_tail.trim().is_empty() {
1064        body.push_str(" It produced no captured output.");
1065    } else {
1066        body.push_str("\n\nOutput tail:\n");
1067        body.push_str(&info.output_tail);
1068    }
1069    body.push_str(&format!(
1070        "\n\nUse BashOutput with bash_id=\"{}\" for the full output, then continue the task.",
1071        info.bash_id
1072    ));
1073    body
1074}
1075
1076/// Loop-facing background-Bash completion delivery (issue #84 Phase 2b
1077/// follow-up). Pushes a completed shell's result into its owning session's loop,
1078/// mirroring how a sub-agent completion reaches its parent — but via the
1079/// running-loop channel, which children never exercise (a parent waiting on
1080/// children is always suspended when one completes; bash is the first completion
1081/// source that can land on a *live, iterating* loop).
1082///
1083/// The push enqueues onto `pending_injected_messages`, the same round-boundary
1084/// steering channel `send_message` uses. That covers every reachable loop state:
1085/// an actively-looping session drains it at its next round
1086/// (`merge_pending_injected_messages`); a session suspended on `waiting_for_bash`
1087/// drains it when the durable end-of-turn poll backstop (`bash_resume_hook`)
1088/// resumes it at round 0. A wired-sink session is never idle-with-a-running-shell
1089/// (ending a turn with one suspends), so no separate idle-wake path is needed —
1090/// keeping the push a pure latency optimization that never races the backstop.
1091/// Enqueue a completed shell's summary as a pending injected message on the
1092/// owning session. Race-safe: `update_runtime_config` loads the freshest session
1093/// under the per-session lock and re-saves, so it can never revert a message the
1094/// live loop appended concurrently — unlike `merge_save_runtime` (which writes
1095/// the caller's whole `messages` snapshot verbatim). Free fn so it is unit-
1096/// testable without constructing a full coordinator. Returns the saved session,
1097/// or `None` if the owning session no longer exists.
1098async fn enqueue_bash_completion_injection(
1099    persistence: &LockedSessionStore,
1100    info: &BashCompletionInfo,
1101) -> std::io::Result<Option<Session>> {
1102    let body = bash_completion_injection_body(info);
1103    let queued = serde_json::json!({
1104        "content": body,
1105        "created_at": Utc::now(),
1106    });
1107    persistence
1108        .update_runtime_config(&info.session_id, move |session| {
1109            let mut pending = session.pending_injected_messages().unwrap_or_default();
1110            pending.push(queued);
1111            session.set_pending_injected_messages(pending);
1112        })
1113        .await
1114}
1115
1116/// Build the hidden, compressible resume message for a completed background
1117/// shell — the same rich notice body used for a live-loop injection
1118/// ([`bash_completion_injection_body`]), but tagged as a resume message so it
1119/// satisfies the `has_pending_user_message` gate that lets a suspended session
1120/// spawn. This is what the **push** appends when it wakes a suspended loop, so
1121/// the model gets the shell's status + output tail in one shot without a
1122/// separate `BashOutput` round-trip.
1123fn bash_resume_message_from_info(info: &BashCompletionInfo) -> Message {
1124    let mut message = Message::user(bash_completion_injection_body(info));
1125    message.metadata = Some(serde_json::json!({
1126        RUNTIME_RESUME_MESSAGE_HIDDEN_KEY: true,
1127        RUNTIME_RESUME_MESSAGE_KIND_KEY: BASH_COMPLETION_RESUME_KIND,
1128    }));
1129    message.never_compress = false;
1130    message
1131}
1132
1133impl ChildCompletionCoordinator {
1134    /// Loop-facing delivery of a completed background shell. Two paths, chosen
1135    /// under the per-session resume lock so we never race the backstop poll or a
1136    /// concurrent child-completion resume:
1137    ///
1138    /// - **Suspended loop** (the model ended its turn with the shell running, so
1139    ///   `waiting_for_bash` is set) AND every waited shell has now finished →
1140    ///   **resume the loop directly**, event-driven, appending the rich completion
1141    ///   notice as the resume message. This is the push's whole point: no polling.
1142    /// - Otherwise (a live/iterating loop, or a suspend still waiting on OTHER
1143    ///   shells) → **enqueue** the notice as a pending injected message, drained at
1144    ///   the next round boundary (a live loop) or folded into the eventual resume
1145    ///   when the last shell finishes.
1146    async fn deliver_bash_completion(&self, info: BashCompletionInfo) {
1147        let guard = session_resume_lock(&info.session_id);
1148        let _held = guard.lock().await;
1149
1150        let Some(session) = self.load_session(&info.session_id).await else {
1151            tracing::warn!(
1152                session_id = %info.session_id,
1153                bash_id = %info.bash_id,
1154                "background bash completion: owning session not found; nothing to notify"
1155            );
1156            return;
1157        };
1158
1159        let waiting = read_runtime_state(&session).waiting_for_bash.is_some();
1160        // The producer flips the shell's `running` flag false BEFORE firing this
1161        // push, so a now-empty per-session registry means every shell the loop was
1162        // waiting on has finished — safe to resume. If OTHER waited shells are
1163        // still running, fall through to the enqueue path and let the last one (or
1164        // the backstop) drive the resume.
1165        let all_shells_done =
1166            bamboo_tools::tools::bash_runtime::running_shells_for_session(&info.session_id)
1167                .is_empty();
1168
1169        if bash_completion_should_resume(waiting, all_shells_done) {
1170            tracing::info!(
1171                session_id = %info.session_id,
1172                bash_id = %info.bash_id,
1173                status = %info.status,
1174                "background bash completion: push-resuming suspended loop (event-driven)"
1175            );
1176            self.perform_bash_resume(&info.session_id, bash_resume_message_from_info(&info))
1177                .await;
1178            return;
1179        }
1180
1181        match enqueue_bash_completion_injection(&self.persistence, &info).await {
1182            Ok(Some(_)) => tracing::info!(
1183                session_id = %info.session_id,
1184                bash_id = %info.bash_id,
1185                status = %info.status,
1186                waiting,
1187                "background bash completion queued for injection at the next round boundary"
1188            ),
1189            Ok(None) => tracing::warn!(
1190                session_id = %info.session_id,
1191                bash_id = %info.bash_id,
1192                "background bash completion: owning session not found; nothing to notify"
1193            ),
1194            Err(error) => tracing::warn!(
1195                session_id = %info.session_id,
1196                bash_id = %info.bash_id,
1197                %error,
1198                "background bash completion: failed to queue injection"
1199            ),
1200        }
1201    }
1202}
1203
1204impl BashCompletionSink for ChildCompletionCoordinator {
1205    fn on_bash_completed(&self, info: BashCompletionInfo) {
1206        // Best-effort, off the shell's completion-poll task: hand the delivery to
1207        // a detached task so the producer is never blocked (mirrors
1208        // `arrange_bash_self_resume`).
1209        let coordinator = Arc::new(self.clone());
1210        tokio::spawn(async move {
1211            coordinator.deliver_bash_completion(info).await;
1212        });
1213    }
1214}
1215
1216#[cfg(test)]
1217mod tests {
1218    use super::*;
1219    use bamboo_agent_core::Message;
1220
1221    fn make_completion(status: &str) -> ChildCompletion {
1222        ChildCompletion {
1223            parent_session_id: "parent-1".to_string(),
1224            child_session_id: "child-1".to_string(),
1225            status: status.to_string(),
1226            error: None,
1227            completed_at: Utc::now(),
1228        }
1229    }
1230
1231    // ── ② derive completed children from the index ──────────────────────
1232
1233    struct StubChildIndex {
1234        children: Vec<(String, Option<String>)>,
1235    }
1236
1237    #[async_trait]
1238    impl Storage for StubChildIndex {
1239        async fn save_session(&self, _session: &Session) -> std::io::Result<()> {
1240            Ok(())
1241        }
1242        async fn load_session(&self, _id: &str) -> std::io::Result<Option<Session>> {
1243            Ok(None)
1244        }
1245        async fn delete_session(&self, _id: &str) -> std::io::Result<bool> {
1246            Ok(false)
1247        }
1248        async fn list_child_run_statuses(
1249            &self,
1250            _parent_session_id: &str,
1251        ) -> std::io::Result<Vec<(String, Option<String>)>> {
1252            Ok(self.children.clone())
1253        }
1254    }
1255
1256    #[tokio::test]
1257    async fn derive_completed_only_includes_terminal_children() {
1258        let storage: Arc<dyn Storage> = Arc::new(StubChildIndex {
1259            children: vec![
1260                ("a".into(), Some("completed".into())),
1261                ("b".into(), Some("running".into())),
1262                ("c".into(), Some("error".into())),
1263                ("d".into(), None),
1264            ],
1265        });
1266        let completed = derive_completed_child_ids(&storage, "parent-1", "b").await;
1267        // Terminal from index: a, c. Plus the just-completed child b folded in.
1268        assert_eq!(
1269            completed,
1270            vec!["a".to_string(), "b".to_string(), "c".to_string()]
1271        );
1272    }
1273
1274    #[tokio::test]
1275    async fn derive_completed_folds_in_just_completed_when_index_lags() {
1276        // Index hasn't caught up — reports the child as still running.
1277        let storage: Arc<dyn Storage> = Arc::new(StubChildIndex {
1278            children: vec![("only".into(), Some("running".into()))],
1279        });
1280        let completed = derive_completed_child_ids(&storage, "parent-1", "only").await;
1281        assert_eq!(completed, vec!["only".to_string()]);
1282    }
1283
1284    #[test]
1285    fn wait_policy_all_uses_derived_completed_set() {
1286        let waited = vec!["a".to_string(), "b".to_string()];
1287        assert!(!wait_policy_satisfied(
1288            ChildWaitPolicy::All,
1289            &waited,
1290            &["a".to_string()],
1291            "completed"
1292        ));
1293        assert!(wait_policy_satisfied(
1294            ChildWaitPolicy::All,
1295            &waited,
1296            &["a".to_string(), "b".to_string()],
1297            "completed"
1298        ));
1299    }
1300
1301    #[test]
1302    fn child_final_assistant_text_returns_last_assistant() {
1303        let mut session = Session::new("child-1", "gpt-4");
1304        session.messages.push(Message::user("hi"));
1305        session
1306            .messages
1307            .push(Message::assistant("first answer", None));
1308        session.messages.push(Message::user("again"));
1309        session
1310            .messages
1311            .push(Message::assistant("final answer", None));
1312
1313        assert_eq!(
1314            child_final_assistant_text(&session).as_deref(),
1315            Some("final answer")
1316        );
1317    }
1318
1319    #[test]
1320    fn child_final_assistant_text_returns_none_when_blank() {
1321        let mut session = Session::new("child-1", "gpt-4");
1322        session.messages.push(Message::assistant("   ", None));
1323        assert!(child_final_assistant_text(&session).is_none());
1324    }
1325
1326    #[test]
1327    fn child_final_assistant_text_returns_none_when_no_assistant() {
1328        let mut session = Session::new("child-1", "gpt-4");
1329        session.messages.push(Message::user("hi"));
1330        assert!(child_final_assistant_text(&session).is_none());
1331    }
1332
1333    #[test]
1334    fn runtime_resume_message_folds_full_response_without_truncation() {
1335        // A very long child final response is folded in verbatim (no 4000-char
1336        // cap, no truncation marker).
1337        let completion = make_completion("completed");
1338        let long: String = "a".repeat(10_000);
1339        let message = runtime_resume_message(&completion, 0, Some(&long));
1340        assert!(message.content.contains(&long));
1341        assert!(!message.content.contains("truncated"));
1342    }
1343
1344    #[test]
1345    fn runtime_resume_message_includes_child_response_when_provided() {
1346        let completion = make_completion("completed");
1347        let message = runtime_resume_message(&completion, 0, Some("the answer is 42"));
1348
1349        assert!(matches!(message.role, Role::User));
1350        // Folded child results are now compressible so the parent context can
1351        // reclaim them under compaction.
1352        assert!(!message.never_compress);
1353        assert!(message.content.contains("Child final response:"));
1354        assert!(message.content.contains("the answer is 42"));
1355
1356        let metadata = message.metadata.expect("metadata present");
1357        assert_eq!(
1358            metadata.get("hidden_from_ui").and_then(|v| v.as_bool()),
1359            Some(true)
1360        );
1361        assert_eq!(
1362            metadata.get("runtime_kind").and_then(|v| v.as_str()),
1363            Some("child_completion_resume")
1364        );
1365        assert_eq!(
1366            metadata
1367                .get("child_final_response_included")
1368                .and_then(|v| v.as_bool()),
1369            Some(true)
1370        );
1371    }
1372
1373    #[test]
1374    fn runtime_resume_message_falls_back_to_error_when_no_response() {
1375        let mut completion = make_completion("error");
1376        completion.error = Some("boom".to_string());
1377
1378        let message = runtime_resume_message(&completion, 1, None);
1379        assert!(message.content.contains("Child error:"));
1380        assert!(message.content.contains("boom"));
1381        let metadata = message.metadata.expect("metadata present");
1382        assert_eq!(
1383            metadata
1384                .get("child_final_response_included")
1385                .and_then(|v| v.as_bool()),
1386            Some(false)
1387        );
1388    }
1389
1390    #[test]
1391    fn runtime_resume_message_minimal_when_no_response_and_no_error() {
1392        let completion = make_completion("completed");
1393        let message = runtime_resume_message(&completion, 2, None);
1394        assert!(!message.content.contains("Child final response:"));
1395        assert!(!message.content.contains("Child error:"));
1396        assert!(message.content.contains("Resume the parent task"));
1397    }
1398
1399    #[test]
1400    fn read_config_snapshot_refreshes_cached_snapshot_from_live_config() {
1401        let runtime = tokio::runtime::Runtime::new().expect("runtime");
1402
1403        runtime.block_on(async {
1404            let config = Arc::new(RwLock::new(Config::default()));
1405            config.write().await.provider = "copilot".to_string();
1406            let cached_config = StdRwLock::new(Config::default());
1407
1408            let snapshot = read_config_snapshot(&config, &cached_config);
1409
1410            assert_eq!(snapshot.provider, "copilot");
1411            assert_eq!(
1412                cached_config.read().expect("cached snapshot lock").provider,
1413                "copilot"
1414            );
1415        });
1416    }
1417
1418    #[test]
1419    fn read_config_snapshot_uses_cached_snapshot_when_live_lock_is_busy() {
1420        let runtime = tokio::runtime::Runtime::new().expect("runtime");
1421
1422        runtime.block_on(async {
1423            let cached_snapshot = Config {
1424                provider: "cached-provider".to_string(),
1425                ..Default::default()
1426            };
1427
1428            let config = Arc::new(RwLock::new(Config::default()));
1429            let cached_config = StdRwLock::new(cached_snapshot);
1430            let _write_guard = config.write().await;
1431
1432            let snapshot = read_config_snapshot(&config, &cached_config);
1433
1434            assert_eq!(snapshot.provider, "cached-provider");
1435        });
1436    }
1437
1438    // ── Bash self-resume (issue #84 Phase 2b): deadline message + clobber-retry ──
1439
1440    #[test]
1441    fn bash_completion_resume_message_normal_announces_completion() {
1442        let ids = vec!["bg-1".to_string(), "bg-2".to_string()];
1443        let message = bash_completion_resume_message(&ids, false);
1444        // Normal path: the shells genuinely finished.
1445        assert!(
1446            message.content.contains("have completed"),
1447            "normal resume message must announce completion: {}",
1448            message.content
1449        );
1450        // Hidden + compressible so the resume gate sees it but the UI hides it.
1451        let metadata = message.metadata.expect("metadata present");
1452        assert_eq!(
1453            metadata
1454                .get(RUNTIME_RESUME_MESSAGE_HIDDEN_KEY)
1455                .and_then(|v| v.as_bool()),
1456            Some(true),
1457            "resume message must be hidden from the UI"
1458        );
1459        assert_eq!(
1460            metadata
1461                .get(RUNTIME_RESUME_MESSAGE_KIND_KEY)
1462                .and_then(|v| v.as_str()),
1463            Some(BASH_COMPLETION_RESUME_KIND),
1464            "resume message must carry the bash-completion kind discriminant"
1465        );
1466    }
1467
1468    #[test]
1469    fn bash_completion_resume_message_deadline_does_not_claim_completion() {
1470        // The 6h+10m deadline force-breaks with shells STILL running. The message
1471        // must NOT say "have completed" — that would let the model assume success
1472        // on a false premise. It must direct the model to verify with BashOutput.
1473        let ids = vec!["bg-long".to_string()];
1474        let message = bash_completion_resume_message(&ids, true);
1475        assert!(
1476            !message.content.contains("have completed"),
1477            "deadline resume message must NOT claim the shells completed: {}",
1478            message.content
1479        );
1480        assert!(
1481            message.content.contains("may still be running"),
1482            "deadline resume message must warn shells may still be running: {}",
1483            message.content
1484        );
1485        assert!(
1486            message.content.contains("BashOutput"),
1487            "deadline resume message must direct verification via BashOutput: {}",
1488            message.content
1489        );
1490        // Same hidden/kind shape so the resume gate is satisfied identically.
1491        let metadata = message.metadata.expect("metadata present");
1492        assert_eq!(
1493            metadata
1494                .get(RUNTIME_RESUME_MESSAGE_KIND_KEY)
1495                .and_then(|v| v.as_str()),
1496            Some(BASH_COMPLETION_RESUME_KIND)
1497        );
1498    }
1499
1500    #[test]
1501    fn bash_resume_should_retry_matrix() {
1502        // The finalize-clobber retry predicate (issue #84 Phase 2b). Retry only
1503        // when the resume did NOT spawn (Completed / AlreadyRunning) AND the
1504        // persisted bash wait is still set on reload — the clobber signature.
1505
1506        // Started: the resume fired — never retry, regardless of persisted state.
1507        assert!(!bash_resume_should_retry(
1508            &ResumeOutcome::Started { run_id: "r".into() },
1509            true
1510        ));
1511        assert!(!bash_resume_should_retry(
1512            &ResumeOutcome::Started { run_id: "r".into() },
1513            false
1514        ));
1515
1516        // NotFound: session gone — never retry.
1517        assert!(!bash_resume_should_retry(&ResumeOutcome::NotFound, true));
1518        assert!(!bash_resume_should_retry(&ResumeOutcome::NotFound, false));
1519
1520        // Completed + persisted wait still set ⇒ finalize-clobber ⇒ retry.
1521        assert!(bash_resume_should_retry(&ResumeOutcome::Completed, true));
1522        // Completed + persisted wait cleared ⇒ handled (our message stuck, or a
1523        // concurrent resume finished) ⇒ stop.
1524        assert!(!bash_resume_should_retry(&ResumeOutcome::Completed, false));
1525
1526        // AlreadyRunning + persisted wait still set ⇒ clobbered while a runner is
1527        // (stale-)active ⇒ retry to re-establish the resume message.
1528        assert!(bash_resume_should_retry(
1529            &ResumeOutcome::AlreadyRunning { run_id: "r".into() },
1530            true
1531        ));
1532        // AlreadyRunning + wait cleared ⇒ a runner owns the session ⇒ stop.
1533        assert!(!bash_resume_should_retry(
1534            &ResumeOutcome::AlreadyRunning { run_id: "r".into() },
1535            false
1536        ));
1537    }
1538
1539    // ── bash completion injection body (Phase 2b follow-up) ──────────────
1540
1541    #[test]
1542    fn injection_body_includes_status_exit_command_and_tail() {
1543        let info = BashCompletionInfo {
1544            session_id: "s".into(),
1545            bash_id: "abc123".into(),
1546            command: "make build".into(),
1547            exit_code: Some(0),
1548            status: "completed".into(),
1549            output_tail: "BUILD OK".into(),
1550        };
1551        let body = bash_completion_injection_body(&info);
1552        assert!(body.contains("abc123"), "body: {body}");
1553        assert!(body.contains("make build"), "body: {body}");
1554        assert!(body.contains("completed"), "body: {body}");
1555        assert!(body.contains("exit code 0"), "body: {body}");
1556        assert!(body.contains("BUILD OK"), "body: {body}");
1557        // The model is pointed at BashOutput for the full log.
1558        assert!(body.contains("BashOutput"), "body: {body}");
1559        assert!(body.contains("bash_id=\"abc123\""), "body: {body}");
1560    }
1561
1562    #[test]
1563    fn injection_body_handles_no_output_and_signal_kill() {
1564        let info = BashCompletionInfo {
1565            session_id: "s".into(),
1566            bash_id: "xyz".into(),
1567            command: "sleep 99".into(),
1568            exit_code: None,
1569            status: "killed".into(),
1570            output_tail: String::new(),
1571        };
1572        let body = bash_completion_injection_body(&info);
1573        assert!(body.contains("killed"), "body: {body}");
1574        assert!(body.contains("none (signal/killed)"), "body: {body}");
1575        assert!(body.contains("no captured output"), "body: {body}");
1576        // No output tail section when there is nothing to show.
1577        assert!(!body.contains("Output tail:"), "body: {body}");
1578    }
1579
1580    async fn temp_store() -> (tempfile::TempDir, Arc<dyn Storage>, LockedSessionStore) {
1581        let temp = tempfile::tempdir().unwrap();
1582        let storage: Arc<dyn Storage> = Arc::new(
1583            bamboo_storage::v2::SessionStoreV2::new(temp.path().to_path_buf())
1584                .await
1585                .expect("storage init"),
1586        );
1587        let persistence = LockedSessionStore::new(storage.clone());
1588        (temp, storage, persistence)
1589    }
1590
1591    #[tokio::test]
1592    async fn enqueue_writes_pending_injection_and_preserves_messages() {
1593        let (_temp, storage, persistence) = temp_store().await;
1594
1595        let mut session = Session::new("sess-enq", "test-model");
1596        session.add_message(Message::user("do the build"));
1597        storage.save_session(&session).await.unwrap();
1598
1599        let info = BashCompletionInfo {
1600            session_id: "sess-enq".into(),
1601            bash_id: "sh-1".into(),
1602            command: "make".into(),
1603            exit_code: Some(0),
1604            status: "completed".into(),
1605            output_tail: "done".into(),
1606        };
1607        let saved = enqueue_bash_completion_injection(&persistence, &info)
1608            .await
1609            .expect("enqueue io ok")
1610            .expect("session exists");
1611
1612        let pending = saved
1613            .pending_injected_messages()
1614            .expect("pending injection present");
1615        assert_eq!(pending.len(), 1);
1616        let content = pending[0].get("content").and_then(|v| v.as_str()).unwrap();
1617        assert!(content.contains("sh-1"), "content: {content}");
1618        assert!(content.contains("make"), "content: {content}");
1619        assert!(content.contains("done"), "content: {content}");
1620        // The pre-existing conversation is untouched (no clobber).
1621        assert_eq!(saved.messages.len(), 1);
1622    }
1623
1624    #[tokio::test]
1625    async fn enqueue_returns_none_for_missing_session() {
1626        let (_temp, _storage, persistence) = temp_store().await;
1627        let info = BashCompletionInfo {
1628            session_id: "does-not-exist".into(),
1629            bash_id: "x".into(),
1630            command: "true".into(),
1631            exit_code: Some(0),
1632            status: "completed".into(),
1633            output_tail: String::new(),
1634        };
1635        let result = enqueue_bash_completion_injection(&persistence, &info)
1636            .await
1637            .expect("io ok");
1638        assert!(result.is_none(), "no session → nothing enqueued");
1639    }
1640
1641    // ── push-driven resume: the state transition + decision the push applies ──
1642
1643    /// A session suspended on `waiting_for_bash`, given the rich completion
1644    /// message, is transitioned to a resumable state: the wait is cleared, the
1645    /// runtime is Idle, the suspend-reason marker is gone, and the resume message
1646    /// is appended. This is exactly what the PUSH does to wake the loop
1647    /// event-driven (vs the old backstop poll).
1648    #[test]
1649    fn apply_bash_resume_transition_clears_wait_and_appends_message() {
1650        use bamboo_domain::session::runtime_state::WaitingForBashState;
1651
1652        let mut session = Session::new("sess-resume", "test-model");
1653        session.add_message(Message::user("kick off the build"));
1654        let mut rt = read_runtime_state(&session);
1655        rt.status = AgentStatusState::Running;
1656        rt.waiting_for_bash = Some(WaitingForBashState::for_bash(vec!["sh-1".into()], Utc::now()));
1657        write_runtime_state(&mut session, &rt);
1658        session.metadata.insert(
1659            "runtime.suspend_reason".to_string(),
1660            "waiting_for_bash".to_string(),
1661        );
1662
1663        let resume = bash_completion_resume_message(&["sh-1".to_string()], false);
1664        let did = apply_bash_resume_transition(&mut session, &resume);
1665
1666        assert!(did, "a suspended session must transition");
1667        let after = read_runtime_state(&session);
1668        assert!(after.waiting_for_bash.is_none(), "bash wait must be cleared");
1669        assert_eq!(after.status, AgentStatusState::Idle, "runtime must be Idle");
1670        assert!(
1671            !session.metadata.contains_key("runtime.suspend_reason"),
1672            "suspend-reason marker must be removed"
1673        );
1674        assert_eq!(session.messages.len(), 2, "resume message must be appended");
1675        assert!(matches!(
1676            session.messages.last().map(|m| &m.role),
1677            Some(Role::User)
1678        ));
1679    }
1680
1681    /// The double-resume guard: a session NOT waiting on bash is a no-op — no
1682    /// message appended, nothing mutated. This is what makes the backstop poll
1683    /// harmlessly yield once the push has already resumed (and vice versa).
1684    #[test]
1685    fn apply_bash_resume_transition_noops_when_not_waiting() {
1686        let mut session = Session::new("sess-live", "test-model");
1687        session.add_message(Message::user("hi"));
1688
1689        let resume = bash_completion_resume_message(&["sh-1".to_string()], false);
1690        let did = apply_bash_resume_transition(&mut session, &resume);
1691
1692        assert!(!did, "a non-waiting session must not transition");
1693        assert_eq!(session.messages.len(), 1, "no resume message appended");
1694    }
1695
1696    /// The resume invariant: push-resume fires ONLY when the loop is suspended on
1697    /// bash AND every waited shell has finished. A still-running sibling shell
1698    /// keeps it on the enqueue path.
1699    #[test]
1700    fn bash_completion_should_resume_only_when_suspended_and_all_done() {
1701        assert!(bash_completion_should_resume(true, true));
1702        assert!(!bash_completion_should_resume(true, false)); // other shells still running
1703        assert!(!bash_completion_should_resume(false, true)); // live loop, not suspended
1704        assert!(!bash_completion_should_resume(false, false));
1705    }
1706
1707    /// The push's resume message carries the shell's identity + status + output
1708    /// tail (so the model needs no `BashOutput` round-trip) and is tagged as a
1709    /// bash-completion resume so it satisfies the `has_pending_user_message` gate.
1710    #[test]
1711    fn bash_resume_message_from_info_carries_bashid_tail_and_kind() {
1712        let info = BashCompletionInfo {
1713            session_id: "s".into(),
1714            bash_id: "sh-42".into(),
1715            command: "cargo test".into(),
1716            exit_code: Some(0),
1717            status: "completed".into(),
1718            output_tail: "test result: ok".into(),
1719        };
1720        let msg = bash_resume_message_from_info(&info);
1721
1722        assert!(matches!(msg.role, Role::User));
1723        assert!(msg.content.contains("sh-42"), "content: {}", msg.content);
1724        assert!(msg.content.contains("cargo test"), "content: {}", msg.content);
1725        assert!(
1726            msg.content.contains("test result: ok"),
1727            "content: {}",
1728            msg.content
1729        );
1730        assert!(msg.content.contains("BashOutput"), "content: {}", msg.content);
1731        let meta = serde_json::to_string(&msg.metadata).unwrap();
1732        assert!(
1733            meta.contains(BASH_COMPLETION_RESUME_KIND),
1734            "resume message must be tagged as a bash-completion resume: {meta}"
1735        );
1736    }
1737}